Skip to content

Commit b659712

Browse files
committed
[Espressif] Fuse linear ReLU and HardTanh patterns
Quantize single-consumer linear followed by ReLU or HardTanh, including ReLU6, at the pattern boundaries. Fold the final output qparams and clamp bounds into the existing ESP-NN fully connected call, preserving per-channel weights and derived int32 bias. Forked and separately quantized activations retain their original quantization boundaries. Read the quantizer and activation fold before the linear lowering and schema/wrapper updates. Shared dialect and S3 cases cover bias/no bias, forks, saturation, vector and higher-rank inputs, and narrower clamp grids with asymmetric, positive-only and negative-only HardTanh bounds. Authored with OpenAI Codex.
1 parent ebcd154 commit b659712

9 files changed

Lines changed: 200 additions & 31 deletions

File tree

‎backends/espressif/README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,11 @@ Linear supports static, contiguous int8 inputs of rank one or higher, constant
5353
per-output-channel symmetric int8 weights, and optional constant int32 bias.
5454
`get_esp_fused_quant_quantizer` derives the bias scales from input and weight scales.
5555
Leading input dimensions are treated as batch rows by ESP-NN's batched fully
56-
connected API. Unsupported linear calls fail at export with the reason;
56+
connected API. Single-consumer linear followed by ReLU or HardTanh (including
57+
ReLU6) is quantized as one pattern. Its final output qparams and clamp bounds
58+
are folded into the linear kernel, preserving the derived int32 bias. Forked
59+
or separately quantized activations retain their boundaries.
60+
Unsupported linear calls fail at export with the reason;
5761
portable linear fallback is not available yet, even when CPU fallback is enabled.
5862
Conv2d supports static 4-D int8 inputs, constant per-output-channel symmetric
5963
weights, optional derived int32 bias, and unit dilation. Ordinary convolution

‎backends/espressif/ops/op_quantized_linear.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Tensor& quantized_linear_int8_out(
3030
const Tensor& output_shift,
3131
int64_t input_offset,
3232
int64_t output_offset,
33+
int64_t activation_min,
34+
int64_t activation_max,
3335
Tensor& out) {
3436
ET_KERNEL_CHECK(ctx, weight.dim() >= 2, InvalidArgument, out);
3537
const auto channels = weight.size(0);
@@ -71,8 +73,8 @@ Tensor& quantized_linear_int8_out(
7173
static_cast<int32_t>(output_offset),
7274
output_shift.const_data_ptr<int32_t>(),
7375
output_multiplier.const_data_ptr<int32_t>(),
74-
std::numeric_limits<int8_t>::min(),
75-
std::numeric_limits<int8_t>::max(),
76+
static_cast<int32_t>(activation_min),
77+
static_cast<int32_t>(activation_max),
7678
static_cast<int32_t>(batches));
7779
return out;
7880
}

‎backends/espressif/ops/operators.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,13 @@
6868
)
6969
lib.define(
7070
"quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, "
71-
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset) -> Tensor"
71+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
72+
"int activation_min, int activation_max) -> Tensor"
7273
)
7374
lib.define(
7475
"quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, "
7576
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
76-
"*, Tensor(a!) out) -> Tensor(a!)"
77+
"int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)"
7778
)
7879

7980
lib.define(
@@ -247,14 +248,30 @@ def quantized_add_int8(
247248

248249
@register_fake("esp_nn::quantized_linear_int8")
249250
def quantized_linear_int8_meta(
250-
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
251+
input,
252+
weight,
253+
bias,
254+
output_multiplier,
255+
output_shift,
256+
input_offset,
257+
output_offset,
258+
activation_min,
259+
activation_max,
251260
):
252261
return input.new_empty((*input.shape[:-1], weight.shape[0]))
253262

254263

255264
@impl(lib, "quantized_linear_int8", "CompositeExplicitAutograd")
256265
def quantized_linear_int8(
257-
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
266+
input,
267+
weight,
268+
bias,
269+
output_multiplier,
270+
output_shift,
271+
input_offset,
272+
output_offset,
273+
activation_min,
274+
activation_max,
258275
):
259276
accumulator = (input.to(torch.int64) + input_offset) @ weight.to(torch.int64).T
260277
if bias is not None:
@@ -268,7 +285,7 @@ def quantized_linear_int8(
268285
],
269286
dim=-1,
270287
)
271-
return (result + output_offset).clamp(-128, 127).to(torch.int8)
288+
return (result + output_offset).clamp(activation_min, activation_max).to(torch.int8)
272289

273290

274291
@register_fake("esp_nn::quantized_conv2d_int8")
@@ -325,6 +342,8 @@ def quantized_conv2d_int8(
325342
output_shift,
326343
0,
327344
output_offset,
345+
-128,
346+
127,
328347
)
329348

330349

‎backends/espressif/ops/operators.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
- arg_meta: null
4141
kernel_name: espressif::quantized_avg_pool2d_int8_out
4242

43-
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, *, Tensor(a!) out) -> Tensor(a!)
43+
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
4444
variants: function
4545
kernels:
4646
- arg_meta: null

‎backends/espressif/passes/fold_activations.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717

1818
class FoldActivations(ExportedProgramPassBase):
19-
"""Fold grouped binary/clamp patterns into the producer's output quantization."""
19+
"""Fold grouped clamps into the producer's output quantization."""
2020

2121
def call(self, exported_program):
2222
graph = exported_program.graph
@@ -43,6 +43,7 @@ def call(self, exported_program):
4343
exir_ops.edge.fused_quant.add.default,
4444
exir_ops.edge.fused_quant.sub.default,
4545
exir_ops.edge.fused_quant.mul.default,
46+
exir_ops.edge.fused_quant.linear.default,
4647
)
4748
or len(producer.users) != 1
4849
or get_qparams_from_node(producer, "out") is not None

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,8 @@ def _linear_kernel_args(program, node, constants):
404404
"input and output must be nonempty, static, contiguous int8 tensors"
405405
)
406406
input_params, output_params = (
407-
_qparams(program, node, name) for name in ("inp", "out")
407+
_qparams(program, node, name, allow_output_clamp=True)
408+
for name in ("inp", "out")
408409
)
409410
if input_params is None or output_params is None:
410411
raise ValueError("input and output require constant per-tensor int8 qparams")
@@ -436,6 +437,8 @@ def _linear_kernel_args(program, node, constants):
436437
shift_node,
437438
-input_params.zp,
438439
output_params.zp,
440+
get_arg(node, "out_quant_min"),
441+
get_arg(node, "out_quant_max"),
439442
)
440443

441444

‎backends/espressif/quantizer/esp_fused_quantizer.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,8 @@ class _EspFusedQuantizer(FusedQuantQuantizer):
106106
Conv2d, and depthwise Conv2d biases use
107107
int32 with scales derived from the input and weight observers.
108108
AvgPool2d without divisor_override shares input and output qparams.
109-
Single-consumer binary/ReLU or HardTanh (including ReLU6) patterns observe
110-
only their boundaries.
109+
Single-consumer binary or linear followed by ReLU or HardTanh (including
110+
ReLU6) patterns observe only their boundaries.
111111
EspPassManager lowers supported Fused Quant operators to ESP-NN and
112112
decomposes the rest for portable execution. Use EspNNQuantizer for the
113113
EspNNBackend delegate instead.
@@ -162,16 +162,19 @@ def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule:
162162
torch.ops.aten.add.Tensor,
163163
torch.ops.aten.sub.Tensor,
164164
torch.ops.aten.mul.Tensor,
165+
torch.ops.aten.linear.default,
165166
)
166167
or len(producer.users) != 1
167-
or not isinstance(get_arg(producer, "other"), torch.fx.Node)
168168
or Q_ANNOTATION_KEY in producer.meta
169169
or Q_ANNOTATION_KEY in activation.meta
170170
):
171171
continue
172-
if (
173-
producer.target != torch.ops.aten.mul.Tensor
174-
and get_arg(producer, "alpha") != 1
172+
if producer.target != torch.ops.aten.linear.default and (
173+
not isinstance(get_arg(producer, "other"), torch.fx.Node)
174+
or (
175+
producer.target != torch.ops.aten.mul.Tensor
176+
and get_arg(producer, "alpha") != 1
177+
)
175178
):
176179
continue
177180
groups.append((producer, activation))

0 commit comments

Comments
 (0)