Skip to content

Commit 4c8e856

Browse files
committed
[Espressif] Lower Fused Quant linear to ESP-NN
Lower statically quantized linear calls to ESP-NN's batched per-channel fully connected API. Contiguous vector and higher-rank inputs use the same row-based kernel without layout copies. Reuse the quantizer's int32 bias directly and prepare constant per-channel multipliers with bounds on integer accumulation and floating reference overflow. Unsupported linear calls now report an export error with the failing requirement, because shared FQ decomposition restores aten.linear without a portable implementation. Other operators retain their existing fallback behavior. This implements the preceding linear quantizer contract; the two commits are intended to land together. The shared dialect and ESP32-S3 implementation cases cover bias and no bias, higher ranks, aligned and tail widths, shared weights and biases, float users, ReLU, chained linear, large bias, and saturation. Normalize returned Parameters to plain tensors when bundling expected test outputs. Validation: 24 linear export/lowering tests and 110 existing operator/pass-manager regressions passed. All 16 shared linear cases passed on the ESP32-S3 emulator with ESP-NN 1.5.0. Lintrunner passed. Authored with OpenAI Codex.
1 parent 05eff02 commit 4c8e856

9 files changed

Lines changed: 569 additions & 1 deletion

File tree

‎backends/espressif/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ idf_component_register(
1010
"ops/op_quantized_add.cpp"
1111
"ops/op_quantized_max_pool2d.cpp"
1212
"ops/op_quantized_avg_pool2d.cpp"
13+
"ops/op_quantized_linear.cpp"
1314
PRIV_REQUIRES
1415
esp-nn
1516
WHOLE_ARCHIVE

‎backends/espressif/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ input/output qparams, floor mode, and no divisor override. Padding is supported
3333
when `count_include_pad=False`.
3434
Its integer kernel rounds ties away from zero; the FQ reference rounds ties to
3535
even, so results can differ by one int8 step.
36+
Linear supports static, contiguous int8 inputs of rank one or higher, constant
37+
per-output-channel symmetric int8 weights, and optional constant int32 bias.
38+
`EspFusedQuantizer` derives the bias scales from input and weight scales.
39+
Leading input dimensions are treated as batch rows by ESP-NN's batched fully
40+
connected API. Unsupported linear calls fail at export with the reason;
41+
portable linear fallback is not available yet, even when CPU fallback is enabled.
3642
The shared Fused Quant layout pass introduces contiguous NHWC tensors for
3743
spatial operators; model inputs and outputs retain their original layout.
3844
Required layout copies are visible in the operator-library report and accepted

‎backends/espressif/lowering.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ def get_esp_fused_quant_recipe(
126126
Use with executorch.export.export(). Supported Fused Quant operators become
127127
direct ESP-NN calls; other operators decompose to portable kernels. Setting
128128
allow_cpu_fallback=False rejects remaining compute, excluding Q/DQ and required NHWC layout copies.
129+
Unsupported linear calls currently raise even when CPU fallback is enabled.
129130
The EspNNBackend delegate continues to use EspNNQuantizer and to_espnn_edge.
130131
"""
131132
quantizer = quantizer if quantizer is not None else EspFusedQuantizer()
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/runtime/kernel/kernel_includes.h>
10+
11+
#include <sdkconfig.h>
12+
13+
#include <esp_nn.h>
14+
15+
#include <cstdint>
16+
#include <limits>
17+
18+
namespace espressif::native {
19+
20+
using executorch::aten::Tensor;
21+
using executorch::runtime::KernelRuntimeContext;
22+
23+
// cppcheck-suppress unusedFunction
24+
Tensor& quantized_linear_int8_out(
25+
KernelRuntimeContext& ctx,
26+
const Tensor& input,
27+
const Tensor& weight,
28+
const std::optional<Tensor>& bias,
29+
const Tensor& output_multiplier,
30+
const Tensor& output_shift,
31+
int64_t input_offset,
32+
int64_t output_offset,
33+
Tensor& out) {
34+
ET_KERNEL_CHECK(ctx, weight.dim() >= 2, InvalidArgument, out);
35+
const auto channels = weight.size(0);
36+
const auto columns = weight.size(1);
37+
ET_KERNEL_CHECK(
38+
ctx,
39+
channels > 0 && channels <= std::numeric_limits<uint16_t>::max() &&
40+
columns > 0 && columns <= std::numeric_limits<uint16_t>::max(),
41+
InvalidArgument,
42+
out);
43+
const auto batches = input.numel() / columns;
44+
ET_KERNEL_CHECK(
45+
ctx,
46+
batches >= 0 && batches <= std::numeric_limits<int32_t>::max(),
47+
InvalidArgument,
48+
out);
49+
const auto rows = static_cast<size_t>(batches);
50+
const auto cols = static_cast<size_t>(columns);
51+
const auto outputs = static_cast<size_t>(channels);
52+
ET_KERNEL_CHECK(
53+
ctx,
54+
rows <= input.nbytes() / cols && rows <= out.nbytes() / outputs &&
55+
outputs <= weight.nbytes() / cols &&
56+
outputs <= output_multiplier.nbytes() / sizeof(int32_t) &&
57+
outputs <= output_shift.nbytes() / sizeof(int32_t) &&
58+
(!bias || outputs <= bias->nbytes() / sizeof(int32_t)),
59+
InvalidArgument,
60+
out);
61+
62+
esp_nn_fully_connected_per_ch_s8_batch(
63+
input.const_data_ptr<int8_t>(),
64+
static_cast<int32_t>(input_offset),
65+
static_cast<uint16_t>(columns),
66+
weight.const_data_ptr<int8_t>(),
67+
0,
68+
bias ? bias->const_data_ptr<int32_t>() : nullptr,
69+
out.mutable_data_ptr<int8_t>(),
70+
static_cast<uint16_t>(channels),
71+
static_cast<int32_t>(output_offset),
72+
output_shift.const_data_ptr<int32_t>(),
73+
output_multiplier.const_data_ptr<int32_t>(),
74+
std::numeric_limits<int8_t>::min(),
75+
std::numeric_limits<int8_t>::max(),
76+
static_cast<int32_t>(batches));
77+
return out;
78+
}
79+
80+
} // namespace espressif::native

‎backends/espressif/ops/operators.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,15 @@
4848
"quantized_avg_pool2d_int8.out(Tensor input, int[2] kernel_size, int[2] stride, "
4949
"int[2] padding, *, Tensor(a!) out) -> Tensor(a!)"
5050
)
51+
lib.define(
52+
"quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, "
53+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset) -> Tensor"
54+
)
55+
lib.define(
56+
"quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, "
57+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
58+
"*, Tensor(a!) out) -> Tensor(a!)"
59+
)
5160

5261

5362
@register_fake("esp_nn::quantized_max_pool2d_int8")
@@ -159,3 +168,29 @@ def quantized_add_int8(
159168
)
160169
result = requantize(total, output_multiplier, output_shift) + output_offset
161170
return result.clamp(-128, 127).to(torch.int8)
171+
172+
173+
@register_fake("esp_nn::quantized_linear_int8")
174+
def quantized_linear_int8_meta(
175+
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
176+
):
177+
return input.new_empty((*input.shape[:-1], weight.shape[0]))
178+
179+
180+
@impl(lib, "quantized_linear_int8", "CompositeExplicitAutograd")
181+
def quantized_linear_int8(
182+
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
183+
):
184+
accumulator = (input.to(torch.int64) + input_offset) @ weight.to(torch.int64).T
185+
if bias is not None:
186+
accumulator = accumulator + bias
187+
result = torch.stack(
188+
[
189+
requantize(accumulator[..., channel], int(multiplier), int(shift))
190+
for channel, (multiplier, shift) in enumerate(
191+
zip(output_multiplier, output_shift)
192+
)
193+
],
194+
dim=-1,
195+
)
196+
return (result + output_offset).clamp(-128, 127).to(torch.int8)

‎backends/espressif/ops/operators.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,9 @@
2727
kernels:
2828
- arg_meta: null
2929
kernel_name: espressif::quantized_avg_pool2d_int8_out
30+
31+
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, *, Tensor(a!) out) -> Tensor(a!)
32+
variants: function
33+
kernels:
34+
- arg_meta: null
35+
kernel_name: espressif::quantized_linear_int8_out

‎backends/espressif/passes/fused_quant_to_esp_pass.py‎

Lines changed: 176 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@
1515
from executorch.backends.espressif.utils.quantization import quantize_multiplier
1616
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
1717
from executorch.backends.transforms.permute_pass_utils import get_arg
18-
from executorch.backends.transforms.utils import get_constant
18+
from executorch.backends.transforms.utils import add_constant, get_constant
1919
from executorch.exir.dialects._ops import ops as exir_ops
2020
from executorch.exir.pass_base import ExportedProgramPassBase, ExportedProgramPassResult
21+
from executorch.exir.passes.constant_prop_pass import get_propagated_const_tensor_dict
22+
from torch.export.graph_signature import InputKind
2123

2224

2325
@dataclass(frozen=True)
@@ -265,15 +267,188 @@ def _lower_avg_pool2d(program, node) -> bool:
265267
return True
266268

267269

270+
def _linear_multipliers(input_params, output_params, weight, scales, bias):
271+
rescales = [
272+
quantize_multiplier(input_params.scale * scale / output_params.scale)
273+
for scale in scales.flatten().tolist()
274+
]
275+
if any(shift < -30 for _, shift in rescales):
276+
raise ValueError("requantization shift is outside the ESP-NN range")
277+
max_input = max(abs(-128 - input_params.zp), abs(127 - input_params.zp))
278+
limits = max_input * weight.to(torch.int64).abs().sum(dim=1)
279+
if bias is not None:
280+
limits += bias.to(torch.int64).abs()
281+
if any(
282+
int(limit) << max(shift, 0)
283+
> torch.iinfo(torch.int32).max - abs(output_params.zp)
284+
for limit, (_, shift) in zip(limits, rescales)
285+
):
286+
raise ValueError("accumulation or requantization may overflow int32")
287+
max_float_input = c_float(max_input * c_float(input_params.scale).value).value
288+
float_weights = weight.to(torch.float32) * scales
289+
max_float_outputs = max_float_input * float_weights.abs().to(torch.float64).sum(
290+
dim=1
291+
)
292+
if bias is not None:
293+
bias_scales = input_params.scale * scales.flatten()
294+
max_float_outputs += (
295+
(bias.to(torch.float32) * bias_scales).abs().to(torch.float64)
296+
)
297+
# Bound float32 accumulation rounding, including the bias addition.
298+
max_float_outputs /= 1 - (weight.shape[1] + 1) * torch.finfo(torch.float32).eps
299+
if (
300+
not math.isfinite(c_float(1.0 / output_params.scale).value)
301+
or not torch.all(torch.isfinite(max_float_outputs))
302+
or torch.any(max_float_outputs > torch.finfo(torch.float32).max)
303+
):
304+
raise ValueError("the float32 reference may overflow")
305+
multipliers, shifts = zip(*rescales)
306+
return torch.tensor(multipliers, dtype=torch.int32), torch.tensor(
307+
shifts, dtype=torch.int32
308+
)
309+
310+
311+
def _linear_kernel_args(program, node, constants):
312+
inp, weight_node, bias_node = (
313+
get_arg(node, name) for name in ("inp", "weight", "bias")
314+
)
315+
input_value, output_value = inp.meta.get("val"), node.meta.get("val")
316+
if not all(
317+
isinstance(value, torch.Tensor)
318+
and value.dtype == torch.int8
319+
and value.dim() >= 1
320+
and all(type(dim) is int for dim in value.shape)
321+
and 0 < value.numel() <= torch.iinfo(torch.int32).max
322+
and value.is_contiguous()
323+
for value in (input_value, output_value)
324+
):
325+
raise ValueError(
326+
"input and output must be nonempty, static, contiguous int8 tensors"
327+
)
328+
input_params, output_params = (
329+
_qparams(program, node, name) for name in ("inp", "out")
330+
)
331+
if input_params is None or output_params is None:
332+
raise ValueError("input and output require constant per-tensor int8 qparams")
333+
weight = constants.get(weight_node)
334+
if (
335+
weight is None
336+
or weight.dtype != torch.int8
337+
or weight.dim() != 2
338+
or not weight.is_contiguous()
339+
or weight.numel() > torch.iinfo(torch.int32).max
340+
or any(not 0 < dim <= 65535 for dim in weight.shape)
341+
):
342+
raise ValueError(
343+
"weight must be a constant contiguous int8 matrix with dimensions in [1, 65535]"
344+
)
345+
if input_value.shape[-1] != weight.shape[1] or output_value.shape != (
346+
*input_value.shape[:-1],
347+
weight.shape[0],
348+
):
349+
raise ValueError("input, weight and output shapes do not match linear")
350+
weight_params = get_qparams_from_node(node, "weight")
351+
if weight_params is None or weight_params.dtype != torch.float32:
352+
raise ValueError(
353+
"weight requires float32 dequantization with per-output-channel scales"
354+
)
355+
scales, zero_points = (
356+
constants.get(arg) for arg in (weight_params.scale, weight_params.zero_point)
357+
)
358+
if (
359+
scales is None
360+
or scales.dtype != torch.float32
361+
or scales.shape != (weight.shape[0], 1)
362+
or not torch.all(torch.isfinite(scales) & (scales > 0))
363+
or zero_points is None
364+
or zero_points.shape != scales.shape
365+
or torch.any(zero_points != 0)
366+
):
367+
raise ValueError(
368+
"weight requires constant per-output-channel scales and zero zero-points"
369+
)
370+
bias = None
371+
if bias_node is not None:
372+
bias = constants.get(bias_node)
373+
bias_params = get_qparams_from_node(node, "bias")
374+
if (
375+
bias is None
376+
or bias.dtype != torch.int32
377+
or not bias.is_contiguous()
378+
or bias.shape != (weight.shape[0],)
379+
or bias_params is None
380+
or bias_params.dtype != torch.float32
381+
):
382+
raise ValueError(
383+
"bias must be a constant int32 vector with derived qparams"
384+
)
385+
bias_scales, bias_zero_points = (
386+
constants.get(arg) for arg in (bias_params.scale, bias_params.zero_point)
387+
)
388+
if (
389+
bias_scales is None
390+
or not torch.equal(bias_scales, input_params.scale * scales.flatten())
391+
or bias_zero_points is None
392+
or bias_zero_points.shape != bias.shape
393+
or torch.any(bias_zero_points != 0)
394+
):
395+
raise ValueError(
396+
"bias scales must equal input scale times weight scales, with zero zero-points"
397+
)
398+
multipliers, shifts = _linear_multipliers(
399+
input_params, output_params, weight, scales, bias
400+
)
401+
multiplier_node, shift_node = (
402+
add_constant(
403+
program, f"{node.name}_{name}", tensor, node, InputKind.CONSTANT_TENSOR
404+
)
405+
for name, tensor in (("multiplier", multipliers), ("shift", shifts))
406+
)
407+
return (
408+
inp,
409+
weight_node,
410+
bias_node,
411+
multiplier_node,
412+
shift_node,
413+
-input_params.zp,
414+
output_params.zp,
415+
)
416+
417+
268418
class FusedQuantToEspPass(ExportedProgramPassBase):
269419
"""Lower supported fused-quant operators to ESP-NN."""
270420

271421
def call(self, exported_program):
272422
module = exported_program.graph_module
273423
graph = module.graph
424+
linear_nodes = graph.find_nodes(
425+
op="call_function", target=exir_ops.edge.fused_quant.linear.default
426+
)
427+
constants = (
428+
get_propagated_const_tensor_dict(exported_program, None)
429+
if linear_nodes
430+
else {}
431+
)
274432

275433
modified = False
276434
for node in list(graph.nodes):
435+
if node.target == exir_ops.edge.fused_quant.linear.default:
436+
try:
437+
args = _linear_kernel_args(exported_program, node, constants)
438+
except ValueError as error:
439+
raise RuntimeError(
440+
f"ESP-NN cannot lower linear '{node.name}': {error}. "
441+
"Portable linear fallback is not available yet."
442+
) from error
443+
with graph.inserting_before(node):
444+
replacement = graph.call_function(
445+
exir_ops.edge.esp_nn.quantized_linear_int8.default, args
446+
)
447+
replacement.meta = node.meta.copy()
448+
node.replace_all_uses_with(replacement)
449+
graph.erase_node(node)
450+
modified = True
451+
continue
277452
if (
278453
node.target
279454
== exir_ops.edge.fused_quant.avg_pool2d_channels_last.default

0 commit comments

Comments
 (0)