Skip to content

Commit 3b908f8

Browse files
committed
[Espressif] Lower Fused Quant linear to ESP-NN
Lower statically quantized linear calls to ESP-NN's batched per-channel fully connected API. Contiguous vector and higher-rank inputs use the same row-based kernel without layout copies. Reuse the quantizer's int32 bias directly and prepare constant per-channel multipliers with bounds on integer accumulation and floating reference overflow. Unsupported linear calls now report an export error with the failing requirement, because shared FQ decomposition restores aten.linear without a portable implementation. Other operators retain their existing fallback behavior. This implements the preceding linear quantizer contract; the two commits are intended to land together. The shared dialect and ESP32-S3 implementation cases cover bias and no bias, higher ranks, aligned and tail widths, shared weights and biases, float users, ReLU, chained linear, large bias, and saturation. Normalize returned Parameters to plain tensors when bundling expected test outputs. Validation: 24 linear export/lowering tests and 110 existing operator/pass-manager regressions passed. All 16 shared linear cases passed on the ESP32-S3 emulator with ESP-NN 1.5.0. Lintrunner passed. Authored with OpenAI Codex.
1 parent 5c3ef5f commit 3b908f8

9 files changed

Lines changed: 569 additions & 1 deletion

File tree

‎backends/espressif/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ idf_component_register(
1010
"ops/op_quantized_add.cpp"
1111
"ops/op_quantized_max_pool2d.cpp"
1212
"ops/op_quantized_avg_pool2d.cpp"
13+
"ops/op_quantized_linear.cpp"
1314
PRIV_REQUIRES
1415
esp-nn
1516
WHOLE_ARCHIVE

‎backends/espressif/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ input/output qparams, floor mode, and no divisor override. Padding is supported
3434
when `count_include_pad=False`.
3535
Its integer kernel rounds ties away from zero; the FQ reference rounds ties to
3636
even, so results can differ by one int8 step.
37+
Linear supports static, contiguous int8 inputs of rank one or higher, constant
38+
per-output-channel symmetric int8 weights, and optional constant int32 bias.
39+
`get_esp_fused_quant_quantizer` derives the bias scales from input and weight scales.
40+
Leading input dimensions are treated as batch rows by ESP-NN's batched fully
41+
connected API. Unsupported linear calls fail at export with the reason;
42+
portable linear fallback is not available yet, even when CPU fallback is enabled.
3743
The shared Fused Quant layout pass introduces contiguous NHWC tensors for
3844
spatial operators; model inputs and outputs retain their original layout.
3945
Required layout copies are visible in the operator-library report and accepted

‎backends/espressif/lowering.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ def get_esp_fused_quant_recipe(
130130
Use with executorch.export.export(). Supported Fused Quant operators become
131131
direct ESP-NN calls; other operators decompose to portable kernels. Setting
132132
allow_cpu_fallback=False rejects remaining compute, excluding Q/DQ and required NHWC layout copies.
133+
Unsupported linear calls currently raise even when CPU fallback is enabled.
133134
Use EspNNQuantizer and to_espnn_edge for the EspNNBackend delegate.
134135
"""
135136
quantizer = quantizer if quantizer is not None else get_esp_fused_quant_quantizer()
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/runtime/kernel/kernel_includes.h>
10+
11+
#include <sdkconfig.h>
12+
13+
#include <esp_nn.h>
14+
15+
#include <cstdint>
16+
#include <limits>
17+
18+
namespace espressif::native {
19+
20+
using executorch::aten::Tensor;
21+
using executorch::runtime::KernelRuntimeContext;
22+
23+
// cppcheck-suppress unusedFunction
24+
Tensor& quantized_linear_int8_out(
25+
KernelRuntimeContext& ctx,
26+
const Tensor& input,
27+
const Tensor& weight,
28+
const std::optional<Tensor>& bias,
29+
const Tensor& output_multiplier,
30+
const Tensor& output_shift,
31+
int64_t input_offset,
32+
int64_t output_offset,
33+
Tensor& out) {
34+
ET_KERNEL_CHECK(ctx, weight.dim() >= 2, InvalidArgument, out);
35+
const auto channels = weight.size(0);
36+
const auto columns = weight.size(1);
37+
ET_KERNEL_CHECK(
38+
ctx,
39+
channels > 0 && channels <= std::numeric_limits<uint16_t>::max() &&
40+
columns > 0 && columns <= std::numeric_limits<uint16_t>::max(),
41+
InvalidArgument,
42+
out);
43+
const auto batches = input.numel() / columns;
44+
ET_KERNEL_CHECK(
45+
ctx,
46+
batches >= 0 && batches <= std::numeric_limits<int32_t>::max(),
47+
InvalidArgument,
48+
out);
49+
const auto rows = static_cast<size_t>(batches);
50+
const auto cols = static_cast<size_t>(columns);
51+
const auto outputs = static_cast<size_t>(channels);
52+
ET_KERNEL_CHECK(
53+
ctx,
54+
rows <= input.nbytes() / cols && rows <= out.nbytes() / outputs &&
55+
outputs <= weight.nbytes() / cols &&
56+
outputs <= output_multiplier.nbytes() / sizeof(int32_t) &&
57+
outputs <= output_shift.nbytes() / sizeof(int32_t) &&
58+
(!bias || outputs <= bias->nbytes() / sizeof(int32_t)),
59+
InvalidArgument,
60+
out);
61+
62+
esp_nn_fully_connected_per_ch_s8_batch(
63+
input.const_data_ptr<int8_t>(),
64+
static_cast<int32_t>(input_offset),
65+
static_cast<uint16_t>(columns),
66+
weight.const_data_ptr<int8_t>(),
67+
0,
68+
bias ? bias->const_data_ptr<int32_t>() : nullptr,
69+
out.mutable_data_ptr<int8_t>(),
70+
static_cast<uint16_t>(channels),
71+
static_cast<int32_t>(output_offset),
72+
output_shift.const_data_ptr<int32_t>(),
73+
output_multiplier.const_data_ptr<int32_t>(),
74+
std::numeric_limits<int8_t>::min(),
75+
std::numeric_limits<int8_t>::max(),
76+
static_cast<int32_t>(batches));
77+
return out;
78+
}
79+
80+
} // namespace espressif::native

‎backends/espressif/ops/operators.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,15 @@
4848
"quantized_avg_pool2d_int8.out(Tensor input, int[2] kernel_size, int[2] stride, "
4949
"int[2] padding, *, Tensor(a!) out) -> Tensor(a!)"
5050
)
51+
lib.define(
52+
"quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, "
53+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset) -> Tensor"
54+
)
55+
lib.define(
56+
"quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, "
57+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
58+
"*, Tensor(a!) out) -> Tensor(a!)"
59+
)
5160

5261

5362
@register_fake("esp_nn::quantized_max_pool2d_int8")
@@ -159,3 +168,29 @@ def quantized_add_int8(
159168
)
160169
result = requantize(total, output_multiplier, output_shift) + output_offset
161170
return result.clamp(-128, 127).to(torch.int8)
171+
172+
173+
@register_fake("esp_nn::quantized_linear_int8")
174+
def quantized_linear_int8_meta(
175+
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
176+
):
177+
return input.new_empty((*input.shape[:-1], weight.shape[0]))
178+
179+
180+
@impl(lib, "quantized_linear_int8", "CompositeExplicitAutograd")
181+
def quantized_linear_int8(
182+
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
183+
):
184+
accumulator = (input.to(torch.int64) + input_offset) @ weight.to(torch.int64).T
185+
if bias is not None:
186+
accumulator = accumulator + bias
187+
result = torch.stack(
188+
[
189+
requantize(accumulator[..., channel], int(multiplier), int(shift))
190+
for channel, (multiplier, shift) in enumerate(
191+
zip(output_multiplier, output_shift)
192+
)
193+
],
194+
dim=-1,
195+
)
196+
return (result + output_offset).clamp(-128, 127).to(torch.int8)

‎backends/espressif/ops/operators.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,9 @@
2727
kernels:
2828
- arg_meta: null
2929
kernel_name: espressif::quantized_avg_pool2d_int8_out
30+
31+
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, *, Tensor(a!) out) -> Tensor(a!)
32+
variants: function
33+
kernels:
34+
- arg_meta: null
35+
kernel_name: espressif::quantized_linear_int8_out

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 176 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@
1515
from executorch.backends.espressif.utils.quantization import quantize_multiplier
1616
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
1717
from executorch.backends.transforms.permute_pass_utils import get_arg
18-
from executorch.backends.transforms.utils import get_constant
18+
from executorch.backends.transforms.utils import add_constant, get_constant
1919
from executorch.exir.dialects._ops import ops as exir_ops
2020
from executorch.exir.pass_base import ExportedProgramPassBase, ExportedProgramPassResult
21+
from executorch.exir.passes.constant_prop_pass import get_propagated_const_tensor_dict
22+
from torch.export.graph_signature import InputKind
2123

2224

2325
@dataclass(frozen=True)
@@ -270,15 +272,188 @@ def _lower_avg_pool2d(program, node) -> bool:
270272
return True
271273

272274

275+
def _linear_multipliers(input_params, output_params, weight, scales, bias):
276+
rescales = [
277+
quantize_multiplier(input_params.scale * scale / output_params.scale)
278+
for scale in scales.flatten().tolist()
279+
]
280+
if any(shift < -30 for _, shift in rescales):
281+
raise ValueError("requantization shift is outside the ESP-NN range")
282+
max_input = max(abs(-128 - input_params.zp), abs(127 - input_params.zp))
283+
limits = max_input * weight.to(torch.int64).abs().sum(dim=1)
284+
if bias is not None:
285+
limits += bias.to(torch.int64).abs()
286+
if any(
287+
int(limit) << max(shift, 0)
288+
> torch.iinfo(torch.int32).max - abs(output_params.zp)
289+
for limit, (_, shift) in zip(limits, rescales)
290+
):
291+
raise ValueError("accumulation or requantization may overflow int32")
292+
max_float_input = c_float(max_input * c_float(input_params.scale).value).value
293+
float_weights = weight.to(torch.float32) * scales
294+
max_float_outputs = max_float_input * float_weights.abs().to(torch.float64).sum(
295+
dim=1
296+
)
297+
if bias is not None:
298+
bias_scales = input_params.scale * scales.flatten()
299+
max_float_outputs += (
300+
(bias.to(torch.float32) * bias_scales).abs().to(torch.float64)
301+
)
302+
# Bound float32 accumulation rounding, including the bias addition.
303+
max_float_outputs /= 1 - (weight.shape[1] + 1) * torch.finfo(torch.float32).eps
304+
if (
305+
not math.isfinite(c_float(1.0 / output_params.scale).value)
306+
or not torch.all(torch.isfinite(max_float_outputs))
307+
or torch.any(max_float_outputs > torch.finfo(torch.float32).max)
308+
):
309+
raise ValueError("the float32 reference may overflow")
310+
multipliers, shifts = zip(*rescales)
311+
return torch.tensor(multipliers, dtype=torch.int32), torch.tensor(
312+
shifts, dtype=torch.int32
313+
)
314+
315+
316+
def _linear_kernel_args(program, node, constants):
317+
inp, weight_node, bias_node = (
318+
get_arg(node, name) for name in ("inp", "weight", "bias")
319+
)
320+
input_value, output_value = inp.meta.get("val"), node.meta.get("val")
321+
if not all(
322+
isinstance(value, torch.Tensor)
323+
and value.dtype == torch.int8
324+
and value.dim() >= 1
325+
and all(type(dim) is int for dim in value.shape)
326+
and 0 < value.numel() <= torch.iinfo(torch.int32).max
327+
and value.is_contiguous()
328+
for value in (input_value, output_value)
329+
):
330+
raise ValueError(
331+
"input and output must be nonempty, static, contiguous int8 tensors"
332+
)
333+
input_params, output_params = (
334+
_qparams(program, node, name) for name in ("inp", "out")
335+
)
336+
if input_params is None or output_params is None:
337+
raise ValueError("input and output require constant per-tensor int8 qparams")
338+
weight = constants.get(weight_node)
339+
if (
340+
weight is None
341+
or weight.dtype != torch.int8
342+
or weight.dim() != 2
343+
or not weight.is_contiguous()
344+
or weight.numel() > torch.iinfo(torch.int32).max
345+
or any(not 0 < dim <= 65535 for dim in weight.shape)
346+
):
347+
raise ValueError(
348+
"weight must be a constant contiguous int8 matrix with dimensions in [1, 65535]"
349+
)
350+
if input_value.shape[-1] != weight.shape[1] or output_value.shape != (
351+
*input_value.shape[:-1],
352+
weight.shape[0],
353+
):
354+
raise ValueError("input, weight and output shapes do not match linear")
355+
weight_params = get_qparams_from_node(node, "weight")
356+
if weight_params is None or weight_params.dtype != torch.float32:
357+
raise ValueError(
358+
"weight requires float32 dequantization with per-output-channel scales"
359+
)
360+
scales, zero_points = (
361+
constants.get(arg) for arg in (weight_params.scale, weight_params.zero_point)
362+
)
363+
if (
364+
scales is None
365+
or scales.dtype != torch.float32
366+
or scales.shape != (weight.shape[0], 1)
367+
or not torch.all(torch.isfinite(scales) & (scales > 0))
368+
or zero_points is None
369+
or zero_points.shape != scales.shape
370+
or torch.any(zero_points != 0)
371+
):
372+
raise ValueError(
373+
"weight requires constant per-output-channel scales and zero zero-points"
374+
)
375+
bias = None
376+
if bias_node is not None:
377+
bias = constants.get(bias_node)
378+
bias_params = get_qparams_from_node(node, "bias")
379+
if (
380+
bias is None
381+
or bias.dtype != torch.int32
382+
or not bias.is_contiguous()
383+
or bias.shape != (weight.shape[0],)
384+
or bias_params is None
385+
or bias_params.dtype != torch.float32
386+
):
387+
raise ValueError(
388+
"bias must be a constant int32 vector with derived qparams"
389+
)
390+
bias_scales, bias_zero_points = (
391+
constants.get(arg) for arg in (bias_params.scale, bias_params.zero_point)
392+
)
393+
if (
394+
bias_scales is None
395+
or not torch.equal(bias_scales, input_params.scale * scales.flatten())
396+
or bias_zero_points is None
397+
or bias_zero_points.shape != bias.shape
398+
or torch.any(bias_zero_points != 0)
399+
):
400+
raise ValueError(
401+
"bias scales must equal input scale times weight scales, with zero zero-points"
402+
)
403+
multipliers, shifts = _linear_multipliers(
404+
input_params, output_params, weight, scales, bias
405+
)
406+
multiplier_node, shift_node = (
407+
add_constant(
408+
program, f"{node.name}_{name}", tensor, node, InputKind.CONSTANT_TENSOR
409+
)
410+
for name, tensor in (("multiplier", multipliers), ("shift", shifts))
411+
)
412+
return (
413+
inp,
414+
weight_node,
415+
bias_node,
416+
multiplier_node,
417+
shift_node,
418+
-input_params.zp,
419+
output_params.zp,
420+
)
421+
422+
273423
class LowerToEsp(ExportedProgramPassBase):
274424
"""Lower supported fused-quant operators to ESP-NN."""
275425

276426
def call(self, exported_program):
277427
module = exported_program.graph_module
278428
graph = module.graph
429+
linear_nodes = graph.find_nodes(
430+
op="call_function", target=exir_ops.edge.fused_quant.linear.default
431+
)
432+
constants = (
433+
get_propagated_const_tensor_dict(exported_program, None)
434+
if linear_nodes
435+
else {}
436+
)
279437

280438
modified = False
281439
for node in list(graph.nodes):
440+
if node.target == exir_ops.edge.fused_quant.linear.default:
441+
try:
442+
args = _linear_kernel_args(exported_program, node, constants)
443+
except ValueError as error:
444+
raise RuntimeError(
445+
f"ESP-NN cannot lower linear '{node.name}': {error}. "
446+
"Portable linear fallback is not available yet."
447+
) from error
448+
with graph.inserting_before(node):
449+
replacement = graph.call_function(
450+
exir_ops.edge.esp_nn.quantized_linear_int8.default, args
451+
)
452+
replacement.meta = node.meta.copy()
453+
node.replace_all_uses_with(replacement)
454+
graph.erase_node(node)
455+
modified = True
456+
continue
282457
if (
283458
node.target
284459
== exir_ops.edge.fused_quant.avg_pool2d_channels_last.default

0 commit comments

Comments
 (0)