Skip to content

Commit 29b470c

Browse files
committed
[Espressif] Lower Fused Quant linear to ESP-NN
Lower supported quantized linear calls to ESP-NN's batched per-channel fully connected API. Contiguous vector and higher-rank inputs use the row-based kernel without layout copies. Reuse the quantizer's int32 bias and prepare constant per-channel multipliers with bounds on integer accumulation and floating reference overflow. Unsupported calls use shared portable decomposition, including runtime weights or bias, oversized dimensions, partial quantization and unsafe accumulators. Strict export rejects those portable operators. The shared dialect and S3 cases cover accelerated and portable linear calls and serialize without remaining Fused Quant or aten.linear operators. Authored with OpenAI Codex.
1 parent 79f0568 commit 29b470c

7 files changed

Lines changed: 619 additions & 1 deletion

File tree

‎backends/espressif/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ idf_component_register(
1111
"ops/op_quantized_add.cpp"
1212
"ops/op_quantized_max_pool2d.cpp"
1313
"ops/op_quantized_avg_pool2d.cpp"
14+
"ops/op_quantized_linear.cpp"
1415
PRIV_REQUIRES
1516
esp-nn
1617
WHOLE_ARCHIVE
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/runtime/kernel/kernel_includes.h>
10+
11+
#include <sdkconfig.h>
12+
13+
#include <esp_nn.h>
14+
15+
#include <cstdint>
16+
#include <limits>
17+
18+
namespace espressif::native {
19+
20+
using executorch::aten::Tensor;
21+
using executorch::runtime::KernelRuntimeContext;
22+
23+
// cppcheck-suppress unusedFunction
24+
Tensor& quantized_linear_int8_out(
25+
KernelRuntimeContext& ctx,
26+
const Tensor& input,
27+
const Tensor& weight,
28+
const std::optional<Tensor>& bias,
29+
const Tensor& output_multiplier,
30+
const Tensor& output_shift,
31+
int64_t input_offset,
32+
int64_t output_offset,
33+
Tensor& out) {
34+
ET_KERNEL_CHECK(ctx, weight.dim() >= 2, InvalidArgument, out);
35+
const auto channels = weight.size(0);
36+
const auto columns = weight.size(1);
37+
ET_KERNEL_CHECK(
38+
ctx,
39+
channels > 0 && channels <= std::numeric_limits<uint16_t>::max() &&
40+
columns > 0 && columns <= std::numeric_limits<uint16_t>::max(),
41+
InvalidArgument,
42+
out);
43+
const auto batches = input.numel() / columns;
44+
ET_KERNEL_CHECK(
45+
ctx,
46+
batches >= 0 && batches <= std::numeric_limits<int32_t>::max(),
47+
InvalidArgument,
48+
out);
49+
const auto rows = static_cast<size_t>(batches);
50+
const auto cols = static_cast<size_t>(columns);
51+
const auto outputs = static_cast<size_t>(channels);
52+
ET_KERNEL_CHECK(
53+
ctx,
54+
rows <= input.nbytes() / cols && rows <= out.nbytes() / outputs &&
55+
outputs <= weight.nbytes() / cols &&
56+
outputs <= output_multiplier.nbytes() / sizeof(int32_t) &&
57+
outputs <= output_shift.nbytes() / sizeof(int32_t) &&
58+
(!bias || outputs <= bias->nbytes() / sizeof(int32_t)),
59+
InvalidArgument,
60+
out);
61+
62+
esp_nn_fully_connected_per_ch_s8_batch(
63+
input.const_data_ptr<int8_t>(),
64+
static_cast<int32_t>(input_offset),
65+
static_cast<uint16_t>(columns),
66+
weight.const_data_ptr<int8_t>(),
67+
0,
68+
bias ? bias->const_data_ptr<int32_t>() : nullptr,
69+
out.mutable_data_ptr<int8_t>(),
70+
static_cast<uint16_t>(channels),
71+
static_cast<int32_t>(output_offset),
72+
output_shift.const_data_ptr<int32_t>(),
73+
output_multiplier.const_data_ptr<int32_t>(),
74+
std::numeric_limits<int8_t>::min(),
75+
std::numeric_limits<int8_t>::max(),
76+
static_cast<int32_t>(batches));
77+
return out;
78+
}
79+
80+
} // namespace espressif::native

‎backends/espressif/ops/operators.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,15 @@
5757
"quantized_avg_pool2d_int8.out(Tensor input, int[2] kernel_size, int[2] stride, "
5858
"int[2] padding, *, Tensor(a!) out) -> Tensor(a!)"
5959
)
60+
lib.define(
61+
"quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, "
62+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset) -> Tensor"
63+
)
64+
lib.define(
65+
"quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, "
66+
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
67+
"*, Tensor(a!) out) -> Tensor(a!)"
68+
)
6069

6170

6271
@register_fake("esp_nn::quantized_max_pool2d_int8")
@@ -182,3 +191,29 @@ def quantized_add_int8(
182191
)
183192
result = requantize(total, output_multiplier, output_shift) + output_offset
184193
return result.clamp(-128, 127).to(torch.int8)
194+
195+
196+
@register_fake("esp_nn::quantized_linear_int8")
197+
def quantized_linear_int8_meta(
198+
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
199+
):
200+
return input.new_empty((*input.shape[:-1], weight.shape[0]))
201+
202+
203+
@impl(lib, "quantized_linear_int8", "CompositeExplicitAutograd")
204+
def quantized_linear_int8(
205+
input, weight, bias, output_multiplier, output_shift, input_offset, output_offset
206+
):
207+
accumulator = (input.to(torch.int64) + input_offset) @ weight.to(torch.int64).T
208+
if bias is not None:
209+
accumulator = accumulator + bias
210+
result = torch.stack(
211+
[
212+
requantize(accumulator[..., channel], int(multiplier), int(shift))
213+
for channel, (multiplier, shift) in enumerate(
214+
zip(output_multiplier, output_shift)
215+
)
216+
],
217+
dim=-1,
218+
)
219+
return (result + output_offset).clamp(-128, 127).to(torch.int8)

‎backends/espressif/ops/operators.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,9 @@
3333
kernels:
3434
- arg_meta: null
3535
kernel_name: espressif::quantized_avg_pool2d_int8_out
36+
37+
- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, *, Tensor(a!) out) -> Tensor(a!)
38+
variants: function
39+
kernels:
40+
- arg_meta: null
41+
kernel_name: espressif::quantized_linear_int8_out

‎backends/espressif/passes/lower_to_esp.py‎

Lines changed: 173 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@
1515
from executorch.backends.espressif.utils.quantization import quantize_multiplier
1616
from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node
1717
from executorch.backends.transforms.permute_pass_utils import get_arg
18-
from executorch.backends.transforms.utils import get_constant
18+
from executorch.backends.transforms.utils import add_constant, get_constant
1919
from executorch.exir.dialects._ops import ops as exir_ops
2020
from executorch.exir.pass_base import ExportedProgramPassBase, ExportedProgramPassResult
21+
from executorch.exir.passes.constant_prop_pass import get_propagated_const_tensor_dict
22+
from torch.export.graph_signature import InputKind
2123

2224

2325
@dataclass(frozen=True)
@@ -272,15 +274,185 @@ def _lower_avg_pool2d(program, node) -> bool:
272274
return True
273275

274276

277+
def _linear_multipliers(input_params, output_params, weight, scales, bias):
278+
rescales = [
279+
quantize_multiplier(input_params.scale * scale / output_params.scale)
280+
for scale in scales.flatten().tolist()
281+
]
282+
if any(shift < -30 for _, shift in rescales):
283+
raise ValueError("requantization shift is outside the ESP-NN range")
284+
max_input = max(abs(-128 - input_params.zp), abs(127 - input_params.zp))
285+
limits = max_input * weight.to(torch.int64).abs().sum(dim=1)
286+
if bias is not None:
287+
limits += bias.to(torch.int64).abs()
288+
if any(
289+
int(limit) << max(shift, 0)
290+
> torch.iinfo(torch.int32).max - abs(output_params.zp)
291+
for limit, (_, shift) in zip(limits, rescales)
292+
):
293+
raise ValueError("accumulation or requantization may overflow int32")
294+
max_float_input = c_float(max_input * c_float(input_params.scale).value).value
295+
float_weights = weight.to(torch.float32) * scales
296+
max_float_outputs = max_float_input * float_weights.abs().to(torch.float64).sum(
297+
dim=1
298+
)
299+
if bias is not None:
300+
bias_scales = input_params.scale * scales.flatten()
301+
max_float_outputs += (
302+
(bias.to(torch.float32) * bias_scales).abs().to(torch.float64)
303+
)
304+
# Bound float32 accumulation rounding, including the bias addition.
305+
max_float_outputs /= 1 - (weight.shape[1] + 1) * torch.finfo(torch.float32).eps
306+
if (
307+
not math.isfinite(c_float(1.0 / output_params.scale).value)
308+
or not torch.all(torch.isfinite(max_float_outputs))
309+
or torch.any(max_float_outputs > torch.finfo(torch.float32).max)
310+
):
311+
raise ValueError("the float32 reference may overflow")
312+
multipliers, shifts = zip(*rescales)
313+
return torch.tensor(multipliers, dtype=torch.int32), torch.tensor(
314+
shifts, dtype=torch.int32
315+
)
316+
317+
318+
def _linear_kernel_args(program, node, constants):
319+
inp, weight_node, bias_node = (
320+
get_arg(node, name) for name in ("inp", "weight", "bias")
321+
)
322+
input_value, output_value = inp.meta.get("val"), node.meta.get("val")
323+
if not all(
324+
isinstance(value, torch.Tensor)
325+
and value.dtype == torch.int8
326+
and value.dim() >= 1
327+
and all(type(dim) is int for dim in value.shape)
328+
and 0 < value.numel() <= torch.iinfo(torch.int32).max
329+
and value.is_contiguous()
330+
for value in (input_value, output_value)
331+
):
332+
raise ValueError(
333+
"input and output must be nonempty, static, contiguous int8 tensors"
334+
)
335+
input_params, output_params = (
336+
_qparams(program, node, name) for name in ("inp", "out")
337+
)
338+
if input_params is None or output_params is None:
339+
raise ValueError("input and output require constant per-tensor int8 qparams")
340+
weight = constants.get(weight_node)
341+
if (
342+
weight is None
343+
or weight.dtype != torch.int8
344+
or weight.dim() != 2
345+
or not weight.is_contiguous()
346+
or weight.numel() > torch.iinfo(torch.int32).max
347+
or any(not 0 < dim <= 65535 for dim in weight.shape)
348+
):
349+
raise ValueError(
350+
"weight must be a constant contiguous int8 matrix with dimensions in [1, 65535]"
351+
)
352+
if input_value.shape[-1] != weight.shape[1] or output_value.shape != (
353+
*input_value.shape[:-1],
354+
weight.shape[0],
355+
):
356+
raise ValueError("input, weight and output shapes do not match linear")
357+
weight_params = get_qparams_from_node(node, "weight")
358+
if weight_params is None or weight_params.dtype != torch.float32:
359+
raise ValueError(
360+
"weight requires float32 dequantization with per-output-channel scales"
361+
)
362+
scales, zero_points = (
363+
constants.get(arg) for arg in (weight_params.scale, weight_params.zero_point)
364+
)
365+
if (
366+
scales is None
367+
or scales.dtype != torch.float32
368+
or scales.shape != (weight.shape[0], 1)
369+
or not torch.all(torch.isfinite(scales) & (scales > 0))
370+
or zero_points is None
371+
or zero_points.shape != scales.shape
372+
or torch.any(zero_points != 0)
373+
):
374+
raise ValueError(
375+
"weight requires constant per-output-channel scales and zero zero-points"
376+
)
377+
bias = None
378+
if bias_node is not None:
379+
bias = constants.get(bias_node)
380+
bias_params = get_qparams_from_node(node, "bias")
381+
if (
382+
bias is None
383+
or bias.dtype != torch.int32
384+
or not bias.is_contiguous()
385+
or bias.shape != (weight.shape[0],)
386+
or bias_params is None
387+
or bias_params.dtype != torch.float32
388+
):
389+
raise ValueError(
390+
"bias must be a constant int32 vector with derived qparams"
391+
)
392+
bias_scales, bias_zero_points = (
393+
constants.get(arg) for arg in (bias_params.scale, bias_params.zero_point)
394+
)
395+
if (
396+
bias_scales is None
397+
or not torch.equal(bias_scales, input_params.scale * scales.flatten())
398+
or bias_zero_points is None
399+
or bias_zero_points.shape != bias.shape
400+
or torch.any(bias_zero_points != 0)
401+
):
402+
raise ValueError(
403+
"bias scales must equal input scale times weight scales, with zero zero-points"
404+
)
405+
multipliers, shifts = _linear_multipliers(
406+
input_params, output_params, weight, scales, bias
407+
)
408+
multiplier_node, shift_node = (
409+
add_constant(
410+
program, f"{node.name}_{name}", tensor, node, InputKind.CONSTANT_TENSOR
411+
)
412+
for name, tensor in (("multiplier", multipliers), ("shift", shifts))
413+
)
414+
return (
415+
inp,
416+
weight_node,
417+
bias_node,
418+
multiplier_node,
419+
shift_node,
420+
-input_params.zp,
421+
output_params.zp,
422+
)
423+
424+
275425
class LowerToEsp(ExportedProgramPassBase):
276426
"""Lower supported fused-quant operators to ESP-NN."""
277427

278428
def call(self, exported_program):
279429
module = exported_program.graph_module
280430
graph = module.graph
431+
linear_nodes = graph.find_nodes(
432+
op="call_function", target=exir_ops.edge.fused_quant.linear.default
433+
)
434+
constants = (
435+
get_propagated_const_tensor_dict(exported_program, None)
436+
if linear_nodes
437+
else {}
438+
)
281439

282440
modified = False
283441
for node in list(graph.nodes):
442+
if node.target == exir_ops.edge.fused_quant.linear.default:
443+
try:
444+
args = _linear_kernel_args(exported_program, node, constants)
445+
except ValueError:
446+
continue
447+
with graph.inserting_before(node):
448+
replacement = graph.call_function(
449+
exir_ops.edge.esp_nn.quantized_linear_int8.default, args
450+
)
451+
replacement.meta = node.meta.copy()
452+
node.replace_all_uses_with(replacement)
453+
graph.erase_node(node)
454+
modified = True
455+
continue
284456
if (
285457
node.target
286458
== exir_ops.edge.fused_quant.avg_pool2d_channels_last.default

0 commit comments

Comments
 (0)