Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backends/espressif/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ idf_component_register(
"ops/op_quantized_add.cpp"
"ops/op_quantized_max_pool2d.cpp"
"ops/op_quantized_avg_pool2d.cpp"
"ops/op_quantized_mean.cpp"
"ops/op_quantized_linear.cpp"
"ops/op_quantized_conv2d.cpp"
"ops/op_quantized_depthwise_conv2d.cpp"
Expand Down
68 changes: 68 additions & 0 deletions backends/espressif/ops/op_quantized_mean.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/

#include <executorch/runtime/kernel/kernel_includes.h>

#include <sdkconfig.h>

#include <esp_nn.h>

#include <cstdint>
#include <limits>

namespace espressif::native {

using executorch::aten::Tensor;
using executorch::runtime::KernelRuntimeContext;

// cppcheck-suppress unusedFunction
Tensor& quantized_mean_int8_out(
KernelRuntimeContext& ctx,
const Tensor& input,
int64_t input_zero_point,
int64_t output_zero_point,
int64_t output_multiplier,
int64_t output_shift,
Tensor& out) {
ET_KERNEL_CHECK(ctx, input.dim() == 4, InvalidArgument, out);
const auto batches = input.size(0);
const auto height = input.size(1);
const auto width = input.size(2);
const auto channels = input.size(3);
ET_KERNEL_CHECK(
ctx,
height > 0 && width > 0 && channels > 0 &&
height <= std::numeric_limits<int32_t>::max() / width &&
height * width <= std::numeric_limits<int32_t>::max() / channels,
InvalidArgument,
out);
const auto count = height * width;
const auto input_bytes = count * channels;
ET_KERNEL_CHECK(
ctx,
count <= std::numeric_limits<int32_t>::max() / 255 && batches >= 0 &&
static_cast<size_t>(batches) <= input.nbytes() / input_bytes &&
static_cast<size_t>(batches) <= out.nbytes() / channels,
InvalidArgument,
out);
for (int64_t batch = 0; batch < batches; ++batch) {
esp_nn_mean_nhwc_s8(
input.const_data_ptr<int8_t>() + batch * input_bytes,
out.mutable_data_ptr<int8_t>() + batch * channels,
static_cast<int32_t>(height),
static_cast<int32_t>(width),
static_cast<int32_t>(channels),
static_cast<int32_t>(input_zero_point),
static_cast<int32_t>(output_zero_point),
static_cast<int32_t>(output_multiplier),
static_cast<int32_t>(output_shift));
}
return out;
}

} // namespace espressif::native
25 changes: 25 additions & 0 deletions backends/espressif/ops/operators.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@
"quantized_avg_pool2d_int8.out(Tensor input, int[2] kernel_size, int[2] stride, "
"int[2] padding, *, Tensor(a!) out) -> Tensor(a!)"
)
lib.define(
"quantized_mean_int8(Tensor input, int input_zero_point, int output_zero_point, "
"int output_multiplier, int output_shift) -> Tensor"
)
lib.define(
"quantized_mean_int8.out(Tensor input, int input_zero_point, int output_zero_point, "
"int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)"
)
lib.define(
"quantized_linear_int8(Tensor input, Tensor weight, Tensor? bias, "
"Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, "
Expand Down Expand Up @@ -154,6 +162,23 @@ def quantized_avg_pool2d_int8(input, kernel_size, stride, padding):
return result


@register_fake("esp_nn::quantized_mean_int8")
def quantized_mean_int8_meta(
input, input_zero_point, output_zero_point, output_multiplier, output_shift
):
batch, _, _, channels = input.shape
return input.new_empty((batch, channels))


@impl(lib, "quantized_mean_int8", "CompositeExplicitAutograd")
def quantized_mean_int8(
input, input_zero_point, output_zero_point, output_multiplier, output_shift
):
total = (input.to(torch.int64) - input_zero_point).sum((1, 2))
result = requantize(total, output_multiplier, output_shift) + output_zero_point
return result.clamp(-128, 127).to(torch.int8)


@register_fake("esp_nn::quantized_mul_int8")
def quantized_mul_int8_meta(
input1,
Expand Down
6 changes: 6 additions & 0 deletions backends/espressif/ops/operators.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@
- arg_meta: null
kernel_name: espressif::quantized_avg_pool2d_int8_out

- func: esp_nn::quantized_mean_int8.out(Tensor input, int input_zero_point, int output_zero_point, int output_multiplier, int output_shift, *, Tensor(a!) out) -> Tensor(a!)
variants: function
kernels:
- arg_meta: null
kernel_name: espressif::quantized_mean_int8_out

- func: esp_nn::quantized_linear_int8.out(Tensor input, Tensor weight, Tensor? bias, Tensor output_multiplier, Tensor output_shift, int input_offset, int output_offset, int activation_min, int activation_max, *, Tensor(a!) out) -> Tensor(a!)
variants: function
kernels:
Expand Down
104 changes: 90 additions & 14 deletions backends/espressif/passes/lower_to_esp.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,85 @@ def _lower_avg_pool2d(program, node) -> bool:
return True


def _mean_multiplier(input_params, output_params, count):
try:
multiplier, shift = quantize_multiplier(
input_params.scale / output_params.scale / count
)
except ValueError:
return None
max_input = max(abs(-128 - input_params.zp), abs(127 - input_params.zp))
if (
shift < -30
or count > torch.iinfo(torch.int32).max // 255
or (count * max_input) << max(shift, 0)
> torch.iinfo(torch.int32).max - abs(output_params.zp)
):
return None
max_float_input = c_float(max_input * c_float(input_params.scale).value).value
rounding_bound = 1 - count * torch.finfo(torch.float32).eps
if (
rounding_bound <= 0
or max_float_input * count / rounding_bound > torch.finfo(torch.float32).max
or not math.isfinite(c_float(1.0 / output_params.scale).value)
):
return None
return multiplier, shift


def _lower_mean(program, node) -> bool:
inp = get_arg(node, "inp", torch.fx.Node)
value = inp.meta.get("val")
if (
not isinstance(value, torch.Tensor)
or value.dim() != 4
or value.dtype != torch.int8
or any(type(dim) is not int or dim <= 0 for dim in value.shape)
or value.numel() > torch.iinfo(torch.int32).max
or not value.is_contiguous()
or get_arg(node, "dtype") not in (None, torch.float32)
):
return False
dims = get_arg(node, "dim")
if (
not isinstance(dims, (tuple, list))
or len(dims) != 2
or any(type(dim) is not int or not -4 <= dim < 4 for dim in dims)
or sorted(dim % 4 for dim in dims) != [2, 3]
):
return False
input_params, output_params = (
_qparams(program, node, prefix) for prefix in ("inp", "out")
)
if input_params is None or output_params is None:
return False
batch, channels, height, width = value.shape
multiplier = _mean_multiplier(input_params, output_params, height * width)
if multiplier is None:
return False
graph = node.graph
with graph.inserting_before(node):
nhwc = graph.call_function(
exir_ops.edge.aten.permute_copy.default, (inp, [0, 2, 3, 1])
)
nhwc.meta["val"] = value.permute(0, 2, 3, 1).contiguous()
replacement = graph.call_function(
exir_ops.edge.esp_nn.quantized_mean_int8.default,
(nhwc, input_params.zp, output_params.zp, *multiplier),
)
replacement.meta = node.meta.copy()
replacement.meta["val"] = value.new_empty((batch, channels))
if get_arg(node, "keepdim"):
replacement = graph.call_function(
exir_ops.edge.aten.view_copy.default,
(replacement, [batch, channels, 1, 1]),
)
replacement.meta = node.meta.copy()
node.replace_all_uses_with(replacement)
graph.erase_node(node)
return True


def _weighted_multipliers(input_params, output_params, weight, scales, bias):
rescales = [
quantize_multiplier(input_params.scale * scale / output_params.scale)
Expand Down Expand Up @@ -682,6 +761,14 @@ def _lower_conv2d(program, node, constants, target, kernel_mode):
return True


_UNARY_LOWERINGS = {
exir_ops.edge.fused_quant.mean.dim: _lower_mean,
exir_ops.edge.fused_quant.avg_pool2d_channels_last.default: _lower_avg_pool2d,
exir_ops.edge.fused_quant.max_pool2d_with_indices_channels_last.default: _lower_max_pool2d,
**dict.fromkeys(_LUT_OPS, _lower_lut),
}


class LowerToEsp(ExportedProgramPassBase):
"""Lower supported fused-quant operators to ESP-NN."""

Expand Down Expand Up @@ -715,8 +802,9 @@ def call(self, exported_program):

modified = False
for node in list(graph.nodes):
if node.target in _LUT_OPS:
modified |= _lower_lut(exported_program, node)
unary_lowering = _UNARY_LOWERINGS.get(node.target)
if unary_lowering is not None:
modified |= unary_lowering(exported_program, node)
continue
if node.target == exir_ops.edge.fused_quant.linear.default:
modified |= _lower_linear(exported_program, node, constants)
Expand All @@ -729,18 +817,6 @@ def call(self, exported_program):
exported_program, node, constants, self.target, self.kernel_mode
)
continue
if (
node.target
== exir_ops.edge.fused_quant.avg_pool2d_channels_last.default
):
modified |= _lower_avg_pool2d(exported_program, node)
continue
if (
node.target
== exir_ops.edge.fused_quant.max_pool2d_with_indices_channels_last.default
):
modified |= _lower_max_pool2d(exported_program, node)
continue
lowering = _BINARY_LOWERINGS.get(node.target)
if lowering is None:
continue
Expand Down
6 changes: 4 additions & 2 deletions backends/espressif/test/models/test_mlperf_tiny.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,16 @@ class ModelCase:
_FQ.relu.default: 7,
_FQ.add.default: 3,
_FQ.linear.default: 1,
_FQ.mean.dim: 1,
},
{
_ESP.quantized_conv2d_int8.default: 9,
_ESP.quantized_add_int8.default: 3,
_ESP.quantized_linear_int8.default: 1,
_ESP.quantized_mean_int8.default: 1,
},
{
"aten::constant_pad_nd": 2,
"aten::mean.dim": 1,
"aten::_softmax": 1,
},
),
Expand All @@ -87,15 +88,16 @@ class ModelCase:
_FQ.convolution.default: 27,
_FQ.relu.default: 27,
_FQ.linear.default: 1,
_FQ.mean.dim: 1,
},
{
_ESP.quantized_conv2d_int8.default: 14,
_ESP.quantized_depthwise_conv2d_int8.default: 13,
_ESP.quantized_linear_int8.default: 1,
_ESP.quantized_mean_int8.default: 1,
},
{
"aten::constant_pad_nd": 5,
"aten::mean.dim": 1,
"aten::_softmax": 1,
},
),
Expand Down
Loading
Loading