Skip to content

Enable 16-bit activations and 8-bit weights in Cadence Quantizer for Conv #15928

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+527 −9
Open

Enable 16-bit activations and 8-bit weights in Cadence Quantizer for Conv #15928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
73cf111
2/n Enable 16-bit activations and 8-bit weights in Cadence Quantizer …
RahulC7 Nov 20, 2025
e064f8e
Enable 16-bit activations and 8-bit weights in Cadence Quantizer for …
RahulC7 Nov 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions backends/cadence/aot/quantizer/quantizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,3 +372,17 @@ def __init__(self, quantizers: Optional[list[Quantizer]] = None) -> None:
# Add 16-bit quantizers for LinearPattern
quantizers.append(CadenceAtenQuantizer(LinearPattern(), qconfig_A16))
super().__init__(quantizers)


class CadenceWith16BitConvActivationsQuantizer(CadenceQuantizer):
"""
Quantizer including A16 conv
"""

def __init__(self, quantizers: Optional[list[Quantizer]] = None) -> None:
if quantizers is None:
quantizers = []
# Add 16-bit quantizers for Conv patterns
quantizers.append(CadenceAtenQuantizer(Conv1dPattern(), qconfig_A16))
quantizers.append(CadenceAtenQuantizer(Conv2dPattern(), qconfig_A16))
super().__init__(quantizers)
49 changes: 49 additions & 0 deletions backends/cadence/hifi/operators/op_quantized_conv2d_nchw_out.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <executorch/backends/cadence/hifi/kernels/kernels.h>
#include <executorch/backends/cadence/hifi/operators/operators.h>
#include <executorch/runtime/kernel/kernel_includes.h>
#include <on_device_ai/Assistant/Jarvis/min_runtime/operators/generic/operators.h>

#define ALIGN_PTR(x, bytes) ((((unsigned)(x)) + (bytes - 1)) & (~(bytes - 1)))

Expand Down Expand Up @@ -532,6 +533,30 @@ void quantized_conv2d_nchw_out(
__ET_UNUSED const Tensor& out_multiplier,
__ET_UNUSED const Tensor& out_shift,
Tensor& out) {
// Handle W8A16 heterogeneous type (int16_t activations, int8_t weights)
if (out.scalar_type() == ::executorch::aten::ScalarType::Short &&
input.scalar_type() == ::executorch::aten::ScalarType::Short &&
weight.scalar_type() == ::executorch::aten::ScalarType::Char) {
::impl::generic::native::quantized_conv2d_nchw_out(
ctx,
input,
weight,
bias,
stride,
padding,
dilation,
groups,
in_zero_point,
weight_zero_point,
bias_scale,
output_scale,
output_zero_point,
out_multiplier,
out_shift,
out);
return;
}

const float bias_scale_float = bias_scale.const_data_ptr<float>()[0];
const int32_t weight_zero_point_int =
weight_zero_point.const_data_ptr<int32_t>()[0];
Expand Down Expand Up @@ -596,6 +621,30 @@ void quantized_conv2d_nchw_per_tensor_out(
__ET_UNUSED int64_t out_multiplier,
__ET_UNUSED int64_t out_shift,
Tensor& out) {
// Handle W8A16 heterogeneous type (int16_t activations, int8_t weights)
if (out.scalar_type() == ::executorch::aten::ScalarType::Short &&
input.scalar_type() == ::executorch::aten::ScalarType::Short &&
weight.scalar_type() == ::executorch::aten::ScalarType::Char) {
::impl::generic::native::quantized_conv2d_nchw_per_tensor_out(
ctx,
input,
weight,
bias,
stride,
padding,
dilation,
groups,
in_zero_point,
weight_zero_point,
bias_scale,
output_scale,
output_zero_point,
out_multiplier,
out_shift,
out);
return;
}

bool optimized = 0;

if ((input.scalar_type() == ScalarType::Char) ||
Expand Down
49 changes: 49 additions & 0 deletions backends/cadence/hifi/operators/op_quantized_conv2d_nhwc_out.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <executorch/backends/cadence/hifi/kernels/kernels.h>
#include <executorch/backends/cadence/hifi/operators/operators.h>
#include <executorch/runtime/kernel/kernel_includes.h>
#include <on_device_ai/Assistant/Jarvis/min_runtime/operators/generic/operators.h>

#define ALIGN_PTR(x, bytes) ((((unsigned)(x)) + (bytes - 1)) & (~(bytes - 1)))

Expand Down Expand Up @@ -438,6 +439,30 @@ void quantized_conv2d_nhwc_out(
__ET_UNUSED const Tensor& out_multiplier,
__ET_UNUSED const Tensor& out_shift,
Tensor& out) {
// Handle W8A16 heterogeneous type (int16_t activations, int8_t weights)
if (out.scalar_type() == ::executorch::aten::ScalarType::Short &&
input.scalar_type() == ::executorch::aten::ScalarType::Short &&
weight.scalar_type() == ::executorch::aten::ScalarType::Char) {
::impl::generic::native::quantized_conv2d_nhwc_out(
ctx,
input,
weight,
bias,
stride,
padding,
dilation,
groups,
in_zero_point,
weight_zero_point,
bias_scale,
output_scale,
output_zero_point,
out_multiplier,
out_shift,
out);
return;
}

const float bias_scale_float = bias_scale.const_data_ptr<float>()[0];
const int32_t weight_zero_point_int =
weight_zero_point.const_data_ptr<int32_t>()[0];
Expand Down Expand Up @@ -502,6 +527,30 @@ void quantized_conv2d_nhwc_per_tensor_out(
__ET_UNUSED int64_t out_multiplier,
__ET_UNUSED int64_t out_shift,
Tensor& out) {
// Handle W8A16 heterogeneous type (int16_t activations, int8_t weights)
if (out.scalar_type() == ::executorch::aten::ScalarType::Short &&
input.scalar_type() == ::executorch::aten::ScalarType::Short &&
weight.scalar_type() == ::executorch::aten::ScalarType::Char) {
::impl::generic::native::quantized_conv2d_nhwc_per_tensor_out(
ctx,
input,
weight,
bias,
stride,
padding,
dilation,
groups,
in_zero_point,
weight_zero_point,
bias_scale,
output_scale,
output_zero_point,
out_multiplier,
out_shift,
out);
return;
}

bool optimized = 0;

if ((input.scalar_type() == ScalarType::Char) ||
Expand Down
43 changes: 39 additions & 4 deletions backends/cadence/hifi/operators/op_quantized_linear_out.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <executorch/backends/cadence/hifi/kernels/kernels.h>
#include <executorch/backends/cadence/hifi/operators/operators.h>
#include <executorch/runtime/kernel/kernel_includes.h>
#include <on_device_ai/Assistant/Jarvis/min_runtime/operators/generic/operators.h>
#include <xa_nnlib_kernels_api.h>
#include <xtensa/tie/xt_datacache.h>
#include <algorithm>
Expand Down Expand Up @@ -207,7 +208,7 @@ void inline _quantized_linear_per_tensor_asym8s(
}

void quantized_linear_out(
__ET_UNUSED KernelRuntimeContext& ctx,
KernelRuntimeContext& ctx,
const Tensor& in,
const Tensor& weight,
const Tensor& bias,
Expand All @@ -216,9 +217,26 @@ void quantized_linear_out(
const Tensor& out_multiplier,
const Tensor& out_shift,
int64_t out_zero_point,
__ET_UNUSED const optional<Tensor>& offset,
const optional<Tensor>& offset,
Tensor& out) {
Copy link

Copilot AI Nov 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a comment before this if-block to explain the purpose, similar to the conv2d implementations. For example: // Handle W8A16 heterogeneous type (int16_t activations, int8_t weights). This improves code consistency and readability.

Suggested change
Tensor& out) {
Tensor& out) {
// Handle W8A16 heterogeneous type (int16_t activations and output, int8_t weights)

Copilot uses AI. Check for mistakes.
if (out.scalar_type() == executorch::aten::ScalarType::Byte) {
if (out.scalar_type() == ::executorch::aten::ScalarType::Short &&
in.scalar_type() == ::executorch::aten::ScalarType::Short &&
weight.scalar_type() == ::executorch::aten::ScalarType::Char) {
::impl::generic::native::quantized_linear_out(
ctx,
in,
weight,
bias,
in_zero_point,
weight_zero_point,
out_multiplier,
out_shift,
out_zero_point,
offset,
out);
}
Copy link

Copilot AI Nov 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing return statement after handling the int16 case. Without a return, execution will fall through to the subsequent else if checks, potentially executing the wrong code path or triggering an incorrect error message. Add return; after line 236.

Suggested change
}
return;

Copilot uses AI. Check for mistakes.

else if (out.scalar_type() == executorch::aten::ScalarType::Byte) {
_quantized_linear_asym8u(
in,
weight,
Expand Down Expand Up @@ -260,7 +278,24 @@ void quantized_linear_per_tensor_out(
int64_t out_zero_point,
__ET_UNUSED const optional<Tensor>& offset,
Tensor& out) {
Copy link

Copilot AI Nov 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a comment before this if-block to explain the purpose, similar to the conv2d implementations. For example: // Handle W8A16 heterogeneous type (int16_t activations, int8_t weights). This improves code consistency and readability.

Suggested change
Tensor& out) {
Tensor& out) {
// Handle W8A16 heterogeneous type (int16_t activations, int8_t weights)

Copilot uses AI. Check for mistakes.
Comment on lines 278 to 280
Copy link

Copilot AI Nov 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The __ET_UNUSED annotations on the ctx parameter (line 270) and offset parameter (line 279) are now incorrect since both are used in the int16 case (lines 285 and 294). Remove the __ET_UNUSED annotations from these parameters.

Copilot uses AI. Check for mistakes.
if (out.scalar_type() == executorch::aten::ScalarType::Byte) {
if (out.scalar_type() == ::executorch::aten::ScalarType::Short &&
in.scalar_type() == ::executorch::aten::ScalarType::Short &&
weight.scalar_type() == ::executorch::aten::ScalarType::Char) {
::impl::generic::native::quantized_linear_per_tensor_out(
ctx,
in,
weight,
bias,
in_zero_point,
weight_zero_point,
out_multiplier,
out_shift,
out_zero_point,
offset,
out);
}
Copy link

Copilot AI Nov 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing return statement after handling the int16 case. Without a return, execution will fall through to the subsequent else if checks, potentially executing the wrong code path or triggering an incorrect error message. Add return; after line 295.

Suggested change
}
return;

Copilot uses AI. Check for mistakes.

else if (out.scalar_type() == executorch::aten::ScalarType::Byte) {
_quantized_linear_per_tensor_asym8u(
in,
weight,
Expand Down
11 changes: 8 additions & 3 deletions backends/cadence/hifi/operators/targets.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ OPERATORS = [
"ne",
"permute_copy",
"pow",
"quantized_conv2d_nchw_out",
"quantized_conv2d_nchw_asym8sxsym8s_asym8s_per_tensor_out",
"quantized_conv2d_nchw_asym8uxsym8u_asym8u_per_tensor_out",
"quantized_conv1d_ncl_asym8sxsym8s_asym8s_per_tensor_out",
Expand All @@ -74,7 +73,6 @@ OPERATORS = [
"quantized_conv2d_nchw_depthwise_asym8uxsym8u_asym8u_per_tensor_out",
"quantized_conv2d_nchw_dilated_asym8sxsym8s_asym8s_per_tensor_out",
"quantized_conv2d_nchw_dilated_asym8uxsym8u_asym8u_per_tensor_out",
"quantized_conv2d_nhwc_out",
"quantized_conv2d_nhwc_asym8sxsym8s_asym8s_per_tensor_out",
"quantized_conv2d_nhwc_asym8uxsym8u_asym8u_per_tensor_out",
"quantized_conv1d_nlc_asym8sxsym8s_asym8s_per_tensor_out",
Expand All @@ -87,7 +85,6 @@ OPERATORS = [
"quantized_fully_connected_asym8sxasym8s_asym8s_per_tensor_out",
"quantized_fully_connected_asym8uxasym8u_asym8u_per_tensor_out",
"quantized_layer_norm",
"quantized_linear_out",
"quantized_linear_asym8sxasym8s_asym8s_per_tensor_out",
"quantized_linear_asym8uxasym8u_asym8u_per_tensor_out",
"quantized_matmul_out",
Expand Down Expand Up @@ -122,3 +119,11 @@ def define_common_targets():
# Define build targets for all operators registered in the tables above.
for op in OPERATORS:
define_operator(op)

# quantized_linear_out and quantized_linear_per_tensor_out needs additional dependency for int16 support
define_operator("quantized_linear_out", deps=["fbcode//on_device_ai/Assistant/Jarvis/min_runtime/operators:quantize_linear_out", "fbcode//on_device_ai/Assistant/Jarvis/min_runtime/operators:headers",])
define_operator("quantized_linear_per_tensor_out", deps=["fbcode//on_device_ai/Assistant/Jarvis/min_runtime/operators:quantize_linear_out", "fbcode//on_device_ai/Assistant/Jarvis/min_runtime/operators:headers",])

# quantized_conv2d_nchw_out and quantized_conv2d_nhwc_out need additional dependency for int16 support
define_operator("quantized_conv2d_nchw_out", deps=["fbcode//on_device_ai/Assistant/Jarvis/min_runtime/operators:quantize_conv2d_out", "fbcode//on_device_ai/Assistant/Jarvis/min_runtime/operators:headers",])
define_operator("quantized_conv2d_nhwc_out", deps=["fbcode//on_device_ai/Assistant/Jarvis/min_runtime/operators:quantize_conv2d_out", "fbcode//on_device_ai/Assistant/Jarvis/min_runtime/operators:headers",])
Loading
Loading