Skip to content

Commit 616a0b4

Browse files
Check quantized embedding qparam dtypes against how the kernels read them (#23912)
### Summary The reference ops in exir/passes/_quant_patterns_and_replacements.py require weight_zero_points to share weight_scales' dtype, and the variants without a dtype argument return the scales' dtype. The kernels checked the zero points against the output dtype instead (while reading them as the scales' type), and the .out variants read the scales as the output type without checking it. So: - embedding_byte/2bit/4bit.dtype_out with fp16 scales and zero points and a fp32 output (valid per the reference) aborts; - fp32 zero points with fp16 scales are accepted and reinterpreted as fp16; - embedding_byte/2bit/4bit.out with fp16 scales and a fp32 output reads past the end of the scales (AddressSanitizer: heap-buffer-overflow). This checks the zero points against the scales, and the scales against the output in the .out variants. Related to #23677 (index bounds check in the same kernels); the two changes are independent and merge cleanly in either order. ### Test plan Built quantized_kernels_test with CMake (Debug, -fsanitize=address). - With the change: 86/86 quantized kernel tests pass (88/88 together with #23677). - Without it, all 6 new tests fail: the two fp16-qparams/fp32-output tests abort, the two zero-point mismatch tests fail to die, and the two .out mismatch tests die in AddressSanitizer (heap-buffer-overflow) instead of the dtype check. - Python reference cross-check: embedding_byte.dtype with fp16 scales and zero points and dtype=float32 returns [[3.5, 2], [5.5, 13]], the value the new test expects; fp32 zero points with fp16 scales raise an assertion. Authored with AI assistance. I reviewed the change and ran the tests myself. cc @larryliu0820 @manuelcandales @JakeStevens @nil-is-all
1 parent 28d872e commit 616a0b4

6 files changed

Lines changed: 205 additions & 6 deletions

File tree

‎exir/passes/_quant_patterns_and_replacements.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,8 @@ def _(
298298
num_embeddings, packed_embedding_dim = weight.shape
299299
embedding_dim = packed_embedding_dim * 4
300300
embedding = torch.nn.Embedding(num_embeddings, embedding_dim, device=weight.device)
301-
return embedding(indices)
301+
# As in the reference op, the variant without dtype returns the scales' dtype.
302+
return embedding(indices).to(weight_scales.dtype)
302303

303304

304305
@register_fake("quantized_decomposed::embedding_2bit.out")
@@ -473,7 +474,8 @@ def _(
473474
num_embeddings, packed_embedding_dim = weight.shape
474475
embedding_dim = packed_embedding_dim * 2
475476
embedding = torch.nn.Embedding(num_embeddings, embedding_dim, device=weight.device)
476-
return embedding(indices)
477+
# As in the reference op, the variant without dtype returns the scales' dtype.
478+
return embedding(indices).to(weight_scales.dtype)
477479

478480

479481
@register_fake("quantized_decomposed::embedding_4bit.out")

‎exir/tests/test_quant_fusion_pass.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
QuantFusionPass,
2020
)
2121
from executorch.exir.tests.common import register_additional_test_aten_ops
22+
from torch._subclasses.fake_tensor import FakeTensorMode
2223
from torch.ao.quantization import ( # @manual
2324
float_qparams_weight_only_qconfig,
2425
get_default_qconfig_mapping,
@@ -381,6 +382,34 @@ def forward(self, indices):
381382
# m.dump_graph_module().code
382383
# )
383384

385+
def test_embedding_xbit_fake_dtype(self) -> None:
386+
# The variants without a dtype argument return the scales' dtype; their
387+
# fake kernels must report the same dtype, or export records the wrong one.
388+
indices = torch.tensor([1, 2, 3])
389+
for op, values_per_byte, quant_min, quant_max in (
390+
(torch.ops.quantized_decomposed.embedding_2bit.default, 4, -2, 1),
391+
(torch.ops.quantized_decomposed.embedding_4bit.default, 2, -8, 7),
392+
):
393+
for scales_dtype in (torch.float16, torch.bfloat16, torch.float32):
394+
with self.subTest(op=str(op), scales_dtype=scales_dtype):
395+
weight = torch.randint(
396+
0, 256, (10, 32 // values_per_byte), dtype=torch.uint8
397+
)
398+
scales = torch.rand(10, dtype=scales_dtype) + 0.5
399+
expected = op(weight, scales, None, quant_min, quant_max, indices)
400+
with FakeTensorMode() as mode:
401+
actual = op(
402+
mode.from_tensor(weight),
403+
mode.from_tensor(scales),
404+
None,
405+
quant_min,
406+
quant_max,
407+
mode.from_tensor(indices),
408+
)
409+
self.assertEqual(expected.dtype, scales_dtype)
410+
self.assertEqual(actual.dtype, expected.dtype)
411+
self.assertEqual(actual.shape, expected.shape)
412+
384413
def test_embedding_torchao(self) -> None:
385414
for bit_width, use_dtype_variant, test_per_group, mapping_type in zip(
386415
[2, 4, 8],

‎kernels/quantized/cpu/embeddingxb.cpp‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,10 @@ void check_embedding_xbit_args(
125125
static_cast<int8_t>(weight_scales.dim()));
126126

127127
ET_CHECK_MSG(
128-
opt_weight_zero_points.value().scalar_type() == out.scalar_type(),
128+
opt_weight_zero_points.value().scalar_type() ==
129+
weight_scales.scalar_type(),
129130
"weight zero points scalar type %" PRId8
130-
" does not match out.scalar_type()",
131+
" does not match weight_scales.scalar_type()",
131132
static_cast<int8_t>(opt_weight_zero_points.value().scalar_type()));
132133

133134
for (int32_t i = 0; i < weight_scales.dim(); ++i) {
@@ -290,6 +291,15 @@ Tensor& quantized_embedding_xbit_out(
290291
out,
291292
weight_nbit);
292293

294+
// Without a dtype argument the output takes the scales' dtype, and the
295+
// kernel reads the scales as the output type.
296+
ET_CHECK_MSG(
297+
weight_scales.scalar_type() == out_type,
298+
"weight_scales.scalar_type() %" PRId8
299+
" does not match out.scalar_type() %" PRId8,
300+
static_cast<int8_t>(weight_scales.scalar_type()),
301+
static_cast<int8_t>(out_type));
302+
293303
constexpr auto name = "quantized_decomposed::embedding_xbit.out";
294304
ScalarType indices_type = indices.scalar_type();
295305
ET_SWITCH_THREE_TYPES(

‎kernels/quantized/cpu/op_embedding.cpp‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,10 @@ void check_embedding_byte_args(
8787
static_cast<int8_t>(weight_scales.dim()));
8888

8989
ET_CHECK_MSG(
90-
opt_weight_zero_points.value().scalar_type() == out.scalar_type(),
90+
opt_weight_zero_points.value().scalar_type() ==
91+
weight_scales.scalar_type(),
9192
"weight zero points scalar type %" PRId8
92-
" does not match out.scalar_type()",
93+
" does not match weight_scales.scalar_type()",
9394
static_cast<int8_t>(opt_weight_zero_points.value().scalar_type()));
9495

9596
for (int32_t i = 0; i < weight_scales.dim(); ++i) {
@@ -259,6 +260,15 @@ Tensor& quantized_embedding_byte_out(
259260
out_type,
260261
out);
261262

263+
// Without a dtype argument the output takes the scales' dtype, and the
264+
// kernel reads the scales as the output type.
265+
ET_CHECK_MSG(
266+
weight_scales.scalar_type() == out_type,
267+
"weight_scales.scalar_type() %" PRId8
268+
" does not match out.scalar_type() %" PRId8,
269+
static_cast<int8_t>(weight_scales.scalar_type()),
270+
static_cast<int8_t>(out_type));
271+
262272
constexpr auto name = "quantized_decomposed::embedding_byte.out";
263273
ET_SWITCH_TWO_TYPES(Byte, Char, w_type, ctx, name, CTYPE_W, [&]() {
264274
ET_SWITCH_THREE_TYPES(

‎kernels/quantized/test/op_embedding4b_test.cpp‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,79 @@ TEST(OpQuantizedEmbedding4bTest, TestGroupWiseQuantizedEmbeddingBFloat16) {
216216
EXPECT_TENSOR_CLOSE(out, expected);
217217
}
218218

219+
TEST(OpQuantizedEmbedding4bTest, DtypeOutHalfQParamsFloatOut) {
220+
et_pal_init();
221+
TensorFactory<ScalarType::Byte> tfb;
222+
TensorFactory<ScalarType::Half> tfh;
223+
TensorFactory<ScalarType::Float> tf;
224+
TensorFactory<ScalarType::Long> tfl;
225+
226+
Tensor weight_scales = tfh.make({3}, {0.5, 1.0, 1.5});
227+
Tensor weight_zero_points = tfh.make({3}, {1, -5, 0});
228+
Tensor qweight = tfb.make({3, 2}, {89, 239, 163, 72, 11, 126});
229+
Tensor indices = tfl.make({3}, {0, 2, 1});
230+
231+
Tensor out = tf.zeros({3, 4});
232+
Tensor expected = tf.make(
233+
{3, 4}, {-2.0, 0.0, 2.5, 3.0, -12.0, 4.5, -1.5, 9.0, 7.0, 0.0, 1.0, 5.0});
234+
235+
quantized_embedding_4bit_dtype_out(
236+
qweight,
237+
weight_scales,
238+
weight_zero_points,
239+
-8,
240+
7,
241+
indices,
242+
ScalarType::Float,
243+
out);
244+
245+
EXPECT_TENSOR_CLOSE(out, expected);
246+
}
247+
248+
TEST(OpQuantizedEmbedding4bTest, DtypeOutZeroPointsScalesMismatchDies) {
249+
et_pal_init();
250+
TensorFactory<ScalarType::Byte> tfb;
251+
TensorFactory<ScalarType::Half> tfh;
252+
TensorFactory<ScalarType::Float> tf;
253+
TensorFactory<ScalarType::Long> tfl;
254+
255+
Tensor weight_scales = tfh.make({3}, {0.5, 1.0, 1.5});
256+
Tensor weight_zero_points = tf.make({3}, {1, -5, 0});
257+
Tensor qweight = tfb.make({3, 2}, {89, 239, 163, 72, 11, 126});
258+
Tensor indices = tfl.make({3}, {0, 2, 1});
259+
260+
Tensor out = tf.zeros({3, 4});
261+
ET_EXPECT_DEATH(
262+
quantized_embedding_4bit_dtype_out(
263+
qweight,
264+
weight_scales,
265+
weight_zero_points,
266+
-8,
267+
7,
268+
indices,
269+
ScalarType::Float,
270+
out),
271+
"does not match weight_scales");
272+
}
273+
274+
TEST(OpQuantizedEmbedding4bTest, OutScalesOutMismatchDies) {
275+
et_pal_init();
276+
TensorFactory<ScalarType::Byte> tfb;
277+
TensorFactory<ScalarType::Half> tfh;
278+
TensorFactory<ScalarType::Float> tf;
279+
TensorFactory<ScalarType::Long> tfl;
280+
281+
Tensor weight_scales = tfh.make({3}, {0.5, 1.0, 1.5});
282+
Tensor qweight = tfb.make({3, 2}, {89, 239, 163, 72, 11, 126});
283+
Tensor indices = tfl.make({3}, {0, 2, 1});
284+
285+
Tensor out = tf.zeros({3, 4});
286+
ET_EXPECT_DEATH(
287+
quantized_embedding_4bit_out(
288+
qweight, weight_scales, std::nullopt, -8, 7, indices, out),
289+
"does not match out.scalar_type");
290+
}
291+
219292
TEST(OpQuantizedEmbedding4bTest, TestGroupWiseQuantizedEmbeddingDeath2) {
220293
et_pal_init();
221294
TensorFactory<ScalarType::Byte> tfb;

‎kernels/quantized/test/op_embedding_test.cpp‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -510,3 +510,78 @@ TEST(OpQuantizedEmbeddingTest, BFloat16Rounding) {
510510

511511
EXPECT_TENSOR_CLOSE_WITH_TOL(out, expected, 1e-2, 1e-2);
512512
}
513+
514+
// Zero points share the scales' dtype, which may differ from the output dtype.
515+
TEST(OpQuantizedEmbeddingTest, DtypeOutHalfQParamsFloatOut) {
516+
et_pal_init();
517+
TensorFactory<ScalarType::Half> tfh;
518+
TensorFactory<ScalarType::Float> tf;
519+
TensorFactory<ScalarType::Byte> tfb;
520+
TensorFactory<ScalarType::Long> tf_l;
521+
522+
Tensor weight_scales = tfh.full({3}, 0.5);
523+
Tensor weight_zero_points = tfh.full({3}, 1);
524+
Tensor qweight = tfb.make({3, 2}, {8, 5, 9, 3, 12, 27});
525+
Tensor indices = tf_l.make({2}, {0, 2});
526+
527+
Tensor out = tf.zeros({2, 2});
528+
Tensor expected = tf.make({2, 2}, {3.5, 2, 5.5, 13});
529+
530+
quantized_embedding_byte_dtype_out(
531+
qweight,
532+
weight_scales,
533+
weight_zero_points,
534+
0,
535+
255,
536+
indices,
537+
ScalarType::Float,
538+
out);
539+
540+
EXPECT_TENSOR_CLOSE(out, expected);
541+
}
542+
543+
TEST(OpQuantizedEmbeddingTest, DtypeOutZeroPointsScalesMismatchDies) {
544+
et_pal_init();
545+
TensorFactory<ScalarType::Half> tfh;
546+
TensorFactory<ScalarType::Float> tf;
547+
TensorFactory<ScalarType::Byte> tfb;
548+
TensorFactory<ScalarType::Long> tf_l;
549+
550+
Tensor weight_scales = tfh.full({3}, 0.5);
551+
Tensor weight_zero_points = tf.full({3}, 1);
552+
Tensor qweight = tfb.make({3, 2}, {8, 5, 9, 3, 12, 27});
553+
Tensor indices = tf_l.make({2}, {0, 2});
554+
555+
Tensor out = tf.zeros({2, 2});
556+
ET_EXPECT_DEATH(
557+
quantized_embedding_byte_dtype_out(
558+
qweight,
559+
weight_scales,
560+
weight_zero_points,
561+
0,
562+
255,
563+
indices,
564+
ScalarType::Float,
565+
out),
566+
"does not match weight_scales");
567+
}
568+
569+
// Without a dtype argument the scales are read as the output type, so a
570+
// narrower scales tensor would be read past its end.
571+
TEST(OpQuantizedEmbeddingTest, OutScalesOutMismatchDies) {
572+
et_pal_init();
573+
TensorFactory<ScalarType::Half> tfh;
574+
TensorFactory<ScalarType::Float> tf;
575+
TensorFactory<ScalarType::Byte> tfb;
576+
TensorFactory<ScalarType::Long> tf_l;
577+
578+
Tensor weight_scales = tfh.full({3}, 0.5);
579+
Tensor qweight = tfb.make({3, 2}, {8, 5, 9, 3, 12, 27});
580+
Tensor indices = tf_l.make({2}, {0, 2});
581+
582+
Tensor out = tf.zeros({2, 2});
583+
ET_EXPECT_DEATH(
584+
quantized_embedding_byte_out(
585+
qweight, weight_scales, std::nullopt, 0, 255, indices, out),
586+
"does not match out.scalar_type");
587+
}

0 commit comments

Comments
 (0)