Skip to content

Commit ca5ea65

Browse files
committed
Address the third round of Claude review findings
[ghstack-poisoned]
1 parent e7a7498 commit ca5ea65

3 files changed

Lines changed: 23 additions & 2 deletions

File tree

‎extension/llm/export/gguf.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,9 @@ def to_exportable_nvfp4_tensor(
458458
``per_tensor_scale`` is the scalar from the GGUF ``<name>.scale``
459459
companion and ``act_per_tensor_scale`` the optional scalar
460460
``<name>.input_scale`` (stored as a Python float, see
461-
``ExportableNVFP4Tensor``); the caller pairs them by name.
461+
``ExportableNVFP4Tensor``); the caller pairs them by name. GGUF stores
462+
both as F32, so tensor scales must still be float32: pair them before
463+
any dtype cast (e.g. a loader's ``dtype=``), which would round them.
462464
463465
GGML packs eight lows followed by eight highs in each K16 group;
464466
torchao packs the even element in the low nibble. Scales E4M3 cannot
@@ -479,6 +481,12 @@ def to_exportable_nvfp4_tensor(
479481
raise ValueError(
480482
"GGML NVFP4 scale 0xff decodes to 480, which cannot be represented losslessly as E4M3"
481483
)
484+
scales = (per_tensor_scale, act_per_tensor_scale)
485+
if any(isinstance(s, Tensor) and s.dtype != torch.float32 for s in scales):
486+
raise ValueError(
487+
"NVFP4 per-tensor scales must be float32 (GGUF F32); a cast "
488+
"before pairing rounds them"
489+
)
482490
pts = torch.as_tensor(
483491
per_tensor_scale, dtype=torch.float32, device=self.raw.device
484492
)
@@ -678,7 +686,8 @@ def iter_gguf(
678686
as plain float tensors in their native dtype. GGUF shapes are reversed to
679687
PyTorch ``(N, K)`` convention. The per-tensor scale of an NVFP4
680688
``<name>.weight`` is yielded as its own ``<name>.scale`` F32 tensor; callers
681-
pair it by name (see :meth:`ExportableGGUFTensor.to_exportable_nvfp4_tensor`).
689+
pair it by name before any dtype cast (see
690+
:meth:`ExportableGGUFTensor.to_exportable_nvfp4_tensor`).
682691
"""
683692
from gguf import GGMLQuantizationType, GGUFReader
684693

‎extension/llm/export/test/BUCK‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ fbcode_target(_kind = runtime.python_test,
4646
"//caffe2:torch",
4747
"//executorch/extension/llm/export:gguf",
4848
"//executorch/extension/llm/export:nvfp4",
49+
"//executorch/extension/llm/export:quant",
4950
"fbsource//third-party/pypi/gguf:gguf",
5051
"fbsource//third-party/pypi/numpy:numpy",
5152
],

‎extension/llm/export/test/test_gguf_nvfp4.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,17 @@ def test_per_tensor_scale_and_dtype(self):
135135
):
136136
with self.assertRaisesRegex(ValueError, "scalars"):
137137
w.to_exportable_nvfp4_tensor(**scales)
138+
# A loader's dtype cast would round the F32 scale before pairing.
139+
for scales in (
140+
{"per_tensor_scale": torch.tensor(0.25, dtype=torch.bfloat16)},
141+
{"per_tensor_scale": ONE, "act_per_tensor_scale": ONE.bfloat16()},
142+
):
143+
with self.assertRaisesRegex(ValueError, "float32"):
144+
w.to_exportable_nvfp4_tensor(**scales)
145+
self.assert_bits_equal(
146+
w.to_exportable_nvfp4_tensor(per_tensor_scale=0.25).dequantize(),
147+
expected * 0.25,
148+
)
138149

139150
def test_act_per_tensor_scale_survives_to(self):
140151
w = ExportableGGUFTensor(

0 commit comments

Comments
 (0)