Skip to content

Commit 3c2f200

Browse files
ethansfngmeta-codesync[bot]
authored andcommitted
Generalize QuantizedOutputWrapper for multi-output models (#19987)
Summary: Pull Request resolved: #19987 Add support for multiple outputs in quantized output wrapper Differential Revision: D107429509
1 parent ff90ade commit 3c2f200

1 file changed

Lines changed: 31 additions & 9 deletions

File tree

backends/cadence/aot/compiler_funcs.py

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -400,33 +400,55 @@ def sink_dequants(program: torch.export.ExportedProgram) -> None:
400400

401401
class QuantizedOutputWrapper(torch.nn.Module):
402402
"""
403-
Wrapper that quantizes a model's output so it produces uint8 tensors.
403+
Wrapper that quantizes a model's output(s) so they produce quantized tensors.
404404
405405
Mirrors QuantizedInputWrapper: the wrapper adds a quantize_per_tensor after
406-
the model's output. When the graph is traced, the dequant (from the model) →
406+
each output. When the graph is traced, the dequant (from the model) →
407407
quant (from the wrapper) pair with matching parameters folds away, leaving
408408
the output in its quantized form.
409409
410410
Args:
411411
module: The module to wrap (may already be a QuantizedInputWrapper).
412-
output_quant_args: (scale, zero_point, qmin, qmax, dtype) for the output.
412+
output_quant_args: Quantization parameters — either a single QuantArgs
413+
tuple or a list with one entry per output.
413414
"""
414415

415416
def __init__(
416417
self,
417418
module: torch.nn.Module,
418-
output_quant_args: QuantArgs,
419+
output_quant_args: Union[QuantArgs, list[QuantArgs | None]],
419420
) -> None:
420421
super().__init__()
421422
self.module: torch.nn.Module = module
422-
self.output_quant_args: QuantArgs = output_quant_args
423+
if isinstance(output_quant_args, list):
424+
self._multi_output: bool = True
425+
self._per_output_args: list[QuantArgs | None] = output_quant_args
426+
else:
427+
self._multi_output = False
428+
self._per_output_args = [output_quant_args]
423429

424430
def forward(self, *args: torch.Tensor) -> Any:
425431
result = self.module(*args)
426-
scale, zp, qmin, qmax, dtype = self.output_quant_args
427-
return torch.ops.quantized_decomposed.quantize_per_tensor.default(
428-
result, scale, zp, qmin, qmax, dtype
429-
)
432+
if not self._multi_output:
433+
qa = self._per_output_args[0]
434+
assert qa is not None
435+
scale, zp, qmin, qmax, dtype = qa
436+
return torch.ops.quantized_decomposed.quantize_per_tensor.default(
437+
result, scale, zp, qmin, qmax, dtype
438+
)
439+
out: list[torch.Tensor] = []
440+
for i, r in enumerate(result):
441+
qa = self._per_output_args[i] if i < len(self._per_output_args) else None
442+
if qa is None:
443+
out.append(r)
444+
continue
445+
scale, zp, qmin, qmax, dtype = qa
446+
out.append(
447+
torch.ops.quantized_decomposed.quantize_per_tensor.default(
448+
r, scale, zp, qmin, qmax, dtype
449+
)
450+
)
451+
return tuple(out)
430452

431453

432454
def _get_transparent_ops() -> set[Any]:

0 commit comments

Comments
 (0)