Skip to content

Commit 3226d83

Browse files
authored
fix: guard the GEMM the cdist converter emits on Turing (#4645)
1 parent 3f8a73c commit 3226d83

2 files changed

Lines changed: 139 additions & 1 deletion

File tree

‎py/torch_tensorrt/dynamo/conversion/aten_ops_converters.py‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3389,7 +3389,55 @@ def aten_ops_convolution(
33893389
)
33903390

33913391

3392-
@dynamo_tensorrt_converter(torch.ops.aten._cdist_forward.default)
3392+
# Above this many rows in either operand, the p == 2 path of
3393+
# impl.normalization.cdist_forward switches from a broadcast-subtract to a matrix
3394+
# multiply. Kept deliberately in sync with the threshold in that converter.
3395+
# aten._cdist_forward(x1, x2, p, compute_mode)
3396+
_CDIST_ARG_P = 2
3397+
_CDIST_ARG_COMPUTE_MODE = 3
3398+
3399+
3400+
def cdist_forward_capability_validator(
3401+
node: Node, settings: Optional[CompilationSettings] = None
3402+
) -> bool:
3403+
"""Reject the cdist variants whose converter emits a GEMM, on Turing (SM 7.5).
3404+
3405+
The GEMM is emitted *inside* the converter, so the graph holds one
3406+
``_cdist_forward`` node and no ``mm``/``bmm`` for ``gemm_capability_validator`` to
3407+
reject, and TensorRT-RTX then fails. Which arguments emit one is decided by
3408+
``cdist_emits_matmul``, the converter's own predicate, so the two cannot drift.
3409+
"""
3410+
if not trt_rtx_targets_turing(settings):
3411+
return True
3412+
3413+
def operand_rows(operand: Argument) -> Optional[int]:
3414+
val = operand.meta.get("val") if hasattr(operand, "meta") else None
3415+
shape = getattr(val, "shape", None)
3416+
if shape is None or len(shape) < 2:
3417+
return None
3418+
rows = shape[-2]
3419+
return rows if isinstance(rows, int) else None
3420+
3421+
operands = node.args[:2] # x1, x2
3422+
if not impl.normalization.ops.cdist_emits_matmul(
3423+
args_bounds_check(node.args, _CDIST_ARG_P, replacement=2.0),
3424+
args_bounds_check(node.args, _CDIST_ARG_COMPUTE_MODE, replacement=None),
3425+
[operand_rows(operand) for operand in operands],
3426+
):
3427+
return True
3428+
3429+
_LOGGER.debug(
3430+
"cdist '%s' computes p=2 as a matrix multiply, which is not supported on "
3431+
"TensorRT-RTX for Turing (SM 7.5). Falling back to PyTorch.",
3432+
node.name,
3433+
)
3434+
return False
3435+
3436+
3437+
@dynamo_tensorrt_converter(
3438+
torch.ops.aten._cdist_forward.default,
3439+
capability_validator=cdist_forward_capability_validator,
3440+
)
33933441
def aten_ops_cdist_forward(
33943442
ctx: ConversionContext,
33953443
target: Target,

‎tests/py/dynamo/models/test_turing_capability_guards.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,30 @@ def forward(self, x):
9797
return self.conv(F.pad(x, (0, 1, 0, 1)))
9898

9999

100+
class Cdist(nn.Module):
101+
"""A GEMM the explicit GEMM guards never see, emitted inside the cdist converter.
102+
103+
The op has to be called directly: torch.cdist does not reach this converter on the
104+
GEMM path, because ATen's cdist calls _euclidean_dist there, which lowers to
105+
aten.matmul and is guarded already. Only _cdist_forward's own p == 2 branch emits a
106+
matmul layer that no mm/bmm node stands for.
107+
"""
108+
109+
def __init__(self, p=2.0, compute_mode=None):
110+
super().__init__()
111+
self.p, self.compute_mode = p, compute_mode
112+
113+
def forward(self, x1, x2):
114+
return torch.ops.aten._cdist_forward.default(x1, x2, self.p, self.compute_mode)
115+
116+
117+
def _cdist_inputs(rows1, rows2, dtype=torch.float32):
118+
return (
119+
torch.randn(4, rows1, 5, dtype=dtype).cuda(),
120+
torch.randn(4, rows2, 5, dtype=dtype).cuda(),
121+
)
122+
123+
100124
@unittest.skipIf(
101125
not ENABLED_FEATURES.tensorrt_rtx,
102126
"Turing capability guards only apply to TensorRT-RTX",
@@ -205,6 +229,59 @@ def test_declared_turing_target_keeps_transposed_conv3d_on_trt(self):
205229
compiled = self._compile(mod, inputs, target_compute_capabilities=[TURING])
206230
self.assertGreater(_trt_submodule_count(compiled), 0)
207231

232+
def test_declared_turing_target_falls_back_cdist_p2_compute_mode_1(self):
233+
# compute_mode=1 is "always use the matrix multiply", whatever the row count.
234+
mod = Cdist(compute_mode=1)
235+
inputs = _cdist_inputs(6, 8)
236+
compiled = self._compile(mod, inputs, target_compute_capabilities=[TURING])
237+
self.assertEqual(_trt_submodule_count(compiled), 0)
238+
self._assert_matches_eager(mod, compiled, inputs)
239+
240+
def test_declared_turing_target_falls_back_cdist_p2_default_compute_mode(self):
241+
# An absent compute_mode means "use the matrix multiply if either operand has
242+
# more than 25 rows". The validator has to normalise None to 0 as the converter does.
243+
mod = Cdist()
244+
inputs = _cdist_inputs(35, 45)
245+
compiled = self._compile(mod, inputs, target_compute_capabilities=[TURING])
246+
self.assertEqual(_trt_submodule_count(compiled), 0)
247+
self._assert_matches_eager(mod, compiled, inputs)
248+
249+
def test_declared_turing_target_falls_back_fp16_cdist_p2(self):
250+
# Unlike every other guard here, this one is not FP32-only: on a T4 the fused
251+
# Matmul_MUL_SUB_SQRT_ pattern fails for FP16 operands too, under every
252+
# enabled_precisions setting, even though a bare FP16 GEMM runs there fine.
253+
mod = Cdist(compute_mode=1)
254+
inputs = _cdist_inputs(6, 8, dtype=torch.half)
255+
compiled = self._compile(mod, inputs, target_compute_capabilities=[TURING])
256+
self.assertEqual(_trt_submodule_count(compiled), 0)
257+
self._assert_matches_eager(mod, compiled, inputs)
258+
259+
def test_declared_turing_target_keeps_cdist_p2_compute_mode_2_on_trt(self):
260+
# compute_mode=2 is "never use the matrix multiply", so there is no GEMM to reject.
261+
mod = Cdist(compute_mode=2)
262+
inputs = _cdist_inputs(35, 45)
263+
compiled = self._compile(mod, inputs, target_compute_capabilities=[TURING])
264+
self.assertGreater(_trt_submodule_count(compiled), 0)
265+
self._assert_matches_eager(mod, compiled, inputs)
266+
267+
def test_declared_turing_target_keeps_cdist_p2_small_rows_on_trt(self):
268+
# At or below 25 rows the default compute_mode emits no GEMM. Rejecting these
269+
# anyway would be worse than useless: PyTorch's cdist_cuda kernel has no Half
270+
# implementation, so for FP16 the fallback raises where TensorRT-RTX succeeds.
271+
mod = Cdist(compute_mode=0)
272+
inputs = _cdist_inputs(6, 8)
273+
compiled = self._compile(mod, inputs, target_compute_capabilities=[TURING])
274+
self.assertGreater(_trt_submodule_count(compiled), 0)
275+
self._assert_matches_eager(mod, compiled, inputs)
276+
277+
def test_declared_turing_target_keeps_cdist_p1_on_trt(self):
278+
# Only p == 2 has a matrix-multiply path at all.
279+
mod = Cdist(p=1.0, compute_mode=0)
280+
inputs = _cdist_inputs(35, 45)
281+
compiled = self._compile(mod, inputs, target_compute_capabilities=[TURING])
282+
self.assertGreater(_trt_submodule_count(compiled), 0)
283+
self._assert_matches_eager(mod, compiled, inputs)
284+
208285
@unittest.skipIf(
209286
_is_turing(), "on Turing the native path is already guarded; see the SM75 tests"
210287
)
@@ -387,6 +464,19 @@ def test_conv3d_falls_back(self):
387464
compiled = self._compile(mod, inputs)
388465
self.assertEqual(_trt_submodule_count(compiled), 0)
389466

467+
def test_cdist_p2_falls_back(self):
468+
# Unguarded this built an engine whose createExecutionContext() then returned
469+
# nullptr: "cuDNN graph compilation failed: No valid engine configs for
470+
# Matmul_MUL_SUB_SQRT_".
471+
mod = Cdist(compute_mode=1)
472+
inputs = _cdist_inputs(6, 8)
473+
compiled = self._compile(mod, inputs)
474+
self.assertEqual(_trt_submodule_count(compiled), 0)
475+
with torch.no_grad():
476+
ref = mod.eval().cuda()(*inputs)
477+
out = compiled(*inputs)
478+
self.assertEqual((ref - out).abs().max().item(), 0.0)
479+
390480
def test_bfloat16_falls_back_without_crashing(self):
391481
# Unguarded, compiling bfloat16 for SM 7.5 segfaulted the process.
392482
class Add(nn.Module):

0 commit comments

Comments
 (0)