Skip to content

Commit 5e6ceb2

Browse files
tp5uiucclaude
andauthored
fix: guard the FP32 GEMMs that reach TensorRT-RTX through linear and attention (#4646)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
1 parent 3226d83 commit 5e6ceb2

8 files changed

Lines changed: 320 additions & 29 deletions

File tree

‎py/torch_tensorrt/dynamo/conversion/aten_ops_converters.py‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4551,7 +4551,11 @@ def aten_ops_nonzero(
45514551
)
45524552

45534553

4554-
@dynamo_tensorrt_converter(torch.ops.aten.linear.default, supports_dynamic_shapes=True)
4554+
@dynamo_tensorrt_converter(
4555+
torch.ops.aten.linear.default,
4556+
capability_validator=gemm_capability_validator,
4557+
supports_dynamic_shapes=True,
4558+
)
45554559
def aten_ops_linear(
45564560
ctx: ConversionContext,
45574561
target: Target,
@@ -4570,9 +4574,43 @@ def aten_ops_linear(
45704574
)
45714575

45724576

4577+
def attention_capability_validator(
4578+
node: Node, settings: Optional[CompilationSettings] = None
4579+
) -> bool:
4580+
"""Reject fused attention TensorRT-RTX cannot serve when Turing (SM 7.5) is a target.
4581+
4582+
A fused attention op carries the same FP32 GEMMs ``gemm_capability_validator``
4583+
rejects, but inside the converter, so the graph holds one node and no ``mm``/``bmm``
4584+
for that guard to see. On Turing TRT-RTX this results in failure. Only the q/k/v
4585+
dtypes matter, so FP16 attention keeps running on TensorRT.
4586+
"""
4587+
if not trt_rtx_targets_turing(settings):
4588+
return True
4589+
4590+
unsupported_dtypes = (torch.float32,)
4591+
4592+
def is_unsupported(operand: Argument) -> bool:
4593+
val = operand.meta.get("val") if hasattr(operand, "meta") else None
4594+
return bool(getattr(val, "dtype", None) in unsupported_dtypes)
4595+
4596+
qkv = node.args[:3] # query, key, value
4597+
if any(map(is_unsupported, qkv)):
4598+
_LOGGER.debug(
4599+
"Attention '%s' is not supported on TensorRT-RTX for Turing (SM 7.5). "
4600+
"Falling back to PyTorch.",
4601+
node.name,
4602+
)
4603+
return False
4604+
4605+
return True
4606+
4607+
45734608
def scaled_dot_product_attention_validator(
45744609
node: Node, settings: Optional[CompilationSettings] = None
45754610
) -> bool:
4611+
if not attention_capability_validator(node, settings):
4612+
return False
4613+
45764614
attn_mask = args_bounds_check(node.args, 3, None)
45774615
is_causal = args_bounds_check(node.args, 5, False)
45784616
if is_causal and attn_mask is not None:
@@ -4671,6 +4709,9 @@ def aten_ops_scaled_dot_product_attention(
46714709
def scaled_dot_product_flash_attention_validator(
46724710
node: Node, settings: Optional[CompilationSettings] = None
46734711
) -> bool:
4712+
if not attention_capability_validator(node, settings):
4713+
return False
4714+
46744715
if args_bounds_check(node.args, 5, False):
46754716
_LOGGER.debug("return_debug_mask is not yet supported.")
46764717
return False
@@ -4765,6 +4806,9 @@ def aten_ops_scaled_dot_product_flash_attention(
47654806
def scaled_dot_product_efficient_attention_validator(
47664807
node: Node, settings: Optional[CompilationSettings] = None
47674808
) -> bool:
4809+
if not attention_capability_validator(node, settings):
4810+
return False
4811+
47684812
if args_bounds_check(node.args, 4, False):
47694813
_LOGGER.debug("compute_log_sumexp is not yet supported.")
47704814
return False
@@ -4851,6 +4895,7 @@ def scaled_dot_product_cudnn_attention_validator(
48514895
_LOGGER.debug("return_debug_mask is not yet supported.")
48524896
return False
48534897

4898+
# Delegating also picks up the Turing (SM 7.5) dtype guard.
48544899
return scaled_dot_product_efficient_attention_validator(node, settings)
48554900

48564901

‎tests/py/dynamo/conversion/test_attention_aten.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from torch.testing._internal.common_utils import run_tests
55
from torch_tensorrt import Input
66

7-
from .harness import DispatchTestCase
7+
from .harness import DispatchTestCase, skip_if_trt_rtx_turing
88

99

1010
class TestScaledDotProductAttention(DispatchTestCase):
@@ -112,6 +112,9 @@ def test_sdpa_bool_mask(
112112
dropout_p=0.0,
113113
enable_gqa=False,
114114
):
115+
if dtype == torch.float32:
116+
skip_if_trt_rtx_turing(self, "FP32 scaled dot-product attention")
117+
115118
class SDPA(nn.Module):
116119
def forward(self, query, key, value, attn_mask=None):
117120
return torch.ops.aten.scaled_dot_product_attention.default(
@@ -247,6 +250,9 @@ def test_sdpa_fp_mask(
247250
dropout_p=0.0,
248251
enable_gqa=False,
249252
):
253+
if dtype == torch.float32:
254+
skip_if_trt_rtx_turing(self, "FP32 scaled dot-product attention")
255+
250256
class SDPA(nn.Module):
251257
def forward(self, query, key, value, attn_mask=None):
252258
return torch.ops.aten.scaled_dot_product_attention.default(
@@ -352,6 +358,9 @@ def test_dynamic_sdpa_fp_mask(
352358
dropout_p=0.0,
353359
enable_gqa=False,
354360
):
361+
if dtype == torch.float32:
362+
skip_if_trt_rtx_turing(self, "FP32 scaled dot-product attention")
363+
355364
class SDPA(nn.Module):
356365
def forward(self, query, key, value, attn_mask=None):
357366
return torch.ops.aten.scaled_dot_product_attention.default(
@@ -484,6 +493,9 @@ def test_efficient_sdpa(
484493
dtype,
485494
dropout_p=0.0,
486495
):
496+
if dtype == torch.float32:
497+
skip_if_trt_rtx_turing(self, "FP32 scaled dot-product attention")
498+
487499
class EfficientSDPA(nn.Module):
488500
def forward(self, query, key, value, attn_bias=None):
489501
attn = torch.ops.aten._scaled_dot_product_efficient_attention.default(
@@ -606,6 +618,9 @@ def test_efficient_sdpa_random_attn_bias(
606618
dtype,
607619
dropout_p=0.0,
608620
):
621+
if dtype == torch.float32:
622+
skip_if_trt_rtx_turing(self, "FP32 scaled dot-product attention")
623+
609624
class EfficientSDPA(nn.Module):
610625
def forward(self, query, key, value, attn_bias=None):
611626
attn = torch.ops.aten._scaled_dot_product_efficient_attention.default(

‎tests/py/dynamo/conversion/test_linear_aten.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from torch.testing._internal.common_utils import run_tests
55
from torch_tensorrt import Input
66

7-
from .harness import DispatchTestCase
7+
from .harness import DispatchTestCase, skip_if_trt_rtx_turing
88

99

1010
class TestLinearConverter(DispatchTestCase):
@@ -17,6 +17,8 @@ class TestLinearConverter(DispatchTestCase):
1717
]
1818
)
1919
def test_linear_converter(self, in_features, out_features):
20+
skip_if_trt_rtx_turing(self, "nn.Linear (an FP32 GEMM)")
21+
2022
class LinearModel(nn.Module):
2123
def __init__(self, in_features, out_features):
2224
super(LinearModel, self).__init__()
@@ -30,6 +32,8 @@ def forward(self, x):
3032
self.run_test(model, inputs, use_dynamo_tracer=True, enable_passes=True)
3133

3234
def test_linear_with_dynamic_shape(self):
35+
skip_if_trt_rtx_turing(self, "aten.linear (an FP32 GEMM)")
36+
3337
class LinearModel(torch.nn.Module):
3438
def forward(self, x, weight, bias):
3539
return torch.ops.aten.linear.default(x, weight, bias)
@@ -50,6 +54,8 @@ def forward(self, x, weight, bias):
5054
)
5155

5256
def test_linear_with_rank_3_input_and_bias(self):
57+
skip_if_trt_rtx_turing(self, "aten.linear (an FP32 GEMM)")
58+
5359
class LinearModel(torch.nn.Module):
5460
def forward(self, x, weight, bias):
5561
return torch.ops.aten.linear.default(x, weight, bias)

‎tests/py/dynamo/models/test_export_kwargs_serde.py‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import torch.nn.functional as F
88
import torch_tensorrt as torchtrt
99
from torch import nn
10+
from torch_tensorrt._utils import trt_rtx_targets_turing
1011
from torch_tensorrt.dynamo._compiler import (
1112
convert_exported_program_to_serialized_trt_engine,
1213
)
@@ -19,6 +20,25 @@
1920
assertions = unittest.TestCase()
2021

2122

23+
def _no_fallback_gemm_dtype() -> torch.dtype:
24+
"""Model dtype for the tests that convert a whole program with no partitioner.
25+
26+
TensorRT-RTX cannot serve an FP32 GEMM on Turing (SM 7.5), so
27+
``gemm_capability_validator`` rejects the models' trailing ``nn.Linear`` there. In a
28+
normal ``torchtrt.dynamo.compile`` the partitioner turns that rejection into a
29+
PyTorch block, but ``convert_exported_program_to_serialized_trt_engine`` emits one
30+
engine for the whole program by design and has nothing to fall back to, so the
31+
rejection raises ``UnsupportedOperatorException`` instead.
32+
33+
These tests are about kwarg plumbing, not about FP32, so follow the capability
34+
rather than switching them off: the guard keys on operand dtype only and Turing has
35+
FP16 GEMM hardware. Keyed off the same predicate the validator uses so the two
36+
cannot drift, and no ``enabled_precisions`` change is needed because the network is
37+
strongly typed.
38+
"""
39+
return torch.float16 if trt_rtx_targets_turing() else torch.float32
40+
41+
2242
@pytest.mark.unit
2343
@pytest.mark.critical
2444
def test_custom_model(tmpdir):
@@ -496,8 +516,9 @@ def forward(self, x, b=5, c=None, d=None):
496516
x = x - d["value"]
497517
return self.fc1(x)
498518

499-
model = net().eval().to("cuda")
500-
args = [torch.rand((1, 3, 224, 224)).to("cuda")]
519+
dtype = _no_fallback_gemm_dtype()
520+
model = net().eval().to("cuda").to(dtype)
521+
args = [torch.rand((1, 3, 224, 224), dtype=dtype).to("cuda")]
501522
kwargs = {
502523
"b": torch.tensor(6).to("cuda"),
503524
"d": {"value": torch.tensor(8).to("cuda")},
@@ -1090,9 +1111,10 @@ def forward(self, x, b=5, c=None, d=None):
10901111
x = x - d["value"]
10911112
return self.fc1(x)
10921113

1093-
model = net().eval().to("cuda")
1114+
dtype = _no_fallback_gemm_dtype()
1115+
model = net().eval().to("cuda").to(dtype)
10941116
kwargs = {
1095-
"x": torch.rand((1, 3, 224, 224)).to("cuda"),
1117+
"x": torch.rand((1, 3, 224, 224), dtype=dtype).to("cuda"),
10961118
"b": torch.tensor(6).to("cuda"),
10971119
"d": {"value": torch.tensor(8).to("cuda")},
10981120
}

‎tests/py/dynamo/models/test_model_refit.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import torch_tensorrt as torch_trt
1212
from torch import nn
1313
from torch_tensorrt._utils import is_tensorrt_rtx_version_supported
14-
from torch_tensorrt.dynamo import refit_module_weights
14+
from torch_tensorrt.dynamo import partitioning, refit_module_weights
1515
from torch_tensorrt.dynamo._refit import (
1616
construct_refit_mapping,
1717
get_engine_from_encoded_engine,
@@ -68,7 +68,26 @@ def test_mapping():
6868
)
6969
new_gm = exp_program2.module()
7070
new_gm = post_lowering(new_gm, settings)
71-
mapping = construct_refit_mapping(new_gm, trt_input, settings)
71+
72+
# construct_refit_mapping interprets whatever module it is handed, so it has to be
73+
# handed the same subgraph the engine above was built from. refit_module_weights
74+
# always partitions first and maps each accelerated submodule; this test was the
75+
# only caller passing a whole, un-partitioned module. That only worked because the
76+
# model happened to be fully convertible: on Turing (SM 7.5) TensorRT-RTX cannot
77+
# serve resnet18's FP32 fc GEMM, so capability partitioning routes it to PyTorch and
78+
# the engine does not contain it -- while the un-partitioned module still does, with
79+
# no partitioner in construct_refit_mapping to fall back to. Partition here the same
80+
# way compile_module does, so the mapping matches the engine on either architecture.
81+
num_supported_ops, total_ops = partitioning.get_graph_converter_support(
82+
new_gm, settings.torch_executed_ops
83+
)
84+
partitioned_gm, _ = partitioning.fast_partition(
85+
new_gm,
86+
min_block_size=min_block_size,
87+
torch_executed_ops=settings.torch_executed_ops,
88+
skip_fusion=(num_supported_ops == total_ops),
89+
)
90+
mapping = construct_refit_mapping(partitioned_gm._run_on_acc_0, trt_input, settings)
7291

7392
refitter = trt.Refitter(engine, TRT_LOGGER)
7493
weight_list = refitter.get_all_weights()

0 commit comments

Comments
 (0)