Skip to content

Commit 4c07205

Browse files
authored
fix(dynamo): gate refittable cumsum on RTX version (#4575)
1 parent 47e8689 commit 4c07205

3 files changed

Lines changed: 90 additions & 32 deletions

File tree

‎py/torch_tensorrt/dynamo/conversion/aten_ops_converters.py‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@
2121
from torch.fx.node import Argument, Node, Target
2222
from torch_tensorrt import ENABLED_FEATURES
2323
from torch_tensorrt._features import needs_not_tensorrt_rtx
24-
from torch_tensorrt._utils import is_tensorrt_version_supported
24+
from torch_tensorrt._utils import (
25+
is_tensorrt_rtx_version_supported,
26+
is_tensorrt_version_supported,
27+
)
2528
from torch_tensorrt.dynamo._settings import CompilationSettings
2629
from torch_tensorrt.dynamo._SourceIR import SourceIR
2730
from torch_tensorrt.dynamo.conversion import impl
@@ -1487,8 +1490,59 @@ def aten_ops_slice(
14871490
)
14881491

14891492

1493+
def cumsum_validator(
1494+
node: Node, settings: Optional[CompilationSettings] = None
1495+
) -> bool:
1496+
# TensorRT-RTX before 1.7 cannot build a refittable cumsum when its loop
1497+
# trip count is a constant. Standard TensorRT and TensorRT-RTX 1.7+ do not
1498+
# have this limitation.
1499+
if (
1500+
is_tensorrt_rtx_version_supported("1.7")
1501+
or settings is None
1502+
or settings.immutable_weights
1503+
):
1504+
return True
1505+
1506+
input_node = node.args[0]
1507+
if not isinstance(input_node, Node):
1508+
return False
1509+
1510+
input_meta = input_node.meta.get("val")
1511+
if input_meta is None:
1512+
input_meta = input_node.meta.get("tensor_meta")
1513+
if input_meta is None:
1514+
_LOGGER.debug(
1515+
f"cumsum node {node.name} has no input shape metadata; falling back "
1516+
"to PyTorch on TensorRT-RTX < 1.7 with refit enabled."
1517+
)
1518+
return False
1519+
1520+
dim = node.kwargs.get("dim", args_bounds_check(node.args, 1))
1521+
dim = get_positive_dim(dim, len(input_meta.shape))
1522+
axis_size = input_meta.shape[dim]
1523+
1524+
# A symbolic dimension is converted to a runtime trip count. DYNAMIC_DIM
1525+
# covers tensor_meta produced by shape propagation instead of export.
1526+
if isinstance(axis_size, torch.SymInt) or axis_size == DYNAMIC_DIM:
1527+
return True
1528+
1529+
if isinstance(axis_size, int) and axis_size >= 0:
1530+
_LOGGER.debug(
1531+
f"cumsum node {node.name} has a static axis {dim} of size {axis_size}; "
1532+
"TensorRT-RTX < 1.7 cannot build its refittable constant loop trip count."
1533+
)
1534+
return False
1535+
1536+
_LOGGER.debug(
1537+
f"cumsum node {node.name} has an unrecognized axis size {axis_size!r}; "
1538+
"falling back to PyTorch on TensorRT-RTX < 1.7 with refit enabled."
1539+
)
1540+
return False
1541+
1542+
14901543
@dynamo_tensorrt_converter(
14911544
torch.ops.aten.cumsum.default,
1545+
capability_validator=cumsum_validator,
14921546
supports_dynamic_shapes=True,
14931547
)
14941548
@enforce_tensor_types(

‎tests/py/dynamo/conversion/test_cumsum_aten.py‎

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
import unittest
2-
31
import torch
42
import torch.nn as nn
53
import torch_tensorrt
64
from parameterized import parameterized
75
from torch.testing._internal.common_utils import run_tests
6+
from torch_tensorrt._utils import is_tensorrt_rtx_version_supported
7+
from torch_tensorrt.dynamo.conversion import UnsupportedOperatorException
88

99
from .harness import DispatchTestCase
1010

@@ -23,10 +23,18 @@ def forward(self, x):
2323
return torch.ops.aten.cumsum.default(x, dim)
2424

2525
inputs = [torch.randn(shape)]
26+
if not is_tensorrt_rtx_version_supported("1.7"):
27+
with self.assertRaises(UnsupportedOperatorException):
28+
self.run_test(
29+
Cumsum(),
30+
inputs,
31+
immutable_weights=False,
32+
use_dynamo_tracer=True,
33+
)
34+
return
35+
2636
self.run_test(
27-
Cumsum(),
28-
inputs,
29-
immutable_weights=False,
37+
Cumsum(), inputs, immutable_weights=False, use_dynamo_tracer=True
3038
)
3139

3240
@parameterized.expand(
@@ -77,6 +85,7 @@ def forward(self, x):
7785
((2, 3), (2, 4), (2, 5), 0),
7886
((2, 3), (3, 4), (4, 5), -1),
7987
((1, 2, 2), (2, 2, 3), (3, 3, 3), 0),
88+
((1, 2, 2), (2, 2, 3), (3, 2, 3), -2),
8089
((1, 2, 2, 3), (2, 2, 3, 4), (3, 3, 4, 5), -3),
8190
((1, 2, 2, 3), (2, 2, 3, 4), (3, 3, 4, 5), -2),
8291
]
@@ -93,36 +102,30 @@ def forward(self, x):
93102
max_shape=max_shape,
94103
),
95104
]
96-
self.run_test_with_dynamic_shape(
97-
Cumsum(),
98-
inputs,
99-
immutable_weights=False,
105+
positive_dim = dims if dims >= 0 else len(min_shape) + dims
106+
has_static_trip_count = (
107+
min_shape[positive_dim]
108+
== opt_shape[positive_dim]
109+
== max_shape[positive_dim]
100110
)
111+
if (
112+
has_static_trip_count
113+
and not is_tensorrt_rtx_version_supported("1.7")
114+
):
115+
with self.assertRaises(UnsupportedOperatorException):
116+
self.run_test_with_dynamic_shape(
117+
Cumsum(),
118+
inputs,
119+
immutable_weights=False,
120+
use_dynamo_tracer=True,
121+
)
122+
return
101123

102-
@unittest.skipIf(
103-
torch_tensorrt.ENABLED_FEATURES.tensorrt_rtx,
104-
"TRT RTX (CPU-only AoT build) cannot run ConstantBuilder, which TRT requires for loops whose trip count is a compile-time constant; occurs here because dim 1 has the same size across all profiles",
105-
)
106-
def test_cumsum_dynamic_shape_static_dim(self):
107-
# min=(1,2,2), opt=(2,2,3), max=(3,2,3), dim=-2=1
108-
# dim 1 is size 2 in all profiles, so TRT uses a constant trip limit
109-
min_shape, opt_shape, max_shape, dims = (1, 2, 2), (2, 2, 3), (3, 2, 3), -2
110-
111-
class Cumsum(nn.Module):
112-
def forward(self, x):
113-
return torch.ops.aten.cumsum.default(x, dims)
114-
115-
inputs = [
116-
torch_tensorrt.Input(
117-
min_shape=min_shape,
118-
opt_shape=opt_shape,
119-
max_shape=max_shape,
120-
),
121-
]
122124
self.run_test_with_dynamic_shape(
123125
Cumsum(),
124126
inputs,
125127
immutable_weights=False,
128+
use_dynamo_tracer=True,
126129
)
127130

128131

‎tests/py/dynamo/models/test_model_refit.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import torch_tensorrt as torchtrt
1111
import torch_tensorrt as torch_trt
1212
from torch import nn
13+
from torch_tensorrt._utils import is_tensorrt_rtx_version_supported
1314
from torch_tensorrt.dynamo import refit_module_weights
1415
from torch_tensorrt.dynamo._refit import (
1516
construct_refit_mapping,
@@ -832,8 +833,8 @@ def forward(self, x):
832833
"Refit feature is not supported in Python 3.13 or higher",
833834
)
834835
@unittest.skipIf(
835-
torch_trt.ENABLED_FEATURES.tensorrt_rtx,
836-
"cumsum is not supported on TensorRT-RTX (build_serialized_network returns None on Linux as well as Windows)",
836+
not is_tensorrt_rtx_version_supported("1.7"),
837+
"Refittable cumsum requires TensorRT-RTX 1.7 or newer",
837838
)
838839
@pytest.mark.unit
839840
def test_refit_cumsum():

0 commit comments

Comments
 (0)