Skip to content

Commit 5ffc2e3

Browse files
narendasanapbose
andauthored
fix(dynamo): handle dynamic scatter and index updates (#4631)
Co-authored-by: apbose <apbose694@gmail.com>
1 parent 5d0ed89 commit 5ffc2e3

24 files changed

Lines changed: 771 additions & 112 deletions

‎py/torch_tensorrt/_Input.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -196,16 +196,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
196196
}
197197
self.shape_mode = Input._ShapeMode.DYNAMIC
198198

199-
# Warn if min_shape has any 0 dimension (empty tensor) - TensorRT doesn't support this
200-
# @apbose: Is this warning necessary?
201-
if any(dim == 0 for dim in self.shape["min_shape"]):
202-
logger.warning(
203-
f"min_shape contains a 0 dimension: {self.shape['min_shape']}. "
204-
"TensorRT does not support dynamic shapes with min dimension of 0 (empty tensors). "
205-
"TensorRT will internally clamp min dimensions to 1, which may cause runtime errors "
206-
"if you try to run inference with empty tensor inputs."
207-
)
208-
209199
# Namedtuple shape API: field names encode per-axis dimension names.
210200
# Convert to shared_dims so _tracer.py needs no changes — axes with the
211201
# same name across inputs become one shared torch.export.Dim.

‎py/torch_tensorrt/dynamo/_compiler.py‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
from torch_tensorrt.dynamo.debug._DebuggerConfig import DebuggerConfig
4141
from torch_tensorrt.dynamo.debug._supports_debugger import fn_supports_debugger
4242
from torch_tensorrt.dynamo.lowering import (
43+
filter_decomposition_table,
4344
get_decompositions,
4445
post_lowering,
4546
pre_export_lowering,
@@ -395,11 +396,14 @@ def cross_compile_for_windows(
395396
logger.info("Compilation Settings: %s\n", settings)
396397
exported_program = pre_export_lowering(exported_program, settings)
397398
exported_program = exported_program.run_decompositions(
398-
get_decompositions(
399-
enable_experimental_decompositions,
400-
decompose_attention,
401-
use_distributed_mode_trace,
402-
use_fp32_acc=use_fp32_acc,
399+
filter_decomposition_table(
400+
get_decompositions(
401+
enable_experimental_decompositions,
402+
decompose_attention,
403+
use_distributed_mode_trace,
404+
use_fp32_acc=use_fp32_acc,
405+
),
406+
exported_program.graph_module,
403407
)
404408
)
405409

@@ -802,11 +806,14 @@ def compile(
802806
logger.info("Compilation Settings: %s\n", settings)
803807
exported_program = pre_export_lowering(exported_program, settings)
804808
exported_program = exported_program.run_decompositions(
805-
get_decompositions(
806-
enable_experimental_decompositions,
807-
decompose_attention,
808-
use_distributed_mode_trace,
809-
use_fp32_acc=use_fp32_acc,
809+
filter_decomposition_table(
810+
get_decompositions(
811+
enable_experimental_decompositions,
812+
decompose_attention,
813+
use_distributed_mode_trace,
814+
use_fp32_acc=use_fp32_acc,
815+
),
816+
exported_program.graph_module,
810817
)
811818
)
812819

@@ -2108,11 +2115,14 @@ def convert_exported_program_to_serialized_trt_engine(
21082115
logger.info("Compilation Settings: %s\n", settings)
21092116
exported_program = pre_export_lowering(exported_program, settings)
21102117
exported_program = exported_program.run_decompositions(
2111-
get_decompositions(
2112-
enable_experimental_decompositions,
2113-
decompose_attention,
2114-
use_distributed_mode_trace,
2115-
use_fp32_acc=use_fp32_acc,
2118+
filter_decomposition_table(
2119+
get_decompositions(
2120+
enable_experimental_decompositions,
2121+
decompose_attention,
2122+
use_distributed_mode_trace,
2123+
use_fp32_acc=use_fp32_acc,
2124+
),
2125+
exported_program.graph_module,
21162126
)
21172127
)
21182128

‎py/torch_tensorrt/dynamo/_refit.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from torch_tensorrt.dynamo.conversion.truncate_double import repair_double_inputs
2626
from torch_tensorrt.dynamo.lowering import (
2727
clean_up_graph_after_modifications,
28+
filter_decomposition_table,
2829
get_decompositions,
2930
post_lowering,
3031
pre_export_lowering,
@@ -257,11 +258,14 @@ def refit_module_weights(
257258
)
258259
new_weight_module = pre_export_lowering(new_weight_module, settings)
259260
new_weight_module = new_weight_module.run_decompositions(
260-
get_decompositions(
261-
settings.enable_experimental_decompositions,
262-
settings.decompose_attention,
263-
settings.use_distributed_mode_trace,
264-
use_fp32_acc=settings.use_fp32_acc,
261+
filter_decomposition_table(
262+
get_decompositions(
263+
settings.enable_experimental_decompositions,
264+
settings.decompose_attention,
265+
settings.use_distributed_mode_trace,
266+
use_fp32_acc=settings.use_fp32_acc,
267+
),
268+
new_weight_module.graph_module,
265269
)
266270
)
267271
new_gm = new_weight_module.module()

‎py/torch_tensorrt/dynamo/backend/backends.py‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from torch_tensorrt.dynamo import CompilationSettings
1616
from torch_tensorrt.dynamo._compiler import compile_module
1717
from torch_tensorrt.dynamo.lowering import (
18+
filter_decomposition_table,
1819
get_decompositions,
1920
post_lowering,
2021
remove_detach,
@@ -81,11 +82,14 @@ def aot_torch_tensorrt_aten_backend(
8182
_pretraced_backend, settings=settings, engine_cache=engine_cache
8283
)
8384
settings_aot_autograd = {}
84-
settings_aot_autograd["decompositions"] = get_decompositions(
85-
settings.enable_experimental_decompositions,
86-
settings.decompose_attention,
87-
settings.use_distributed_mode_trace,
88-
use_fp32_acc=settings.use_fp32_acc,
85+
settings_aot_autograd["decompositions"] = filter_decomposition_table(
86+
get_decompositions(
87+
settings.enable_experimental_decompositions,
88+
settings.decompose_attention,
89+
settings.use_distributed_mode_trace,
90+
use_fp32_acc=settings.use_fp32_acc,
91+
),
92+
gm,
8993
)
9094
# This is added since detach lowering leads to alias nodes
9195
# Error - View operation returned a tensor that is the same as the input base tensor
@@ -132,10 +136,13 @@ def aot_torch_tensorrt_aten_backend(
132136
_pretraced_backend_autograd = functools.partial(
133137
_pretraced_backend, settings=aot_settings, engine_cache=engine_cache
134138
)
135-
aot_decomps = get_decompositions(
136-
settings.enable_experimental_decompositions,
137-
settings.decompose_attention,
138-
use_fp32_acc=settings.use_fp32_acc,
139+
aot_decomps = filter_decomposition_table(
140+
get_decompositions(
141+
settings.enable_experimental_decompositions,
142+
settings.decompose_attention,
143+
use_fp32_acc=settings.use_fp32_acc,
144+
),
145+
gm,
139146
)
140147
# Remove detach decompositions to avoid alias node errors.
141148
to_delete = {k for k in aot_decomps if "detach" in k._name}
@@ -333,11 +340,14 @@ def _pretraced_backend(
333340
gm,
334341
sample_inputs,
335342
trace_joint=False,
336-
decompositions=get_decompositions(
337-
settings.enable_experimental_decompositions,
338-
settings.decompose_attention,
339-
settings.use_distributed_mode_trace,
340-
use_fp32_acc=settings.use_fp32_acc,
343+
decompositions=filter_decomposition_table(
344+
get_decompositions(
345+
settings.enable_experimental_decompositions,
346+
settings.decompose_attention,
347+
settings.use_distributed_mode_trace,
348+
use_fp32_acc=settings.use_fp32_acc,
349+
),
350+
gm,
341351
),
342352
)
343353

‎py/torch_tensorrt/dynamo/conversion/_ConverterRegistry.py‎

Lines changed: 32 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -138,63 +138,56 @@ def _has_dynamic_shapes(
138138
node, torch.fx.Node
139139
), "Inputs to validator functions must be FX Nodes"
140140

141+
def _contains_dynamic_value(value: Any) -> bool:
142+
"""Recursively checks FX metadata for symbolic values or dimensions."""
143+
if isinstance(value, (SymFloat, SymInt, SymBool)):
144+
return True
145+
if isinstance(value, dict):
146+
return any(_contains_dynamic_value(item) for item in value.values())
147+
if isinstance(value, (list, tuple)):
148+
return any(_contains_dynamic_value(item) for item in value)
149+
if value is None or isinstance(value, (int, float, bool)):
150+
return False
151+
if hasattr(value, "size"):
152+
return any(_contains_dynamic_value(dim) for dim in value.size())
153+
return False
154+
141155
def _is_subnode_dynamic(subnode: torch.fx.Node) -> bool:
142-
"""Checks if a node itself has Dynamic properties"""
143-
_has_symbolic_sizes_strides, is_shape_dynamic = False, False
144-
if "val" in subnode.meta:
145-
_has_symbolic_sizes_strides = getattr(
146-
subnode.meta["val"], "_has_symbolic_sizes_strides", False
147-
)
148-
meta_val = subnode.meta["val"]
149-
if isinstance(meta_val, (list, tuple)):
150-
for val in meta_val:
151-
if val is None:
152-
continue
153-
if isinstance(val, (SymFloat, SymInt, SymBool)):
154-
is_shape_dynamic = True
155-
break
156-
if not hasattr(val, "size"):
157-
continue
158-
shape = val.size()
159-
if any(
160-
isinstance(dim, (SymFloat, SymInt, SymBool)) for dim in shape
161-
):
162-
is_shape_dynamic = True
163-
break
164-
elif isinstance(meta_val, (SymFloat, SymInt, SymBool)):
165-
is_shape_dynamic = True
166-
elif isinstance(meta_val, (int, float, bool)):
167-
is_shape_dynamic = False
168-
else:
169-
shape = subnode.meta["val"].size()
170-
is_shape_dynamic = any(
171-
isinstance(dim, (SymFloat, SymInt, SymBool)) for dim in shape
172-
)
156+
"""Checks if a node itself has Dynamic properties."""
157+
meta_val = subnode.meta.get("val", subnode.meta.get("example_value"))
158+
return bool(
159+
getattr(meta_val, "_has_symbolic_sizes_strides", False)
160+
or _contains_dynamic_value(meta_val)
161+
)
173162

174-
return _has_symbolic_sizes_strides or is_shape_dynamic
163+
def _is_argument_dynamic(argument: Argument) -> bool:
164+
"""Checks nodes nested inside FX container arguments."""
165+
if isinstance(argument, torch.fx.Node):
166+
return _is_subnode_dynamic(argument)
167+
if isinstance(argument, dict):
168+
return any(_is_argument_dynamic(item) for item in argument.values())
169+
if isinstance(argument, (list, tuple)):
170+
return any(_is_argument_dynamic(item) for item in argument)
171+
return False
175172

176173
# Check node value itself
177174
if arg_positions_to_check is None and _is_subnode_dynamic(node):
178175
return True
179176

180177
# Check node arguments individually
181178
if arg_positions_to_check is None and any(
182-
_is_subnode_dynamic(arg) for arg in node.args if isinstance(arg, torch.fx.Node)
179+
_is_argument_dynamic(arg) for arg in node.args
183180
):
184181
return True
185182
# Check specific arg positions if the caller has specified positions to check
186183
elif arg_positions_to_check is not None and any(
187-
_is_subnode_dynamic(node.args[i])
188-
for i in arg_positions_to_check
189-
if isinstance(node.args[i], torch.fx.Node)
184+
_is_argument_dynamic(node.args[i]) for i in arg_positions_to_check
190185
):
191186
return True
192187

193188
# Check node keyword arguments individually
194189
if arg_positions_to_check is None and any(
195-
_is_subnode_dynamic(kwarg)
196-
for kwarg in node.kwargs.values()
197-
if isinstance(kwarg, torch.fx.Node)
190+
_is_argument_dynamic(kwarg) for kwarg in node.kwargs.values()
198191
):
199192
return True
200193

‎py/torch_tensorrt/dynamo/conversion/impl/arange.py‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
from typing import Optional, Union
22

3+
import numpy as np
34
import tensorrt as trt
45
import torch
56
from tensorrt import ITensor as TRTTensor
6-
from torch._subclasses.fake_tensor import unset_fake_temporarily
77
from torch.fx.node import Target
88
from torch_tensorrt import _enums
99
from torch_tensorrt.dynamo.conversion import impl
@@ -38,7 +38,7 @@ def _sequence_dtype(
3838
if isinstance(x, float):
3939
return trt.DataType.FLOAT
4040

41-
return trt.DataType.INT64
41+
return trt.DataType.INT64
4242

4343

4444
def arange(
@@ -65,6 +65,9 @@ def arange(
6565
start_rank_0 = get_trt_tensor(
6666
ctx, start, name + "_start_rank_0", value_dtype, min_rank=0
6767
)
68+
start_rank_0 = cast_trt_tensor(
69+
ctx, start_rank_0, value_dtype, name + "_start_rank_0_casted"
70+
)
6871
# LINSPACE's start input requires rank 0; if the upstream ITensor came in
6972
# as rank-1 (e.g. a SymInt materialized by a sym_size op), reshape it.
7073
if len(start_rank_0.shape) > 0:
@@ -80,6 +83,11 @@ def arange(
8083
)
8184
end = get_trt_tensor(ctx, end, name + "_end", value_dtype, min_rank=1)
8285
step = get_trt_tensor(ctx, step, name + "_step", value_dtype, min_rank=1)
86+
start_rank_1 = cast_trt_tensor(
87+
ctx, start_rank_1, value_dtype, name + "_start_rank_1_casted"
88+
)
89+
end = cast_trt_tensor(ctx, end, value_dtype, name + "_end_casted")
90+
step = cast_trt_tensor(ctx, step, value_dtype, name + "_step_casted")
8391

8492
# The number of elements is ceil((end - start) / step), computed as
8593
# -floor((start - end) / step) so that the whole expression stays in the
@@ -112,10 +120,26 @@ def arange(
112120

113121
else:
114122
# All arguments are static, so evaluate the sequence eagerly and freeze it
115-
# into the engine as a constant. Letting torch pick the dtype preserves
116-
# PyTorch's promotion rules (float result if any argument is a float).
117-
with unset_fake_temporarily():
118-
values = torch.arange(start, end, step, dtype=dtype)
119-
if values.dtype == torch.int64:
120-
values = values.to(torch.int32)
121-
return get_trt_tensor(ctx, values, f"{name}_arange_const")
123+
# into the engine as a constant. NumPy avoids creating a FakeTensor when
124+
# conversion runs inside torch.compile's active FakeTensorMode.
125+
resolved_dtype = dtype
126+
if resolved_dtype is None and any(
127+
isinstance(value, float) for value in (start, end, step)
128+
):
129+
resolved_dtype = torch.get_default_dtype()
130+
constant_dtype = None
131+
if resolved_dtype is not None:
132+
try:
133+
np_dtype = _enums.dtype._from(resolved_dtype).to(np.dtype)
134+
except TypeError:
135+
# Some TensorRT dtypes, such as BF16, have no NumPy
136+
# representation. Build the sequence in NumPy's inferred dtype
137+
# and let constant creation cast it to the requested dtype.
138+
np_dtype = None
139+
constant_dtype = resolved_dtype
140+
else:
141+
np_dtype = None
142+
values = np.arange(start, end, step, dtype=np_dtype)
143+
if values.dtype == np.int64:
144+
values = values.astype(np.int32)
145+
return get_trt_tensor(ctx, values, f"{name}_arange_const", dtype=constant_dtype)

‎py/torch_tensorrt/dynamo/conversion/impl/select.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1005,6 +1005,29 @@ def index_put_converter(
10051005
values_permuted,
10061006
expected_shape,
10071007
)
1008+
elif K == 1 and len(values.shape) == 1 and F and I[0] > max(F):
1009+
# For data[..., idx] = values, a 1-D values tensor carries the
1010+
# index extent and broadcasts across the preceding free dims.
1011+
# The converter's internal layout is (N, *F), so make that
1012+
# index axis explicit before expanding. This is essential when N
1013+
# is dynamic: using -1 for both axes would keep the original N
1014+
# extent instead of broadcasting the free dimensions.
1015+
values_reshaped = impl.shuffle.reshape(
1016+
ctx,
1017+
target,
1018+
source_ir,
1019+
f"{name}_reshape_index_values",
1020+
values,
1021+
(N,) + (1,) * len(F),
1022+
)
1023+
values_expanded = impl.slice.expand(
1024+
ctx,
1025+
target,
1026+
source_ir,
1027+
f"{name}_expand_values",
1028+
values_reshaped,
1029+
expected_shape,
1030+
)
10081031
elif (
10091032
K > 0
10101033
and N in values_shape
@@ -1089,6 +1112,13 @@ def index_put_converter(
10891112
values_expanded,
10901113
(-1,),
10911114
)
1115+
if flattened_values.dtype != input_tensor.dtype:
1116+
flattened_values = cast_trt_tensor(
1117+
ctx,
1118+
flattened_values,
1119+
input_tensor.dtype,
1120+
f"{name}_values_cast",
1121+
)
10921122
indices_cat = cast_trt_tensor(ctx, indices_cat, trt.int32, f"{name}_idx_int32")
10931123
scatter_layer = ctx.net.add_scatter(
10941124
input_tensor,

0 commit comments

Comments
 (0)