Skip to content

Commit bed4401

Browse files
fix: handle empty one-dimensional tensors in concatenation (#4701)
Co-authored-by: shoumikhin <shoumikhin@users.noreply.github.com> Co-authored-by: PyTorch Bot <pytorchbot@users.noreply.github.com>
1 parent 8618e36 commit bed4401

3 files changed

Lines changed: 213 additions & 17 deletions

File tree

‎py/torch_tensorrt/dynamo/conversion/aten_ops_converters.py‎

Lines changed: 88 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
# mypy: disallow-untyped-decorators=False
55

6+
import functools
67
import logging
78
import operator
89
from typing import (
@@ -21,8 +22,9 @@
2122
import numpy as np
2223
import torch
2324
from tensorrt import ITensor as TRTTensor
25+
from torch.fx.experimental.symbolic_shapes import statically_known_true
2426
from torch.fx.node import Argument, Node, Target
25-
from torch_tensorrt import ENABLED_FEATURES
27+
from torch_tensorrt import ENABLED_FEATURES, _enums
2628
from torch_tensorrt._utils import (
2729
is_tensorrt_rtx_version_supported,
2830
is_tensorrt_version_supported,
@@ -269,16 +271,46 @@ def parse_cat_args(
269271
return input_tensors, dim
270272

271273

274+
def _is_rank1_empty_shape(shape: Sequence[int]) -> bool:
275+
"""Is this the shape of a rank-1 empty tensor, as torch.tensor([]) produces?"""
276+
return len(shape) == 1 and statically_known_true(shape[0] == 0)
277+
278+
279+
def _is_rank1_empty(value: Any) -> bool:
280+
"""Is this operand a rank-1 empty tensor, as torch.tensor([]) produces?"""
281+
shape = getattr(value, "shape", None)
282+
return shape is not None and _is_rank1_empty_shape(shape)
283+
284+
285+
def _promoted_dtype(operands: Sequence[Any]) -> Optional[_enums.dtype]:
286+
"""The dtype torch.cat would produce over these operands.
287+
288+
A rank-1 empty operand contributes nothing to the shape but still takes part in
289+
dtype promotion, so this has to run over the operands before any are dropped.
290+
Returns None when they already agree, leaving the existing behaviour alone.
291+
"""
292+
dtypes = []
293+
for operand in operands:
294+
dtype = getattr(operand, "dtype", None)
295+
if dtype is None:
296+
return None
297+
dtypes.append(_enums.dtype._from(dtype).to(torch.dtype))
298+
if not dtypes or all(dtype == dtypes[0] for dtype in dtypes):
299+
return None
300+
return _enums.dtype._from(functools.reduce(torch.promote_types, dtypes))
301+
302+
272303
def cat_validator(node: Node, settings: Optional[CompilationSettings] = None) -> bool:
273304
"""
274305
Validator for torch.cat operation with empty tensor handling.
275306
276307
PyTorch allows torch.tensor([]) (shape (0,)) to be concatenated with higher-dimensional
277-
tensors, but TensorRT requires all inputs to have the same rank. This validator catches
278-
this specific edge case.
308+
tensors, but TensorRT requires all inputs to have the same rank. A rank-1 empty operand
309+
holds no elements, so aten_ops_cat leaves it out and the ranks then agree.
279310
280-
Example valid case: cat([(3, 4), (0, 4)], dim=0) - same rank, properly shaped empty tensor for TRT
281-
Example invalid case: cat([(3, 4), (0,)], dim=0) - torch.tensor([]) with rank mismatch
311+
Example valid case: cat([(3, 4), (0,)], dim=0) - the empty operand is dropped
312+
Example invalid case: cat([(0,), (2, 3), (2, 3, 4)], dim=0) - the operands that hold
313+
elements still disagree on rank after the empty one is dropped
282314
"""
283315
# Use parse_cat_args to properly extract inputs (handles both args and kwargs patterns)
284316
inputs, _ = parse_cat_args(node.args, node.kwargs)
@@ -288,10 +320,12 @@ def cat_validator(node: Node, settings: Optional[CompilationSettings] = None) ->
288320

289321
# Collect metadata for all inputs
290322
input_metas = []
323+
input_dtypes = []
291324
for inp in inputs:
292325
if isinstance(inp, TRTTensor):
293326
# TRTTensor has shape directly
294327
input_metas.append(inp.shape)
328+
input_dtypes.append(inp.dtype)
295329
else:
296330
# For nodes, get metadata
297331
meta = getattr(inp, "meta", {}).get("tensor_meta")
@@ -300,25 +334,46 @@ def cat_validator(node: Node, settings: Optional[CompilationSettings] = None) ->
300334
return True
301335
shape = tuple(meta.shape)
302336
input_metas.append(shape)
337+
input_dtypes.append(meta.dtype)
338+
339+
# Dropping an empty operand also drops it from dtype promotion, so aten_ops_cat works
340+
# the promoted dtype out over every operand and passes it through. TensorRT has no
341+
# float64, so if the promotion lands there and truncate_double is not set, the build
342+
# fails on a graph that used to fall back. Refuse it here instead.
343+
if any(_is_rank1_empty_shape(tuple(shape)) for shape in input_metas):
344+
try:
345+
promoted = functools.reduce(torch.promote_types, input_dtypes)
346+
except TypeError:
347+
promoted = None
348+
if promoted == torch.float64 and not (
349+
settings is not None and settings.truncate_double
350+
):
351+
_LOGGER.debug(
352+
"Concatenation rejected by TRT, dropping the empty operand promotes to "
353+
"float64, which needs truncate_double. Falling back to PyTorch"
354+
)
355+
return False
303356

304357
# Check for the specific problematic case:
305358
# 1D empty tensor (0,) being concatenated with higher-dimensional tensors
306359
ranks = [len(shape) for shape in input_metas]
307360
# If all ranks are the same, it's fine (PyTorch and TensorRT both handle this)
308361
if len(set(ranks)) == 1:
309362
return True
310-
# If ranks differ, check if we have a 1D empty tensor (0,) in the mix
311-
# This is the torch.tensor([]) case that PyTorch allows but TensorRT doesn't
312-
for i, shape in enumerate(input_metas):
313-
if shape == (0,) or (len(shape) == 1 and shape[0] == 0):
314-
# Found a 1D empty tensor with rank mismatch
315-
_LOGGER.debug(
316-
f"Concatenation rejected by TRT, torch.tensor([]) or 1D empty tensor at position {i} "
317-
f"PyTorch allows this but TensorRT requires all inputs to have the same rank. "
318-
f"Use torch.empty((0, ...)) with explicit dimensions matching other inputs instead. Falling back to Pytorch"
319-
)
320-
return False
321-
return True
363+
# A rank-1 empty operand holds no elements, so leaving it out of the
364+
# concatenation gives the same result. Accept the node when the operands that
365+
# do hold elements agree on rank; aten_ops_cat drops the empty ones before
366+
# building the layer.
367+
non_empty = [shape for shape in input_metas if not _is_rank1_empty_shape(shape)]
368+
if len({len(shape) for shape in non_empty}) == 1:
369+
return True
370+
# Reaching here means the operands that survive still disagree on rank, which
371+
# TensorRT cannot concatenate, so name those rather than the empty one.
372+
_LOGGER.debug(
373+
f"Concatenation rejected by TRT, operands {non_empty} do not all have the same "
374+
f"rank. TensorRT requires every operand to have the same rank. Falling back to PyTorch"
375+
)
376+
return False
322377

323378

324379
@dynamo_tensorrt_converter(
@@ -334,13 +389,29 @@ def aten_ops_cat(
334389
name: str,
335390
) -> Union[TRTTensor, Sequence[TRTTensor]]:
336391
inputs, dim = parse_cat_args(args, kwargs)
392+
# TensorRT requires every operand to have the same rank, so a rank-1 empty operand
393+
# has to go. It holds no elements, so leaving it out does not change the result,
394+
# but torch.cat still counts it when it promotes the output dtype: concatenating
395+
# torch.tensor([]), which is float32, onto a float16 tensor produces float32. Work
396+
# the promoted dtype out over every operand and pass it through, or the engine
397+
# builds the concatenation in the survivors' narrower type and overflows.
398+
non_empty = [operand for operand in inputs if not _is_rank1_empty(operand)]
399+
cast_dtype = None
400+
if non_empty and len(non_empty) != len(inputs):
401+
cast_dtype = _promoted_dtype(inputs)
402+
# TensorRT has no float64. The validator only lets float64 reach here when
403+
# truncate_double is set, which means the caller accepts float32, so map it.
404+
if cast_dtype is not None and cast_dtype.to(torch.dtype) == torch.float64:
405+
cast_dtype = _enums.dtype._from(torch.float32)
406+
inputs = non_empty
337407
return impl.cat.cat(
338408
ctx,
339409
target,
340410
SourceIR.ATEN,
341411
name,
342412
input=inputs,
343413
dim=dim,
414+
cast_dtype=cast_dtype,
344415
)
345416

346417

‎tests/py/dynamo/conversion/test_cat_aten.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,69 @@ def forward(self, x):
378378
inputs,
379379
)
380380

381+
@parameterized.expand(
382+
[
383+
("leading_negative_dim", 0, -2, torch.float32),
384+
("middle_negative_dim", 1, -2, torch.float32),
385+
("trailing_positive_dim", 2, 2, torch.float32),
386+
# The empty operand is float32 while the rest are not, so torch.cat
387+
# promotes the result. Dropping the operand must not drop the promotion.
388+
("promotes_from_float16", 0, -2, torch.float16),
389+
("promotes_from_int32", 0, -2, torch.int32),
390+
]
391+
)
392+
def test_cat_rank1_empty_operand(self, _, position, dim, dtype):
393+
"""A rank-1 empty operand should not push the concatenation out of TensorRT.
394+
395+
transformers emits this on the first write to every layer of a DynamicCache, so
396+
refusing it splits the graph once per layer.
397+
398+
The arithmetic after the concatenation is deliberate. With the concatenation as
399+
the graph output the engine boundary casts the result back, which hides a wrong
400+
internal dtype; the multiply makes it visible.
401+
"""
402+
403+
class CatRank1Empty(nn.Module):
404+
def forward(self, x, y):
405+
empty = torch.tensor([], dtype=torch.float32, device=x.device)
406+
operands = [x, y]
407+
operands.insert(position, empty)
408+
return torch.ops.aten.cat.default(tuple(operands), dim) * 1000
409+
410+
inputs = [
411+
torch.full((1, 2, 8, 4), 300, dtype=dtype, device="cuda"),
412+
torch.full((1, 2, 8, 4), 300, dtype=dtype, device="cuda"),
413+
]
414+
self.run_test(
415+
CatRank1Empty(),
416+
inputs,
417+
use_dynamo_tracer=True,
418+
enable_passes=True,
419+
)
420+
421+
def test_cat_rank1_empty_float64_with_truncation(self):
422+
"""A float32 tensor concatenated with a float64 empty constant. torch.cat promotes
423+
to float64, and dropping the empty operand carries that promoted dtype through, but
424+
TensorRT has no float64. The converter test harness sets truncate_double, so the
425+
caller accepts float32 and this has to build rather than raise. All operands are
426+
rank 1, so the validator always accepted this shape and TensorRT compiled it before
427+
the empty-operand change."""
428+
429+
class CatFloat64Empty(nn.Module):
430+
def forward(self, x):
431+
empty = torch.tensor([], dtype=torch.float64, device=x.device)
432+
# Cast to float32 so the reference is float32 too: with truncate_double the
433+
# engine returns float32, and the test compares dtype as well as values.
434+
return torch.ops.aten.cat.default((empty, x), 0).to(torch.float32)
435+
436+
inputs = [torch.tensor([0.0, 1.0, 2.0], dtype=torch.float32, device="cuda")]
437+
self.run_test(
438+
CatFloat64Empty(),
439+
inputs,
440+
use_dynamo_tracer=True,
441+
enable_passes=True,
442+
)
443+
381444

382445
if __name__ == "__main__":
383446
run_tests()
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import unittest
2+
from types import SimpleNamespace
3+
from unittest.mock import MagicMock
4+
5+
import torch
6+
from torch.testing._internal.common_utils import run_tests
7+
from torch_tensorrt.dynamo.conversion.aten_ops_converters import cat_validator
8+
9+
10+
def _make_cat_node(shapes, dim=0):
11+
"""A cat node whose operands carry only the metadata the validator reads."""
12+
operands = []
13+
for shape in shapes:
14+
operand = MagicMock()
15+
operand.meta = {
16+
"tensor_meta": SimpleNamespace(shape=torch.Size(shape), dtype=torch.float32)
17+
}
18+
operands.append(operand)
19+
node = MagicMock()
20+
node.args = (operands, dim)
21+
node.kwargs = {}
22+
return node
23+
24+
25+
class TestCatValidator(unittest.TestCase):
26+
"""Metadata-only checks need no GPU and live outside the converter harness."""
27+
28+
def test_rank1_empty_with_agreeing_survivors_is_accepted(self):
29+
"""The DynamicCache shape. TensorRT cannot take mixed ranks, but the empty
30+
operand holds nothing, so dropping it leaves operands that agree."""
31+
self.assertTrue(cat_validator(_make_cat_node([(0,), (1, 2, 8, 4)])))
32+
self.assertTrue(cat_validator(_make_cat_node([(0,), (2, 3), (5, 3)])))
33+
34+
def test_disagreeing_survivors_are_refused(self):
35+
"""Dropping the empty operand must not paper over a real mismatch."""
36+
self.assertFalse(cat_validator(_make_cat_node([(0,), (2, 3), (2, 3, 4)])))
37+
38+
def test_uniform_ranks_are_accepted_unchanged(self):
39+
"""Nothing is dropped here, so the answer must not depend on this change."""
40+
self.assertTrue(cat_validator(_make_cat_node([(2, 3), (5, 3)])))
41+
self.assertTrue(cat_validator(_make_cat_node([(0,), (3,)])))
42+
43+
def test_single_operand_is_accepted(self):
44+
self.assertTrue(cat_validator(_make_cat_node([(0,)])))
45+
46+
def test_unbacked_rank1_operands_do_not_guard(self):
47+
"""Data-dependent lengths must not force a guard during validation."""
48+
49+
class M(torch.nn.Module):
50+
def forward(self, x):
51+
a = torch.nonzero(x).flatten().float()
52+
return torch.cat([a, a * 2])
53+
54+
ep = torch.export.export(M(), (torch.tensor([0, 1, 1]),), strict=False)
55+
cat = next(n for n in ep.graph.nodes if n.target is torch.ops.aten.cat.default)
56+
for operand in cat.args[0]:
57+
self.assertIsInstance(operand.meta["tensor_meta"].shape[0], torch.SymInt)
58+
self.assertTrue(cat_validator(cat))
59+
60+
61+
if __name__ == "__main__":
62+
run_tests()

0 commit comments

Comments
 (0)