33
44# mypy: disallow-untyped-decorators=False
55
6+ import functools
67import logging
78import operator
89from typing import (
2122import numpy as np
2223import torch
2324from tensorrt import ITensor as TRTTensor
25+ from torch .fx .experimental .symbolic_shapes import statically_known_true
2426from torch .fx .node import Argument , Node , Target
25- from torch_tensorrt import ENABLED_FEATURES
27+ from torch_tensorrt import ENABLED_FEATURES , _enums
2628from torch_tensorrt ._utils import (
2729 is_tensorrt_rtx_version_supported ,
2830 is_tensorrt_version_supported ,
@@ -269,16 +271,46 @@ def parse_cat_args(
269271 return input_tensors , dim
270272
271273
274+ def _is_rank1_empty_shape (shape : Sequence [int ]) -> bool :
275+ """Is this the shape of a rank-1 empty tensor, as torch.tensor([]) produces?"""
276+ return len (shape ) == 1 and statically_known_true (shape [0 ] == 0 )
277+
278+
279+ def _is_rank1_empty (value : Any ) -> bool :
280+ """Is this operand a rank-1 empty tensor, as torch.tensor([]) produces?"""
281+ shape = getattr (value , "shape" , None )
282+ return shape is not None and _is_rank1_empty_shape (shape )
283+
284+
285+ def _promoted_dtype (operands : Sequence [Any ]) -> Optional [_enums .dtype ]:
286+ """The dtype torch.cat would produce over these operands.
287+
288+ A rank-1 empty operand contributes nothing to the shape but still takes part in
289+ dtype promotion, so this has to run over the operands before any are dropped.
290+ Returns None when they already agree, leaving the existing behaviour alone.
291+ """
292+ dtypes = []
293+ for operand in operands :
294+ dtype = getattr (operand , "dtype" , None )
295+ if dtype is None :
296+ return None
297+ dtypes .append (_enums .dtype ._from (dtype ).to (torch .dtype ))
298+ if not dtypes or all (dtype == dtypes [0 ] for dtype in dtypes ):
299+ return None
300+ return _enums .dtype ._from (functools .reduce (torch .promote_types , dtypes ))
301+
302+
272303def cat_validator (node : Node , settings : Optional [CompilationSettings ] = None ) -> bool :
273304 """
274305 Validator for torch.cat operation with empty tensor handling.
275306
276307 PyTorch allows torch.tensor([]) (shape (0,)) to be concatenated with higher-dimensional
277- tensors, but TensorRT requires all inputs to have the same rank. This validator catches
278- this specific edge case .
308+ tensors, but TensorRT requires all inputs to have the same rank. A rank-1 empty operand
309+ holds no elements, so aten_ops_cat leaves it out and the ranks then agree .
279310
280- Example valid case: cat([(3, 4), (0, 4)], dim=0) - same rank, properly shaped empty tensor for TRT
281- Example invalid case: cat([(3, 4), (0,)], dim=0) - torch.tensor([]) with rank mismatch
311+ Example valid case: cat([(3, 4), (0,)], dim=0) - the empty operand is dropped
312+ Example invalid case: cat([(0,), (2, 3), (2, 3, 4)], dim=0) - the operands that hold
313+ elements still disagree on rank after the empty one is dropped
282314 """
283315 # Use parse_cat_args to properly extract inputs (handles both args and kwargs patterns)
284316 inputs , _ = parse_cat_args (node .args , node .kwargs )
@@ -288,10 +320,12 @@ def cat_validator(node: Node, settings: Optional[CompilationSettings] = None) ->
288320
289321 # Collect metadata for all inputs
290322 input_metas = []
323+ input_dtypes = []
291324 for inp in inputs :
292325 if isinstance (inp , TRTTensor ):
293326 # TRTTensor has shape directly
294327 input_metas .append (inp .shape )
328+ input_dtypes .append (inp .dtype )
295329 else :
296330 # For nodes, get metadata
297331 meta = getattr (inp , "meta" , {}).get ("tensor_meta" )
@@ -300,25 +334,46 @@ def cat_validator(node: Node, settings: Optional[CompilationSettings] = None) ->
300334 return True
301335 shape = tuple (meta .shape )
302336 input_metas .append (shape )
337+ input_dtypes .append (meta .dtype )
338+
339+ # Dropping an empty operand also drops it from dtype promotion, so aten_ops_cat works
340+ # the promoted dtype out over every operand and passes it through. TensorRT has no
341+ # float64, so if the promotion lands there and truncate_double is not set, the build
342+ # fails on a graph that used to fall back. Refuse it here instead.
343+ if any (_is_rank1_empty_shape (tuple (shape )) for shape in input_metas ):
344+ try :
345+ promoted = functools .reduce (torch .promote_types , input_dtypes )
346+ except TypeError :
347+ promoted = None
348+ if promoted == torch .float64 and not (
349+ settings is not None and settings .truncate_double
350+ ):
351+ _LOGGER .debug (
352+ "Concatenation rejected by TRT, dropping the empty operand promotes to "
353+ "float64, which needs truncate_double. Falling back to PyTorch"
354+ )
355+ return False
303356
304357 # Check for the specific problematic case:
305358 # 1D empty tensor (0,) being concatenated with higher-dimensional tensors
306359 ranks = [len (shape ) for shape in input_metas ]
307360 # If all ranks are the same, it's fine (PyTorch and TensorRT both handle this)
308361 if len (set (ranks )) == 1 :
309362 return True
310- # If ranks differ, check if we have a 1D empty tensor (0,) in the mix
311- # This is the torch.tensor([]) case that PyTorch allows but TensorRT doesn't
312- for i , shape in enumerate (input_metas ):
313- if shape == (0 ,) or (len (shape ) == 1 and shape [0 ] == 0 ):
314- # Found a 1D empty tensor with rank mismatch
315- _LOGGER .debug (
316- f"Concatenation rejected by TRT, torch.tensor([]) or 1D empty tensor at position { i } "
317- f"PyTorch allows this but TensorRT requires all inputs to have the same rank. "
318- f"Use torch.empty((0, ...)) with explicit dimensions matching other inputs instead. Falling back to Pytorch"
319- )
320- return False
321- return True
363+ # A rank-1 empty operand holds no elements, so leaving it out of the
364+ # concatenation gives the same result. Accept the node when the operands that
365+ # do hold elements agree on rank; aten_ops_cat drops the empty ones before
366+ # building the layer.
367+ non_empty = [shape for shape in input_metas if not _is_rank1_empty_shape (shape )]
368+ if len ({len (shape ) for shape in non_empty }) == 1 :
369+ return True
370+ # Reaching here means the operands that survive still disagree on rank, which
371+ # TensorRT cannot concatenate, so name those rather than the empty one.
372+ _LOGGER .debug (
373+ f"Concatenation rejected by TRT, operands { non_empty } do not all have the same "
374+ f"rank. TensorRT requires every operand to have the same rank. Falling back to PyTorch"
375+ )
376+ return False
322377
323378
324379@dynamo_tensorrt_converter (
@@ -334,13 +389,29 @@ def aten_ops_cat(
334389 name : str ,
335390) -> Union [TRTTensor , Sequence [TRTTensor ]]:
336391 inputs , dim = parse_cat_args (args , kwargs )
392+ # TensorRT requires every operand to have the same rank, so a rank-1 empty operand
393+ # has to go. It holds no elements, so leaving it out does not change the result,
394+ # but torch.cat still counts it when it promotes the output dtype: concatenating
395+ # torch.tensor([]), which is float32, onto a float16 tensor produces float32. Work
396+ # the promoted dtype out over every operand and pass it through, or the engine
397+ # builds the concatenation in the survivors' narrower type and overflows.
398+ non_empty = [operand for operand in inputs if not _is_rank1_empty (operand )]
399+ cast_dtype = None
400+ if non_empty and len (non_empty ) != len (inputs ):
401+ cast_dtype = _promoted_dtype (inputs )
402+ # TensorRT has no float64. The validator only lets float64 reach here when
403+ # truncate_double is set, which means the caller accepts float32, so map it.
404+ if cast_dtype is not None and cast_dtype .to (torch .dtype ) == torch .float64 :
405+ cast_dtype = _enums .dtype ._from (torch .float32 )
406+ inputs = non_empty
337407 return impl .cat .cat (
338408 ctx ,
339409 target ,
340410 SourceIR .ATEN ,
341411 name ,
342412 input = inputs ,
343413 dim = dim ,
414+ cast_dtype = cast_dtype ,
344415 )
345416
346417
0 commit comments