Skip to content

Commit c4ce919

Browse files
authored
fix: keep random operations in PyTorch (#4711)
Co-authored-by: shoumikhin <shoumikhin@users.noreply.github.com>
1 parent c9a36c6 commit c4ce919

4 files changed

Lines changed: 129 additions & 215 deletions

File tree

‎docsrc/indices/supported_ops.rst‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ Operators Supported
1414
ATen Core Ops — Converted
1515
-------------------------
1616

17-
*156 operators with native Dynamo converters.*
17+
*153 operators with native Dynamo converters.*
1818

1919
- aten._adaptive_avg_pool2d(Tensor self, SymInt[2] output_size) -> Tensor
2020
- aten._adaptive_avg_pool3d(Tensor self, SymInt[3] output_size) -> Tensor
@@ -133,9 +133,6 @@ ATen Core Ops — Converted
133133
- aten.pow.Tensor_Tensor(Tensor self, Tensor exponent) -> Tensor
134134
- aten.prod(Tensor self, \*, ScalarType? dtype=None) -> Tensor
135135
- aten.prod.dim_int(Tensor self, int dim, bool keepdim=False, \*, ScalarType? dtype=None) -> Tensor
136-
- aten.rand(SymInt[] size, \*, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor
137-
- aten.randn(SymInt[] size, \*, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor
138-
- aten.randperm(SymInt n, \*, ScalarType? dtype=long, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor
139136
- aten.reflection_pad1d(Tensor self, SymInt[2] padding) -> Tensor
140137
- aten.reflection_pad2d(Tensor self, SymInt[4] padding) -> Tensor
141138
- aten.reflection_pad3d(Tensor self, SymInt[6] padding) -> Tensor

‎py/torch_tensorrt/dynamo/conversion/ops_evaluators.py‎

Lines changed: 0 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -58,79 +58,6 @@ def generic_evaluator(
5858
return target(*args)
5959

6060

61-
def rand_validator(rand_node: Node, settings: CompilationSettings = None) -> bool:
62-
dtype = rand_node.kwargs.get("dtype", None)
63-
layout = rand_node.kwargs.get("layout", None)
64-
if dtype is not None:
65-
_LOGGER.debug(
66-
f"Currently we don't support specifying output dtype, got {dtype}."
67-
)
68-
return False
69-
if layout is not None:
70-
_LOGGER.debug(f"Currently we don't support specifying layout, got {layout}.")
71-
return False
72-
return True
73-
74-
75-
@dynamo_tensorrt_converter(
76-
torch.ops.aten.rand.default, capability_validator=rand_validator
77-
)
78-
def aten_ops_rand(
79-
ctx: ConversionContext,
80-
target: Target,
81-
args: Tuple[Argument, ...],
82-
kwargs: Dict[str, Argument],
83-
name: str,
84-
) -> Union[TRTTensor, Sequence[TRTTensor]]:
85-
return np.random.rand(*args[0])
86-
87-
88-
@dynamo_tensorrt_converter(
89-
torch.ops.aten.randn.default, capability_validator=rand_validator
90-
)
91-
def aten_ops_randn(
92-
ctx: ConversionContext,
93-
target: Target,
94-
args: Tuple[Argument, ...],
95-
kwargs: Dict[str, Argument],
96-
name: str,
97-
) -> Union[TRTTensor, Sequence[TRTTensor]]:
98-
return np.random.randn(*args[0])
99-
100-
101-
def randperm_validator(
102-
randperm_node: Node, settings: CompilationSettings = None
103-
) -> bool:
104-
dtype = randperm_node.kwargs.get("dtype", None)
105-
layout = randperm_node.kwargs.get("layout", None)
106-
input = randperm_node.args[0]
107-
if not isinstance(input, int):
108-
_LOGGER.error("Input should be of type int.")
109-
return False
110-
if dtype is not None:
111-
_LOGGER.debug(
112-
f"Currently we don't support specifying output dtype, got {dtype}."
113-
)
114-
return False
115-
if layout is not None:
116-
_LOGGER.debug(f"Currently we don't support specifying layout, got {layout}.")
117-
return False
118-
return True
119-
120-
121-
@dynamo_tensorrt_converter(
122-
torch.ops.aten.randperm.default, capability_validator=randperm_validator
123-
)
124-
def aten_ops_randperm(
125-
ctx: ConversionContext,
126-
target: Target,
127-
args: Tuple[Argument, ...],
128-
kwargs: Dict[str, Argument],
129-
name: str,
130-
) -> Union[TRTTensor, Sequence[TRTTensor]]:
131-
return np.random.permutation(args[0])
132-
133-
13461
def empty_validator(empty_node: Node, settings: CompilationSettings = None) -> bool:
13562
device = empty_node.kwargs.get("device", None)
13663
if device is not None:

‎tests/py/dynamo/conversion/test_rand_aten.py‎

Lines changed: 0 additions & 138 deletions
This file was deleted.
Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: BSD-3-Clause
3+
4+
import torch
5+
import torch_tensorrt
6+
from torch.testing._internal.common_utils import TestCase, run_tests
7+
8+
9+
class TestRandomOperatorsFallBack(TestCase):
10+
"""rand, randn and randperm used to have converters that called numpy during engine
11+
construction and froze one sample into the engine, so a compiled model returned the
12+
same values on every call. They now fall back to PyTorch like the rest of the family,
13+
which is the correct behaviour for an operator TensorRT cannot express.
14+
"""
15+
16+
@staticmethod
17+
def _module_with(random_call):
18+
class WithRandom(torch.nn.Module):
19+
def __init__(self):
20+
super().__init__()
21+
self.layers = torch.nn.ModuleList(
22+
[torch.nn.Linear(64, 64) for _ in range(6)]
23+
)
24+
25+
def forward(self, x):
26+
out = x
27+
for layer in self.layers:
28+
out = torch.relu(layer(out))
29+
return out + random_call(out)
30+
31+
return WithRandom().eval().cuda()
32+
33+
def _assert_not_frozen(self, module, inputs):
34+
compiled = torch_tensorrt.dynamo.compile(
35+
torch.export.export(module, tuple(inputs)),
36+
inputs=list(inputs),
37+
enabled_precisions={torch.float32},
38+
truncate_double=True,
39+
)
40+
first = compiled(*inputs).clone()
41+
second = compiled(*inputs).clone()
42+
self.assertFalse(
43+
torch.equal(first, second),
44+
"a compiled model containing a random operator returned identical values "
45+
"on two calls, so the values were frozen into the engine",
46+
)
47+
48+
def test_randn_is_not_frozen(self):
49+
inputs = [torch.randn(8, 64, device="cuda")]
50+
module = self._module_with(
51+
lambda out: torch.randn(out.shape, device=out.device)
52+
)
53+
self._assert_not_frozen(module, inputs)
54+
55+
def test_rand_is_not_frozen(self):
56+
inputs = [torch.randn(8, 64, device="cuda")]
57+
module = self._module_with(lambda out: torch.rand(out.shape, device=out.device))
58+
self._assert_not_frozen(module, inputs)
59+
60+
def test_manual_seed_is_honored(self):
61+
"""Resetting torch.manual_seed has to reproduce the output. A frozen value also
62+
passes this, so it only checks the fallback path."""
63+
inputs = [torch.randn(8, 64, device="cuda")]
64+
module = self._module_with(
65+
lambda out: torch.randn(out.shape, device=out.device)
66+
)
67+
compiled = torch_tensorrt.dynamo.compile(
68+
torch.export.export(module, tuple(inputs)),
69+
inputs=list(inputs),
70+
enabled_precisions={torch.float32},
71+
truncate_double=True,
72+
)
73+
torch.manual_seed(0)
74+
first = compiled(*inputs).clone()
75+
torch.manual_seed(0)
76+
second = compiled(*inputs).clone()
77+
torch.testing.assert_close(first, second)
78+
79+
def test_randperm_is_a_permutation(self):
80+
"""randperm's correctness is exact, so this needs no tolerance: the output has to be
81+
a rearrangement of 0..n-1."""
82+
83+
class WithRandperm(torch.nn.Module):
84+
def forward(self, x):
85+
return torch.randperm(16, device=x.device) + (x.sum() * 0).long()
86+
87+
inputs = [torch.randn(8, 64, device="cuda")]
88+
module = WithRandperm().eval().cuda()
89+
compiled = torch_tensorrt.dynamo.compile(
90+
torch.export.export(module, tuple(inputs)),
91+
inputs=list(inputs),
92+
min_block_size=1,
93+
enabled_precisions={torch.float32},
94+
truncate_double=True,
95+
)
96+
out = compiled(*inputs)
97+
self.assertEqual(out.dtype, module(*inputs).dtype)
98+
torch.testing.assert_close(
99+
out.sort().values, torch.arange(16, device=out.device, dtype=out.dtype)
100+
)
101+
102+
def test_random_operators_are_not_registered(self):
103+
"""The direct check.
104+
105+
Whether a graph ends up with a frozen value depends on the partitioner, which on
106+
some stacks declines the surrounding block for unrelated reasons, so an end to end
107+
test can pass either way. What this change actually does is take these three
108+
operators out of the registry, so assert that.
109+
"""
110+
from torch_tensorrt.dynamo.conversion._ConverterRegistry import (
111+
DYNAMO_CONVERTERS,
112+
)
113+
114+
for target in (
115+
torch.ops.aten.rand.default,
116+
torch.ops.aten.randn.default,
117+
torch.ops.aten.randperm.default,
118+
):
119+
self.assertNotIn(
120+
target,
121+
DYNAMO_CONVERTERS,
122+
f"{target} is still registered, so it can be folded into an engine at "
123+
"build time instead of running fresh on every call",
124+
)
125+
126+
127+
if __name__ == "__main__":
128+
run_tests()

0 commit comments

Comments
 (0)