Skip to content

Commit b4e80d2

Browse files
authored
fix: skip const-fold for arange for attention mask (#4452)
1 parent 56691b7 commit b4e80d2

6 files changed

Lines changed: 549 additions & 4 deletions

File tree

‎py/torch_tensorrt/dynamo/_compiler.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,8 @@ def cross_compile_for_windows(
219219
Torch-TensorRT rules to disable. Rules are registered by lowering
220220
implementations and enabled by default; compilation users do not
221221
need to register them. Disabling a rule makes its matching nodes
222-
foldable. Default is empty.
222+
foldable. Supported IDs: ``"attention_mask_arange"``.
223+
Default is empty.
223224
attn_bias_is_causal (bool): Whether the attn_bias in efficient SDPA is causal. Default is True. This can accelerate models from HF because attn_bias is always a causal mask in HF. If you want to use non-causal attn_bias, you can set this to False.
224225
fallback_data_dependent_ops (bool): If True, operators whose converters require a TensorRT output allocator (i.e. data-dependent output shapes, such as nonzero) are added to torch_executed_ops and run in PyTorch instead of being lowered into a TensorRT engine. This is useful when targeting runtimes that cannot consume a TensorRT output allocator. Default is False.
225226
**kwargs: Any,
@@ -613,7 +614,8 @@ def compile(
613614
Torch-TensorRT rules to disable. Rules are registered by lowering
614615
implementations and enabled by default; compilation users do not
615616
need to register them. Disabling a rule makes its matching nodes
616-
foldable. Default is empty.
617+
foldable. Supported IDs: ``"attention_mask_arange"``.
618+
Default is empty.
617619
attn_bias_is_causal (bool): Whether the attn_bias in efficient SDPA is causal. Default is True. This can accelerate models from HF because attn_bias is always a causal mask in HF. If you want to use non-causal attn_bias, you can set this to False.
618620
fallback_data_dependent_ops (bool): If True, operators whose converters require a TensorRT output allocator (i.e. data-dependent output shapes, such as nonzero) are added to torch_executed_ops and run in PyTorch instead of being lowered into a TensorRT engine. This is useful when targeting runtimes that cannot consume a TensorRT output allocator. Default is False.
619621
**kwargs: Any,
@@ -2016,7 +2018,8 @@ def convert_exported_program_to_serialized_trt_engine(
20162018
Torch-TensorRT rules to disable. Rules are registered by lowering
20172019
implementations and enabled by default; compilation users do not
20182020
need to register them. Disabling a rule makes its matching nodes
2019-
foldable. Default is empty.
2021+
foldable. Supported IDs: ``"attention_mask_arange"``.
2022+
Default is empty.
20202023
attn_bias_is_causal (bool): Whether the attn_bias in efficient SDPA is causal. Default is True. This can accelerate models from HF because attn_bias is always a causal mask in HF. If you want to use non-causal attn_bias, you can set this to False.
20212024
**kwargs: Any,
20222025
Returns:

‎py/torch_tensorrt/dynamo/_settings.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,8 @@ class CompilationSettings:
146146
Torch-TensorRT rules to disable. Rules are registered by lowering
147147
implementations and enabled by default; compilation users do not
148148
need to register them. Disabling a rule makes its matching nodes
149-
foldable. Default is empty.
149+
foldable. Supported IDs: ``"attention_mask_arange"``.
150+
Default is empty.
150151
attn_bias_is_causal (bool): Whether the attn_bias in efficient SDPA is causal. Default is True. This can accelerate models from HF because attn_bias is always a causal mask in HF. If you want to use non-causal attn_bias, you can set this to False.
151152
fallback_data_dependent_ops (bool): If True, operators whose converters require a TensorRT output allocator (i.e. data-dependent output shapes, such as nonzero) are added to torch_executed_ops and run in PyTorch instead of being lowered into a TensorRT engine. This is useful when targeting runtimes that cannot consume a TensorRT output allocator. Default is False.
152153
"""

‎py/torch_tensorrt/dynamo/lowering/_decompositions.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@
2525
torch_disabled_decompositions,
2626
torch_enabled_decompositions,
2727
)
28+
from .constant_fold_exclusions.attention_mask import (
29+
exclude_attn_mask_aranges_from_constant_fold,
30+
)
2831

2932
logger = logging.getLogger(__name__)
3033

@@ -497,6 +500,9 @@ def scaled_dot_product_attention_decomposition(
497500
attn_bias = attn_bias.masked_fill(temp_mask.logical_not(), float("-inf"))
498501

499502
if attn_mask is not None:
503+
# Unconditional: mark_constant_fold_exclusions revokes these marks when the
504+
# rule is disabled, so the decompositions do not need the setting.
505+
exclude_attn_mask_aranges_from_constant_fold(attn_mask)
500506
if attn_mask.dtype == torch.bool:
501507
attn_bias = attn_bias.masked_fill(attn_mask.logical_not(), float("-inf"))
502508
else:

‎py/torch_tensorrt/dynamo/lowering/constant_fold_exclusions/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@
88
unregister_constant_fold_exclusion_rule,
99
validate_disabled_constant_fold_exclusions,
1010
)
11+
from .attention_mask import ATTENTION_MASK_ARANGE_RULE_ID
1112

1213
__all__ = [
14+
"ATTENTION_MASK_ARANGE_RULE_ID",
1315
"CONSTANT_FOLD_EXCLUSION_META_KEY",
1416
"ConstantFoldExclusionRule",
1517
"register_constant_fold_exclusion_rule",
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: BSD-3-Clause
3+
4+
from typing import Any, Iterable
5+
6+
import torch
7+
from torch._subclasses.functional_tensor import mb_unwrap_functional_tensor
8+
from torch.fx.experimental.proxy_tensor import get_proxy_mode, get_proxy_slot
9+
10+
from ._core import (
11+
_mark_constant_fold_exclusion,
12+
register_constant_fold_exclusion_rule,
13+
)
14+
15+
ATTENTION_MASK_ARANGE_RULE_ID = "attention_mask_arange"
16+
17+
18+
def _is_arange_node(node: torch.fx.Node) -> bool:
19+
return (
20+
node.op == "call_function"
21+
and getattr(node.target, "overloadpacket", None) is torch.ops.aten.arange
22+
)
23+
24+
25+
def _find_attention_mask_aranges(
26+
mask_node: torch.fx.Node,
27+
) -> list[torch.fx.Node]:
28+
"""Find INT32/INT64 position aranges feeding an attention mask.
29+
30+
Other range dtypes, including unknown dtypes, retain normal constant
31+
folding. Integer ranges also remain foldable when mask construction uses
32+
FP64 intermediates: TensorRT cannot represent those tensors, and truncating
33+
them can change position comparisons.
34+
"""
35+
nodes_to_visit: list[torch.fx.Node] = [mask_node]
36+
visited: set[torch.fx.Node] = set()
37+
matching_nodes: list[torch.fx.Node] = []
38+
39+
while nodes_to_visit:
40+
current = nodes_to_visit.pop()
41+
if current in visited:
42+
continue
43+
44+
visited.add(current)
45+
value = current.meta.get("val")
46+
dtype = getattr(value, "dtype", current.kwargs.get("dtype"))
47+
if dtype is torch.float64:
48+
# Leave the whole constructor foldable, including integer aranges
49+
# compared with FP64 values or cast to FP64 after construction.
50+
return []
51+
52+
if _is_arange_node(current):
53+
if dtype not in (torch.int32, torch.int64):
54+
# Keep mixed-dtype constructors foldable as a whole, rather
55+
# than preserving only their integer ranges.
56+
return []
57+
matching_nodes.append(current)
58+
59+
nodes_to_visit.extend(current.all_input_nodes)
60+
61+
return matching_nodes
62+
63+
64+
def exclude_attn_mask_aranges_from_constant_fold(attn_mask: torch.Tensor) -> None:
65+
"""Mark aranges behind an attention mask while tracing a decomposition.
66+
67+
``run_decompositions`` invokes decompositions once for functionalization and
68+
again while proxy tracing. Only the proxy-tracing invocation has an FX graph
69+
on which metadata can be set.
70+
"""
71+
proxy_mode = get_proxy_mode()
72+
if proxy_mode is None:
73+
return
74+
75+
unwrapped_mask = mb_unwrap_functional_tensor(attn_mask)
76+
tracked_mask = get_proxy_slot(
77+
unwrapped_mask,
78+
proxy_mode.tracer,
79+
default=None,
80+
)
81+
proxy = getattr(tracked_mask, "proxy", tracked_mask)
82+
mask_node = getattr(proxy, "node", None)
83+
84+
if isinstance(mask_node, torch.fx.Node):
85+
_mark_constant_fold_exclusion(
86+
_find_attention_mask_aranges(mask_node),
87+
ATTENTION_MASK_ARANGE_RULE_ID,
88+
)
89+
90+
91+
@register_constant_fold_exclusion_rule(ATTENTION_MASK_ARANGE_RULE_ID)
92+
def _attention_mask_arange_rule(node: torch.fx.Node) -> Iterable[torch.fx.Node]:
93+
"""Select aranges feeding an attention mask."""
94+
# Every SDPA overload that takes a mask keeps it at positional index 3.
95+
# _scaled_dot_product_flash_attention is absent because it has no mask.
96+
attention_mask_args: dict[Any, tuple[int, str]] = {
97+
torch.ops.aten.scaled_dot_product_attention: (3, "attn_mask"),
98+
torch.ops.aten._scaled_dot_product_efficient_attention: (3, "attn_bias"),
99+
torch.ops.aten._scaled_dot_product_cudnn_attention: (3, "attn_bias"),
100+
}
101+
102+
if node.op != "call_function":
103+
return ()
104+
105+
overload_packet = getattr(node.target, "overloadpacket", None)
106+
mask_arg = attention_mask_args.get(overload_packet)
107+
if mask_arg is None:
108+
return ()
109+
110+
mask_index, mask_kwarg = mask_arg
111+
mask = node.kwargs.get(
112+
mask_kwarg,
113+
node.args[mask_index] if len(node.args) > mask_index else None,
114+
)
115+
if not isinstance(mask, torch.fx.Node):
116+
return ()
117+
118+
return _find_attention_mask_aranges(mask)

0 commit comments

Comments
 (0)