|
| 1 | +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# SPDX-License-Identifier: BSD-3-Clause |
| 3 | + |
| 4 | +from typing import Any, Iterable |
| 5 | + |
| 6 | +import torch |
| 7 | +from torch._subclasses.functional_tensor import mb_unwrap_functional_tensor |
| 8 | +from torch.fx.experimental.proxy_tensor import get_proxy_mode, get_proxy_slot |
| 9 | + |
| 10 | +from ._core import ( |
| 11 | + _mark_constant_fold_exclusion, |
| 12 | + register_constant_fold_exclusion_rule, |
| 13 | +) |
| 14 | + |
| 15 | +ATTENTION_MASK_ARANGE_RULE_ID = "attention_mask_arange" |
| 16 | + |
| 17 | + |
| 18 | +def _is_arange_node(node: torch.fx.Node) -> bool: |
| 19 | + return ( |
| 20 | + node.op == "call_function" |
| 21 | + and getattr(node.target, "overloadpacket", None) is torch.ops.aten.arange |
| 22 | + ) |
| 23 | + |
| 24 | + |
| 25 | +def _find_attention_mask_aranges( |
| 26 | + mask_node: torch.fx.Node, |
| 27 | +) -> list[torch.fx.Node]: |
| 28 | + """Find INT32/INT64 position aranges feeding an attention mask. |
| 29 | +
|
| 30 | + Other range dtypes, including unknown dtypes, retain normal constant |
| 31 | + folding. Integer ranges also remain foldable when mask construction uses |
| 32 | + FP64 intermediates: TensorRT cannot represent those tensors, and truncating |
| 33 | + them can change position comparisons. |
| 34 | + """ |
| 35 | + nodes_to_visit: list[torch.fx.Node] = [mask_node] |
| 36 | + visited: set[torch.fx.Node] = set() |
| 37 | + matching_nodes: list[torch.fx.Node] = [] |
| 38 | + |
| 39 | + while nodes_to_visit: |
| 40 | + current = nodes_to_visit.pop() |
| 41 | + if current in visited: |
| 42 | + continue |
| 43 | + |
| 44 | + visited.add(current) |
| 45 | + value = current.meta.get("val") |
| 46 | + dtype = getattr(value, "dtype", current.kwargs.get("dtype")) |
| 47 | + if dtype is torch.float64: |
| 48 | + # Leave the whole constructor foldable, including integer aranges |
| 49 | + # compared with FP64 values or cast to FP64 after construction. |
| 50 | + return [] |
| 51 | + |
| 52 | + if _is_arange_node(current): |
| 53 | + if dtype not in (torch.int32, torch.int64): |
| 54 | + # Keep mixed-dtype constructors foldable as a whole, rather |
| 55 | + # than preserving only their integer ranges. |
| 56 | + return [] |
| 57 | + matching_nodes.append(current) |
| 58 | + |
| 59 | + nodes_to_visit.extend(current.all_input_nodes) |
| 60 | + |
| 61 | + return matching_nodes |
| 62 | + |
| 63 | + |
| 64 | +def exclude_attn_mask_aranges_from_constant_fold(attn_mask: torch.Tensor) -> None: |
| 65 | + """Mark aranges behind an attention mask while tracing a decomposition. |
| 66 | +
|
| 67 | + ``run_decompositions`` invokes decompositions once for functionalization and |
| 68 | + again while proxy tracing. Only the proxy-tracing invocation has an FX graph |
| 69 | + on which metadata can be set. |
| 70 | + """ |
| 71 | + proxy_mode = get_proxy_mode() |
| 72 | + if proxy_mode is None: |
| 73 | + return |
| 74 | + |
| 75 | + unwrapped_mask = mb_unwrap_functional_tensor(attn_mask) |
| 76 | + tracked_mask = get_proxy_slot( |
| 77 | + unwrapped_mask, |
| 78 | + proxy_mode.tracer, |
| 79 | + default=None, |
| 80 | + ) |
| 81 | + proxy = getattr(tracked_mask, "proxy", tracked_mask) |
| 82 | + mask_node = getattr(proxy, "node", None) |
| 83 | + |
| 84 | + if isinstance(mask_node, torch.fx.Node): |
| 85 | + _mark_constant_fold_exclusion( |
| 86 | + _find_attention_mask_aranges(mask_node), |
| 87 | + ATTENTION_MASK_ARANGE_RULE_ID, |
| 88 | + ) |
| 89 | + |
| 90 | + |
| 91 | +@register_constant_fold_exclusion_rule(ATTENTION_MASK_ARANGE_RULE_ID) |
| 92 | +def _attention_mask_arange_rule(node: torch.fx.Node) -> Iterable[torch.fx.Node]: |
| 93 | + """Select aranges feeding an attention mask.""" |
| 94 | + # Every SDPA overload that takes a mask keeps it at positional index 3. |
| 95 | + # _scaled_dot_product_flash_attention is absent because it has no mask. |
| 96 | + attention_mask_args: dict[Any, tuple[int, str]] = { |
| 97 | + torch.ops.aten.scaled_dot_product_attention: (3, "attn_mask"), |
| 98 | + torch.ops.aten._scaled_dot_product_efficient_attention: (3, "attn_bias"), |
| 99 | + torch.ops.aten._scaled_dot_product_cudnn_attention: (3, "attn_bias"), |
| 100 | + } |
| 101 | + |
| 102 | + if node.op != "call_function": |
| 103 | + return () |
| 104 | + |
| 105 | + overload_packet = getattr(node.target, "overloadpacket", None) |
| 106 | + mask_arg = attention_mask_args.get(overload_packet) |
| 107 | + if mask_arg is None: |
| 108 | + return () |
| 109 | + |
| 110 | + mask_index, mask_kwarg = mask_arg |
| 111 | + mask = node.kwargs.get( |
| 112 | + mask_kwarg, |
| 113 | + node.args[mask_index] if len(node.args) > mask_index else None, |
| 114 | + ) |
| 115 | + if not isinstance(mask, torch.fx.Node): |
| 116 | + return () |
| 117 | + |
| 118 | + return _find_attention_mask_aranges(mask) |
0 commit comments