|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the BSD-style license found in the |
| 5 | +# LICENSE file in the root directory of this source tree. |
| 6 | +import logging |
| 7 | +from typing import Dict, Set |
| 8 | + |
| 9 | +import torch |
| 10 | +from executorch.backends.qualcomm.serialization.qc_schema_serialize import ( |
| 11 | + flatbuffer_to_option, |
| 12 | +) |
| 13 | +from executorch.backends.qualcomm.utils.constants import ( |
| 14 | + QCOM_FALLBACK_NODE, |
| 15 | + QCOM_QUANT_ATTRS, |
| 16 | +) |
| 17 | +from executorch.exir.dialects._ops import ops as exir_ops |
| 18 | +from executorch.exir.pass_base import ExportPass, PassResult |
| 19 | +from torch.fx.experimental.symbolic_shapes import statically_known_true |
| 20 | + |
| 21 | +logger = logging.getLogger(__name__) |
| 22 | + |
| 23 | +_SIN_COS_OPS = {exir_ops.edge.aten.sin.default, exir_ops.edge.aten.cos.default} |
| 24 | + |
| 25 | +# Ops that only move or cast values. An angle that is a graph input moved this |
| 26 | +# way alone is not computed by the graph, so the caller can keep it small. |
| 27 | +_VIEW_OPS = { |
| 28 | + exir_ops.edge.aten._to_copy.default, |
| 29 | + exir_ops.edge.aten.cat.default, |
| 30 | + exir_ops.edge.aten.clone.default, |
| 31 | + exir_ops.edge.aten.expand_copy.default, |
| 32 | + exir_ops.edge.aten.permute_copy.default, |
| 33 | + exir_ops.edge.aten.select_copy.int, |
| 34 | + exir_ops.edge.aten.slice_copy.Tensor, |
| 35 | + exir_ops.edge.aten.squeeze_copy.dims, |
| 36 | + exir_ops.edge.aten.unsqueeze_copy.default, |
| 37 | + exir_ops.edge.aten.view_copy.default, |
| 38 | + exir_ops.edge.dim_order_ops._to_dim_order_copy.default, |
| 39 | +} |
| 40 | + |
| 41 | +# The cheap elementwise and shape ops a positional angle, like t * freqs, |
| 42 | +# positions / 10000 ** (2 * arange(n) / d) or t * exp(-log(10000) * arange(n) |
| 43 | +# / n), is built from. where comes from a decomposed linspace. |
| 44 | +_ANGLE_OPS = _VIEW_OPS | { |
| 45 | + exir_ops.edge.aten.add.Tensor, |
| 46 | + exir_ops.edge.aten.arange.start_step, |
| 47 | + exir_ops.edge.aten.div.Tensor, |
| 48 | + exir_ops.edge.aten.exp.default, |
| 49 | + exir_ops.edge.aten.log.default, |
| 50 | + exir_ops.edge.aten.mul.Tensor, |
| 51 | + exir_ops.edge.aten.neg.default, |
| 52 | + exir_ops.edge.aten.pow.Tensor_Tensor, |
| 53 | + exir_ops.edge.aten.reciprocal.default, |
| 54 | + exir_ops.edge.aten.sub.Tensor, |
| 55 | + exir_ops.edge.aten.where.self, |
| 56 | +} |
| 57 | + |
| 58 | +# Positions are also built from masks, like attention_mask.cumsum(-1), |
| 59 | +# mask.sum(-1) + ones(n).cumsum(0) or arange(n) < k. |
| 60 | +_POSITION_OPS = _ANGLE_OPS | { |
| 61 | + exir_ops.edge.aten.cumsum.default, |
| 62 | + exir_ops.edge.aten.full.default, |
| 63 | + exir_ops.edge.aten.sum.dim_IntList, |
| 64 | + exir_ops.edge.aten.eq.Scalar, |
| 65 | + exir_ops.edge.aten.eq.Tensor, |
| 66 | + exir_ops.edge.aten.ge.Scalar, |
| 67 | + exir_ops.edge.aten.ge.Tensor, |
| 68 | + exir_ops.edge.aten.gt.Scalar, |
| 69 | + exir_ops.edge.aten.gt.Tensor, |
| 70 | + exir_ops.edge.aten.le.Scalar, |
| 71 | + exir_ops.edge.aten.le.Tensor, |
| 72 | + exir_ops.edge.aten.lt.Scalar, |
| 73 | + exir_ops.edge.aten.lt.Tensor, |
| 74 | + exir_ops.edge.aten.ne.Scalar, |
| 75 | + exir_ops.edge.aten.ne.Tensor, |
| 76 | +} |
| 77 | + |
| 78 | + |
| 79 | +def _is_exact(node: torch.fx.Node) -> bool: |
| 80 | + val = node.meta.get("val") |
| 81 | + return isinstance(val, torch.SymInt) or ( |
| 82 | + isinstance(val, torch.Tensor) and not val.is_floating_point() |
| 83 | + ) |
| 84 | + |
| 85 | + |
| 86 | +class TagPositionalSinCos(ExportPass): |
| 87 | + """ |
| 88 | + On HTP in float16 precision, tag the unquantized float32 sin and cos nodes |
| 89 | + whose angle the graph computes from its inputs, constants and integer |
| 90 | + positions with only the ops in _ANGLE_OPS, as a timestep or rotary |
| 91 | + embedding does, and that are larger than each input they read, together |
| 92 | + with those ops, so that the partitioner leaves them on CPU. |
| 93 | +
|
| 94 | + float16 cannot hold such an angle precisely: its values are 1.0 apart from |
| 95 | + 1024 to 2048, where the angles of a timestep embedding can reach, and HTP's |
| 96 | + float16 Sin and Cos lose more on top of that for large angles. On CPU |
| 97 | + these nodes compute in float32, as the model does. A sin or cos whose angle |
| 98 | + reads an activation, even through integers, is left alone, so no conv or |
| 99 | + linear moves to CPU and the delegate is not split. |
| 100 | +
|
| 101 | + The tagged ops then need CPU kernels at runtime. To delegate them as |
| 102 | + before, take passes_job = get_qnn_pass_manager_cls().get_capture_program_passes() |
| 103 | + and set passes_job[TagPositionalSinCos][QCOM_PASS_ACTIVATE_KEY] = False. |
| 104 | + """ |
| 105 | + |
| 106 | + def __init__(self, edge_program, compiler_specs=None): |
| 107 | + super(TagPositionalSinCos, self).__init__() |
| 108 | + self.user_inputs = set(edge_program.graph_signature.user_inputs) |
| 109 | + self.htp_fp16 = False |
| 110 | + if compiler_specs is not None: |
| 111 | + # Imported here, since partition depends on _passes. |
| 112 | + from executorch.backends.qualcomm.partition.utils import ( |
| 113 | + generate_qnn_executorch_option, |
| 114 | + is_htp_fp16, |
| 115 | + ) |
| 116 | + |
| 117 | + self.htp_fp16 = is_htp_fp16( |
| 118 | + flatbuffer_to_option( |
| 119 | + generate_qnn_executorch_option(compiler_specs) |
| 120 | + ).backend_options |
| 121 | + ) |
| 122 | + |
| 123 | + def call(self, graph_module: torch.fx.GraphModule): |
| 124 | + if not self.htp_fp16: |
| 125 | + return PassResult(graph_module, False) |
| 126 | + nodes, positions = set(), {} |
| 127 | + for node in graph_module.graph.nodes: |
| 128 | + if ( |
| 129 | + node.target in _SIN_COS_OPS |
| 130 | + and QCOM_QUANT_ATTRS not in node.meta |
| 131 | + and node.meta["val"].dtype == torch.float32 |
| 132 | + ): |
| 133 | + angle_nodes = self._get_angle_nodes(node.args[0], positions) |
| 134 | + if any( |
| 135 | + n.target not in _VIEW_OPS for n in angle_nodes |
| 136 | + ) and self._broadcasts_inputs(node, angle_nodes): |
| 137 | + nodes |= angle_nodes | {node} |
| 138 | + for node in nodes: |
| 139 | + node.meta[QCOM_FALLBACK_NODE] = True |
| 140 | + if nodes: |
| 141 | + logger.warning( |
| 142 | + f"TagPositionalSinCos keeps {len(nodes)} ops of positional sin " |
| 143 | + "and cos on CPU in float32, which needs their CPU kernels. " |
| 144 | + "To delegate them, set passes_job[TagPositionalSinCos]" |
| 145 | + "[QCOM_PASS_ACTIVATE_KEY] = False in the passes_job from " |
| 146 | + "get_qnn_pass_manager_cls().get_capture_program_passes()." |
| 147 | + ) |
| 148 | + return PassResult(graph_module, bool(nodes)) |
| 149 | + |
| 150 | + def _broadcasts_inputs( |
| 151 | + self, node: torch.fx.Node, angle_nodes: Set[torch.fx.Node] |
| 152 | + ) -> bool: |
| 153 | + # A positional angle spreads a few positions or timesteps over a |
| 154 | + # frequency axis, so it is larger than every user input and integer |
| 155 | + # tensor its float ops read. An angle as large as one, like an image |
| 156 | + # times a scale, stays delegated. Integer positions may read larger |
| 157 | + # masks. |
| 158 | + numel = node.meta["val"].numel() |
| 159 | + return all( |
| 160 | + statically_known_true(arg.meta["val"].numel() < numel) |
| 161 | + for angle_node in angle_nodes |
| 162 | + if not _is_exact(angle_node) |
| 163 | + for arg in angle_node.all_input_nodes |
| 164 | + if isinstance(arg.meta.get("val"), torch.Tensor) |
| 165 | + and (arg.name in self.user_inputs or _is_exact(arg)) |
| 166 | + ) |
| 167 | + |
| 168 | + def _get_angle_nodes( |
| 169 | + self, angle: torch.fx.Node, positions: Dict[torch.fx.Node, Set[torch.fx.Node]] |
| 170 | + ) -> Set[torch.fx.Node]: |
| 171 | + """ |
| 172 | + The nodes computing `angle` from graph inputs, constants and integer |
| 173 | + positions, or an empty set if another op takes part. |
| 174 | + """ |
| 175 | + nodes, stack = set(), [angle] |
| 176 | + while stack: |
| 177 | + node = stack.pop() |
| 178 | + if node in nodes or node.op != "call_function": |
| 179 | + continue |
| 180 | + if _is_exact(node): |
| 181 | + position_nodes = self._get_position_nodes(node, positions) |
| 182 | + if not position_nodes and not isinstance( |
| 183 | + node.meta["val"], torch.SymInt |
| 184 | + ): |
| 185 | + return set() |
| 186 | + nodes |= position_nodes |
| 187 | + continue |
| 188 | + if node.target not in _ANGLE_OPS or QCOM_QUANT_ATTRS in node.meta: |
| 189 | + return set() |
| 190 | + nodes.add(node) |
| 191 | + stack.extend(node.all_input_nodes) |
| 192 | + return nodes |
| 193 | + |
| 194 | + def _get_position_nodes( |
| 195 | + self, node: torch.fx.Node, positions: Dict[torch.fx.Node, Set[torch.fx.Node]] |
| 196 | + ) -> Set[torch.fx.Node]: |
| 197 | + """ |
| 198 | + `node` and the integer nodes it is computed from, if they are built |
| 199 | + from graph inputs, constants and sizes with the ops in _POSITION_OPS |
| 200 | + only, or an empty set otherwise. |
| 201 | +
|
| 202 | + Integers are exact, so they only need to move with the angle when they |
| 203 | + read nothing else. Then the casts I64toI32 puts after int64 producers |
| 204 | + move together with those producers. |
| 205 | + """ |
| 206 | + if node not in positions: |
| 207 | + positions[node] = set() |
| 208 | + nodes = {node} |
| 209 | + for arg in node.all_input_nodes: |
| 210 | + if arg.op != "call_function" or isinstance( |
| 211 | + arg.meta.get("val"), torch.SymInt |
| 212 | + ): |
| 213 | + continue |
| 214 | + arg_nodes = ( |
| 215 | + self._get_position_nodes(arg, positions) |
| 216 | + if _is_exact(arg) |
| 217 | + else set() |
| 218 | + ) |
| 219 | + if not arg_nodes: |
| 220 | + nodes = set() |
| 221 | + break |
| 222 | + nodes |= arg_nodes |
| 223 | + if node.target not in _POSITION_OPS or QCOM_QUANT_ATTRS in node.meta: |
| 224 | + nodes = set() |
| 225 | + positions[node] = nodes |
| 226 | + return positions[node] |
0 commit comments