Skip to content

Commit caf79c5

Browse files
authored
Qualcomm AI Engine Direct - Support Constant Propagation (#22867)
### Summary - Support Constant Folding through edge pass. ### Test plan `pytest backends/qualcomm/tests/rework/passes/test.py -k "test_constant_folding"`
1 parent 80df2ee commit caf79c5

10 files changed

Lines changed: 710 additions & 10 deletions

File tree

‎backends/qualcomm/_passes/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from .annotate_unbind import AnnotateUnbind
1212
from .build_quant_io import BuildQuantIo
1313
from .canonicalize_conv import CanonicalizeConv
14+
from .constant_folding import ConstantFolding
1415
from .convert_bmm_to_matmul import ConvertBmmToMatmul
1516
from .convert_complex_stft import ConvertComplexStft
1617
from .convert_linear_to_conv2d import ConvertLinearToConv2d
@@ -84,6 +85,7 @@
8485
AnnotateUnbind,
8586
BuildQuantIo,
8687
CanonicalizeConv,
88+
ConstantFolding,
8789
ConvertBmmToMatmul,
8890
ConvertComplexStft,
8991
ConvertLinearToConv2d,
Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
# Copyright (c) Qualcomm Innovation Center, Inc.
2+
# All rights reserved
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
import logging
7+
from typing import Any
8+
9+
import torch
10+
from executorch.backends.qualcomm._passes.utils import copy_meta
11+
from executorch.backends.qualcomm.builders.utils import (
12+
get_parameter,
13+
is_mutable_buffer_input,
14+
is_parameter,
15+
)
16+
from executorch.exir.operator.util import _QUANT_PRIMITIVES
17+
from executorch.exir.pass_base import ExportPass, PassResult
18+
from executorch.exir.passes import dead_code_elimination_pass
19+
from executorch.exir.passes.constant_prop_pass import _PRIMITIVE_TYPES
20+
from executorch.exir.passes.replace_aten_with_edge_pass import aten_to_edge
21+
from torch._guards import detect_fake_mode
22+
from torch.utils import _pytree as pytree
23+
from torchao.quantization.pt2e.utils import get_new_attr_name_with_prefix
24+
25+
26+
# These are qdq edge targets
27+
_EDGE_QDQ_TARGETS = set(_QUANT_PRIMITIVES) | {
28+
aten_to_edge(op) for op in _QUANT_PRIMITIVES
29+
}
30+
31+
32+
class ConstantFolding(ExportPass):
33+
"""
34+
executorch/exir/passes/const_prop_pass.py runs at to_executorch stage, and it won't
35+
work if we run the pass at edge stage. This pass is to address the issue.
36+
One of biggest reason for this pass is that some ops are not supported by QNN and will
37+
fail during op validation. If the op can be constant folded, then there won't be partitions.
38+
39+
40+
Folds subgraphs whose leaves are parameters, lifted tensor constants, or
41+
*non-mutated* buffers. Mutated buffers are intentionally excluded — view ops
42+
over them would alias mutable storage if folded, and downstream passes /
43+
`run_decompositions` mis-handle that aliasing.
44+
"""
45+
46+
_TENSOR_CONSTANT_PREFIX = "_prop_tensor_constant_"
47+
48+
def __init__(self, edge_program: torch.export.ExportedProgram):
49+
super().__init__()
50+
# Run decomposition is required so graph_signature stores information about mutable buffer.
51+
self.decomposed_aten_program = edge_program.run_decompositions({})
52+
53+
def _get_const_placeholders(
54+
self, graph_module: torch.fx.GraphModule
55+
) -> dict[torch.fx.Node, torch.Tensor]:
56+
"""
57+
Find all constant_tensor and store it in a dict {node : tensor}
58+
The tensor in dict has actual tensor with values, not fake tensor.
59+
"""
60+
61+
node_to_tensor = {}
62+
for node in graph_module.graph.nodes:
63+
if node.op != "placeholder":
64+
continue
65+
# Don't fold mutable buffer
66+
if is_mutable_buffer_input(node, self.decomposed_aten_program):
67+
continue
68+
if not is_parameter(node, self.decomposed_aten_program):
69+
continue
70+
node_to_tensor[node] = get_parameter(node, self.decomposed_aten_program)
71+
return node_to_tensor
72+
73+
def _propagate( # noqa: C901
74+
self,
75+
graph_module: torch.fx.GraphModule,
76+
node_to_tensor: dict[torch.fx.Node, Any],
77+
) -> None:
78+
"""
79+
Iterate the call_function node. compute the output of that node if it can be constant folded,
80+
and save it in node_to_tensor. Based on torch/fx/graph.py, graph.nodes should always return
81+
nodes in topological order.
82+
Example:
83+
constant_value_1 -> upsample_bicubic2d ----\
84+
add_1 --------------------------\
85+
constant_value_2 ----/ add_2 ----> output
86+
user_input(non-constant)------/
87+
88+
With graph above, when first enter this method, node_to_tensor dict looks like:
89+
node_to_tensor = {
90+
constant_value_1_node : constant_value_1_tensor,
91+
constant_value_2_node : constant_value_2_tensor,
92+
}
93+
94+
At the end of the method, node_to_tensor dict looks like following:
95+
node_to_tensor = {
96+
constant_value_1_node : constant_value_1_tensor,
97+
constant_value_2_node : constant_value_2_tensor,
98+
upsample_bicubic2d_node : upsample_bicubic2d_output_tensor,
99+
add_1_node : add_1_output_tensor,
100+
}
101+
102+
Tensor in dict will be actual values instead of fake tensor.
103+
For example, add_1_output_tensor value would be the result of adding upsample_bicubic2d_output_tensor and constant_value_2.
104+
"""
105+
106+
# If previous node is visited, this function won't recursive search all the way back to input node.
107+
# It should fall into the base case.
108+
# This prevents recursive search back to input source node everytime.
109+
def _is_const(arg, node_to_tensor):
110+
111+
# For args case.
112+
if isinstance(arg, (tuple, list)):
113+
return all(_is_const(x, node_to_tensor) for x in arg)
114+
115+
# For kwargs case
116+
if isinstance(arg, dict):
117+
return all(_is_const(x, node_to_tensor) for x in arg.values())
118+
119+
# If a tensor is optional and not provided, it will be None.
120+
# Primitive_types is for constants like integers. These should be able to be folded.
121+
if arg is None or isinstance(arg, _PRIMITIVE_TYPES):
122+
return True
123+
124+
if isinstance(arg, torch.fx.Node):
125+
# Base case
126+
return arg in node_to_tensor
127+
else:
128+
# If there are some unexpected args that doesn't know how to handle, just return False to be safe.
129+
return False
130+
131+
# `pytree.tree_map` flattens containers and treats Node as a leaf, so this
132+
# only ever receives leaves.
133+
def _get_data(arg, node_to_tensor):
134+
if arg is None or isinstance(arg, _PRIMITIVE_TYPES):
135+
return arg
136+
if isinstance(arg, torch.fx.Node):
137+
return node_to_tensor.get(arg)
138+
return None
139+
140+
for node in graph_module.graph.nodes:
141+
if node.op != "call_function":
142+
continue
143+
# Avoid folding nodes like torch.rand since we shouldn't precompute.
144+
if node.is_impure():
145+
continue
146+
if not _is_const(node.args, node_to_tensor):
147+
continue
148+
if not _is_const(node.kwargs, node_to_tensor):
149+
continue
150+
151+
# Copied from executorch/exir/passes/constant_prop_pass.py
152+
# Retrieves args and kwargs required for the node to perform inference.
153+
args_data, kwargs_data = pytree.tree_map(
154+
lambda x: _get_data(x, node_to_tensor),
155+
(node.args, node.kwargs),
156+
)
157+
# Perform node inference
158+
with torch.no_grad():
159+
try:
160+
result = node.target(*args_data, **kwargs_data)
161+
except Exception:
162+
logging.warning(
163+
f"Unable to fold the node {node.name}. Skip folding.",
164+
exc_info=True,
165+
)
166+
continue
167+
168+
# Save the node's result to the map.
169+
node_to_tensor[node] = result
170+
171+
def _materialize_as_buffer(
172+
self,
173+
graph_module: torch.fx.GraphModule,
174+
node: torch.fx.Node,
175+
tensor: torch.Tensor,
176+
) -> None:
177+
buffer_name = get_new_attr_name_with_prefix(self._TENSOR_CONSTANT_PREFIX)(
178+
graph_module
179+
)
180+
tensor = tensor.detach().clone(memory_format=torch.contiguous_format)
181+
graph_module.register_buffer(buffer_name, tensor)
182+
val = node.meta.get("val")
183+
fake_mode = detect_fake_mode(val) if val is not None else None
184+
with graph_module.graph.inserting_before(node):
185+
get_attr_node = graph_module.graph.get_attr(buffer_name)
186+
get_attr_node.meta = copy_meta(
187+
node.meta,
188+
lambda m: (
189+
{
190+
**m,
191+
"val": fake_mode.fake_tensor_converter.from_real_tensor(
192+
fake_mode, tensor
193+
),
194+
}
195+
if fake_mode is not None
196+
else m
197+
),
198+
)
199+
# Replace node's user with const node as input
200+
node.replace_all_uses_with(get_attr_node)
201+
202+
def _materialize(
203+
self,
204+
graph_module: torch.fx.GraphModule,
205+
node_to_tensor: dict[torch.fx.Node, Any],
206+
) -> None:
207+
"""
208+
The term "boundary" here refers to where node can no longer be folded.
209+
Boundary will be before add for this graph is const -> relu1 -> sqrt --- > add -> output
210+
input _|
211+
212+
Rules:
213+
1) When creating buffer, start with reverse order, so just create the buffer before boundary.
214+
2) Only the boundary buffer will be created, won't recursive trace args and create unused buffer.
215+
3) For nodes like conv2d with quantizer, preserve dq node right after weight and bias.
216+
4) A graph output is not replaced by a constant, since its output spec would not
217+
follow the constant once it is lifted. Its args become the boundary instead.
218+
"""
219+
220+
# Reverse order: process later (more-derived) constants first, so a
221+
# chain collapses to a single buffer at its boundary.
222+
# Align with rule 1.
223+
for node, tensor in reversed(list(node_to_tensor.items())):
224+
if node.op == "placeholder":
225+
continue
226+
227+
# Aligns with rule 4
228+
if any(user.op == "output" for user in node.users):
229+
del node_to_tensor[node]
230+
continue
231+
232+
# A folded value can be a scalar, SymInt or tuple (`_local_scalar_dense`,
233+
# `sym_size`, multi-output ops). Only a tensor can become a buffer.
234+
if not isinstance(tensor, torch.Tensor):
235+
continue
236+
237+
# If all users can be constant folded, then don't need to fold at this level.
238+
# This behavior aligns with rule 2.
239+
if all(user in node_to_tensor for user in node.users):
240+
continue
241+
242+
# Guarding cases like weight -> dq -> conv2d. Don't fold dq here.
243+
# Aligns with rule 3
244+
if node.target in _EDGE_QDQ_TARGETS:
245+
continue
246+
247+
self._materialize_as_buffer(graph_module, node, tensor)
248+
249+
def call(self, graph_module: torch.fx.GraphModule):
250+
node_to_tensor = self._get_const_placeholders(graph_module)
251+
if len(node_to_tensor) == 0:
252+
return PassResult(graph_module, False)
253+
self._propagate(graph_module, node_to_tensor)
254+
self._materialize(graph_module, node_to_tensor)
255+
dead_code_elimination_pass(graph_module)
256+
graph_module.recompile()
257+
return PassResult(graph_module, True)

‎backends/qualcomm/_passes/qnn_pass_manager.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
AnnotateStack,
1717
AnnotateUnbind,
1818
CanonicalizeConv,
19+
ConstantFolding,
1920
ConvertBmmToMatmul,
2021
ConvertComplexStft,
2122
ConvertLinearToConv2d,
@@ -133,6 +134,7 @@ def get_default_pass_activations(cls):
133134
(AnnotateStack, True),
134135
(AnnotateUnbind, True),
135136
(CanonicalizeConv, True),
137+
(ConstantFolding, True),
136138
(ConvertBmmToMatmul, False),
137139
(ConvertLinearToConv2d, False),
138140
(DecomposeAcos, True),
@@ -299,6 +301,7 @@ def get_passes_dependency_for_capture_program(cls):
299301
AnnotateStack: [RemoveRedundancy],
300302
AnnotateUnbind: [RemoveRedundancy],
301303
CanonicalizeConv: [FoldQDQ, FuseBatchNormWithConv],
304+
ConstantFolding: [LayoutTransform],
302305
ConvertBmmToMatmul: [RecomposePixelUnshuffle],
303306
ConvertLinearToConv2d: [FoldQDQ],
304307
DecomposeAcos: [RemoveRedundancy],

‎backends/qualcomm/builders/node_visitor.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,11 @@ def get_quant_tensor_value(
387387
scale = quant_attrs[QCOM_SCALE]
388388
zero_point = quant_attrs[QCOM_ZERO_POINT]
389389
tensor = (
390-
tensor.div(scale).add(zero_point).round().to(quant_configs[QCOM_DTYPE])
390+
tensor.div(scale)
391+
.add(zero_point)
392+
.round()
393+
.clamp(quant_attrs[QCOM_QUANT_MIN], quant_attrs[QCOM_QUANT_MAX])
394+
.to(quant_configs[QCOM_DTYPE])
391395
)
392396
# Since we're using torch.int32 to store 16bit data
393397
# need to make it compact here for QNN to correctly retrieve data

‎backends/qualcomm/builders/utils.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,15 +104,15 @@ def is_graph_input(
104104

105105

106106
def is_mutable_buffer_input(
107-
tensor: torch.fx.Node, edge_program: torch.export.ExportedProgram
107+
node: torch.fx.Node, edge_program: torch.export.ExportedProgram
108108
) -> bool:
109109
"""
110110
Check if the given tensor is a mutable buffer input
111111
Args:
112-
tensor: EdgeIR Tensor that is being checked for mutable buffer input
112+
node: EdgeIR node that is being checked for mutable buffer input
113113
"""
114-
if tensor.op == "placeholder" and is_buffer(edge_program, tensor):
115-
fqn = edge_program.graph_signature.inputs_to_buffers[tensor.target]
114+
if node.op == "placeholder" and is_buffer(edge_program, node):
115+
fqn = edge_program.graph_signature.inputs_to_buffers[node.target]
116116
# if the buffer is mutated then record that
117117
return fqn in edge_program.graph_signature.buffers_to_mutate.values()
118118

‎backends/qualcomm/quantizer/validators.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,14 @@ def _get_input_qspecs_from_node(
197197
return found_qspecs
198198

199199

200+
def _is_tensor_node(arg) -> bool:
201+
# Scalar args such as the SymFloat of a traced .item() are op parameters,
202+
# not tensor inputs, so they have no dtype to validate.
203+
return isinstance(arg, Node) and isinstance(
204+
arg.meta.get("val"), (torch.Tensor, tuple, list)
205+
)
206+
207+
200208
def _node_port_dtypes(node) -> Tuple[List[str], List[str]]:
201209
"""
202210
Extract input/output dtypes from FX node meta.
@@ -212,11 +220,11 @@ def _node_port_dtypes(node) -> Tuple[List[str], List[str]]:
212220
output_qspec = _get_output_qspec_from_node(quantization_annotation)
213221

214222
for arg in node.args:
215-
if isinstance(arg, Node) and "val" in arg.meta:
223+
if _is_tensor_node(arg):
216224
in_dtypes.append(_get_qnn_datatype(arg, input_qspecs.get(arg, None)))
217225
elif isinstance(arg, (tuple, list)): # for concat op
218226
for item in arg:
219-
if isinstance(item, Node) and "val" in item.meta:
227+
if _is_tensor_node(item):
220228
in_dtypes.append(
221229
_get_qnn_datatype(item, input_qspecs.get(item, None))
222230
)

‎backends/qualcomm/tests/rework/passes/test.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,18 @@ def test_canonicalize_conv(request, kwargs):
6060
CanonicalizeConv.test(request, kwargs) # noqa: F405
6161

6262

63+
@enumerate_backends()
64+
@repack_pass_fixtures
65+
def test_constant_folding(request, kwargs):
66+
ConstantFolding.test(request, kwargs) # noqa: F405
67+
68+
69+
@enumerate_backends()
70+
@repack_pass_fixtures
71+
def test_constant_folding_idempotent(request, kwargs):
72+
ConstantFolding.test_idempotent(request, kwargs) # noqa: F405
73+
74+
6375
@enumerate_backends()
6476
@repack_pass_fixtures
6577
def test_convert_bmm_to_matmul(request, kwargs):

0 commit comments

Comments
 (0)