Skip to content

Commit 3a344c3

Browse files
committed
Export hf_static float-cache programs with a registered HF runner
Move backend-neutral static-cache rewriting into the LLM extension and keep QNN partitioning and tuning in the backend. Detect sliding layers from model configuration, write every cache and counter in place on the caller buffer (export fails otherwise, since a missed write becomes full-cache CPU copies), and register a versioned runner that handles fixed chunks, padding, counters and reset. Quantized caches, sha attention and full-length sliding mode are excluded. Authored with AI assistance (Metamate / agentcloud).
1 parent 5e8859d commit 3a344c3

17 files changed

Lines changed: 4267 additions & 23 deletions

File tree

‎backends/qualcomm/_passes/convert_linear_to_conv2d.py‎

Lines changed: 34 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ def call(self, graph_module: GraphModule):
176176
# The set tracks whether the weights associated with a linear node have been preprocessed,
177177
# and is used in cases where multiple linear nodes share the same weights.
178178
preprocessed_linear_weights_set = set()
179+
calibrated_inputs = {}
179180

180181
for node in list(graph.nodes):
181182
if node.target in self._linear_targets and self._is_selected(node):
@@ -202,30 +203,42 @@ def call(self, graph_module: GraphModule):
202203
# rank = 2: Reshape (dim, C) to (1, dim, C, 1), and permute to (1, C, 1, dim)
203204
# rank = 3: Reshape (N, dim, C) to (N, dim, C, 1), and permute to (N, C, 1, dim)
204205
# rank = 4: Permute (N, H, W, C) to (N, C, H, W)
205-
if rank <= 3:
206-
shape = (
207-
(1, *input_meta_val.shape, 1)
208-
if rank == 2
209-
else (*input_meta_val.shape, 1)
210-
)
211-
cur_meta_val = cur_meta_val.reshape(shape)
212-
reshape_node = self._create_node(
206+
# Linears reading the same input (q/k/v, gate/up) share one layout
207+
# change: each extra copy is an op HTP runs.
208+
# Share layout changes only within the same parent quantizer
209+
# scope. Otherwise a helper could inherit another module's recipe.
210+
parent_stack = tuple(
211+
list(node.meta.get("nn_module_stack", {}).items())[:-1]
212+
)
213+
input_key = (input_node, is_edge, parent_stack)
214+
if input_key in calibrated_inputs:
215+
permute_node, cur_meta_val = calibrated_inputs[input_key]
216+
else:
217+
if rank <= 3:
218+
shape = (
219+
(1, *input_meta_val.shape, 1)
220+
if rank == 2
221+
else (*input_meta_val.shape, 1)
222+
)
223+
cur_meta_val = cur_meta_val.reshape(shape)
224+
reshape_node = self._create_node(
225+
graph_module,
226+
ops["view"],
227+
(input_node, shape),
228+
cur_meta_val,
229+
)
230+
helpers.append(reshape_node)
231+
232+
order = (0, 3, 1, 2) if rank == 4 else (0, 2, 3, 1)
233+
cur_meta_val = cur_meta_val.permute(order)
234+
permute_node = self._create_node(
213235
graph_module,
214-
ops["view"],
215-
(input_node, shape),
236+
ops["permute"],
237+
(reshape_node, order) if rank <= 3 else (input_node, order),
216238
cur_meta_val,
217239
)
218-
helpers.append(reshape_node)
219-
220-
order = (0, 3, 1, 2) if rank == 4 else (0, 2, 3, 1)
221-
cur_meta_val = cur_meta_val.permute(order)
222-
permute_node = self._create_node(
223-
graph_module,
224-
ops["permute"],
225-
(reshape_node, order) if rank <= 3 else (input_node, order),
226-
cur_meta_val,
227-
)
228-
helpers.append(permute_node)
240+
helpers.append(permute_node)
241+
calibrated_inputs[input_key] = (permute_node, cur_meta_val)
229242

230243
# Step 2: reshape weight
231244
if weight_placeholder_node.name not in preprocessed_linear_weights_set:

‎backends/qualcomm/quantizer/custom_annotation.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -269,9 +269,14 @@ def annotate_matmul_input1(node: Node, is_qat: str):
269269
if (
270270
node.op == "call_function"
271271
and node.target == torch.ops.aten.matmul.default
272-
and all(arg.op == "call_function" for arg in node.args)
272+
and all(
273+
arg.op == "call_function"
274+
or (arg.op == "placeholder" and arg.meta["val"].is_floating_point())
275+
for arg in node.args
276+
)
273277
):
274-
# Only apply custom annotation on Q @ K^T @ V
278+
# Only apply custom annotation on Q @ K^T @ V (activations, or a
279+
# cache that is a graph input), not on matmuls with constants.
275280
annotate_matmul(node, quantization_config_16a8w)
276281
annotate_matmul_input1(node.args[1], is_qat=is_qat)
277282

0 commit comments

Comments
 (0)