|
5 | 5 | # LICENSE file in the root directory of this source tree. |
6 | 6 |
|
7 | 7 | """Compare GGML LLM runner prefill and decoding with a tiny eager HF Llama, |
8 | | -including a prompt longer than the method's token input.""" |
| 8 | +including a prompt longer than the method's token input, for in-graph and |
| 9 | +off-graph KV cache packages.""" |
9 | 10 |
|
10 | 11 | import argparse |
11 | 12 | import base64 |
| 13 | +import copy |
12 | 14 | import re |
13 | 15 | import subprocess |
14 | 16 | import tempfile |
15 | 17 | from pathlib import Path |
16 | 18 |
|
17 | 19 | import torch |
18 | | -from executorch.backends.native.examples.hf_llm.export_hf_llm import LastTokenLM |
| 20 | +from executorch.backends.native.examples.hf_llm.export_hf_llm import ( |
| 21 | + LastTokenLM, |
| 22 | + OffGraphLastTokenLM, |
| 23 | +) |
19 | 24 | from executorch.exir.native import to_native |
| 25 | +from executorch.extension.llm.cache.hf_attention import register_offgraph_attention |
20 | 26 | from transformers import LlamaConfig, LlamaForCausalLM |
21 | 27 |
|
22 | 28 |
|
23 | 29 | def export_package( |
24 | | - model: LlamaForCausalLM, max_seq_length: int, max_tokens: int, path: Path |
| 30 | + wrapper: torch.nn.Module, |
| 31 | + max_tokens: int, |
| 32 | + path: Path, |
| 33 | + metadata: dict | None = None, |
| 34 | + selected: bool = False, |
25 | 35 | ) -> None: |
26 | | - """A last-token package whose method takes at most `max_tokens` tokens.""" |
27 | | - wrapper = LastTokenLM(model, max_seq_length).eval() |
| 36 | + """A package of `wrapper` whose method takes at most `max_tokens` tokens, |
| 37 | + and with `selected` the token indices whose logits to return.""" |
28 | 38 | seq = torch.export.Dim("seq", min=1, max=max_tokens) |
| 39 | + inputs = [torch.zeros((1, 6), dtype=torch.long), torch.arange(6)] |
| 40 | + dynamic_shapes = {"input_ids": {1: seq}, "cache_position": {0: seq}} |
| 41 | + if selected: |
| 42 | + # Trace two rows: export specializes a size of 1. |
| 43 | + inputs.append(torch.tensor([0, 5])) |
| 44 | + dynamic_shapes["logits_to_keep"] = { |
| 45 | + 0: torch.export.Dim("rows", min=1, max=max_tokens) |
| 46 | + } |
29 | 47 | exported = torch.export.export( |
30 | | - wrapper, |
31 | | - (torch.zeros((1, 6), dtype=torch.long), torch.arange(6)), |
32 | | - dynamic_shapes={"input_ids": {1: seq}, "cache_position": {0: seq}}, |
33 | | - strict=True, |
| 48 | + wrapper, tuple(inputs), dynamic_shapes=dynamic_shapes, strict=True |
34 | 49 | ) |
35 | | - to_native(exported).save(str(path)) |
| 50 | + to_native(exported, metadata=metadata).save(str(path)) |
36 | 51 |
|
37 | 52 |
|
38 | 53 | def check_runner( |
@@ -121,12 +136,34 @@ def main() -> None: |
121 | 136 | f"{base64.b64encode(bytes([i])).decode()} {i}\n" for i in range(256) |
122 | 137 | ) |
123 | 138 | ) |
| 139 | + in_graph = LastTokenLM(model, max_seq_length).eval() |
124 | 140 | full = root / "llama.ptn" |
125 | | - export_package(model, max_seq_length, max_seq_length - 1, full) |
| 141 | + export_package(in_graph, max_seq_length - 1, full) |
126 | 142 | # A 20-token prompt prefills in three calls of at most 7 tokens. |
127 | 143 | chunked = root / "llama_chunked.ptn" |
128 | | - export_package(model, max_seq_length, 7, chunked) |
129 | | - for package, prompt in ((full, "abc"), (chunked, "the quick brown fox")): |
| 144 | + export_package(in_graph, 7, chunked) |
| 145 | + # The runner supplies the KV cache, sized from the package's metadata. |
| 146 | + register_offgraph_attention("offgraph") |
| 147 | + offgraph_model = copy.deepcopy(model) |
| 148 | + offgraph_model.config._attn_implementation = "offgraph" |
| 149 | + offgraph = OffGraphLastTokenLM(offgraph_model, max_seq_length).eval() |
| 150 | + offgraph_chunked = root / "llama_offgraph_chunked.ptn" |
| 151 | + export_package(offgraph, 7, offgraph_chunked, offgraph.metadata()) |
| 152 | + # A batching export; the runner selects the last token's logits. |
| 153 | + selected = OffGraphLastTokenLM( |
| 154 | + offgraph_model, max_seq_length, selected=True |
| 155 | + ).eval() |
| 156 | + selected_chunked = root / "llama_selected_chunked.ptn" |
| 157 | + export_package( |
| 158 | + selected, 7, selected_chunked, selected.metadata(), selected=True |
| 159 | + ) |
| 160 | + long_prompt = "the quick brown fox" |
| 161 | + for package, prompt in ( |
| 162 | + (full, "abc"), |
| 163 | + (chunked, long_prompt), |
| 164 | + (offgraph_chunked, long_prompt), |
| 165 | + (selected_chunked, long_prompt), |
| 166 | + ): |
130 | 167 | check_runner( |
131 | 168 | args.runner, |
132 | 169 | model, |
|
0 commit comments