Skip to content

Commit 8b1f12e

Browse files
committed
[UPDATE] Address the three review comments on #25336: package dflash2.py for Buck, restore GGUF draft capacity with regression coverage, and clarify native test oracles. Rebase the remaining stack after #25335 merged.
[ghstack-poisoned]
2 parents 8815e5e + 737b243 commit 8b1f12e

5 files changed

Lines changed: 107 additions & 6 deletions

File tree

‎examples/models/muse-glimmer/loaders/dflash_loader.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@ def _validate_no_meta(model: nn.Module) -> None:
8484
def read_dflash_gguf_config(gguf_path: str) -> DFlashConfig:
8585
"""Read and validate draft metadata without loading checkpoint weights."""
8686
from executorch.examples.models.muse_glimmer.model.dflash_model import DFlashConfig
87-
8887
from gguf import GGUFReader
8988

9089
reader = GGUFReader(gguf_path)
@@ -116,7 +115,8 @@ def load_dflash_gguf(
116115
"""Load a DFlash draft GGUF, remap keys, and return (model, config).
117116
118117
Streams weights through ``load_checkpoint`` with a DFlash key remap. The
119-
caller applies backend-specific packing after loading.
118+
caller applies backend-specific packing after loading. ``max_seq_len`` sizes
119+
runtime buffers; the config retains the checkpoint's context capacity.
120120
121121
Returns ``(model, config)`` where config is a ``DFlashConfig``.
122122
"""
@@ -127,7 +127,6 @@ def load_dflash_gguf(
127127
from executorch.extension.llm.export.quant import to_default
128128

129129
config = read_dflash_gguf_config(gguf_path)
130-
config.max_seq_len = max_seq_len
131130

132131
print(
133132
f"DFlash config: {config.n_layers} layers, dim={config.dim}, "

‎examples/models/muse-glimmer/model/BUCK‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ fbcode_target(
3030
fbcode_target(
3131
_kind = runtime.python_library,
3232
name = "dflash_model",
33-
srcs = ["dflash_model.py"],
33+
srcs = [
34+
"dflash2.py",
35+
"dflash_model.py",
36+
],
3437
_is_external_target = True,
3538
base_module = "executorch.examples.models.muse_glimmer.model",
3639
visibility = ["PUBLIC"],

‎examples/models/muse-glimmer/tests/README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ ctest --test-dir cmake-out/examples/models/muse-glimmer-batching \
2323
-R '^muse_glimmer_mlx_dflash$' --output-on-failure --no-tests=error
2424
```
2525

26-
The tiny PTEs cover rejection, cache lifecycle, packed owners, and legacy
27-
artifacts against eager references. Seeded continuations are compared across
26+
Random and quantized PTE fixtures compare draft numerics against eager references.
27+
Controlled rejection and cache-lifecycle checks assert expected tokens and
28+
acceptance counts; continuation checks compare with target-only execution,
29+
including legacy artifacts. Seeded continuations are compared across
2830
prompt chunking and mixed prefill/decode batches for both draft formats.
2931
Routing-sensitive fixtures also compare per-round tokens and acceptance counts
3032
between solo and packed execution with unequal prompts and reordered owners.

‎examples/models/muse-glimmer/tests/test_cuda_dflash_packed.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import copy
1010
import importlib.resources
1111
import json
12+
import tempfile
1213
import unittest
1314
import unittest.mock
1415
from dataclasses import replace
@@ -90,6 +91,35 @@ def update_and_fetch(self, layer, k, v, positions):
9091

9192

9293
class PackedDraftMathTest(unittest.TestCase):
94+
def test_gguf_context_capacity_survives_sequence_export(self):
95+
from executorch.examples.models.muse_glimmer.loaders.dflash_loader import (
96+
load_dflash_gguf,
97+
)
98+
from executorch.examples.models.muse_glimmer.tests.test_dflash2 import (
99+
write_original_dflash_gguf,
100+
)
101+
102+
with tempfile.TemporaryDirectory() as directory:
103+
path = Path(directory) / "draft.gguf"
104+
write_original_dflash_gguf(path)
105+
draft, config = load_dflash_gguf(str(path), backend="cuda", max_seq_len=32)
106+
target, _ = models()
107+
target.config.max_seq_len = 32
108+
_, manifest, methods = prepare_cuda_packed_draft(
109+
target,
110+
target.config,
111+
draft,
112+
max_step_tokens=8,
113+
max_cells=64,
114+
max_owners=1,
115+
kv_layout="sequence",
116+
max_kv_seqs=2,
117+
)
118+
capacity = methods["get_draft_offgraph_kv_seq_capacity"]
119+
self.assertEqual(capacity, 64)
120+
self.assertEqual(json.loads(manifest)["maximum_capacity"], capacity)
121+
self.assertGreaterEqual(capacity - config.block_size, target.config.max_seq_len)
122+
93123
def test_uneven_context_nonidentity_order_and_zero_new_context(self):
94124
target, draft = models()
95125
refs = [copy.deepcopy(draft), copy.deepcopy(draft)]

‎examples/models/muse-glimmer/tests/test_dflash2.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,53 @@
3030
)
3131

3232

33+
def write_original_dflash_gguf(path: Path) -> None:
34+
from gguf import GGUFWriter
35+
36+
metadata = {
37+
"embedding_length": 32,
38+
"block_count": 1,
39+
"attention.head_count": 4,
40+
"attention.head_count_kv": 2,
41+
"attention.key_length": 8,
42+
"feed_forward_length": 64,
43+
"block_size": 4,
44+
"context_length": 64,
45+
}
46+
config = DFlashConfig.from_gguf_metadata(
47+
{"dflash." + key: value for key, value in metadata.items()}
48+
| {"dflash.target_layers": [0, 1], "tokenizer.ggml.mask_token_id": 0}
49+
)
50+
weights = dict(DFlashDraftModel(config).named_parameters())
51+
names = {
52+
"fc.weight": "fc.weight",
53+
"enc.output_norm.weight": "hidden_norm.weight",
54+
"output_norm.weight": "norm.weight",
55+
"blk.0.attn_q.weight": "layers.0.self_attn.q_proj.weight",
56+
"blk.0.attn_k.weight": "layers.0.self_attn.k_proj.weight",
57+
"blk.0.attn_v.weight": "layers.0.self_attn.v_proj.weight",
58+
"blk.0.attn_output.weight": "layers.0.self_attn.o_proj.weight",
59+
"blk.0.attn_q_norm.weight": "layers.0.self_attn.q_norm.weight",
60+
"blk.0.attn_k_norm.weight": "layers.0.self_attn.k_norm.weight",
61+
"blk.0.attn_norm.weight": "layers.0.input_layernorm.weight",
62+
"blk.0.ffn_norm.weight": "layers.0.post_attention_layernorm.weight",
63+
"blk.0.ffn_gate.weight": "layers.0.mlp.gate_proj.weight",
64+
"blk.0.ffn_up.weight": "layers.0.mlp.up_proj.weight",
65+
"blk.0.ffn_down.weight": "layers.0.mlp.down_proj.weight",
66+
}
67+
writer = GGUFWriter(str(path), "dflash")
68+
for key, value in metadata.items():
69+
writer.add_uint32("dflash." + key, value)
70+
writer.add_array("dflash.target_layers", [0, 1])
71+
writer.add_uint32("tokenizer.ggml.mask_token_id", 0)
72+
for name, parameter in names.items():
73+
writer.add_tensor(name, weights[parameter].detach().numpy())
74+
writer.write_header_to_file()
75+
writer.write_kv_data_to_file()
76+
writer.write_tensors_to_file()
77+
writer.close()
78+
79+
3380
class ReferenceTarget(torch.nn.Module):
3481
def __init__(self):
3582
super().__init__()
@@ -214,6 +261,26 @@ def old_uniform_mask(total_context, block, capacity, window):
214261

215262

216263
class DFlash2OperationsTest(TestCase):
264+
def test_gguf_preserves_checkpoint_capacity(self):
265+
from executorch.examples.models.muse_glimmer.loaders.dflash_loader import (
266+
load_dflash_gguf,
267+
)
268+
269+
with tempfile.TemporaryDirectory() as directory:
270+
path = Path(directory) / "draft.gguf"
271+
write_original_dflash_gguf(path)
272+
for backend in ("cuda", "mlx"):
273+
for max_seq_len in (32, 128):
274+
with self.subTest(backend=backend, max_seq_len=max_seq_len):
275+
draft, config = load_dflash_gguf(
276+
str(path), backend=backend, max_seq_len=max_seq_len
277+
)
278+
self.assertEqual(config.max_seq_len, 64)
279+
self.assertEqual(
280+
draft.layers[0].self_attn.kv_cache.k_cache.shape[2],
281+
max_seq_len,
282+
)
283+
217284
def test_invalid_metadata(self):
218285
for metadata in (
219286
{"conv_kernel_size": 2},

0 commit comments

Comments
 (0)