Skip to content
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
17a9d33
[INITIAL] Submit reviewed MLX foundation, GGUF empty-context fix, DFl…
mergennachin Nov 20, 2026
7f19a62
[INITIAL] Submit reviewed MLX foundation, GGUF empty-context fix, DFl…
mergennachin Nov 20, 2026
7d151c0
[INITIAL] Submit reviewed MLX foundation, GGUF empty-context fix, DFl…
mergennachin Nov 20, 2026
c5ea2a2
[INITIAL] Submit reviewed MLX foundation, GGUF empty-context fix, DFl…
mergennachin Nov 20, 2026
5af9907
[INITIAL] Update
mergennachin Nov 20, 2026
a0865d8
[UPDATE] Address DFlash2 review feedback and add routing regression c…
mergennachin Nov 20, 2026
c856627
[UPDATE] Address DFlash2 review feedback and add routing regression c…
mergennachin Nov 20, 2026
a785e4e
[UPDATE] Fix CI lint, preserve MLX runtime exports in shared builds a…
mergennachin Nov 20, 2026
d96c545
[UPDATE] Fix CI lint, preserve MLX runtime exports in shared builds a…
mergennachin Nov 20, 2026
aa11aaa
[UPDATE] Fix CI lint, preserve MLX runtime exports in shared builds a…
mergennachin Nov 20, 2026
c81c667
[UPDATE] Fix CI lint, preserve MLX runtime exports in shared builds a…
mergennachin Nov 20, 2026
e22879b
[UPDATE] Fix CI lint, preserve MLX runtime exports in shared builds a…
mergennachin Nov 20, 2026
5bca98d
[UPDATE] Rebase the open stack after its foundation merged and save t…
mergennachin Nov 21, 2026
d5deba0
[UPDATE] Rebase the open stack after its foundation merged and save t…
mergennachin Nov 21, 2026
8815e5e
[UPDATE] Rebase the open stack after its foundation merged and save t…
mergennachin Nov 21, 2026
737b243
[UPDATE] Address the three review comments on #25336: package dflash2…
mergennachin Nov 21, 2026
8b1f12e
[UPDATE] Address the three review comments on #25336: package dflash2…
mergennachin Nov 21, 2026
fc68946
[UPDATE] Update
mergennachin Nov 21, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions .github/workflows/mlx.yml
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ jobs:
examples/models/gemma4_31b/tests/test_mlx_pipeline.py \
examples/models/muse-glimmer/tests/test_mlx_pipeline.py \
examples/models/muse-glimmer/tests/test_mlx_offgraph.py \
examples/models/muse-glimmer/tests/test_dflash_attention.py \
examples/models/muse-glimmer/tests/test_dflash2.py \
examples/models/muse-glimmer/tests/test_checkpoint_context.py \
-v
echo "::endgroup::"
Expand Down Expand Up @@ -376,10 +378,16 @@ jobs:
${CONDA_RUN} make muse-glimmer-batching-mlx
BATCHING_BIN_DIR="${PWD}/cmake-out/examples/models/muse-glimmer-batching"
${CONDA_RUN} cmake -S "${PWD}/examples/models/muse-glimmer" -B "${BATCHING_BIN_DIR}" \
-DMUSE_GLIMMER_BUILD_BATCHING_TESTS=ON
${CONDA_RUN} cmake --build "${BATCHING_BIN_DIR}" --target muse_glimmer_batching_test
-DMUSE_GLIMMER_BUILD_BATCHING_TESTS=ON \
-DMUSE_GLIMMER_MLX_TEST_FIXTURES=/tmp/muse-glimmer-mlx-fixtures
${CONDA_RUN} python -m examples.models.muse_glimmer.tests.export_mlx_dflash_fixture \
/tmp/muse-glimmer-mlx-fixtures
${CONDA_RUN} cmake --build "${BATCHING_BIN_DIR}" \
--target muse_glimmer_batching_test muse_glimmer_mlx_dflash_test sampling_test
${CONDA_RUN} ctest --test-dir "${BATCHING_BIN_DIR}" \
-R '^muse_glimmer_batching$' --output-on-failure --no-tests=error
-R '^muse_glimmer_(batching|sampling)$' --output-on-failure --no-tests=error
${CONDA_RUN} ctest --test-dir "${BATCHING_BIN_DIR}" \
-R '^muse_glimmer_mlx_dflash$' --output-on-failure --no-tests=error
test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_runner"
test -x "${BATCHING_BIN_DIR}/muse_glimmer_batching_worker"
test -f "${BATCHING_BIN_DIR}/mlx.metallib"
Expand Down
11 changes: 10 additions & 1 deletion backends/mlx/runtime/MLXSequenceCache.h
Original file line number Diff line number Diff line change
Expand Up @@ -686,7 +686,16 @@ class DFlash final : public KVAttentionPolicy {
::mlx::core::concatenate(values, 2, s)};
}();
return ::mlx::core::fast::scaled_dot_product_attention(
q, kv.k, kv.v, scale, -1, "", std::nullopt, std::nullopt, false, s);
q,
kv.k,
kv.v,
scale,
window_.value_or(-1),
"",
std::nullopt,
std::nullopt,
false,
s);
}

const int block_size_;
Expand Down
125 changes: 110 additions & 15 deletions backends/mlx/test/mlx_batch_control_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -711,9 +711,18 @@ TEST_P(MLXBatchedCacheRegistryTest, GeometrySelectsPolicies) {
for (int layer = 0; layer < 2; ++layer) {
std::vector<float> means;
if (block >= 0) {
// Uniform, noncausal attention over history + block; layer 1 retains
// only context values 3 and 4, but still attends to the entire block.
means.assign(queries, (static_cast<float>(n) + (layer ? 3 : 1)) / 2);
for (int row = 0; row < queries; ++row) {
const int query_position = n - queries + row;
float sum = 0;
int visible = 0;
for (int key_position = 0; key_position < n; ++key_position) {
if (!layer || std::abs(query_position - key_position) < 2) {
sum += key_position + 1;
++visible;
}
}
means.push_back(sum / visible);
}
} else {
means = layer ? std::vector<float>{1, 1.5f, 2.5f, 3.5f}
: std::vector<float>{1, 1.5f, 2, 2.5f};
Expand Down Expand Up @@ -760,6 +769,76 @@ INSTANTIATE_TEST_SUITE_P(
MLXBatchedCacheRegistryTest,
::testing::Values(-1, 0, 1, 2));

TEST(MLXDFlashWindowTest, AbsolutePositionsAcrossRingWraps) {
using namespace ::mlx::core;
for (int window : {1, 3, 7}) {
for (int block : {1, 4, 16}) {
SCOPED_TRACE(
::testing::Message() << "window=" << window << " block=" << block);
auto args = ring_config(
32, window, 2, 1, 128, static_cast<int>(ScalarType::Float));
args.geometry.dflash_block_size = block;
auto owner = cache::CacheFactory::global().build(
kMLXBackendId,
cache::kind::kBatchedSequence,
args.geometry,
args.config);
ASSERT_TRUE(owner.ok());
auto* batch = (*owner)->as<cache::BatchControl>();
auto* attention = (*owner)->as<MLXCache>();
const auto id = batch->seq_new();
ASSERT_TRUE(id);
std::vector<float> history;
for (int context : {0, 1, 2, 2, 2, 2, 0}) {
const int start = static_cast<int>(history.size());
std::vector<float> incoming;
for (int i = 0; i < context; ++i) {
incoming.push_back(static_cast<float>(start + i + 1) / 16);
history.push_back(incoming.back());
}
for (int i = 0; i < block; ++i) {
incoming.push_back(static_cast<float>(32 + start + i) / 16);
}
auto all_values = history;
all_values.insert(
all_values.end(), incoming.begin() + context, incoming.end());
std::vector<float> expected;
for (int row = 0; row < block; ++row) {
const int query_position = static_cast<int>(history.size()) + row;
double sum = 0;
int visible = 0;
for (int key_position = 0;
key_position < static_cast<int>(all_values.size());
++key_position) {
if (std::abs(query_position - key_position) < window) {
sum += all_values[key_position];
++visible;
}
}
expected.push_back(static_cast<float>(sum / visible));
}
const int n = context + block;
std::vector<int32_t> positions(n);
std::iota(positions.begin(), positions.end(), start);
ASSERT_TRUE(batch->declare_step(std::vector<int32_t>(n, *id)));
const auto out = attention->attend(
0,
positions,
zeros({1, 1, block, 128}),
zeros({1, 1, n, 128}),
broadcast_to(array(incoming.data(), {1, 1, n, 1}), {1, 1, n, 128}),
1,
{});
const auto want = broadcast_to(
array(expected.data(), {1, 1, block, 1}), {1, 1, block, 128});
EXPECT_TRUE(allclose(out, want, 1e-5f))
<< "context end=" << history.size();
EXPECT_EQ(batch->pos(*id), static_cast<int>(history.size()));
}
}
}
}

class MLXDFlashBatchedCacheTest
: public ::testing::TestWithParam<std::tuple<std::string, int, int>> {
protected:
Expand Down Expand Up @@ -819,27 +898,40 @@ class MLXDFlashBatchedCacheTest
const int n = context + block();
std::vector<std::vector<float>> incoming;
std::vector<int> starts;
std::vector<float> means;
std::vector<std::vector<float>> means;
for (auto* row : rows) {
starts.push_back(static_cast<int>(row->history.size()));
std::vector<float> values;
for (int p = 0; p < context; ++p) {
values.push_back(
100 * (row->id + 1) + 10 * generation_ + starts.back() + p);
(100 * (row->id + 1) + 10 * generation_ + starts.back() + p) /
1024.0f);
}
row->history.insert(row->history.end(), values.begin(), values.end());
// Only persistent context enters the host oracle's history.
const int retained = window()
? std::min(window(), static_cast<int>(row->history.size()))
: static_cast<int>(row->history.size());
float sum = std::accumulate(
row->history.end() - retained, row->history.end(), 0.0f);
auto history_and_block = row->history;
for (int p = 0; p < block(); ++p) {
const float temporary = 1000 + 100 * generation_ + 10 * row->id + p;
const float temporary =
(1000 + 100 * generation_ + 10 * row->id + p) / 1024.0f;
values.push_back(temporary);
sum += temporary;
history_and_block.push_back(temporary);
}
means.push_back(sum / (retained + block()));
std::vector<float> row_means;
for (int p = 0; p < std::max(block(), 1); ++p) {
const int query_position =
static_cast<int>(row->history.size()) + (block() ? p : -1);
double sum = 0;
int visible = 0;
for (int key_position = 0;
key_position < static_cast<int>(history_and_block.size());
++key_position) {
if (!window() || std::abs(query_position - key_position) < window()) {
sum += history_and_block[key_position];
++visible;
}
}
row_means.push_back(static_cast<float>(sum / visible));
}
means.push_back(std::move(row_means));
incoming.push_back(std::move(values));
}
++generation_;
Expand All @@ -854,7 +946,7 @@ class MLXDFlashBatchedCacheTest
positions.push_back(starts[r] + p);
values.push_back(incoming[r][p]);
if (block() ? p >= context : p == n - 1) {
expected.push_back(means[r]);
expected.push_back(means[r][block() ? p - context : 0]);
}
}
}
Expand Down Expand Up @@ -977,6 +1069,9 @@ TEST_P(MLXDFlashBatchedCacheTest, InterleavedQueriesKeepTheirOwnWeights) {
queries.push_back(query);
float weighted = 0, weights = 0;
for (int j = 0; j < n; ++j) {
if (window() && std::abs(p - j) >= window()) {
continue;
}
const float weight = std::exp(query * 0.25f * (j + 1) * (r + 1));
weighted += weight * (100 * r + 7 * j + 1);
weights += weight;
Expand Down
28 changes: 20 additions & 8 deletions backends/mlx/test/mlx_sequence_control_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -449,30 +449,42 @@ TEST_P(MLXSequenceControlTest, DFlashRejectsCellOrPersistsOnlyContext) {
std::vector<float> context,
float temporary,
float flat_mean,
float swa_mean) {
std::vector<float> swa_means) {
if (block > 0) {
context.insert(context.end(), {temporary, temporary + 100});
}
forward(
position,
context,
{std::vector<float>(static_cast<size_t>(queries), flat_mean),
std::vector<float>(static_cast<size_t>(queries), swa_mean)},
std::move(swa_means)},
queries);
};
ASSERT_NO_FATAL_FAILURE(
step(0, {1, 3}, 100, block ? 76 : 2, block ? 76 : 2));
ASSERT_NO_FATAL_FAILURE(step(
0,
{1, 3},
100,
block ? 76 : 2,
block ? std::vector<float>{101, 150} : std::vector<float>{2}));
EXPECT_EQ(length(), 2);
ASSERT_NO_FATAL_FAILURE(
step(2, {5, 7}, 300, block ? 716.0f / 6 : 4, block ? 178 : 6));
ASSERT_NO_FATAL_FAILURE(step(
2,
{5, 7},
300,
block ? 716.0f / 6 : 4,
block ? std::vector<float>{707.0f / 3, 350} : std::vector<float>{6}));
EXPECT_EQ(length(), 4); // temporary tokens never consume capacity
EXPECT_FALSE(control_->can_extend());
EXPECT_FALSE(sequence_->can_admit(block + 1));
control_->clear();
EXPECT_EQ(length(), 0);
EXPECT_TRUE(control_->can_extend(4));
ASSERT_NO_FATAL_FAILURE(
step(0, {9, 11}, 500, block ? 280 : 10, block ? 280 : 10));
ASSERT_NO_FATAL_FAILURE(step(
0,
{9, 11},
500,
block ? 280 : 10,
block ? std::vector<float>{1111.0f / 3, 550} : std::vector<float>{10}));
EXPECT_EQ(length(), 2);
}
}
Expand Down
53 changes: 50 additions & 3 deletions examples/models/muse-glimmer/BATCHING_SERVING.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,59 @@ an error. The three relevant limits are independent:
The DFlash exporter accepts `--backend mlx --use-offgraph-kv-cache`.
Serve that artifact with `--dflash`. Draft proposals are always greedy;
temperature, top-p, and top-k apply only to the target distribution.
Set `--dflash-n-draft N` at startup to propose N tokens using block size N+1.
Set `--dflash-n-draft N` at startup to propose N tokens. On MLX, original DFlash uses
block size N+1; DFlash2 keeps its trained block size as described below.
Omitted or zero uses the exported maximum minus one; larger values are rejected.
Re-export older fixed-block artifacts before using this option. One updated
artifact supports different draft counts without another export. Draft tokens
have separate `[owners, block_size]` axes; each block ranges from 2 to the
exported maximum. Torch infers the concatenated context-plus-block K/V length.
have separate `[owners, block_size]` axes. Original DFlash blocks range from 2
to the exported maximum.

Both draft formats use a per-query sliding window with the official PyTorch
DFlash rule `abs(query - key) < window`, including within the draft block.

### DFlash2 on Apple Silicon

DFlash2 GGUF drafts use the same exporter and serving flags. For example:

```sh
python -m executorch.examples.models.muse_glimmer.export.export_dflash \
--backend mlx --use-offgraph-kv-cache \
--target-gguf /path/to/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf \
--draft-gguf /path/to/Muse-Glimmer-30B-DFlash2-Q4_K_M.gguf \
--activation-dtype bfloat16 \
--output-dir /path/to/muse-dflash2
```

Serve the resulting artifact with the batching worker and `--dflash`. DFlash2
requires the MLX off-graph path; legacy runners and CUDA export are not yet
supported. Its GGUF must explicitly set `dflash.attention.causal` to false.
Existing DFlash artifacts keep their logits ABI.

DFlash2 runs its full trained block (16 positions for this checkpoint), even
with `--dflash-n-draft 3`; this option limits the output projection and
verification to three proposals. Keeping the full block preserves its draft
conditioning; this is an execution policy, not a correctness requirement.
Unlike reference paths that shorten the draft block, this path falls back to
the target near the context limit when a full block does not fit.
Each session's convolution stays within its own block.

The artifact identifies the draft through `get_dflash_selector_top_k`: absent
or zero selects the existing DFlash logits ABI; a positive value selects
DFlash2. The exporter reads this value from the GGUF's `dflash.selector_top_k`
metadata. Renaming a model file does not change its behavior.

The shared PyTorch model computes convolution, top-k candidates, and conditional
candidate scores. DFlash2's `draft_forward` has a fifth, length-N selector input;
only its length is used to select the first N proposals. Its values are ignored.
It returns token IDs `[owners, N, K]` and float scores `[owners, N, K, K]`.
The host walks the candidates conditioned
on the previously selected token. MLX keeps greedy draft proposals and the
existing target acceptance and residual sampling rules, including for nonzero
target temperature.

CUDA DFlash2 is future work. Its backend transformations must preserve the
shared model's convolutions and candidate selector before export is enabled.

## Build

Expand Down
20 changes: 20 additions & 0 deletions examples/models/muse-glimmer/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -350,12 +350,32 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS)
message(FATAL_ERROR "MG batching tests require extension_llm_serving")
endif()
enable_testing()
add_test(NAME muse_glimmer_sampling COMMAND sampling_test)
add_executable(muse_glimmer_batching_test tests/batching_runtime_test.cpp)
target_link_libraries(
muse_glimmer_batching_test PRIVATE muse_glimmer_batching_preparation
extension_llm_serving Threads::Threads
)
add_test(NAME muse_glimmer_batching COMMAND muse_glimmer_batching_test)
if(MUSE_GLIMMER_BUILD_BATCHING AND _batching_backend STREQUAL "mlx")
add_executable(
muse_glimmer_mlx_dflash_test tests/mlx_dflash_native_test.cpp
)
target_link_libraries(
muse_glimmer_mlx_dflash_test PRIVATE muse_glimmer_batching_runtime
)
executorch_target_copy_mlx_metallib(muse_glimmer_mlx_dflash_test)
set(MUSE_GLIMMER_MLX_TEST_FIXTURES
""
CACHE PATH "Directory containing exported MLX DFlash test fixtures"
)
if(MUSE_GLIMMER_MLX_TEST_FIXTURES)
add_test(NAME muse_glimmer_mlx_dflash
COMMAND muse_glimmer_mlx_dflash_test
${MUSE_GLIMMER_MLX_TEST_FIXTURES}
)
endif()
endif()
if(MUSE_GLIMMER_BUILD_BATCHING AND _batching_backend STREQUAL "cuda")
# Tokenizer include paths contain regex.h; keep GoogleTest and its users
# on the same simple-regex ABI instead of the shadowed POSIX header.
Expand Down
Loading
Loading