Skip to content

Commit 5bfbb59

Browse files
authored
[MLX] Respect each DFlash query's sliding attention window (#25335)
DFlash attention currently exposes the retained context and entire draft block to every query. Later queries can therefore attend to keys outside their configured window. Pass the window to MLX attention and make the eager Muse Glimmer mask use the absolute distance between each query and key, excluding unfilled ring slots. Use independent absolute-position oracles for sequence and packed cache tests, including wrapped windows, partial context, and blocks wider than the window. Add an asset-independent Python mask regression to MLX CI; it runs without DFlash2 model additions. The native window regression fails before the correction and passes afterward. Cache lifecycle tests pass with strict FP32, and the isolated Python window test passes all 54 parameter subtests. The official PyTorch noncausal mask also agrees across 72 independent cases, including wrapped caches and windows smaller than the block. Both downloaded original/DFlash2 checkpoints use block 16 and window 2048, where official DFlash, MLX, sglang, and llama.cpp agree on the strict past-window bound. Transformers uses an inclusive boundary for the original draft, a known one-key difference. For synthetic blocks wider than the window, this change follows official PyTorch symmetric masking rather than the MLX/llama.cpp whole-block convention. This correction intentionally changes original DFlash draft numerics, including existing off-graph artifacts without re-export, while preserving their ABI. It depends on the MLX pin/API update below it. The eager mask is also used by CUDA DFlash, so CUDA SWA draft numerics change. CUDA execution was not tested on this Apple Silicon host; run the CUDA DFlash checks in CI or on a CUDA host before landing. Native builds of each staged layer remain for CI; local native results cover the final implementation. Tracked in #23011. Authored with assistance from OpenAI Codex.
1 parent 8cae65d commit 5bfbb59

6 files changed

Lines changed: 184 additions & 36 deletions

File tree

‎.github/workflows/mlx.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ jobs:
138138
examples/models/gemma4_31b/tests/test_mlx_pipeline.py \
139139
examples/models/muse-glimmer/tests/test_mlx_pipeline.py \
140140
examples/models/muse-glimmer/tests/test_mlx_offgraph.py \
141+
examples/models/muse-glimmer/tests/test_dflash_attention.py \
141142
examples/models/muse-glimmer/tests/test_checkpoint_context.py \
142143
-v
143144
echo "::endgroup::"

‎backends/mlx/runtime/MLXSequenceCache.h‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -686,7 +686,16 @@ class DFlash final : public KVAttentionPolicy {
686686
::mlx::core::concatenate(values, 2, s)};
687687
}();
688688
return ::mlx::core::fast::scaled_dot_product_attention(
689-
q, kv.k, kv.v, scale, -1, "", std::nullopt, std::nullopt, false, s);
689+
q,
690+
kv.k,
691+
kv.v,
692+
scale,
693+
window_.value_or(-1),
694+
"",
695+
std::nullopt,
696+
std::nullopt,
697+
false,
698+
s);
690699
}
691700

692701
const int block_size_;

‎backends/mlx/test/mlx_batch_control_test.cpp‎

Lines changed: 110 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -711,9 +711,18 @@ TEST_P(MLXBatchedCacheRegistryTest, GeometrySelectsPolicies) {
711711
for (int layer = 0; layer < 2; ++layer) {
712712
std::vector<float> means;
713713
if (block >= 0) {
714-
// Uniform, noncausal attention over history + block; layer 1 retains
715-
// only context values 3 and 4, but still attends to the entire block.
716-
means.assign(queries, (static_cast<float>(n) + (layer ? 3 : 1)) / 2);
714+
for (int row = 0; row < queries; ++row) {
715+
const int query_position = n - queries + row;
716+
float sum = 0;
717+
int visible = 0;
718+
for (int key_position = 0; key_position < n; ++key_position) {
719+
if (!layer || std::abs(query_position - key_position) < 2) {
720+
sum += key_position + 1;
721+
++visible;
722+
}
723+
}
724+
means.push_back(sum / visible);
725+
}
717726
} else {
718727
means = layer ? std::vector<float>{1, 1.5f, 2.5f, 3.5f}
719728
: std::vector<float>{1, 1.5f, 2, 2.5f};
@@ -760,6 +769,76 @@ INSTANTIATE_TEST_SUITE_P(
760769
MLXBatchedCacheRegistryTest,
761770
::testing::Values(-1, 0, 1, 2));
762771

772+
TEST(MLXDFlashWindowTest, AbsolutePositionsAcrossRingWraps) {
773+
using namespace ::mlx::core;
774+
for (int window : {1, 3, 7}) {
775+
for (int block : {1, 4, 16}) {
776+
SCOPED_TRACE(
777+
::testing::Message() << "window=" << window << " block=" << block);
778+
auto args = ring_config(
779+
32, window, 2, 1, 128, static_cast<int>(ScalarType::Float));
780+
args.geometry.dflash_block_size = block;
781+
auto owner = cache::CacheFactory::global().build(
782+
kMLXBackendId,
783+
cache::kind::kBatchedSequence,
784+
args.geometry,
785+
args.config);
786+
ASSERT_TRUE(owner.ok());
787+
auto* batch = (*owner)->as<cache::BatchControl>();
788+
auto* attention = (*owner)->as<MLXCache>();
789+
const auto id = batch->seq_new();
790+
ASSERT_TRUE(id);
791+
std::vector<float> history;
792+
for (int context : {0, 1, 2, 2, 2, 2, 0}) {
793+
const int start = static_cast<int>(history.size());
794+
std::vector<float> incoming;
795+
for (int i = 0; i < context; ++i) {
796+
incoming.push_back(static_cast<float>(start + i + 1) / 16);
797+
history.push_back(incoming.back());
798+
}
799+
for (int i = 0; i < block; ++i) {
800+
incoming.push_back(static_cast<float>(32 + start + i) / 16);
801+
}
802+
auto all_values = history;
803+
all_values.insert(
804+
all_values.end(), incoming.begin() + context, incoming.end());
805+
std::vector<float> expected;
806+
for (int row = 0; row < block; ++row) {
807+
const int query_position = static_cast<int>(history.size()) + row;
808+
double sum = 0;
809+
int visible = 0;
810+
for (int key_position = 0;
811+
key_position < static_cast<int>(all_values.size());
812+
++key_position) {
813+
if (std::abs(query_position - key_position) < window) {
814+
sum += all_values[key_position];
815+
++visible;
816+
}
817+
}
818+
expected.push_back(static_cast<float>(sum / visible));
819+
}
820+
const int n = context + block;
821+
std::vector<int32_t> positions(n);
822+
std::iota(positions.begin(), positions.end(), start);
823+
ASSERT_TRUE(batch->declare_step(std::vector<int32_t>(n, *id)));
824+
const auto out = attention->attend(
825+
0,
826+
positions,
827+
zeros({1, 1, block, 128}),
828+
zeros({1, 1, n, 128}),
829+
broadcast_to(array(incoming.data(), {1, 1, n, 1}), {1, 1, n, 128}),
830+
1,
831+
{});
832+
const auto want = broadcast_to(
833+
array(expected.data(), {1, 1, block, 1}), {1, 1, block, 128});
834+
EXPECT_TRUE(allclose(out, want, 1e-5f))
835+
<< "context end=" << history.size();
836+
EXPECT_EQ(batch->pos(*id), static_cast<int>(history.size()));
837+
}
838+
}
839+
}
840+
}
841+
763842
class MLXDFlashBatchedCacheTest
764843
: public ::testing::TestWithParam<std::tuple<std::string, int, int>> {
765844
protected:
@@ -819,27 +898,40 @@ class MLXDFlashBatchedCacheTest
819898
const int n = context + block();
820899
std::vector<std::vector<float>> incoming;
821900
std::vector<int> starts;
822-
std::vector<float> means;
901+
std::vector<std::vector<float>> means;
823902
for (auto* row : rows) {
824903
starts.push_back(static_cast<int>(row->history.size()));
825904
std::vector<float> values;
826905
for (int p = 0; p < context; ++p) {
827906
values.push_back(
828-
100 * (row->id + 1) + 10 * generation_ + starts.back() + p);
907+
(100 * (row->id + 1) + 10 * generation_ + starts.back() + p) /
908+
1024.0f);
829909
}
830910
row->history.insert(row->history.end(), values.begin(), values.end());
831-
// Only persistent context enters the host oracle's history.
832-
const int retained = window()
833-
? std::min(window(), static_cast<int>(row->history.size()))
834-
: static_cast<int>(row->history.size());
835-
float sum = std::accumulate(
836-
row->history.end() - retained, row->history.end(), 0.0f);
911+
auto history_and_block = row->history;
837912
for (int p = 0; p < block(); ++p) {
838-
const float temporary = 1000 + 100 * generation_ + 10 * row->id + p;
913+
const float temporary =
914+
(1000 + 100 * generation_ + 10 * row->id + p) / 1024.0f;
839915
values.push_back(temporary);
840-
sum += temporary;
916+
history_and_block.push_back(temporary);
841917
}
842-
means.push_back(sum / (retained + block()));
918+
std::vector<float> row_means;
919+
for (int p = 0; p < std::max(block(), 1); ++p) {
920+
const int query_position =
921+
static_cast<int>(row->history.size()) + (block() ? p : -1);
922+
double sum = 0;
923+
int visible = 0;
924+
for (int key_position = 0;
925+
key_position < static_cast<int>(history_and_block.size());
926+
++key_position) {
927+
if (!window() || std::abs(query_position - key_position) < window()) {
928+
sum += history_and_block[key_position];
929+
++visible;
930+
}
931+
}
932+
row_means.push_back(static_cast<float>(sum / visible));
933+
}
934+
means.push_back(std::move(row_means));
843935
incoming.push_back(std::move(values));
844936
}
845937
++generation_;
@@ -854,7 +946,7 @@ class MLXDFlashBatchedCacheTest
854946
positions.push_back(starts[r] + p);
855947
values.push_back(incoming[r][p]);
856948
if (block() ? p >= context : p == n - 1) {
857-
expected.push_back(means[r]);
949+
expected.push_back(means[r][block() ? p - context : 0]);
858950
}
859951
}
860952
}
@@ -977,6 +1069,9 @@ TEST_P(MLXDFlashBatchedCacheTest, InterleavedQueriesKeepTheirOwnWeights) {
9771069
queries.push_back(query);
9781070
float weighted = 0, weights = 0;
9791071
for (int j = 0; j < n; ++j) {
1072+
if (window() && std::abs(p - j) >= window()) {
1073+
continue;
1074+
}
9801075
const float weight = std::exp(query * 0.25f * (j + 1) * (r + 1));
9811076
weighted += weight * (100 * r + 7 * j + 1);
9821077
weights += weight;

‎backends/mlx/test/mlx_sequence_control_test.cpp‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -449,30 +449,42 @@ TEST_P(MLXSequenceControlTest, DFlashRejectsCellOrPersistsOnlyContext) {
449449
std::vector<float> context,
450450
float temporary,
451451
float flat_mean,
452-
float swa_mean) {
452+
std::vector<float> swa_means) {
453453
if (block > 0) {
454454
context.insert(context.end(), {temporary, temporary + 100});
455455
}
456456
forward(
457457
position,
458458
context,
459459
{std::vector<float>(static_cast<size_t>(queries), flat_mean),
460-
std::vector<float>(static_cast<size_t>(queries), swa_mean)},
460+
std::move(swa_means)},
461461
queries);
462462
};
463-
ASSERT_NO_FATAL_FAILURE(
464-
step(0, {1, 3}, 100, block ? 76 : 2, block ? 76 : 2));
463+
ASSERT_NO_FATAL_FAILURE(step(
464+
0,
465+
{1, 3},
466+
100,
467+
block ? 76 : 2,
468+
block ? std::vector<float>{101, 150} : std::vector<float>{2}));
465469
EXPECT_EQ(length(), 2);
466-
ASSERT_NO_FATAL_FAILURE(
467-
step(2, {5, 7}, 300, block ? 716.0f / 6 : 4, block ? 178 : 6));
470+
ASSERT_NO_FATAL_FAILURE(step(
471+
2,
472+
{5, 7},
473+
300,
474+
block ? 716.0f / 6 : 4,
475+
block ? std::vector<float>{707.0f / 3, 350} : std::vector<float>{6}));
468476
EXPECT_EQ(length(), 4); // temporary tokens never consume capacity
469477
EXPECT_FALSE(control_->can_extend());
470478
EXPECT_FALSE(sequence_->can_admit(block + 1));
471479
control_->clear();
472480
EXPECT_EQ(length(), 0);
473481
EXPECT_TRUE(control_->can_extend(4));
474-
ASSERT_NO_FATAL_FAILURE(
475-
step(0, {9, 11}, 500, block ? 280 : 10, block ? 280 : 10));
482+
ASSERT_NO_FATAL_FAILURE(step(
483+
0,
484+
{9, 11},
485+
500,
486+
block ? 280 : 10,
487+
block ? std::vector<float>{1111.0f / 3, 550} : std::vector<float>{10}));
476488
EXPECT_EQ(length(), 2);
477489
}
478490
}

‎examples/models/muse-glimmer/model/dflash_model.py‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -101,20 +101,15 @@ def build_dflash_swa_mask(
101101
buf_size: int,
102102
window_size: int,
103103
) -> torch.Tensor:
104-
"""Build boolean validity mask for ring-buffer context + block K/V.
105-
106-
The SWA ring buffer evicts entries beyond ``window_size``. This mask
107-
marks which ring slots are filled — uniform for all Q positions (the
108-
draft model's cross-attention is non-causal). Block entries are always
109-
valid.
110-
111-
Returns [1, 1, 1, buf_size + block_len] boolean mask (True = attend).
112-
"""
104+
"""Per-query noncausal window over ring-buffer context and draft K/V."""
113105
slots = torch.arange(buf_size, dtype=torch.long)
114106
ring_pos = slots + ((total_ctx - 1 - slots) // buf_size) * buf_size
115-
valid = (ring_pos >= 0) & (ring_pos >= total_ctx - window_size)
116-
blk_valid = torch.ones(block_len, dtype=torch.bool)
117-
return torch.cat([valid, blk_valid]).unsqueeze(0).unsqueeze(0).unsqueeze(0)
107+
query_pos = total_ctx + torch.arange(block_len, dtype=torch.long)
108+
key_pos = torch.cat([ring_pos, query_pos])
109+
visible = (key_pos >= 0) & (
110+
(query_pos[:, None] - key_pos[None, :]).abs() < window_size
111+
)
112+
return visible[None, None]
118113

119114

120115
class DFlashAttention(nn.Module):
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Position-based oracle for draft sliding-window attention."""
8+
9+
from unittest import TestCase
10+
11+
import torch
12+
from executorch.examples.models.muse_glimmer.model.dflash_model import (
13+
build_dflash_swa_mask,
14+
)
15+
16+
17+
class DFlashAttentionTest(TestCase):
18+
def test_window_mask_uses_absolute_query_positions(self):
19+
for window in (1, 3, 8):
20+
for context in (0, 1, window - 1, window, window + 1, 3 * window):
21+
for block in (1, 4, 16):
22+
with self.subTest(window=window, context=context, block=block):
23+
slots = [-1] * window
24+
for position in range(context):
25+
slots[position % window] = position
26+
keys = slots + list(range(context, context + block))
27+
expected = torch.tensor(
28+
[
29+
[key >= 0 and abs(query - key) < window for key in keys]
30+
for query in range(context, context + block)
31+
]
32+
)[None, None]
33+
torch.testing.assert_close(
34+
build_dflash_swa_mask(context, block, window, window),
35+
expected,
36+
)

0 commit comments

Comments
 (0)