diff --git a/examples/models/muse-glimmer/runtime/mlx_executor.cpp b/examples/models/muse-glimmer/runtime/mlx_executor.cpp index 3b42a689708..672bf043517 100644 --- a/examples/models/muse-glimmer/runtime/mlx_executor.cpp +++ b/examples/models/muse-glimmer/runtime/mlx_executor.cpp @@ -659,7 +659,6 @@ std::optional MuseGlimmerMLXExecutor::sample_row( int row, batching::SessionId session, batching::Position prediction_position, - std::vector* probabilities, bool draft) { const auto it = sessions_.find(session); if (it == sessions_.end() || !it->second.sampling || row < 0 || @@ -675,18 +674,10 @@ std::optional MuseGlimmerMLXExecutor::sample_row( })) { return std::nullopt; } - if (draft) { + if (draft || params.temperature == 0) { return muse_glimmer::argmax_index(values, spec_->vocab_size); } - std::vector scratch; - auto& p = probabilities ? *probabilities : scratch; - if (params.temperature == 0) { - const auto token = muse_glimmer::argmax_index(values, spec_->vocab_size); - p.assign(spec_->vocab_size, 0.0f); - p[token] = 1.0f; - return token; - } - p = muse_glimmer::sampling_probabilities( + const auto p = muse_glimmer::sampling_probabilities( values, spec_->vocab_size, params.temperature, @@ -879,8 +870,8 @@ bool MuseGlimmerMLXExecutor::draft( } } else { for (int j = 1; j < v; ++j) { - auto token = sample_row( - logits, i * b + j, owners[i], ends[i] + j, nullptr, true); + auto token = + sample_row(logits, i * b + j, owners[i], ends[i] + j, true); if (!token) { return false; } @@ -903,7 +894,8 @@ bool MuseGlimmerMLXExecutor::draft( bool MuseGlimmerMLXExecutor::speculate( const batching::BatchInput& batch, - batching::BatchOutput& out) { + batching::BatchOutput& out, + bool input_anchors) { const int b = dflash_->verify_block; size_t cursor = 0; while (cursor < batch.inputs.size()) { @@ -914,7 +906,8 @@ bool MuseGlimmerMLXExecutor::speculate( while (cursor < batch.inputs.size()) { const auto& input = batch.inputs[cursor]; auto& state = sessions_.at(input.sid); - const int end = input.position + input.offset + input.size; + const int end = input.position + input.offset + input.size - + static_cast(input_anchors); if (!out.outputs[cursor] || end != ctl_->pos(state.seq_id) || end > spec_->max_context_length - dflash_->block) { ++cursor; @@ -930,7 +923,10 @@ bool MuseGlimmerMLXExecutor::speculate( } context += count; owners.push_back(input.sid); - anchors.push_back(out.outputs[cursor]->tokens.front()); + anchors.push_back( + input_anchors ? (*std::get( + input.payload))[input.offset] + : out.outputs[cursor]->tokens.front()); indices.push_back(cursor++); } if (owners.empty()) { @@ -940,9 +936,8 @@ bool MuseGlimmerMLXExecutor::speculate( if (!draft(owners, anchors, dflash_->context, &verification)) { return false; } - Probabilities p(verification.inputs.size()); batching::BatchOutput predictions; - if (!execute_impl(verification, predictions, &p)) { + if (!execute_impl(verification, predictions, true)) { return false; } for (size_t i = 0; i < owners.size(); ++i) { @@ -951,28 +946,18 @@ bool MuseGlimmerMLXExecutor::speculate( const auto& input = verification.inputs[i * b]; const auto& block = *std::get(input.payload); for (int j = 0; j < b; ++j) { - auto& target = p[i * b + j]; - const int position = input.position + j + 1; - if (j + 1 < b) { - const auto proposal = block[j + 1]; - auto rng = draw_rng(state.seed, position, 2); - if (muse_glimmer::accept_with_probability(rng, target[proposal])) { - output.push_back(proposal); - continue; - } + const auto& prediction = predictions.outputs[i * b + j]; + if (!prediction || prediction->tokens.size() != 1) { + return false; } - auto rng = draw_rng(state.seed, position, j + 1 == b ? 4 : 3); - batching::Token correction; - if (state.sampling->temperature == 0) { - correction = muse_glimmer::argmax_index(target.data(), target.size()); - } else if (j + 1 == b) { - correction = muse_glimmer::categorical_sample( - rng, target.data(), target.size()); - } else { - correction = muse_glimmer::sample_excluding_token_in_place( - rng, target, block[j + 1]); + const auto token = prediction->tokens.front(); + const int position = input.position + j + 1; + // For a greedy draft, matching a target draw accepts with p(proposal). + // Reuse its per-position RNG so batching/chunking cannot change draws. + output.push_back(token); + if (j + 1 < b && token == block[j + 1]) { + continue; } - output.push_back(correction); // Leave the returned correction/bonus uncommitted. if (!rewind(state, position)) { return false; @@ -987,13 +972,13 @@ bool MuseGlimmerMLXExecutor::speculate( bool MuseGlimmerMLXExecutor::execute( const batching::BatchInput& batch, batching::BatchOutput& out) { - return execute_impl(batch, out, nullptr); + return execute_impl(batch, out, false); } bool MuseGlimmerMLXExecutor::execute_impl( const batching::BatchInput& batch, batching::BatchOutput& out, - Probabilities* probabilities) { + bool verification) { #if ET_HAS_EXCEPTIONS try { #endif @@ -1014,6 +999,42 @@ bool MuseGlimmerMLXExecutor::execute_impl( } batching::BatchOutput pending; pending.outputs.resize(batch.inputs.size()); + // Verification can commit a single input token as the draft anchor. + if (dflash_ && !verification && !batch.inputs.empty()) { + std::vector owners; + bool input_anchors = true; + for (const auto& input : batch.inputs) { + const auto& state = sessions_.at(input.sid); + const int start = input.position + input.offset; + // Older draft artifacts require at least one new context row. + if (!input.produce_output || input.size != 1 || + !std::holds_alternative(input.payload) || + start != ctl_->pos(state.seq_id) || state.hidden.empty() || + start > spec_->max_context_length - dflash_->block || + std::find(owners.begin(), owners.end(), input.sid) != + owners.end()) { + input_anchors = false; + break; + } + owners.push_back(input.sid); + } + if (input_anchors) { + for (size_t i = 0; i < batch.inputs.size(); ++i) { + pending.outputs[i] = batching::Output{batch.inputs[i].sid, {}}; + } + if (!speculate(batch, pending, true) || + std::any_of( + pending.outputs.begin(), + pending.outputs.end(), + [](const auto& output) { + return !output || output->tokens.empty(); + })) { + return false; + } + out = std::move(pending); + return true; + } + } size_t input_index = 0; size_t consumed = 0; while (input_index < batch.inputs.size()) { @@ -1055,7 +1076,7 @@ bool MuseGlimmerMLXExecutor::execute_impl( selectors.push_back(n - 1); } const auto rows = static_cast(selectors.size()); - if (dflash_ && !probabilities) { + if (dflash_ && !verification) { for (const auto& slice : slices) { if (!drain(slice.sid)) { return false; @@ -1119,12 +1140,8 @@ bool MuseGlimmerMLXExecutor::execute_impl( const auto& input = batch.inputs[index]; const auto end = static_cast( static_cast(input.position) + input.offset + input.size); - const auto token = sample_row( - logits, - static_cast(row), - input.sid, - end, - probabilities ? &probabilities->at(index) : nullptr); + const auto token = + sample_row(logits, static_cast(row), input.sid, end); if (!token) { return false; } @@ -1132,7 +1149,7 @@ bool MuseGlimmerMLXExecutor::execute_impl( pending.outputs[index] = batching::Output{input.sid, {*token}}; } } - if (dflash_ && !probabilities && !speculate(batch, pending)) { + if (dflash_ && !verification && !speculate(batch, pending)) { return false; } out = std::move(pending); diff --git a/examples/models/muse-glimmer/runtime/mlx_executor.h b/examples/models/muse-glimmer/runtime/mlx_executor.h index 6b0a47a7a00..3d7050b43ef 100644 --- a/examples/models/muse-glimmer/runtime/mlx_executor.h +++ b/examples/models/muse-glimmer/runtime/mlx_executor.h @@ -46,7 +46,6 @@ class MuseGlimmerMLXExecutor final : public batching::Executor { private: struct DFlash; - using Probabilities = std::vector>; struct SessionState { int32_t seq_id; std::optional sampling; @@ -75,17 +74,19 @@ class MuseGlimmerMLXExecutor final : public batching::Executor { const std::vector& anchors, int context_limit, batching::BatchInput* verification); - bool speculate(const batching::BatchInput& batch, batching::BatchOutput& out); + bool speculate( + const batching::BatchInput& batch, + batching::BatchOutput& out, + bool input_anchors = false); bool execute_impl( const batching::BatchInput& batch, batching::BatchOutput& out, - Probabilities* probabilities); + bool verification); std::optional sample_row( aten::Tensor& logits, int row, batching::SessionId session, batching::Position prediction_position, - std::vector* probabilities = nullptr, bool draft = false); // Reverse destruction: vision before its module/mutex, then the module before diff --git a/examples/models/muse-glimmer/tests/README.md b/examples/models/muse-glimmer/tests/README.md index 12991a65346..f48bfed30ce 100644 --- a/examples/models/muse-glimmer/tests/README.md +++ b/examples/models/muse-glimmer/tests/README.md @@ -26,9 +26,11 @@ ctest --test-dir cmake-out/examples/models/muse-glimmer-batching \ Random and quantized PTE fixtures compare draft numerics against eager references. Controlled rejection and cache-lifecycle checks assert expected tokens and acceptance counts; continuation checks compare with target-only execution, -including legacy artifacts. +including legacy artifacts. Seeded continuations are compared across +prompt chunking and mixed prefill/decode batches for both draft formats. Routing-sensitive fixtures also compare per-round tokens and acceptance counts between solo and packed execution with unequal prompts and reordered owners. +Input-sensitive continuations check shared-payload slices against target-only decoding. A vendored stateless official reference separately checks DFlash2 model semantics, including sliding-window boundaries. Native sampling checks target-only, original DFlash, and DFlash2 at temperature 0.8 with top-k/top-p filtering; diff --git a/examples/models/muse-glimmer/tests/export_mlx_dflash_fixture.py b/examples/models/muse-glimmer/tests/export_mlx_dflash_fixture.py index 6e6b354150f..8637a6ef61e 100644 --- a/examples/models/muse-glimmer/tests/export_mlx_dflash_fixture.py +++ b/examples/models/muse-glimmer/tests/export_mlx_dflash_fixture.py @@ -377,7 +377,7 @@ def export_fixture( ) directory.mkdir(parents=True, exist_ok=True) - if not controlled: + if not controlled and not legacy: write_draft_reference(directory, target, draft, config, draft_config) original_constants = export_dflash._dflash_constant_methods @@ -413,6 +413,7 @@ def constants(**kwargs): write_nonconstant_selector_fixture(args.output) export_fixture(args.output / "legacy", -1, legacy=True) export_fixture(args.output / "random", -1, controlled=False) + export_fixture(args.output / "random_legacy", -1, legacy=True, controlled=False) export_fixture(args.output / "owner_sensitive", -1, owner_sensitive=True) export_fixture(args.output / "single_proposal", -1, block_size=2) export_fixture(args.output / "routing", -1, routing_sensitive=True) diff --git a/examples/models/muse-glimmer/tests/mlx_dflash_native_test.cpp b/examples/models/muse-glimmer/tests/mlx_dflash_native_test.cpp index 73eb5cf63e4..f948c2c923b 100644 --- a/examples/models/muse-glimmer/tests/mlx_dflash_native_test.cpp +++ b/examples/models/muse-glimmer/tests/mlx_dflash_native_test.cpp @@ -91,9 +91,11 @@ void check_cycles(const std::string& path, int proposals, int rejected) { while (position + static_cast(tokens.size()) <= 32) { const int end = position + static_cast(tokens.size()); auto output = forward(*executor, *sid, position, tokens); - if (end <= 28) { + const bool input_anchor = position > 0 && tokens.size() == 1; + if (end - static_cast(input_anchor) <= 28) { require( - output.size() == expected, "wrong accepted prefix/correction count"); + output.size() == expected - static_cast(input_anchor), + "wrong accepted prefix/correction count"); ++speculative_cycles; } else { require(output.size() == 1, "context-tail fallback did not run"); @@ -132,7 +134,7 @@ void check_packed_replay(const std::string& path) { "packed owner mismatch"); } // Discard a speculative suffix as a token-budget stop would, then resume. - require(forward(*executor, *a, 4, {7}).size() == 5, "rewind/replay failed"); + require(forward(*executor, *a, 4, {7}).size() == 4, "rewind/replay failed"); const auto fork = executor->clone(*a, 8); require(fork.has_value(), "prefix clone failed"); const auto source_output = forward(*executor, *a, 8, {7}); @@ -141,12 +143,63 @@ void check_packed_replay(const std::string& path) { "clone continuation differs"); executor->close_session(*a); require( - forward(*executor, *b, 7, {7}).size() == 5, + forward(*executor, *b, 7, {7}).size() == 4, "closing another owner changed state"); } -void check_temperature(const std::string& path, bool speculative) { +void check_input_anchors(const std::string& path) { + auto executor = create(path, 3); + const auto a = executor->open_session(), b = executor->open_session(); + require(a.has_value() && b.has_value(), "anchor session allocation failed"); + executor->set_sampling(*a, {}, 42); + executor->set_sampling(*b, {}, 42); + b::BatchInput batch; + batch.inputs = {input(*a, 0, {1, 2, 3}), input(*b, 0, {4, 5})}; + for (auto& item : batch.inputs) { + item.produce_output = false; + } + b::BatchOutput output; + require(executor->execute(batch, output), "anchor prefill failed"); + require(!output.outputs[0] && !output.outputs[1], "prefill emitted tokens"); + + batch.inputs = {input(*a, 2, {99, 5}), input(*b, 2, {6})}; + batch.inputs[0].offset = 1; + batch.inputs[0].size = 1; + require(executor->execute(batch, output), "packed input anchors failed"); + for (const auto& result : output.outputs) { + require( + result && result->tokens == Tokens(4, 7), + "input anchor was emitted or verification lost a prediction"); + } + + const auto fork = executor->clone(*a, 7); + require(fork.has_value(), "anchor prefix clone failed"); + executor->set_sampling(*fork, {}, 42); + batch.inputs = { + input(*a, 7, {8}), input(*fork, 7, {8}), input(*b, 6, {3, 4})}; + batch.inputs[2].produce_output = false; + require( + executor->execute(batch, output), "mixed anchor/prefill batch failed"); + require( + output.outputs[0] && output.outputs[1] && !output.outputs[2] && + output.outputs[0]->tokens == Tokens(5, 7) && + output.outputs[1]->tokens == output.outputs[0]->tokens, + "mixed batch fallback changed a cloned continuation"); + executor->close_session(*a); + executor->close_session(*b); + require( + forward(*executor, *fork, 7, {5}) == Tokens(5, 7), + "empty-context fallback changed after rewind"); + executor->close_session(*fork); +} + +void check_temperature( + const std::string& path, + bool speculative, + bool input_anchor = false) { auto executor = create(path, 3, speculative); + const size_t maximum = speculative ? (input_anchor ? 4 : 5) : 1; + const size_t minimum = speculative && !input_anchor ? 2 : 1; for (int variant = 0; variant < 4; ++variant) { const int trials = variant == 3 ? 128 : 1024; b::SamplingParams sampling; @@ -182,6 +235,15 @@ void check_temperature(const std::string& path, bool speculative) { batch.inputs.push_back(input(*sid, 0, {1, 2})); } b::BatchOutput output; + if (input_anchor) { + for (auto& item : batch.inputs) { + item.produce_output = false; + } + require(executor->execute(batch, output), "temperature prefill failed"); + for (size_t owner = 0; owner < sessions.size(); ++owner) { + batch.inputs[owner] = input(sessions[owner], 2, {5}); + } + } require(executor->execute(batch, output), "temperature execution failed"); require(output.outputs.size() == 4, "temperature batch lost an owner"); for (size_t owner = 0; owner < 4; ++owner) { @@ -189,8 +251,7 @@ void check_temperature(const std::string& path, bool speculative) { output.outputs[owner].has_value(), "missing temperature output"); const auto& tokens = output.outputs[owner]->tokens; require( - speculative ? tokens.size() >= 2 && tokens.size() <= 5 - : tokens.size() == 1, + tokens.size() >= minimum && tokens.size() <= maximum, "unexpected temperature output length"); for (size_t position = 0; position < tokens.size(); ++position) { const auto token = tokens[position]; @@ -204,28 +265,30 @@ void check_temperature(const std::string& path, bool speculative) { ++draws[position]; } if (speculative) { - for (size_t position = 1; position + 1 < tokens.size(); ++position) { + for (size_t position = input_anchor ? 0 : 1; + position + 1 < tokens.size(); + ++position) { require( tokens[position] == 7, "accepted draft proposal was not greedy"); } - if (tokens.size() < 5) { + if (tokens.size() < maximum) { require( tokens.back() != 7, "correction sampled the rejected proposal"); } - rejections += tokens.size() == 2; + rejections += tokens.size() == minimum; } executor->close_session(sessions[owner]); } } if (speculative && variant != 3) { require( - draws[4] > 50 && rejections > 50, + draws[maximum - 1] > 50 && rejections > 50, "temperature test missed acceptance or rejection"); } // Constant target logits make every reached position independently have p, // including anchors, later rejection corrections, and all-accepted bonuses. - for (int position = 0; position < (speculative ? 5 : 1); ++position) { + for (size_t position = 0; position < maximum; ++position) { require( draws[position] > 0, "temperature test missed an output position"); for (size_t token = 0; token < 3; ++token) { @@ -239,15 +302,87 @@ void check_temperature(const std::string& path, bool speculative) { } } std::cout << "temperature=" << sampling.temperature - << " speculative=" << speculative << " variant=" << variant + << " speculative=" << speculative + << " input_anchor=" << input_anchor << " variant=" << variant << " anchor_counts=" << counts[0][0] << ',' << counts[0][1] << ',' - << counts[0][2] << " bonuses=" << draws[4] + << counts[0][2] << " bonuses=" << draws[maximum - 1] << " first_rejections=" << rejections << '\n'; } } -std::vector -generate(b::Executor& executor, const std::vector& prompts, int count) { +void check_sampling_schedules(const std::string& path) { + for (int filter = 0; filter < 3; ++filter) { + for (uint64_t seed : {0, 1, 42, 123}) { + Tokens expected; + for (int schedule = 0; schedule < 4; ++schedule) { + auto executor = create(path, 3); + const auto sid = executor->open_session(); + require(sid.has_value(), "sampling schedule session failed"); + b::SamplingParams sampling; + sampling.temperature = 0.8f; + sampling.top_k = filter == 1 ? 2 : 0; + sampling.top_p = filter == 2 ? 0.7f : 1.0f; + executor->set_sampling(*sid, sampling, seed); + int position = 0; + Tokens pending{1, 2, 5}, generated; + b::BatchOutput output; + if (schedule == 1 || schedule == 2) { + auto prefix = input(*sid, 0, {1, 2}); + prefix.produce_output = false; + b::BatchInput prefill; + prefill.inputs = {prefix}; + require(executor->execute(prefill, output), "chunked prefill failed"); + position = 2; + pending = {5}; + } + bool first = true; + while (generated.size() < 12) { + b::BatchInput batch; + batch.inputs = {input(*sid, position, pending)}; + size_t row = 0; + if (first && schedule == 2) { + const auto other = executor->open_session(); + require(other.has_value(), "mixed sampling session failed"); + auto prefix = input(*other, 0, {6, 4}); + prefix.produce_output = false; + batch.inputs.push_back(prefix); + } else if (first && schedule == 3) { + auto prefix = input(*sid, 0, {1, 2}); + prefix.produce_output = false; + batch.inputs = {prefix, input(*sid, 2, {5})}; + row = 1; + } + require(executor->execute(batch, output), "sampling schedule failed"); + require(output.outputs[row].has_value(), "missing sampled output"); + const auto& tokens = output.outputs[row]->tokens; + require(!tokens.empty(), "empty sampled output"); + position += static_cast(pending.size() + tokens.size() - 1); + pending = {tokens.back()}; + generated.insert(generated.end(), tokens.begin(), tokens.end()); + first = false; + } + generated.resize(12); + if (schedule == 0) { + expected = generated; + } else { + if (generated != expected) { + std::cerr << "seed=" << seed << " filter=" << filter + << " schedule=" << schedule << '\n'; + } + require( + generated == expected, + "seeded continuation depends on batching or prompt chunking"); + } + } + } + } +} + +std::vector generate( + b::Executor& executor, + const std::vector& prompts, + int count, + bool slice_prompts = false) { std::vector sessions; std::vector positions(prompts.size(), 0); std::vector pending = prompts, completions(prompts.size()); @@ -257,12 +392,31 @@ generate(b::Executor& executor, const std::vector& prompts, int count) { executor.set_sampling(*sid, {}, 42); sessions.push_back(*sid); } + b::BatchInput sliced; + if (slice_prompts) { + for (size_t i = 0; i < prompts.size(); ++i) { + auto prefix = input(sessions[i], 0, prompts[i]); + prefix.produce_output = false; + --prefix.size; + sliced.inputs.push_back(prefix); + } + b::BatchOutput output; + require(executor.execute(sliced, output), "sliced prefill failed"); + for (auto& suffix : sliced.inputs) { + suffix.produce_output = true; + suffix.offset = suffix.size; + suffix.size = 1; + } + } while (true) { b::BatchInput batch; std::vector active; for (size_t i = 0; i < prompts.size(); ++i) { if (completions[i].size() < static_cast(count)) { - batch.inputs.push_back(input(sessions[i], positions[i], pending[i])); + batch.inputs.push_back( + slice_prompts && completions[i].empty() + ? sliced.inputs[i] + : input(sessions[i], positions[i], pending[i])); active.push_back(i); } } @@ -335,6 +489,9 @@ void check_random_continuations( require( generate(*executor, prompts, 20) == expected, "reset/replay changed random continuation"); + require( + generate(*executor, prompts, 20, true) == expected, + "sliced input-anchor continuation differs from target-only"); const auto source = executor->open_session(); require(source.has_value(), "replay session allocation failed"); @@ -709,6 +866,8 @@ int main(int argc, char** argv) { "fixed single-proposal selector lost its input ABI"); executor->close_session(*sid); } + check_sampling_schedules(root + "/reject_-1/model.pte"); + check_sampling_schedules(root + "/legacy/model.pte"); for (int rejected : {-1, 0, 1, 2}) { const auto path = root + "/reject_" + std::to_string(rejected) + "/model.pte"; @@ -717,6 +876,8 @@ int main(int argc, char** argv) { } } check_packed_replay(root + "/reject_-1/model.pte"); + check_input_anchors(root + "/reject_-1/model.pte"); + check_input_anchors(root + "/legacy/model.pte"); check_owner_proposals(root + "/owner_sensitive/model.pte"); check_draft_routing(root + "/routing/model.pte"); check_draft_routing(root + "/routing_legacy/model.pte"); @@ -724,6 +885,7 @@ int main(int argc, char** argv) { check_draft_numerics(root + "/random", proposals); } check_random_continuations(root + "/random/model.pte", {1, 2, 3}); + check_random_continuations(root + "/random_legacy/model.pte", {1, 2, 3}); for (int proposals = 1; proposals < 16; ++proposals) { check_draft_numerics(root + "/quantized", proposals); } @@ -732,6 +894,8 @@ int main(int argc, char** argv) { check_temperature(root + "/reject_-1/model.pte", false); check_temperature(root + "/reject_-1/model.pte", true); check_temperature(root + "/legacy/model.pte", true); + check_temperature(root + "/reject_-1/model.pte", true, true); + check_temperature(root + "/legacy/model.pte", true, true); auto target = create(root + "/reject_-1/model.pte", 0, false); const auto sid = target->open_session(); require(sid.has_value(), "target session allocation failed");