From 80c51fc6843bf9dc3d4cbd372465afb5ab24ccca Mon Sep 17 00:00:00 2001 From: annika-modal Date: Wed, 7 Oct 2026 19:17:17 -0400 Subject: [PATCH] fix: filter_long_prompt crashes on chat-templated multimodal prompts With apply_chat_template, Dataset.__init__ stores the templated string in sample.prompt and the parsed media in sample.multimodal_inputs. The multimodal branch of filter_long_prompt re-ran process_vision_info on that string, which expects a list of messages, so any image dataset with --rollout-max-prompt-len (or --rollout-max-context-len) set failed with AttributeError: 'str' object has no attribute 'get'. Reuse sample.multimodal_inputs instead. --- slime/utils/data.py | 7 ++--- tests/test_filter_long_prompt.py | 49 +++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/slime/utils/data.py b/slime/utils/data.py index bc7c855b3..84c6cf3b1 100644 --- a/slime/utils/data.py +++ b/slime/utils/data.py @@ -111,11 +111,10 @@ def filter_long_prompt(origin_samples: list[Sample], tokenizer, processor, max_l if len(input_ids) <= max_length: kept.append((position, sample)) if multimodal: - from slime.utils.processing_utils import process_vision_info - for position, sample in multimodal: - multimodal_inputs = process_vision_info(sample.prompt, processor) - processor_output = processor(text=sample.prompt, **multimodal_inputs) + # sample.prompt is already chat-templated text, so reuse the media + # Dataset.__init__ parsed from the message list. + processor_output = processor(text=sample.prompt, **sample.multimodal_inputs) input_ids = processor_output["input_ids"][0] if len(input_ids) <= max_length: kept.append((position, sample)) diff --git a/tests/test_filter_long_prompt.py b/tests/test_filter_long_prompt.py index 9ec946233..23c155a16 100644 --- a/tests/test_filter_long_prompt.py +++ b/tests/test_filter_long_prompt.py @@ -25,19 +25,6 @@ NUM_GPUS = 0 -@pytest.fixture -def stub_process_vision_info(monkeypatch): - """`slime.utils.processing_utils` pulls in transformers + PIL. - - The multimodal branch imports it lazily, and only `process_vision_info` is - needed here, so stub the module rather than requiring transformers on the - CPU image. - """ - module = types.ModuleType("slime.utils.processing_utils") - module.process_vision_info = lambda prompt, processor: {"images": None} - monkeypatch.setitem(sys.modules, "slime.utils.processing_utils", module) - - class _Tokenizer: """Batched tokenizer stand-in: prompt "pN:len" tokenizes to `len` ids.""" @@ -67,7 +54,7 @@ def _make_samples(specs): @pytest.mark.unit -def test_preserves_order_when_nothing_is_filtered(stub_process_vision_info): +def test_preserves_order_when_nothing_is_filtered(): # Alternating modality, every prompt well under the limit. samples = _make_samples([(i % 2 == 0, 5) for i in range(6)]) @@ -77,7 +64,7 @@ def test_preserves_order_when_nothing_is_filtered(stub_process_vision_info): @pytest.mark.unit -def test_preserves_order_when_some_are_filtered(stub_process_vision_info): +def test_preserves_order_when_some_are_filtered(): specs = [ (True, 5), # p0 multimodal, keep (False, 500), # p1 text-only, drop @@ -95,7 +82,7 @@ def test_preserves_order_when_some_are_filtered(stub_process_vision_info): @pytest.mark.unit @pytest.mark.parametrize("all_multimodal", [True, False]) -def test_single_modality_batches_are_unchanged(stub_process_vision_info, all_multimodal): +def test_single_modality_batches_are_unchanged(all_multimodal): samples = _make_samples([(all_multimodal, 5)] * 4) kept = filter_long_prompt(samples, _Tokenizer(), _Processor(), max_length=100) @@ -112,5 +99,35 @@ def test_no_processor_path_still_preserves_order(): assert [s.prompt for s in kept] == ["p0:5", "p2:5"] +@pytest.mark.unit +def test_multimodal_length_reuses_parsed_inputs(monkeypatch): + """With apply_chat_template, sample.prompt is a string, not a message list. + + Re-parsing it with process_vision_info crashes (the parser expects message + dicts), so the media parsed by Dataset.__init__ must be passed through. + """ + + def reparse(prompt, processor): + raise AssertionError("filter_long_prompt re-parsed the chat-templated prompt") + + module = types.ModuleType("slime.utils.processing_utils") + module.process_vision_info = reparse + monkeypatch.setitem(sys.modules, "slime.utils.processing_utils", module) + + calls = [] + + class _RecordingProcessor(_Processor): + def __call__(self, text=None, **kwargs): + calls.append(kwargs) + return super().__call__(text=text) + + samples = _make_samples([(True, 5), (True, 500)]) + + kept = filter_long_prompt(samples, _Tokenizer(), _RecordingProcessor(), max_length=100) + + assert [s.prompt for s in kept] == ["p0:5"] + assert calls == [{"images": ["img"]}, {"images": ["img"]}] + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__]))