Skip to content

Commit ea7ba02

Browse files
tom-armtom-arm
committed
Arm backend: Add xlarge pytest decorator
* Add xlarge decorator for tests that use a lot of memory * Add full Qwen 3 VL model tests Signed-off-by: Tom Allsop <tom.allsop@arm.com> Change-Id: I8debb090cb8e2bdb39a05d0b1c5d4e0d8ef6b779
1 parent 124b677 commit ea7ba02

3 files changed

Lines changed: 95 additions & 31 deletions

File tree

‎backends/arm/test/models/Qwen3_VL/test_qwen3_vl_model.py‎

Lines changed: 89 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ def _make_pixel_values(config, device: torch.device) -> torch.Tensor:
8888

8989
class Qwen3VLModelTestModule(torch.nn.Module):
9090
@classmethod
91-
def prepare_model_and_inputs(cls):
91+
def prepare_model_and_inputs(cls, config_factory=_make_qwen3_vl_e2e_test_config):
9292
raise NotImplementedError
9393

9494

@@ -130,9 +130,9 @@ def forward(
130130
return outputs.last_hidden_state
131131

132132
@classmethod
133-
def prepare_model_and_inputs(cls):
133+
def prepare_model_and_inputs(cls, config_factory=_make_qwen3_vl_e2e_test_config):
134134
torch.manual_seed(0)
135-
config = _make_qwen3_vl_e2e_test_config()
135+
config = config_factory()
136136
model = cls(config).eval()
137137
input_ids = torch.randint(0, 128, (2, 8), dtype=torch.long)
138138
attention_mask = torch.ones_like(input_ids)
@@ -193,9 +193,9 @@ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
193193
return hidden_states + deepstack_residual
194194

195195
@classmethod
196-
def prepare_model_and_inputs(cls):
196+
def prepare_model_and_inputs(cls, config_factory=_make_qwen3_vl_e2e_test_config):
197197
torch.manual_seed(0)
198-
config = _make_qwen3_vl_e2e_test_config()
198+
config = config_factory()
199199
model = cls(config).eval()
200200
pixel_values = _make_pixel_values(config, torch.device("cpu"))
201201
return model, (pixel_values,)
@@ -232,10 +232,11 @@ class Qwen3VLModelTestCase:
232232
}
233233

234234

235-
@pytest.mark.slow
236-
@common.parametrize("test_case", TOSA_FP_TEST_CASES)
237-
def test_qwen3_vl_full_models_tosa_FP(test_case: Qwen3VLModelTestCase):
238-
model, inputs = test_case.model_cls.prepare_model_and_inputs()
235+
def _test_qwen3_vl_full_models_tosa_FP(
236+
test_case: Qwen3VLModelTestCase,
237+
config_factory=_make_qwen3_vl_e2e_test_config,
238+
):
239+
model, inputs = test_case.model_cls.prepare_model_and_inputs(config_factory)
239240
with torch.no_grad():
240241
pipeline = TosaPipelineFP[input_t](
241242
model,
@@ -248,10 +249,11 @@ def test_qwen3_vl_full_models_tosa_FP(test_case: Qwen3VLModelTestCase):
248249
pipeline.run()
249250

250251

251-
@pytest.mark.slow
252-
@common.parametrize("test_case", TOSA_FP_TEST_CASES)
253-
def test_qwen3_vl_full_models_tosa_FP_bf16(test_case: Qwen3VLModelTestCase):
254-
model, inputs = test_case.model_cls.prepare_model_and_inputs()
252+
def _test_qwen3_vl_full_models_tosa_FP_bf16(
253+
test_case: Qwen3VLModelTestCase,
254+
config_factory=_make_qwen3_vl_e2e_test_config,
255+
):
256+
model, inputs = test_case.model_cls.prepare_model_and_inputs(config_factory)
255257
model, inputs = _to_bfloat16_model_and_floating_inputs(model, inputs)
256258
# Slightly higher atol for TOSA BF16 on aarch64 (MLETORCH-2048: numeric mismatch)
257259
atol = (
@@ -273,11 +275,11 @@ def test_qwen3_vl_full_models_tosa_FP_bf16(test_case: Qwen3VLModelTestCase):
273275
pipeline.run()
274276

275277

276-
@pytest.mark.slow
277-
@common.SkipIfNoModelConverter
278-
@common.parametrize("test_case", VGF_NO_QUANT_TEST_CASES)
279-
def test_qwen3_vl_full_models_vgf_no_quant(test_case: Qwen3VLModelTestCase):
280-
model, inputs = test_case.model_cls.prepare_model_and_inputs()
278+
def _test_qwen3_vl_full_models_vgf_no_quant(
279+
test_case: Qwen3VLModelTestCase,
280+
config_factory=_make_qwen3_vl_e2e_test_config,
281+
):
282+
model, inputs = test_case.model_cls.prepare_model_and_inputs(config_factory)
281283
with torch.no_grad():
282284
pipeline = VgfPipeline[input_t](
283285
model,
@@ -290,11 +292,11 @@ def test_qwen3_vl_full_models_vgf_no_quant(test_case: Qwen3VLModelTestCase):
290292
pipeline.run()
291293

292294

293-
@pytest.mark.slow
294-
@common.SkipIfNoModelConverter
295-
@common.parametrize("test_case", VGF_NO_QUANT_TEST_CASES)
296-
def test_qwen3_vl_full_models_vgf_no_quant_bf16(test_case: Qwen3VLModelTestCase):
297-
model, inputs = test_case.model_cls.prepare_model_and_inputs()
295+
def _test_qwen3_vl_full_models_vgf_no_quant_bf16(
296+
test_case: Qwen3VLModelTestCase,
297+
config_factory=_make_qwen3_vl_e2e_test_config,
298+
):
299+
model, inputs = test_case.model_cls.prepare_model_and_inputs(config_factory)
298300
model, inputs = _to_bfloat16_model_and_floating_inputs(model, inputs)
299301
with torch.no_grad():
300302
pipeline = VgfPipeline[input_t](
@@ -309,10 +311,11 @@ def test_qwen3_vl_full_models_vgf_no_quant_bf16(test_case: Qwen3VLModelTestCase)
309311
pipeline.run()
310312

311313

312-
@pytest.mark.slow
313-
def test_qwen3_vl_text_model_tosa_mxfp8_bf16():
314+
def _test_qwen3_vl_text_model_tosa_mxfp8_bf16(
315+
config_factory=_make_qwen3_vl_e2e_test_config,
316+
):
314317
# The Qwen 3 VL FP8 model only quantizes the TextModel
315-
model, inputs = TextModelWrapper.prepare_model_and_inputs()
318+
model, inputs = TextModelWrapper.prepare_model_and_inputs(config_factory)
316319
model, inputs = _to_bfloat16_model_and_floating_inputs(model, inputs)
317320
mxfp_config = MXFPOpConfig(weight_dtype=torch.float8_e4m3fn)
318321
with torch.no_grad():
@@ -339,3 +342,63 @@ def test_qwen3_vl_text_model_tosa_mxfp8_bf16():
339342
suffix="mxfp_linear",
340343
)
341344
pipeline.run()
345+
346+
347+
@pytest.mark.slow
348+
@common.parametrize("test_case", TOSA_FP_TEST_CASES)
349+
def test_qwen3_vl_full_models_tosa_FP(test_case: Qwen3VLModelTestCase):
350+
_test_qwen3_vl_full_models_tosa_FP(test_case)
351+
352+
353+
@pytest.mark.slow
354+
@common.parametrize("test_case", TOSA_FP_TEST_CASES)
355+
def test_qwen3_vl_full_models_tosa_FP_bf16(test_case: Qwen3VLModelTestCase):
356+
_test_qwen3_vl_full_models_tosa_FP_bf16(test_case)
357+
358+
359+
@pytest.mark.slow
360+
@common.SkipIfNoModelConverter
361+
@common.parametrize("test_case", VGF_NO_QUANT_TEST_CASES)
362+
def test_qwen3_vl_full_models_vgf_no_quant(test_case: Qwen3VLModelTestCase):
363+
_test_qwen3_vl_full_models_vgf_no_quant(test_case)
364+
365+
366+
@pytest.mark.slow
367+
@common.SkipIfNoModelConverter
368+
@common.parametrize("test_case", VGF_NO_QUANT_TEST_CASES)
369+
def test_qwen3_vl_full_models_vgf_no_quant_bf16(test_case: Qwen3VLModelTestCase):
370+
_test_qwen3_vl_full_models_vgf_no_quant_bf16(test_case)
371+
372+
373+
@pytest.mark.slow
374+
def test_qwen3_vl_text_model_tosa_mxfp8_bf16():
375+
_test_qwen3_vl_text_model_tosa_mxfp8_bf16()
376+
377+
378+
@pytest.mark.slow
379+
@pytest.mark.xlarge
380+
@common.parametrize("test_case", TOSA_FP_TEST_CASES)
381+
def test_qwen3_vl_2b_instruct_full_models_tosa_FP_bf16(
382+
test_case: Qwen3VLModelTestCase,
383+
):
384+
_test_qwen3_vl_full_models_tosa_FP_bf16(
385+
test_case, _make_qwen3_vl_2b_instruct_layer_config
386+
)
387+
388+
389+
@pytest.mark.slow
390+
@pytest.mark.xlarge
391+
@common.SkipIfNoModelConverter
392+
@common.parametrize("test_case", VGF_NO_QUANT_TEST_CASES)
393+
def test_qwen3_vl_2b_instruct_full_models_vgf_no_quant_bf16(
394+
test_case: Qwen3VLModelTestCase,
395+
):
396+
_test_qwen3_vl_full_models_vgf_no_quant_bf16(
397+
test_case, _make_qwen3_vl_2b_instruct_layer_config
398+
)
399+
400+
401+
@pytest.mark.slow
402+
@pytest.mark.xlarge
403+
def test_qwen3_vl_2b_instruct_text_model_tosa_mxfp8_bf16():
404+
_test_qwen3_vl_text_model_tosa_mxfp8_bf16(_make_qwen3_vl_2b_instruct_layer_config)

‎backends/arm/test/pytest.ini‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,6 @@ timeout = 1800
33
addopts = --strict-markers
44
markers =
55
slow: Tests that take long time
6+
xlarge: Tests that are known to use a lot of memory
67
flaky: Tests that are known to be flaky/intermittent
78
tosa_ref_model: Tests that use TOSA reference model # Temporary!

‎backends/arm/test/test_arm_backend.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ test_pytest_models_no_target() {
9090
source backends/arm/scripts/install_models_for_test.sh
9191

9292
# Run arm baremetal pytest tests without FVP
93-
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k "${EXCLUDE_TARGET_EXPR}"
93+
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k "${EXCLUDE_TARGET_EXPR}" -m "not xlarge"
9494
echo "${TEST_SUITE_NAME}: PASS"
9595
}
9696

@@ -110,7 +110,7 @@ test_pytest_models_tosa() {
110110
# Install model dependencies for pytest
111111
source backends/arm/scripts/install_models_for_test.sh
112112

113-
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k tosa
113+
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k tosa -m "not xlarge"
114114
echo "${TEST_SUITE_NAME}: PASS"
115115
}
116116

@@ -146,7 +146,7 @@ test_pytest_models_ethos_u55() {
146146
# Install model dependencies for pytest
147147
source backends/arm/scripts/install_models_for_test.sh
148148

149-
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k u55
149+
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k u55 -m "not xlarge"
150150
echo "${TEST_SUITE_NAME}: PASS"
151151
}
152152

@@ -204,7 +204,7 @@ test_pytest_models_ethos_u85() {
204204
# Install model dependencies for pytest
205205
source backends/arm/scripts/install_models_for_test.sh
206206

207-
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k u85
207+
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k u85 -m "not xlarge"
208208
echo "${TEST_SUITE_NAME}: PASS"
209209
}
210210

@@ -251,7 +251,7 @@ test_pytest_models_vkml() {
251251
# Install model dependencies for pytest
252252
source backends/arm/scripts/install_models_for_test.sh
253253

254-
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k _vgf_
254+
pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k _vgf_ -m "not xlarge"
255255
echo "${TEST_SUITE_NAME}: PASS"
256256
}
257257

0 commit comments

Comments
 (0)