Skip to content

Commit 8e28aa7

Browse files
YufengShi-duduYufengShi-dudu
committed
Arm backend: Add run-only TOSA ref model test mode
Allow Arm tester pipelines to run the lowered artifact without comparing against eager reference outputs. This lets FP8 tests validate that the serialized TOSA graph runs in the reference model even when eager CPU FP8 execution is unavailable. Use the new mode for FP8 TOSA tests that previously disabled run_on_tosa_ref_model, preserving the default output comparison behavior for existing tests. Change-Id: I237333eb2d9cb83fe386581e351fe009925e5968 Signed-off-by: Yufeng Shi <yufeng.shi@arm.com>
1 parent 9898bc5 commit 8e28aa7

12 files changed

Lines changed: 139 additions & 21 deletions

‎backends/arm/test/misc/test_debug_feats.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,28 @@ def test_compare_initial_to_quantized_tosa_INT(test_data: input_t1):
9999
pipeline.run()
100100

101101

102+
@common.parametrize("test_data", Linear.inputs)
103+
def test_tosa_FP_can_skip_ref_model_output_comparison(test_data: input_t1):
104+
default_pipeline = TosaPipelineFP[input_t1](Linear(), test_data, [], [])
105+
default_compare_stage = default_pipeline._stages[
106+
default_pipeline.find_pos("run_method_and_compare_outputs")
107+
]
108+
assert default_compare_stage.kwargs["compare_outputs"] is True
109+
110+
validation_pipeline = TosaPipelineFP[input_t1](
111+
Linear(),
112+
test_data,
113+
[],
114+
[],
115+
compare_tosa_ref_model_outputs=False,
116+
)
117+
validation_compare_stage = validation_pipeline._stages[
118+
validation_pipeline.find_pos("run_method_and_compare_outputs")
119+
]
120+
assert validation_compare_stage.kwargs["compare_outputs"] is False
121+
assert validation_compare_stage.kwargs["inputs"] == test_data
122+
123+
102124
@common.parametrize("test_data", Linear.inputs)
103125
def test_artifact_tosa_FP(test_data: input_t1):
104126
model = Linear()

‎backends/arm/test/ops/test_adaptive_avg_pool2d.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ def test_adaptive_avg_pool2d_tosa_FP_fp8(test_module):
149149
aten_op=[],
150150
exir_op=exir_op,
151151
tosa_extensions=[tosa_extension],
152-
run_on_tosa_ref_model=False, # torch.avg_pool2d() has no eager CPU FP8 implementation, so eager reference execution fails.
152+
compare_tosa_ref_model_outputs=False,
153153
)
154154
pipeline.count_tosa_ops({"AVG_POOL2D": 4})
155155
pipeline.run()

‎backends/arm/test/ops/test_avg_pool2d.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ def test_avg_pool2d_tosa_FP_fp8(test_module):
187187
aten_op,
188188
exir_op,
189189
tosa_extensions=[tosa_extension],
190-
run_on_tosa_ref_model=False, # torch.avg_pool2d() has no eager CPU FP8 implementation, so eager reference execution fails.
190+
compare_tosa_ref_model_outputs=False,
191191
)
192192
pipeline.count_tosa_ops({"AVG_POOL2D": 1})
193193
pipeline.run()

‎backends/arm/test/ops/test_conv2d.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,9 @@ def conv2d_fp16_1x1():
553553
"fp8e5m2",
554554
),
555555
}
556+
_fp8_conv2d_tosa_ref_model_xfails = {
557+
name: "MLETORCH-2238: Fix invalid FP8 CONV TOSA graphs" for name in test_data_FP_fp8
558+
}
556559

557560
# Generate a new test set paired with per_channel_quant=True/False.
558561
test_data_INT = {
@@ -608,15 +611,17 @@ def test_convolution_2d_tosa_FP(test_data):
608611
pipeline.run()
609612

610613

611-
@common.parametrize("test_data", test_data_FP_fp8)
614+
@common.parametrize(
615+
"test_data", test_data_FP_fp8, xfails=_fp8_conv2d_tosa_ref_model_xfails
616+
)
612617
def test_convolution_2d_tosa_FP_fp8(test_data):
613618
model, tosa_extension = test_data()
614619
pipeline = TosaPipelineFP[input_t](
615620
model,
616621
model.get_inputs(),
617622
aten_op,
618623
exir_op,
619-
run_on_tosa_ref_model=False, # torch.conv2d() has no eager CPU FP8 implementation, so eager reference execution fails.
624+
compare_tosa_ref_model_outputs=False,
620625
tosa_extensions=[tosa_extension],
621626
)
622627
pipeline.count_tosa_ops({"CONV2D": 1, "CAST": 1})

‎backends/arm/test/ops/test_conv3d.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,9 @@ def forward(self, x):
515515
"fp8e5m2",
516516
),
517517
}
518+
_fp8_conv3d_tosa_ref_model_xfails = {
519+
name: "MLETORCH-2238: Fix invalid FP8 CONV TOSA graphs" for name in test_data_FP_fp8
520+
}
518521

519522
test_data_FP_bf16 = {
520523
"bf16_3x3": lambda: Conv3d(
@@ -608,15 +611,17 @@ def test_convolution_3d_tosa_FP(test_data):
608611
pipeline.run()
609612

610613

611-
@common.parametrize("test_data", test_data_FP_fp8)
614+
@common.parametrize(
615+
"test_data", test_data_FP_fp8, xfails=_fp8_conv3d_tosa_ref_model_xfails
616+
)
612617
def test_convolution_3d_tosa_FP_fp8(test_data):
613618
model, tosa_extension = test_data()
614619
pipeline = TosaPipelineFP[input_t](
615620
model,
616621
model.get_inputs(),
617622
aten_op,
618623
exir_op,
619-
run_on_tosa_ref_model=False, # torch.conv3d() has no eager CPU FP8 implementation, so eager reference execution fails.
624+
compare_tosa_ref_model_outputs=False,
620625
tosa_extensions=[tosa_extension],
621626
)
622627
pipeline.count_tosa_ops({"CONV3D": 1, "CAST": 1})

‎backends/arm/test/ops/test_depthwise_conv.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,10 @@
227227
"fp8e5m2",
228228
),
229229
}
230+
_fp8_depthwise_conv_tosa_ref_model_xfails = {
231+
name: "MLETORCH-2238: Fix invalid FP8 CONV TOSA graphs"
232+
for name in test_data_conv2d_FP_fp8
233+
}
230234

231235
# Generate a new test set paired with per_channel_quant=True/False.
232236
test_data_conv2d_INT = {
@@ -289,15 +293,19 @@ def test_convolution_2d_tosa_FP_depthwise(test_data: torch.nn.Module):
289293
pipeline.run()
290294

291295

292-
@common.parametrize("test_data", test_data_conv2d_FP_fp8)
296+
@common.parametrize(
297+
"test_data",
298+
test_data_conv2d_FP_fp8,
299+
xfails=_fp8_depthwise_conv_tosa_ref_model_xfails,
300+
)
293301
def test_convolution_2d_tosa_FP_fp8_depthwise(test_data):
294302
model, tosa_extension = test_data()
295303
pipeline = TosaPipelineFP[input_t](
296304
model,
297305
model.get_inputs(),
298306
aten_op=[],
299307
exir_op=exir_op,
300-
run_on_tosa_ref_model=False, # torch.conv2d() has no eager CPU FP8 implementation, so eager reference execution fails.
308+
compare_tosa_ref_model_outputs=False,
301309
tosa_extensions=[tosa_extension],
302310
)
303311
pipeline.count_tosa_ops({"DEPTHWISE_CONV2D": 1, "CAST": 1})

‎backends/arm/test/ops/test_gather.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ def test_gather_tosa_FP_fp8(test_data: tuple[input_params, str]):
186186
transform_passes=[
187187
InsertInt32CastsAfterInt64PlaceholdersPass(),
188188
], # int64 index are not currently supported and need to be cast to int32
189-
run_on_tosa_ref_model=False, # torch.gather() has no eager CPU FP8 implementation here, so eager reference execution fails.
189+
compare_tosa_ref_model_outputs=False,
190190
tosa_extensions=[tosa_extension],
191191
)
192192
pipeline.run()

‎backends/arm/test/ops/test_matmul.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,9 @@ def forward(self, x1: torch.Tensor, x2: torch.Tensor, x3: torch.Tensor):
375375
(exir_op_mm_2d, exir_op_mm_2d),
376376
),
377377
}
378+
fp8_xfails = {
379+
name: "MLETORCH-2239: Fix invalid FP8 MATMUL TOSA graphs" for name in test_suite_fp8
380+
}
378381

379382
xfails = {
380383
"double_input_randn_rand_1d_1d": "aten.dot.default is not supported",
@@ -398,7 +401,7 @@ def test_matmul_tosa_FP(test_case: test_case_t):
398401
pipeline.run()
399402

400403

401-
@common.parametrize("test_case", test_suite_fp8)
404+
@common.parametrize("test_case", test_suite_fp8, xfails=fp8_xfails)
402405
def test_matmul_tosa_FP_fp8(test_case: test_case_t):
403406
test_data = test_case()
404407
input_dtype = test_data.input_factory()[0].dtype
@@ -409,7 +412,7 @@ def test_matmul_tosa_FP_fp8(test_case: test_case_t):
409412
aten_op_mm,
410413
list(test_data.exir_ops),
411414
tosa_extensions=[tosa_extension],
412-
run_on_tosa_ref_model=False,
415+
compare_tosa_ref_model_outputs=False,
413416
)
414417
pipeline.count_tosa_ops(
415418
{"MATMUL": len(test_data.exir_ops), "CAST": len(test_data.exir_ops)}

‎backends/arm/test/ops/test_max_pool.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ def test_max_pool2d_tosa_FP_fp8(test_data: torch.Tensor):
191191
aten_op,
192192
exir_op,
193193
tosa_extensions=[tosa_extension],
194-
run_on_tosa_ref_model=False, # torch.max_pool2d() has no eager CPU FP8 implementation, so eager reference execution fails.
194+
compare_tosa_ref_model_outputs=False,
195195
)
196196
pipeline.count_tosa_ops({"MAX_POOL2D": 1})
197197
pipeline.run()
@@ -352,7 +352,7 @@ def test_max_pool2d_tosa_FP_fp8_dilation(test_data):
352352
aten_op,
353353
exir_op,
354354
tosa_extensions=[tosa_extension],
355-
run_on_tosa_ref_model=False, # torch.max_pool2d() has no eager CPU FP8 implementation, so eager reference execution fails.
355+
compare_tosa_ref_model_outputs=False,
356356
)
357357
pipeline.count_tosa_ops({"MAX_POOL2D": 1})
358358
pipeline.run()

‎backends/arm/test/ops/test_transpose_conv2d.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,9 @@ def _get_per_channel_observers(module: torch.nn.Module):
267267
"fp8e5m2",
268268
),
269269
}
270+
_fp8_transpose_conv_tosa_ref_model_xfails = {
271+
name: "MLETORCH-2238: Fix invalid FP8 CONV TOSA graphs" for name in test_data_FP8
272+
}
270273

271274

272275
@common.parametrize("test_data", test_data_FP | test_data_FP_fp16 | test_data_BF16)
@@ -284,15 +287,17 @@ def test_conv_transpose2d_tosa_FP(test_data):
284287
pipeline.run()
285288

286289

287-
@common.parametrize("test_data", test_data_FP8)
290+
@common.parametrize(
291+
"test_data", test_data_FP8, xfails=_fp8_transpose_conv_tosa_ref_model_xfails
292+
)
288293
def test_conv_transpose2d_tosa_FP_fp8(test_data):
289294
model, tosa_extension = test_data()
290295
pipeline = TosaPipelineFP[input_t](
291296
model,
292297
model.get_inputs(),
293298
aten_op,
294299
exir_op,
295-
run_on_tosa_ref_model=False, # torch.conv_transpose2d() has no eager CPU FP8 implementation, so eager reference execution fails.
300+
compare_tosa_ref_model_outputs=False,
296301
tosa_extensions=[tosa_extension],
297302
)
298303
pipeline.count_tosa_ops({"TRANSPOSE_CONV2D": 1, "CAST": 1})

0 commit comments

Comments
 (0)