From 34f396a6b46263419f8b12c07ca86ddd4952593b Mon Sep 17 00:00:00 2001 From: Urvashi Tiwari Date: Mon, 2 Mar 2026 14:56:25 -0500 Subject: [PATCH 1/4] Fixed the issue while parsing the output in Transfer bench test with latest Rocm versions Signed-off-by: Urvashi Tiwari --- cvs/tests/health/transferbench_cvs.py | 54 ++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/cvs/tests/health/transferbench_cvs.py b/cvs/tests/health/transferbench_cvs.py index 7fbb7f9b3..b45cf01ad 100644 --- a/cvs/tests/health/transferbench_cvs.py +++ b/cvs/tests/health/transferbench_cvs.py @@ -60,9 +60,12 @@ def parse_tb_a2a_bw(out_dict, exp_dict): for node in out_dict.keys(): print(exp_dict) rtotal_list = re.findall( - 'RTotal\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+', + r'│\s+RTotal\s+│\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)', out_dict[node], ) + if not rtotal_list: + fail_test(f"RTotal row not found in TransferBench a2a output on node {node}") + continue for gpu_bw in list(rtotal_list[0]): if float(gpu_bw) < float(exp_dict['gpu_to_gpu_a2a_rtotal']): fail_test( @@ -142,8 +145,14 @@ def parse_tb_example_test_results(out_dict, exp_dict): for node in out_dict.keys(): # Test 1 results parse match = re.search('(Test\s1:\n.*\n.*\n.*\n)', out_dict[node]) + if not match: + fail_test(f"Test 1 output section not found in TransferBench output on node {node}") + continue test1_out = match.group(1) - match = re.search('Transfer\s+[0-9]+\s+\|\s+([0-9\.]+)\sGB\/s', test1_out, re.I) + match = re.search(r'Executor:\s+GPU\s+[0-9]+\s+│\s*([0-9\.]+)\s+GB/s', test1_out, re.I) + if not match: + fail_test(f"Transfer bandwidth pattern not found in Test 1 output on node {node}. Output: {test1_out}") + continue test1_res = match.group(1) if float(test1_res) < float(exp_dict['test1']): fail_test( @@ -152,26 +161,41 @@ def parse_tb_example_test_results(out_dict, exp_dict): # Test 2 results parse match = re.search('(Test\s2:\n.*\n.*\n.*\n)', out_dict[node]) + if not match: + fail_test(f"Test 2 output section not found in TransferBench output on node {node}") + continue test2_out = match.group(1) - match = re.search('Transfer\s+[0-9]+\s+\|\s+([0-9\.]+)\sGB\/s', test2_out, re.I) - test2_res = match.group(1) + match = re.search(r'Executor:\s+(GPU|DMA)\s+[0-9]+\s+│\s*([0-9\.]+)\s+GB/s', test2_out, re.I) + if not match: + fail_test(f"Transfer bandwidth pattern not found in Test 2 output on node {node}. Output: {test2_out}") + continue + test2_res = match.group(2) if float(test2_res) < float(exp_dict['test2']): fail_test( f"Transfer Bench example test2 failed actual value {test2_res} is less than expected {exp_dict['test2']}" ) # Test 3 results parse - match = re.search('(Test\s3:\n.*\n.*\n.*\n.*\n.*\n)', out_dict[node]) + match = re.search('(Test\s3:.*?)(?=Test\s[456]:|##|$)', out_dict[node], re.DOTALL) + if not match: + fail_test(f"Test 3 output section not found in TransferBench output on node {node}") + continue test3_out = match.group(1) - match = re.search('Transfer\s+[0-9]+\s+\|\s+([0-9\.]+)\sGB\/s[\s0-9\.\|a-z]+\s+G0 \-\>', test3_out, re.I) + match = re.search(r'Executor:\s+GPU\s+00\s+│\s*([0-9\.]+)\s+GB/s', test3_out, re.I) + if not match: + fail_test(f"G0->G1 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}") + continue test3_res_0_1 = match.group(1) if float(test3_res_0_1) < float(exp_dict['test3_0_to_1']): fail_test( f"Transfer Bench example test3 failed actual value {test3_res_0_1} is less than expected {exp_dict['test3_0_to_1']}" ) - match = re.search('Transfer\s+[0-9]+\s+\|\s+([0-9\.]+)\sGB\/s[\s0-9\.\|a-z]+\s+G1 \-\>', test3_out, re.I) + match = re.search(r'Executor:\s+GPU\s+01\s+│\s*([0-9\.]+)\s+GB/s', test3_out, re.I) + if not match: + fail_test(f"G1->G0 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}") + continue test3_res_1_0 = match.group(1) if float(test3_res_1_0) < float(exp_dict['test3_1_to_0']): fail_test( @@ -180,8 +204,14 @@ def parse_tb_example_test_results(out_dict, exp_dict): # Test 4 results parse match = re.search('(Test\s4:\n.*\n.*\n.*\n)', out_dict[node]) + if not match: + fail_test(f"Test 4 output section not found in TransferBench output on node {node}") + continue test4_out = match.group(1) - match = re.search('Transfer\s+[0-9]+\s+\|\s+([0-9\.]+)\sGB\/s', test4_out, re.I) + match = re.search(r'Executor:\s+GPU\s+[0-9]+\s+│\s*([0-9\.]+)\s+GB/s', test4_out, re.I) + if not match: + fail_test(f"Transfer bandwidth pattern not found in Test 4 output on node {node}. Output: {test4_out}") + continue test4_res = match.group(1) if float(test4_res) < float(exp_dict['test4']): fail_test( @@ -191,8 +221,14 @@ def parse_tb_example_test_results(out_dict, exp_dict): # Test 5 CPU only .. skip # Test 6 results parse match = re.search('(Test\s6:\n.*\n.*\n.*\n)', out_dict[node]) + if not match: + fail_test(f"Test 6 output section not found in TransferBench output on node {node}") + continue test6_out = match.group(1) - match = re.search('Transfer\s+[0-9]+\s+\|\s+([0-9\.]+)\sGB\/s', test6_out, re.I) + match = re.search(r'Executor:\s+GPU\s+[0-9]+\s+│\s*([0-9\.]+)\s+GB/s', test6_out, re.I) + if not match: + fail_test(f"Transfer bandwidth pattern not found in Test 6 output on node {node}. Output: {test6_out}") + continue test6_res = match.group(1) if float(test6_res) < float(exp_dict['test6']): fail_test( From e1bd0e7e81de00efa5a24f570006f02af5301067 Mon Sep 17 00:00:00 2001 From: Urvashi Tiwari Date: Mon, 2 Mar 2026 15:22:26 -0500 Subject: [PATCH 2/4] Fixed the CI issue Signed-off-by: Urvashi Tiwari --- cvs/tests/health/transferbench_cvs.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cvs/tests/health/transferbench_cvs.py b/cvs/tests/health/transferbench_cvs.py index b45cf01ad..429908651 100644 --- a/cvs/tests/health/transferbench_cvs.py +++ b/cvs/tests/health/transferbench_cvs.py @@ -184,7 +184,9 @@ def parse_tb_example_test_results(out_dict, exp_dict): match = re.search(r'Executor:\s+GPU\s+00\s+│\s*([0-9\.]+)\s+GB/s', test3_out, re.I) if not match: - fail_test(f"G0->G1 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}") + fail_test( + f"G0->G1 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}" + ) continue test3_res_0_1 = match.group(1) if float(test3_res_0_1) < float(exp_dict['test3_0_to_1']): @@ -194,7 +196,9 @@ def parse_tb_example_test_results(out_dict, exp_dict): match = re.search(r'Executor:\s+GPU\s+01\s+│\s*([0-9\.]+)\s+GB/s', test3_out, re.I) if not match: - fail_test(f"G1->G0 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}") + fail_test( + f"G1->G0 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}" + ) continue test3_res_1_0 = match.group(1) if float(test3_res_1_0) < float(exp_dict['test3_1_to_0']): From 015220b00f705def2553b5715499d3611eb716c9 Mon Sep 17 00:00:00 2001 From: Urvashi Tiwari Date: Thu, 5 Mar 2026 19:32:44 -0500 Subject: [PATCH 3/4] Added backward compatibility Signed-off-by: Urvashi Tiwari --- cvs/tests/health/transferbench_cvs.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cvs/tests/health/transferbench_cvs.py b/cvs/tests/health/transferbench_cvs.py index 429908651..b09d29e5b 100644 --- a/cvs/tests/health/transferbench_cvs.py +++ b/cvs/tests/health/transferbench_cvs.py @@ -60,7 +60,8 @@ def parse_tb_a2a_bw(out_dict, exp_dict): for node in out_dict.keys(): print(exp_dict) rtotal_list = re.findall( - r'│\s+RTotal\s+│\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)', + r'(?:│\s+)?RTotal\s+(?:│\s+)?([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s*', + out_dict[node], ) if not rtotal_list: @@ -149,7 +150,7 @@ def parse_tb_example_test_results(out_dict, exp_dict): fail_test(f"Test 1 output section not found in TransferBench output on node {node}") continue test1_out = match.group(1) - match = re.search(r'Executor:\s+GPU\s+[0-9]+\s+│\s*([0-9\.]+)\s+GB/s', test1_out, re.I) + match = re.search(r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s', test1_out, re.I) if not match: fail_test(f"Transfer bandwidth pattern not found in Test 1 output on node {node}. Output: {test1_out}") continue @@ -165,11 +166,11 @@ def parse_tb_example_test_results(out_dict, exp_dict): fail_test(f"Test 2 output section not found in TransferBench output on node {node}") continue test2_out = match.group(1) - match = re.search(r'Executor:\s+(GPU|DMA)\s+[0-9]+\s+│\s*([0-9\.]+)\s+GB/s', test2_out, re.I) + match = re.search(r'(?:Transfer|Executor:\s+(?:GPU|DMA))\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s', test2_out, re.I) if not match: fail_test(f"Transfer bandwidth pattern not found in Test 2 output on node {node}. Output: {test2_out}") continue - test2_res = match.group(2) + test2_res = match.group(1) if float(test2_res) < float(exp_dict['test2']): fail_test( f"Transfer Bench example test2 failed actual value {test2_res} is less than expected {exp_dict['test2']}" @@ -182,7 +183,7 @@ def parse_tb_example_test_results(out_dict, exp_dict): continue test3_out = match.group(1) - match = re.search(r'Executor:\s+GPU\s+00\s+│\s*([0-9\.]+)\s+GB/s', test3_out, re.I) + match = re.search(r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s(?:[\s0-9\.\|a-z]+\s+G0\s*-\>)?', test3_out, re.I) if not match: fail_test( f"G0->G1 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}" @@ -194,7 +195,7 @@ def parse_tb_example_test_results(out_dict, exp_dict): f"Transfer Bench example test3 failed actual value {test3_res_0_1} is less than expected {exp_dict['test3_0_to_1']}" ) - match = re.search(r'Executor:\s+GPU\s+01\s+│\s*([0-9\.]+)\s+GB/s', test3_out, re.I) + match = re.search(r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s(?:[\s0-9\.\|a-z]+\s+G1\s*-\>)?', test3_out, re.I) if not match: fail_test( f"G1->G0 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}" @@ -212,7 +213,7 @@ def parse_tb_example_test_results(out_dict, exp_dict): fail_test(f"Test 4 output section not found in TransferBench output on node {node}") continue test4_out = match.group(1) - match = re.search(r'Executor:\s+GPU\s+[0-9]+\s+│\s*([0-9\.]+)\s+GB/s', test4_out, re.I) + match = re.search(r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s', test4_out, re.I) if not match: fail_test(f"Transfer bandwidth pattern not found in Test 4 output on node {node}. Output: {test4_out}") continue From ae7411dc16ec90e8448a78812d950d1fa6a97bfe Mon Sep 17 00:00:00 2001 From: Urvashi Tiwari Date: Sat, 7 Mar 2026 01:14:22 -0500 Subject: [PATCH 4/4] Fixed CI error Signed-off-by: Urvashi Tiwari --- cvs/tests/health/transferbench_cvs.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cvs/tests/health/transferbench_cvs.py b/cvs/tests/health/transferbench_cvs.py index b09d29e5b..75e098f56 100644 --- a/cvs/tests/health/transferbench_cvs.py +++ b/cvs/tests/health/transferbench_cvs.py @@ -61,7 +61,6 @@ def parse_tb_a2a_bw(out_dict, exp_dict): print(exp_dict) rtotal_list = re.findall( r'(?:│\s+)?RTotal\s+(?:│\s+)?([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s*', - out_dict[node], ) if not rtotal_list: @@ -183,7 +182,11 @@ def parse_tb_example_test_results(out_dict, exp_dict): continue test3_out = match.group(1) - match = re.search(r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s(?:[\s0-9\.\|a-z]+\s+G0\s*-\>)?', test3_out, re.I) + match = re.search( + r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s(?:[\s0-9\.\|a-z]+\s+G0\s*-\>)?', + test3_out, + re.I, + ) if not match: fail_test( f"G0->G1 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}" @@ -195,7 +198,11 @@ def parse_tb_example_test_results(out_dict, exp_dict): f"Transfer Bench example test3 failed actual value {test3_res_0_1} is less than expected {exp_dict['test3_0_to_1']}" ) - match = re.search(r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s(?:[\s0-9\.\|a-z]+\s+G1\s*-\>)?', test3_out, re.I) + match = re.search( + r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s(?:[\s0-9\.\|a-z]+\s+G1\s*-\>)?', + test3_out, + re.I, + ) if not match: fail_test( f"G1->G0 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}"