From 244f067ba50bff2425018a02909aba2f1363268d Mon Sep 17 00:00:00 2001 From: speriaswamy-amd Date: Wed, 13 May 2026 21:49:22 -0400 Subject: [PATCH 1/2] dmesg pattern fix --- cvs/lib/rccl_lib.py | 14 ++++++++------ cvs/lib/verify_lib.py | 6 +++++- cvs/tests/rccl/rccl_perf.py | 6 +++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/cvs/lib/rccl_lib.py b/cvs/lib/rccl_lib.py index 1240f5698..2a1891c46 100644 --- a/cvs/lib/rccl_lib.py +++ b/cvs/lib/rccl_lib.py @@ -643,10 +643,11 @@ def rccl_regression( for node in vpc_node_list: host_file_params = f'{host_file_params}{node} slots={proc_per_node}\n' - cmd = 'rm -f /tmp/rccl_hosts_file.txt' + hosts_file_path = f'/tmp/rccl_hosts_file_{os.environ.get("USER", "cvs")}.txt' + cmd = f'rm -f {hosts_file_path}' shdl.exec(cmd) - cmd = f'echo "{host_file_params}" > /tmp/rccl_hosts_file.txt' + cmd = f'echo "{host_file_params}" > {hosts_file_path}' shdl.exec(cmd) # Determine PML (Point-to-Point Messaging Layer) based on user config or auto-detection @@ -700,7 +701,7 @@ def rccl_regression( cmd = f'''{mpi_dir}/bin/mpirun \ --allow-run-as-root \ -np {no_of_global_ranks} \ - --hostfile /tmp/rccl_hosts_file.txt \ + --hostfile {hosts_file_path} \ --bind-to numa \ {ucx_params} \ --mca btl ^vader,openib \ @@ -826,10 +827,11 @@ def rccl_perf( for node in vpc_node_list: host_file_params = f'{host_file_params}' + f'{node} slots={proc_per_node}\n' - cmd = 'rm -f /tmp/rccl_hosts_file.txt' + hosts_file_path = f'/tmp/rccl_hosts_file_{os.environ.get("USER", "cvs")}.txt' + cmd = f'rm -f {hosts_file_path}' shdl.exec(cmd) - cmd = f'echo "{host_file_params}" > /tmp/rccl_hosts_file.txt' + cmd = f'echo "{host_file_params}" > {hosts_file_path}' shdl.exec(cmd) # Determine PML (Point-to-Point Messaging Layer) based on user config or auto-detection @@ -887,7 +889,7 @@ def rccl_perf( # Build mpirun command cmd = f'''{mpi_dir}/bin/mpirun --np {no_of_global_ranks} \ --allow-run-as-root \ - --hostfile /tmp/rccl_hosts_file.txt \ + --hostfile {hosts_file_path} \ --bind-to numa \ {ucx_params} \ --mca btl ^vader,openib \ diff --git a/cvs/lib/verify_lib.py b/cvs/lib/verify_lib.py index 95074ac11..7614b6ac9 100644 --- a/cvs/lib/verify_lib.py +++ b/cvs/lib/verify_lib.py @@ -17,7 +17,11 @@ 'crash': 'crashed|Traceback|cut here|Bug:|Call Trace|RIP:|end trace|amdgpu: Fatal error|segfault|show_stack|dump_stack|fault ', 'test_fail': 'Test failure', 'fault': 'no-retry page fault|Illegal register access|PROTECTION_FAULT_STATUS', - 'driver': 'Queue preemption failed for queue|Failed to evict process queues|Runlist is getting oversubscribed|No more SDMA queue to allocate|Expect reduced ROCm performance|amdgpu: process pid', + # Note: 'Runlist is getting oversubscribed' and 'Expect reduced ROCm performance' + # are amdgpu kernel info-level messages (not errors). They fire routinely on + # large multi-rank RCCL runs whenever HSA queue count exceeds the runlist + # size, even when the run itself is healthy. Excluded from failure matching. + 'driver': 'Queue preemption failed for queue|Failed to evict process queues|No more SDMA queue to allocate|amdgpu: process pid', 'hardware': 'hardware error|hardware fail|ras error|uncorrectable|correctable err', 'network': 'NIC Link is Down|link is down|ib_uverb|CQE|queue catastrophic|CQ error', } diff --git a/cvs/tests/rccl/rccl_perf.py b/cvs/tests/rccl/rccl_perf.py index a28e535d7..75f57b6b0 100644 --- a/cvs/tests/rccl/rccl_perf.py +++ b/cvs/tests/rccl/rccl_perf.py @@ -350,7 +350,11 @@ def test_rccl_perf(phdl, shdl, cluster_dict, config_dict, rccl_collective): end_time = phdl.exec('date +"%a %b %e %H:%M"') if can_use_sudo: - verify_dmesg_for_errors(phdl, start_time, end_time, till_end_flag=True) + # Bound dmesg scan to this test's own start..end window (per-test). + # till_end_flag=True scans from start_time to the end of the dmesg + # buffer, which causes earlier-test kernel events (e.g. a scatter_perf + # segfault) to repeatedly fail every subsequent parametrized test. + verify_dmesg_for_errors(phdl, start_time, end_time, till_end_flag=False) # Get new cluster snapshot and compare .. if can_use_sudo and re.search( From 04fe2fc770300159bec2a4b9820240c7c1b82534 Mon Sep 17 00:00:00 2001 From: speriaswamy-amd Date: Tue, 19 May 2026 12:05:45 -0400 Subject: [PATCH 2/2] verify_lib: surface oversubscription as WARN, not silent Per AMD docs the runlist/VM-context oversubscription messages are real perf-degrading conditions (HW scheduler round-robins queues, blocking the GPU for ms-scale windows), not benign info. Earlier patch dropped them from the failure regex entirely, which suppressed the signal: a perf run in a degraded HW state would now be marked PASS and its bandwidth deltas silently trusted. Add a warn_patterns_dict alongside err_patterns_dict; verify_dmesg_for_errors now scans both. Err matches still fail_test (no behavior change). Warn matches emit log.warning with a "perf numbers may not be trustworthy" note so the run is visibly flagged in std.log without failing the collective itself. Backward compatible: function signature and return shape unchanged, all other callers untouched. Ref: https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/conceptual/oversubscription.html Co-authored-by: Cursor --- cvs/lib/verify_lib.py | 46 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/cvs/lib/verify_lib.py b/cvs/lib/verify_lib.py index 7614b6ac9..e49b3635e 100644 --- a/cvs/lib/verify_lib.py +++ b/cvs/lib/verify_lib.py @@ -17,16 +17,31 @@ 'crash': 'crashed|Traceback|cut here|Bug:|Call Trace|RIP:|end trace|amdgpu: Fatal error|segfault|show_stack|dump_stack|fault ', 'test_fail': 'Test failure', 'fault': 'no-retry page fault|Illegal register access|PROTECTION_FAULT_STATUS', - # Note: 'Runlist is getting oversubscribed' and 'Expect reduced ROCm performance' - # are amdgpu kernel info-level messages (not errors). They fire routinely on - # large multi-rank RCCL runs whenever HSA queue count exceeds the runlist - # size, even when the run itself is healthy. Excluded from failure matching. + # Note: amdgpu oversubscription messages ('Runlist is getting oversubscribed', + # 'Expect reduced ROCm performance') are perf-degrading but not test failures — + # they're matched as warnings via warn_patterns_dict below. See AMD docs: + # https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/conceptual/oversubscription.html 'driver': 'Queue preemption failed for queue|Failed to evict process queues|No more SDMA queue to allocate|amdgpu: process pid', 'hardware': 'hardware error|hardware fail|ras error|uncorrectable|correctable err', 'network': 'NIC Link is Down|link is down|ib_uverb|CQE|queue catastrophic|CQ error', } +# warn_patterns_dict captures kernel events that indicate the GPU entered a +# perf-degrading state but the workload itself still completes correctly. Matches +# emit a WARN to the run log (no fail_test) so reviewers know the perf numbers +# from this run were measured under a degraded HW state and should not be trusted +# blindly for regression comparisons. +warn_patterns_dict = { + # GPU runlist / VM context oversubscription. Per AMD docs, when this fires the + # HW scheduler is round-robining queues and inactive queues can block the GPU + # for millisecond-scale windows. Triggers: >24 user-mode compute queues per + # GPU, >11 VM context slots, or >1 process using cooperative workgroups. + # https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/conceptual/oversubscription.html + 'oversubscribed': 'Runlist is getting oversubscribed|Expect reduced ROCm performance', +} + + err_stats_pattern = 'err|drop|discard|overflow|fcs|nak|uncorrect|loss' warn_stats_pattern = 'retry|timeout|exceeded|ooo|retransmit' threshold_stats_pattern = 'cnp|ecn' @@ -178,7 +193,8 @@ def verify_gpu_pcie_errors(phdl): def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag=True): """ Scan kernel logs (dmesg) between given start and end timestamps across nodes - and fail if any known error patterns are detected. + and fail if any known error patterns are detected. Lines matching warn-only + patterns are logged via log.warning but do not fail the test. Parameters: phdl: pssh handle that can execute remote shell commands via .exec(cmd) -> dict. @@ -189,17 +205,19 @@ def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag= - Extracts a human-readable timestamp prefix (e.g., 'Mon Jan 2 03:04:05') from provided times. - Uses dmesg -T (human-readable timestamps) piped to awk to slice the log from start to end. - Filters out lines containing 'ALLOWED' or 'DENIED' (non-fatal/noisy) via egrep -v. - - Scans each line against a set of known error regex patterns (err_patterns_dict). - - Immediately fails the test via fail_test if any error pattern is seen. + - Scans each line against err_patterns_dict (fail_test on match) and warn_patterns_dict + (log.warning only on match; test still passes but run carries a visible WARN). Assumptions: - - err_patterns_dict is defined in scope: {name: regex_pattern, ...}. + - err_patterns_dict and warn_patterns_dict are defined in scope: {name: regex_pattern, ...}. - phdl.exec(cmd) returns a dict: { node: stdout_str }. - Input timestamps contain a prefix matching the regex used here. - sudo is available and does not prompt for a password when running dmesg. Notes: - This function fails fast on the first detected error to shorten feedback cycles. + - Warn-bucket matches are non-fatal but should be reviewed before trusting any + perf measurements from the run (e.g. RCCL bandwidth, training step time). - If start/end times are not aligned with dmesg -T formatting, the awk range may be empty. - Consider handling cases where regex extraction fails (no match) to avoid attribute errors. """ @@ -237,13 +255,23 @@ def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag= for node in output_dict.keys(): err_dict[node] = [] - # Iterate through each node's sliced dmesg and scan for known error patterns + # Iterate through each node's sliced dmesg and scan for known error/warn patterns. + # err patterns -> fail_test (existing behavior). + # warn patterns -> log.warning only; the test still passes but the run carries a + # visible WARN so callers / reviewers know the GPU was in a degraded state and + # any perf numbers from this run should not be trusted blindly. for node in output_dict.keys(): for line in output_dict[node].split("\n"): for err_key in err_patterns_dict.keys(): if re.search(f'{err_patterns_dict[err_key]}', line, re.I): fail_test(f'ERROR - Failue pattern ** {line} ** seen in Dmesg') err_dict[node].append(line) + for warn_key in warn_patterns_dict.keys(): + if re.search(f'{warn_patterns_dict[warn_key]}', line, re.I): + log.warning( + f'WARN - GPU in degraded state ({warn_key}) on {node}: {line.strip()} ' + f'-- perf numbers from this run may not be trustworthy' + ) return err_dict