diff --git a/cvs/lib/rccl_lib.py b/cvs/lib/rccl_lib.py index 1240f5698..2a1891c46 100644 --- a/cvs/lib/rccl_lib.py +++ b/cvs/lib/rccl_lib.py @@ -643,10 +643,11 @@ def rccl_regression( for node in vpc_node_list: host_file_params = f'{host_file_params}{node} slots={proc_per_node}\n' - cmd = 'rm -f /tmp/rccl_hosts_file.txt' + hosts_file_path = f'/tmp/rccl_hosts_file_{os.environ.get("USER", "cvs")}.txt' + cmd = f'rm -f {hosts_file_path}' shdl.exec(cmd) - cmd = f'echo "{host_file_params}" > /tmp/rccl_hosts_file.txt' + cmd = f'echo "{host_file_params}" > {hosts_file_path}' shdl.exec(cmd) # Determine PML (Point-to-Point Messaging Layer) based on user config or auto-detection @@ -700,7 +701,7 @@ def rccl_regression( cmd = f'''{mpi_dir}/bin/mpirun \ --allow-run-as-root \ -np {no_of_global_ranks} \ - --hostfile /tmp/rccl_hosts_file.txt \ + --hostfile {hosts_file_path} \ --bind-to numa \ {ucx_params} \ --mca btl ^vader,openib \ @@ -826,10 +827,11 @@ def rccl_perf( for node in vpc_node_list: host_file_params = f'{host_file_params}' + f'{node} slots={proc_per_node}\n' - cmd = 'rm -f /tmp/rccl_hosts_file.txt' + hosts_file_path = f'/tmp/rccl_hosts_file_{os.environ.get("USER", "cvs")}.txt' + cmd = f'rm -f {hosts_file_path}' shdl.exec(cmd) - cmd = f'echo "{host_file_params}" > /tmp/rccl_hosts_file.txt' + cmd = f'echo "{host_file_params}" > {hosts_file_path}' shdl.exec(cmd) # Determine PML (Point-to-Point Messaging Layer) based on user config or auto-detection @@ -887,7 +889,7 @@ def rccl_perf( # Build mpirun command cmd = f'''{mpi_dir}/bin/mpirun --np {no_of_global_ranks} \ --allow-run-as-root \ - --hostfile /tmp/rccl_hosts_file.txt \ + --hostfile {hosts_file_path} \ --bind-to numa \ {ucx_params} \ --mca btl ^vader,openib \ diff --git a/cvs/lib/verify_lib.py b/cvs/lib/verify_lib.py index 95074ac11..e49b3635e 100644 --- a/cvs/lib/verify_lib.py +++ b/cvs/lib/verify_lib.py @@ -17,12 +17,31 @@ 'crash': 'crashed|Traceback|cut here|Bug:|Call Trace|RIP:|end trace|amdgpu: Fatal error|segfault|show_stack|dump_stack|fault ', 'test_fail': 'Test failure', 'fault': 'no-retry page fault|Illegal register access|PROTECTION_FAULT_STATUS', - 'driver': 'Queue preemption failed for queue|Failed to evict process queues|Runlist is getting oversubscribed|No more SDMA queue to allocate|Expect reduced ROCm performance|amdgpu: process pid', + # Note: amdgpu oversubscription messages ('Runlist is getting oversubscribed', + # 'Expect reduced ROCm performance') are perf-degrading but not test failures — + # they're matched as warnings via warn_patterns_dict below. See AMD docs: + # https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/conceptual/oversubscription.html + 'driver': 'Queue preemption failed for queue|Failed to evict process queues|No more SDMA queue to allocate|amdgpu: process pid', 'hardware': 'hardware error|hardware fail|ras error|uncorrectable|correctable err', 'network': 'NIC Link is Down|link is down|ib_uverb|CQE|queue catastrophic|CQ error', } +# warn_patterns_dict captures kernel events that indicate the GPU entered a +# perf-degrading state but the workload itself still completes correctly. Matches +# emit a WARN to the run log (no fail_test) so reviewers know the perf numbers +# from this run were measured under a degraded HW state and should not be trusted +# blindly for regression comparisons. +warn_patterns_dict = { + # GPU runlist / VM context oversubscription. Per AMD docs, when this fires the + # HW scheduler is round-robining queues and inactive queues can block the GPU + # for millisecond-scale windows. Triggers: >24 user-mode compute queues per + # GPU, >11 VM context slots, or >1 process using cooperative workgroups. + # https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/conceptual/oversubscription.html + 'oversubscribed': 'Runlist is getting oversubscribed|Expect reduced ROCm performance', +} + + err_stats_pattern = 'err|drop|discard|overflow|fcs|nak|uncorrect|loss' warn_stats_pattern = 'retry|timeout|exceeded|ooo|retransmit' threshold_stats_pattern = 'cnp|ecn' @@ -174,7 +193,8 @@ def verify_gpu_pcie_errors(phdl): def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag=True): """ Scan kernel logs (dmesg) between given start and end timestamps across nodes - and fail if any known error patterns are detected. + and fail if any known error patterns are detected. Lines matching warn-only + patterns are logged via log.warning but do not fail the test. Parameters: phdl: pssh handle that can execute remote shell commands via .exec(cmd) -> dict. @@ -185,17 +205,19 @@ def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag= - Extracts a human-readable timestamp prefix (e.g., 'Mon Jan 2 03:04:05') from provided times. - Uses dmesg -T (human-readable timestamps) piped to awk to slice the log from start to end. - Filters out lines containing 'ALLOWED' or 'DENIED' (non-fatal/noisy) via egrep -v. - - Scans each line against a set of known error regex patterns (err_patterns_dict). - - Immediately fails the test via fail_test if any error pattern is seen. + - Scans each line against err_patterns_dict (fail_test on match) and warn_patterns_dict + (log.warning only on match; test still passes but run carries a visible WARN). Assumptions: - - err_patterns_dict is defined in scope: {name: regex_pattern, ...}. + - err_patterns_dict and warn_patterns_dict are defined in scope: {name: regex_pattern, ...}. - phdl.exec(cmd) returns a dict: { node: stdout_str }. - Input timestamps contain a prefix matching the regex used here. - sudo is available and does not prompt for a password when running dmesg. Notes: - This function fails fast on the first detected error to shorten feedback cycles. + - Warn-bucket matches are non-fatal but should be reviewed before trusting any + perf measurements from the run (e.g. RCCL bandwidth, training step time). - If start/end times are not aligned with dmesg -T formatting, the awk range may be empty. - Consider handling cases where regex extraction fails (no match) to avoid attribute errors. """ @@ -233,13 +255,23 @@ def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag= for node in output_dict.keys(): err_dict[node] = [] - # Iterate through each node's sliced dmesg and scan for known error patterns + # Iterate through each node's sliced dmesg and scan for known error/warn patterns. + # err patterns -> fail_test (existing behavior). + # warn patterns -> log.warning only; the test still passes but the run carries a + # visible WARN so callers / reviewers know the GPU was in a degraded state and + # any perf numbers from this run should not be trusted blindly. for node in output_dict.keys(): for line in output_dict[node].split("\n"): for err_key in err_patterns_dict.keys(): if re.search(f'{err_patterns_dict[err_key]}', line, re.I): fail_test(f'ERROR - Failue pattern ** {line} ** seen in Dmesg') err_dict[node].append(line) + for warn_key in warn_patterns_dict.keys(): + if re.search(f'{warn_patterns_dict[warn_key]}', line, re.I): + log.warning( + f'WARN - GPU in degraded state ({warn_key}) on {node}: {line.strip()} ' + f'-- perf numbers from this run may not be trustworthy' + ) return err_dict diff --git a/cvs/tests/rccl/rccl_perf.py b/cvs/tests/rccl/rccl_perf.py index a28e535d7..75f57b6b0 100644 --- a/cvs/tests/rccl/rccl_perf.py +++ b/cvs/tests/rccl/rccl_perf.py @@ -350,7 +350,11 @@ def test_rccl_perf(phdl, shdl, cluster_dict, config_dict, rccl_collective): end_time = phdl.exec('date +"%a %b %e %H:%M"') if can_use_sudo: - verify_dmesg_for_errors(phdl, start_time, end_time, till_end_flag=True) + # Bound dmesg scan to this test's own start..end window (per-test). + # till_end_flag=True scans from start_time to the end of the dmesg + # buffer, which causes earlier-test kernel events (e.g. a scatter_perf + # segfault) to repeatedly fail every subsequent parametrized test. + verify_dmesg_for_errors(phdl, start_time, end_time, till_end_flag=False) # Get new cluster snapshot and compare .. if can_use_sudo and re.search(