cvs/rccl: surface oversubscription as WARN, per-test dmesg window, per-user hostfile

cvs/lib/rccl_lib.py

@@ -643,10 +643,11 @@ def rccl_regression(
     for node in vpc_node_list:
         host_file_params = f'{host_file_params}{node} slots={proc_per_node}\n'
 
-    cmd = 'rm -f /tmp/rccl_hosts_file.txt'
+    hosts_file_path = f'/tmp/rccl_hosts_file_{os.environ.get("USER", "cvs")}.txt'
+    cmd = f'rm -f {hosts_file_path}'
     shdl.exec(cmd)
 
-    cmd = f'echo "{host_file_params}" > /tmp/rccl_hosts_file.txt'
+    cmd = f'echo "{host_file_params}" > {hosts_file_path}'
     shdl.exec(cmd)
 
     # Determine PML (Point-to-Point Messaging Layer) based on user config or auto-detection
@@ -700,7 +701,7 @@ def rccl_regression(
     cmd = f'''{mpi_dir}/bin/mpirun \
         --allow-run-as-root \
         -np {no_of_global_ranks} \
-        --hostfile /tmp/rccl_hosts_file.txt \
+        --hostfile {hosts_file_path} \
         --bind-to numa \
         {ucx_params} \
         --mca btl ^vader,openib \
@@ -826,10 +827,11 @@ def rccl_perf(
     for node in vpc_node_list:
         host_file_params = f'{host_file_params}' + f'{node} slots={proc_per_node}\n'
 
-    cmd = 'rm -f /tmp/rccl_hosts_file.txt'
+    hosts_file_path = f'/tmp/rccl_hosts_file_{os.environ.get("USER", "cvs")}.txt'
+    cmd = f'rm -f {hosts_file_path}'
     shdl.exec(cmd)
 
-    cmd = f'echo "{host_file_params}" > /tmp/rccl_hosts_file.txt'
+    cmd = f'echo "{host_file_params}" > {hosts_file_path}'
     shdl.exec(cmd)
 
     # Determine PML (Point-to-Point Messaging Layer) based on user config or auto-detection
@@ -887,7 +889,7 @@ def rccl_perf(
         # Build mpirun command
         cmd = f'''{mpi_dir}/bin/mpirun --np {no_of_global_ranks} \
         --allow-run-as-root \
-        --hostfile /tmp/rccl_hosts_file.txt \
+        --hostfile {hosts_file_path} \
         --bind-to numa \
         {ucx_params} \
         --mca btl ^vader,openib \

cvs/lib/verify_lib.py

@@ -17,12 +17,31 @@
     'crash': 'crashed|Traceback|cut here|Bug:|Call Trace|RIP:|end trace|amdgpu: Fatal error|segfault|show_stack|dump_stack|fault ',
     'test_fail': 'Test failure',
     'fault': 'no-retry page fault|Illegal register access|PROTECTION_FAULT_STATUS',
-    'driver': 'Queue preemption failed for queue|Failed to evict process queues|Runlist is getting oversubscribed|No more SDMA queue to allocate|Expect reduced ROCm performance|amdgpu: process pid',
+    # Note: amdgpu oversubscription messages ('Runlist is getting oversubscribed',
+    # 'Expect reduced ROCm performance') are perf-degrading but not test failures —
+    # they're matched as warnings via warn_patterns_dict below. See AMD docs:
+    # https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/conceptual/oversubscription.html
+    'driver': 'Queue preemption failed for queue|Failed to evict process queues|No more SDMA queue to allocate|amdgpu: process pid',
     'hardware': 'hardware error|hardware fail|ras error|uncorrectable|correctable err',
     'network': 'NIC Link is Down|link is down|ib_uverb|CQE|queue catastrophic|CQ error',
 }
 
 
+# warn_patterns_dict captures kernel events that indicate the GPU entered a
+# perf-degrading state but the workload itself still completes correctly. Matches
+# emit a WARN to the run log (no fail_test) so reviewers know the perf numbers
+# from this run were measured under a degraded HW state and should not be trusted
+# blindly for regression comparisons.
+warn_patterns_dict = {
+    # GPU runlist / VM context oversubscription. Per AMD docs, when this fires the
+    # HW scheduler is round-robining queues and inactive queues can block the GPU
+    # for millisecond-scale windows. Triggers: >24 user-mode compute queues per
+    # GPU, >11 VM context slots, or >1 process using cooperative workgroups.
+    # https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/conceptual/oversubscription.html
+    'oversubscribed': 'Runlist is getting oversubscribed|Expect reduced ROCm performance',
+}
+
+
 err_stats_pattern = 'err|drop|discard|overflow|fcs|nak|uncorrect|loss'
 warn_stats_pattern = 'retry|timeout|exceeded|ooo|retransmit'
 threshold_stats_pattern = 'cnp|ecn'
@@ -174,7 +193,8 @@ def verify_gpu_pcie_errors(phdl):
 def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag=True):
     """
     Scan kernel logs (dmesg) between given start and end timestamps across nodes
-    and fail if any known error patterns are detected.
+    and fail if any known error patterns are detected. Lines matching warn-only
+    patterns are logged via log.warning but do not fail the test.
 
     Parameters:
       phdl: pssh handle that can execute remote shell commands via .exec(cmd) -> dict.
@@ -185,17 +205,19 @@ def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag=
       - Extracts a human-readable timestamp prefix (e.g., 'Mon Jan  2 03:04:05') from provided times.
       - Uses dmesg -T (human-readable timestamps) piped to awk to slice the log from start to end.
       - Filters out lines containing 'ALLOWED' or 'DENIED' (non-fatal/noisy) via egrep -v.
-      - Scans each line against a set of known error regex patterns (err_patterns_dict).
-      - Immediately fails the test via fail_test if any error pattern is seen.
+      - Scans each line against err_patterns_dict (fail_test on match) and warn_patterns_dict
+        (log.warning only on match; test still passes but run carries a visible WARN).
 
     Assumptions:
-      - err_patterns_dict is defined in scope: {name: regex_pattern, ...}.
+      - err_patterns_dict and warn_patterns_dict are defined in scope: {name: regex_pattern, ...}.
       - phdl.exec(cmd) returns a dict: { node: stdout_str }.
       - Input timestamps contain a prefix matching the regex used here.
       - sudo is available and does not prompt for a password when running dmesg.
 
     Notes:
       - This function fails fast on the first detected error to shorten feedback cycles.
+      - Warn-bucket matches are non-fatal but should be reviewed before trusting any
+        perf measurements from the run (e.g. RCCL bandwidth, training step time).
       - If start/end times are not aligned with dmesg -T formatting, the awk range may be empty.
       - Consider handling cases where regex extraction fails (no match) to avoid attribute errors.
     """
@@ -233,13 +255,23 @@ def verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict, till_end_flag=
     for node in output_dict.keys():
         err_dict[node] = []
 
-    # Iterate through each node's sliced dmesg and scan for known error patterns
+    # Iterate through each node's sliced dmesg and scan for known error/warn patterns.
+    # err patterns -> fail_test (existing behavior).
+    # warn patterns -> log.warning only; the test still passes but the run carries a
+    # visible WARN so callers / reviewers know the GPU was in a degraded state and
+    # any perf numbers from this run should not be trusted blindly.
     for node in output_dict.keys():
         for line in output_dict[node].split("\n"):
             for err_key in err_patterns_dict.keys():
                 if re.search(f'{err_patterns_dict[err_key]}', line, re.I):
                     fail_test(f'ERROR - Failue pattern ** {line} ** seen in Dmesg')
                     err_dict[node].append(line)
+            for warn_key in warn_patterns_dict.keys():
+                if re.search(f'{warn_patterns_dict[warn_key]}', line, re.I):
+                    log.warning(
+                        f'WARN - GPU in degraded state ({warn_key}) on {node}: {line.strip()} '
+                        f'-- perf numbers from this run may not be trustworthy'
+                    )
 
     return err_dict
 

cvs/tests/rccl/rccl_perf.py

@@ -350,7 +350,11 @@ def test_rccl_perf(phdl, shdl, cluster_dict, config_dict, rccl_collective):
 
     end_time = phdl.exec('date +"%a %b %e %H:%M"')
     if can_use_sudo:
-        verify_dmesg_for_errors(phdl, start_time, end_time, till_end_flag=True)
+        # Bound dmesg scan to this test's own start..end window (per-test).
+        # till_end_flag=True scans from start_time to the end of the dmesg
+        # buffer, which causes earlier-test kernel events (e.g. a scatter_perf
+        # segfault) to repeatedly fail every subsequent parametrized test.
+        verify_dmesg_for_errors(phdl, start_time, end_time, till_end_flag=False)
 
     # Get new cluster snapshot and compare ..
     if can_use_sudo and re.search(