diff --git a/cvs/cli_plugins/exec_plugin.py b/cvs/cli_plugins/exec_plugin.py index 6e9b0367d..2466b61fb 100644 --- a/cvs/cli_plugins/exec_plugin.py +++ b/cvs/cli_plugins/exec_plugin.py @@ -62,8 +62,9 @@ def run(self, args): # Create Pssh instance (Pssh requires a logger; it calls log.debug/info/warning) log = logging.getLogger(__name__) + env_vars = cluster.get('env_vars') try: - pssh = Pssh(log=log, host_list=hosts, user=username, pkey=pkey, stop_on_errors=False) + pssh = Pssh(log=log, host_list=hosts, user=username, pkey=pkey, stop_on_errors=False, env_vars=env_vars) except Exception as e: print(f"Error initializing Pssh: {e}") sys.exit(1) diff --git a/cvs/input/config_file/preflight/README_preflight_config.md b/cvs/input/config_file/preflight/README_preflight_config.md new file mode 100644 index 000000000..d6268cd7f --- /dev/null +++ b/cvs/input/config_file/preflight/README_preflight_config.md @@ -0,0 +1,363 @@ +# Preflight Configuration Guide + +This document explains how to configure the GPU cluster preflight checks system. + +## Overview + +The preflight checks system validates essential cluster health before running performance tests like IB performance tests, RCCL training, and inference workloads. It performs four key validations: + +1. **GID Consistency** - Ensures RDMA interfaces have valid GID entries +2. **RDMA Connectivity** - Tests node-to-node RDMA communication using ibv_rc_pingpong +3. **ROCm Version Consistency** - Verifies consistent ROCm versions across nodes +4. **Interface Name Consistency** - Validates RDMA interface naming patterns + +## Configuration File Structure + +The preflight configuration file follows this structure: + +```json +{ + "preflight": { + "debug": { + "scriptlet": false + }, + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.2.0", + "rdma_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + }, + "connectivity_check": { + "rdma": { + "connectivity_mode": "basic", + "nodes_per_full_mesh_group": 128, + "ibv_test_timeout": 90, + "ibv_test_port_range": "10000-50000", + "inter_full_mesh_group_pairs_per_wave": "auto" + } + }, + "reporting": { + "generate_html_report": "true", + "artifacts_root_dir": "/tmp/{user-id}/preflight", + "generate_rdma_pairs_csv": "true" + } + } +} +``` + +## Configuration Structure + +The preflight configuration uses a **nested structure organized by execution phase** for better organization and future extensibility: + +### Structure Overview + +``` +preflight/ +├── debug/ # Debug and troubleshooting options +├── node_check/ # Individual node validation parameters +├── connectivity_check/ # Inter-node connectivity tests +│ └── rdma/ # RDMA-specific parameters (including nodes_per_full_mesh_group) +│ └── ifoe/ # (Future: IFoE parameters) +└── reporting/ # Output and report generation +``` + +### Execution Flow +1. **node_check** - Validate individual nodes in parallel +2. **connectivity_check.rdma.nodes_per_full_mesh_group** - Configure RDMA batching resources +3. **connectivity_check** - Test inter-node connectivity by protocol +4. **reporting** - Generate reports and outputs + +## Configuration Parameters + +### Complete Parameter Reference + +All parameters below are optional and have sensible defaults. The sample configuration file includes all available parameters with their default values and inline comments explaining their purpose. + +### Important Update: RDMA Connectivity Testing + +**As of this version, preflight checks now use `ibv_rc_pingpong` instead of `rping` for RDMA connectivity testing.** + +**Why this change?** +- `rping` uses RDMA Connection Manager (`rdma_cm`) which is more forgiving of network issues +- `ibv_rc_pingpong` uses direct InfiniBand verbs (same as RCCL) which is more strict +- This change allows preflight checks to detect the same connectivity issues that cause RCCL failures +- **Result**: Better correlation between preflight results and actual RCCL performance + +**Updated parameter names**: Configuration parameters now use accurate names (`ibv_test_timeout`, `ibv_test_port_range`) that reflect the use of `ibv_rc_pingpong` for testing. + +### RDMA Batching (`connectivity_check.rdma`) + +- **`nodes_per_full_mesh_group`** (default: 128) + - Group size for parallel RDMA connectivity testing (2-512 nodes per group) + - Smaller groups use fewer resources per node but require more rounds + - Adjust based on cluster size and resource constraints + +### Debug Settings (`debug`) + +- **`scriptlet`** (default: false) + - Enable ScriptLet debug mode: preserve generated scripts/logs on remote nodes + - For RDMA connectivity, wraps each ibv_rc_pingpong server in strace + - Creates per-test traces under /tmp/preflight/strace_server__.log + - **Warning**: Can be expensive at scale due to strace overhead + +### Node Check Settings (`node_check`) + +- **`gid_index`** (default: "3") + - GID index to check on all RDMA interfaces + - Typically "3" for RoCE (RDMA over Converged Ethernet) + - Must be a valid GID index for your InfiniBand/RoCE setup + +- **`expected_rocm_version`** (default: "6.2.0") + - Expected ROCm version across all cluster nodes + - Must match the output of `amd-smi version` on all nodes + - Format: "major.minor.patch" (e.g., "6.2.0", "5.7.1") + +- **`rdma_interfaces`** (default: ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"]) + - List of specific RDMA interface names that should be present on all cluster nodes + - Examples: + - `["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"]` - Standard 4-interface setup + - `["mlx5_0", "mlx5_1"]` - Mellanox 2-interface setup + - `["ib0", "ib1", "ib2", "ib3"]` - Generic InfiniBand setup + +### Connectivity Check Settings (`connectivity_check`) + +#### RDMA Settings (`connectivity_check.rdma`) + +- **`connectivity_mode`** (default: "basic") + - **"basic"**: Test adjacent node pairs (fast, ~14% coverage for 8 nodes) + - **"full_mesh"**: Test all possible node pairs (comprehensive, 100% coverage) + - **"skip"**: Skip RDMA connectivity testing entirely + +- **`ibv_test_timeout`** (default: 90) + - Timeout in seconds for each ibv_rc_pingpong connectivity test + - Integer value (seconds), used directly as configured + - Uses `ibv_rc_pingpong` (direct InfiniBand verbs) for RCCL-compatible testing + - Increase for slower networks or high-latency connections + +- **`ibv_test_port_range`** (default: "10000-50000") + - Port range for ibv_rc_pingpong tests to avoid conflicts + - Format: "start-end" (e.g., "10000-50000", "10000-10999") + - Ensure ports are not blocked by firewalls + +- **`inter_full_mesh_group_pairs_per_wave`** (default: "auto") + - Max ordered group-pairs (Gi→Gj keys) per wave during inter-group RDMA testing + - "auto" calculates as max(1, num_groups - 1) + - Can be set to a specific integer to control wave size and reduce memory/CPU load + +- **`prune_failure_threshold`** (default: 0.5) + - Prune nodes whose fraction of peers with ≥1 FAIL intra test is ≥ this value + - Range: 0.0 to 1.0 (0.5 = 50% failure threshold) + - Helps remove problematic nodes before inter-group testing + - Lower values (0.2-0.3) are more aggressive at removing problematic nodes + +- **`port_retry_max`** (default: 3) + - Max retry attempts for pairs whose logs show PORT_LISTEN_FAILED + - Range: 0-10 retries with new TCP ports after each wave + - Helps handle port conflicts during large-scale testing + +- **`port_retry_gap`** (default: 1000) + - Port gap when remapping ports for PORT_LISTEN_FAILED retries + - Range: 1-65535 + - Starts at (max port in batch) + this gap to reduce overlap with ephemeral ports + +- **`exclude_failed_interface_nodes`** (default: "true") + - Legacy hint for reporting: preflight now prunes interface/GID-failed nodes automatically + - Interface failures are excluded from mesh testing regardless of this flag + +### Reporting Settings (`reporting`) + +- **`generate_html_report`** (default: "true") + - Whether to generate detailed HTML report + - Set to "false" to disable HTML report generation + +- **`artifacts_root_dir`** (default: "/tmp/{user-id}/preflight") + - Root directory where preflight artifacts are saved + - Includes HTML reports and RDMA full_mesh workspace logs under `rdma_connectivity_workspace/` + - Must be writable by the user running the tests + +- **`generate_rdma_pairs_csv`** (default: "true") + - Whether to generate CSV file with failed RDMA pairs alongside HTML report + - Set to "false" to disable CSV generation + +## Usage Examples + +### Basic 8-Node Cluster Check + +```json +{ + "preflight": { + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.2.0", + "rdma_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + }, + "connectivity_check": { + "rdma": { + "connectivity_mode": "basic", + "ibv_test_timeout": 90, + "ibv_test_port_range": "10000-50000" + } + } + } +} +``` + +### Comprehensive Full-Mesh Testing + +```json +{ + "preflight": { + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.2.0", + "rdma_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + }, + "connectivity_check": { + "rdma": { + "connectivity_mode": "full_mesh", + "ibv_test_timeout": 120, + "ibv_test_port_range": "10000-50000" + } + } + } +} +``` + +### Configuration-Only Validation (Skip Connectivity) + +```json +{ + "preflight": { + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.2.0", + "rdma_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + }, + "connectivity_check": { + "rdma": { + "connectivity_mode": "skip" + } + } + } +} +``` + +### Advanced Configuration with Debug and Tuning + +```json +{ + "preflight": { + "debug": { + "scriptlet": true + }, + "node_check": { + "gid_index": "3", + "expected_rocm_version": "7.2.0", + "rdma_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0", "rocep158s0", "rocep190s0", "rocep206s0", "rocep222s0"] + }, + "connectivity_check": { + "rdma": { + "connectivity_mode": "full_mesh", + "nodes_per_full_mesh_group": 32, + "ibv_test_timeout": 180, + "ibv_test_port_range": "15000-20000", + "inter_full_mesh_group_pairs_per_wave": 4, + "prune_failure_threshold": 0.3, + "port_retry_max": 5, + "port_retry_gap": 2000 + } + }, + "reporting": { + "generate_html_report": "true", + "artifacts_root_dir": "/tmp/{user-id}/preflight", + "generate_rdma_pairs_csv": "true" + } + } +} +``` + +## Running Preflight Checks + +```bash +# Basic usage with default config +cvs run preflight_checks --cluster_file cluster.json --config_file preflight_config.json + +# With custom HTML output +cvs run preflight_checks \ + --cluster_file cluster.json \ + --config_file preflight_config.json \ + --html /path/to/preflight_report.html \ + --self-contained-html +``` + +## Troubleshooting + +### Common Issues + +1. **GID Check Failures** + - Ensure RDMA drivers are loaded: `lsmod | grep rdma` + - Check interface status: `rdma link show` + - Verify GID entries: `cat /sys/class/infiniband/*/ports/1/gids/3` + +2. **RDMA Connectivity Failures** + - Check firewall settings: `sudo ufw status` + - Verify ibv_rc_pingpong is available: `which ibv_rc_pingpong` + - Enable debug mode: set `"scriptlet": true` for detailed logs + - Check port conflicts in the specified `ibv_test_port_range` + - Test manual connectivity: `ibv_rc_pingpong -d -g ` + +3. **ROCm Version Mismatches** + - Check ROCm installation: `amd-smi version` + - Verify consistent installation across nodes + - Update expected_rocm_version in config + +4. **Missing RDMA Interfaces** + - List interfaces: `ls /sys/class/infiniband/` + - Update rdma_interfaces list to match your cluster setup + - Ensure all expected interfaces are present on each node + +### Performance Considerations + +**RDMA Connectivity Testing Times:** +- **Basic mode**: ~30 seconds for 8 nodes +- **Full mesh mode**: ~5-10 minutes for 8 nodes +- **Skip mode**: fastest path when validating only node-local checks + +**Parallel Processing Impact:** +- **Small nodes_per_full_mesh_group (16-32)**: More rounds, less resource usage per node, better for resource-constrained environments +- **Large nodes_per_full_mesh_group (128+)**: Fewer rounds, more resource usage per node, faster overall completion +- **Debug mode impact**: `scriptlet=true` adds 10-30% overhead due to strace logging + +**Advanced Parameter Tuning:** +- **inter_full_mesh_group_pairs_per_wave**: Lower values (2-4) reduce memory/CPU load but increase test time +- **prune_failure_threshold**: Lower values (0.2-0.3) are more aggressive at removing problematic nodes +- **Port retry settings**: Higher retry counts help in congested network environments but increase test time + +### Network Requirements + +- **Ports**: Ensure ibv_test_port_range is not blocked by firewalls +- **RDMA**: InfiniBand or RoCE interfaces must be active +- **SSH**: Passwordless SSH access to all cluster nodes +- **Privileges**: Some checks may require sudo access for system information + +## Integration with Performance Tests + +The preflight checks are designed to run before: + +1. **IB Performance Tests** (`ib_perf_bw_test`) +2. **RCCL Training Tests** (`rccl_multinode_*`) +3. **Inference Workloads** (PyTorch, JAX, etc.) + +A typical workflow: + +```bash +# 1. Run preflight checks +cvs run preflight_checks --cluster_file cluster.json --config_file preflight_config.json + +# 2. If preflight passes, run performance tests +cvs run ib_perf_bw_test --cluster_file cluster.json --config_file ib_config.json + +# 3. Run RCCL training tests +cvs run rccl_multinode_default_cvs --cluster_file cluster.json --config_file rccl_config.json +``` + +This ensures your cluster is healthy before running resource-intensive performance tests. \ No newline at end of file diff --git a/cvs/input/config_file/preflight/preflight_config.json b/cvs/input/config_file/preflight/preflight_config.json new file mode 100644 index 000000000..d40ae95fa --- /dev/null +++ b/cvs/input/config_file/preflight/preflight_config.json @@ -0,0 +1,74 @@ +{ + "preflight": { + "node_check": { + "_comment": "Individual node validation (parallel across nodes)", + + "_example_gid_index": "3", + "gid_index": "", + "_comment_gid_index": "GID index to check on all RDMA interfaces. Typically '3' for RoCE (RDMA over Converged Ethernet). Must be a valid GID index for your InfiniBand/RoCE setup.", + + "_example_expected_rocm_version": "6.2.0", + "expected_rocm_version": "", + "_comment_expected_rocm_version": "Expected ROCm version across all cluster nodes. Must match the output of 'amd-smi version' on all nodes. Format: 'major.minor.patch' (e.g., '6.2.0', '5.7.1').", + + "_example_rdma_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0", "rocep158s0", "rocep190s0", "rocep206s0", "rocep222s0"], + "rdma_interfaces": "[]", + "_comment_rdma_interfaces": "List of specific RDMA interface names that should be present on all cluster nodes. Examples: ['rocep28s0', 'rocep62s0'] for standard setup, ['mlx5_0', 'mlx5_1'] for Mellanox, ['ib0', 'ib1'] for generic InfiniBand." + }, + + "connectivity_check": { + "_comment": "Inter-node connectivity tests by protocol", + + "rdma": { + "_comment": "RDMA connectivity testing parameters using ibv_rc_pingpong (direct IB verbs) for RCCL-compatible testing", + + "connectivity_mode": "basic", + "_comment_connectivity_mode": "RDMA connectivity testing mode. Options: 'basic' (test adjacent node pairs, fast, ~14% coverage for 8 nodes), 'full_mesh' (test all possible node pairs, comprehensive, 100% coverage), 'skip' (skip RDMA connectivity testing entirely).", + + "nodes_per_full_mesh_group": 32, + "_comment_nodes_per_full_mesh_group": "Number of nodes in each full-mesh partition group for parallel RDMA testing (2-128). Smaller groups use fewer resources per node but require more rounds. Adjust based on cluster size and resource constraints.", + + "ibv_test_timeout": 90, + "_comment_ibv_test_timeout": "Timeout in seconds for each ibv_rc_pingpong connectivity test. Uses ibv_rc_pingpong (direct IB verbs) instead of rping for RCCL-compatible testing. Increase for slower networks or high-latency connections.", + + "ibv_test_port_range": "10000-50000", + "_comment_ibv_test_port_range": "Port range for ibv_rc_pingpong tests to avoid conflicts. Format: 'start-end' (e.g., '10000-50000', '10000-10999'). Ensure ports are not blocked by firewalls.", + + "inter_full_mesh_group_pairs_per_wave": "auto", + "_comment_inter_full_mesh_group_pairs_per_wave": "Max ordered full-mesh group-pairs (Gi→Gj keys) per wave during inter-group RDMA testing. 'auto' calculates as max(1, num_groups - 1). Can be set to a specific integer to control wave size and reduce memory/CPU load.", + + "prune_failure_threshold": 0.5, + "_comment_prune_failure_threshold": "Prune nodes whose fraction of peers with ≥1 FAIL intra test is ≥ this value. Range: 0.0 to 1.0 (0.5 = 50% failure threshold). Helps remove problematic nodes before inter-group testing. Lower values (0.2-0.3) are more aggressive.", + + "port_retry_max": 3, + "_comment_port_retry_max": "Max retry attempts for pairs whose logs show PORT_LISTEN_FAILED. Range: 0-10 retries with new TCP ports after each wave. Helps handle port conflicts during large-scale testing.", + + "port_retry_gap": 1000, + "_comment_port_retry_gap": "Port gap when remapping ports for PORT_LISTEN_FAILED retries. Range: 1-65535. Starts at (max port in batch) + this gap to reduce overlap with ephemeral ports.", + + "exclude_failed_interface_nodes": "true", + "_comment_exclude_failed_interface_nodes": "Legacy hint for reporting: preflight now prunes interface/GID-failed nodes from the SSH host list automatically. Interface failures are excluded from mesh testing regardless of this flag." + } + }, + + "reporting": { + "_comment": "Post-test reporting and output", + + "generate_html_report": "true", + "_comment_generate_html_report": "Whether to generate detailed HTML report. Set to 'false' to disable HTML report generation.", + + "artifacts_root_dir": "/tmp/{user-id}/preflight", + "_comment_artifacts_root_dir": "Root directory for preflight artifacts. HTML reports are saved here, and RDMA full_mesh ScriptLet logs use /rdma_connectivity_workspace/// on each node (NFS-friendly). Must be writable by the user running the tests.", + + "generate_rdma_pairs_csv": "true", + "_comment_generate_rdma_pairs_csv": "Whether to generate CSV file with failed RDMA pairs alongside HTML report. Set to 'false' to disable CSV generation. Useful for analyzing connectivity patterns and failures." + }, + + "debug": { + "_comment": "Debug and troubleshooting options", + + "scriptlet": false, + "_comment_scriptlet": "Enable ScriptLet debug mode: preserve generated scripts/logs on remote nodes. For RDMA connectivity, wraps each ibv_rc_pingpong server in strace with per-test traces under /tmp/preflight/strace_server__.log. WARNING: Can be expensive at scale due to strace overhead." + } + } +} \ No newline at end of file diff --git a/cvs/lib/linux_utils.py b/cvs/lib/linux_utils.py index 45f2348d9..9d01dcf8c 100644 --- a/cvs/lib/linux_utils.py +++ b/cvs/lib/linux_utils.py @@ -226,7 +226,7 @@ def get_rdma_nic_dict(phdl): rdma_dict = {} # Run 'rdma link' across node(s); expect a dict: { node_name: "", ... } - out_dict = phdl.exec('sudo rdma link') + out_dict = phdl.exec('rdma link') # gid_dict_t = phdl.exec('sudo show_gids | grep -i v2 --color=never') for node in out_dict.keys(): rdma_dict[node] = {} @@ -281,7 +281,7 @@ def get_active_rdma_nic_dict(phdl): """ rdma_dict = {} - out_dict = phdl.exec('sudo rdma link') + out_dict = phdl.exec('rdma link') # gid_dict_t = phdl.exec('sudo show_gids | grep -i v2 --color=never') for node in out_dict.keys(): rdma_dict[node] = {} diff --git a/cvs/lib/parallel/interfaces.py b/cvs/lib/parallel/interfaces.py index f98c7e707..21dccd2cb 100644 --- a/cvs/lib/parallel/interfaces.py +++ b/cvs/lib/parallel/interfaces.py @@ -31,6 +31,11 @@ def upload_file(self, local_file, remote_file, recurse=False): """Upload file via SFTP - must support sharding for performance.""" pass + @abstractmethod + def upload_file_list(self, node_path_map): + """Upload different files to different hosts - must support sharding for performance.""" + pass + @abstractmethod def download_file(self, remote_file, local_file, recurse=False, suffix_separator='_'): """Download file via SFTP - must support sharding for performance.""" diff --git a/cvs/lib/parallel/multiprocess_pssh.py b/cvs/lib/parallel/multiprocess_pssh.py index 4d9ce9fb4..74af8d876 100644 --- a/cvs/lib/parallel/multiprocess_pssh.py +++ b/cvs/lib/parallel/multiprocess_pssh.py @@ -37,6 +37,7 @@ def __init__( stop_on_errors=True, env_vars=None, config=None, + **ssh_client_kwargs, ): # Initialize configuration self.config = config or ParallelConfig.from_env() @@ -45,14 +46,19 @@ def __init__( # Always ensure self.log is set for backward compatibility self.log = global_log + # Store ssh_client_kwargs for forwarding to shard workers + self.ssh_client_kwargs = ssh_client_kwargs + n = len(host_list) if host_list is not None else 0 use_mp = hosts_per_shard > 0 and n > hosts_per_shard if use_mp: # Initialize for multi-process sharding - no Pssh instance needed - self._init_sharded(log, host_list, user, password, pkey, host_key_check, stop_on_errors, env_vars) + self._init_sharded( + log, host_list, user, password, pkey, host_key_check, stop_on_errors, env_vars, **ssh_client_kwargs + ) + self.sharder = PsshSharder(self.config) - self._use_process_sharding = True self.pssh = None # No Pssh instance needed for sharded mode else: # No sharding - create Pssh instance for delegation @@ -66,9 +72,12 @@ def __init__( stop_on_errors, env_vars, process_output=True, # Default to True for compatibility + **ssh_client_kwargs, ) - self._use_process_sharding = False # Ensure attributes needed by _shard_init_kwargs are available + self.host_list = host_list + self.reachable_hosts = list(host_list) # Initialize with all hosts + self.unreachable_hosts = [] self.env_vars = env_vars def _init_sharded( @@ -81,6 +90,7 @@ def _init_sharded( host_key_check, stop_on_errors, env_vars, + **ssh_client_kwargs, ): """Initialize for sharded multi-process execution.""" # Always use global logger but maintain self.log for backward compatibility @@ -94,10 +104,10 @@ def _init_sharded( self.stop_on_errors = stop_on_errors self.unreachable_hosts = [] self.env_vars = env_vars + self.ssh_client_kwargs = ssh_client_kwargs self.env_prefix = build_env_prefix(env_vars) self.process_output = True # Default to True for compatibility self.client = None - self._use_process_sharding = True self.log.debug(f"Environ vars: {self.env_prefix}") def _shard_init_kwargs(self): @@ -113,8 +123,16 @@ def _shard_init_kwargs(self): 'host_key_check': self.host_key_check, 'stop_on_errors': self.stop_on_errors, 'env_vars': self.env_vars, + **self.ssh_client_kwargs, # Forward all ssh client kwargs to shard workers } + def _sync_pssh_state(self): + """Sync wrapper reachability state from Single-process Pssh (non-sharded mode).""" + if self.pssh is None: + return + self.reachable_hosts = list(self.pssh.reachable_hosts) + self.unreachable_hosts = list(self.pssh.unreachable_hosts) + def _merge_shard_returns(self, shard_returns, merge_unreachable=True): """ Update reachable/unreachable host lists from shard workers and merge results to cmd_output dict. @@ -147,11 +165,11 @@ def _merge_shard_returns(self, shard_returns, merge_unreachable=True): return cmd_output - def _print_merged_outputs(self, cmd_output, cmd=None, cmd_list=None, print_console=True): + def _print_merged_outputs(self, cmd_output, cmd=None, cmd_list=None, cmd_hosts=None, print_console=True): """Print command outputs in original host order.""" if not print_console: return - idx = {h: i for i, h in enumerate(self.host_list)} + idx = {h: i for i, h in enumerate(cmd_hosts)} if cmd_hosts is not None else {} for host in self.host_list: if host not in cmd_output: continue @@ -167,8 +185,10 @@ def _print_merged_outputs(self, cmd_output, cmd=None, cmd_list=None, print_conso def exec(self, cmd, timeout=None, print_console=True, detailed=False): """Execute command with automatic sharding if needed.""" - if not self._use_process_sharding: - return self.pssh.exec(cmd, timeout=timeout, print_console=print_console, detailed=detailed) + if self.pssh is not None: + result = self.pssh.exec(cmd, timeout=timeout, print_console=print_console, detailed=detailed) + self._sync_pssh_state() + return result if self.env_prefix: full_cmd = f"{self.env_prefix} ; {cmd}" @@ -207,28 +227,35 @@ def exec(self, cmd, timeout=None, print_console=True, detailed=False): def exec_cmd_list(self, cmd_list, timeout=None, print_console=True): """Execute command list with automatic sharding if needed.""" - if not self._use_process_sharding: - return self.pssh.exec_cmd_list(cmd_list, timeout=timeout, print_console=print_console) - - if len(cmd_list) != len(self.host_list): - raise ValueError('cmd_list length must match host_list length') + if self.pssh is not None: + result = self.pssh.exec_cmd_list(cmd_list, timeout=timeout, print_console=print_console) + self._sync_pssh_state() + return result + + # Keep host_list as full inventory; accept command lists aligned either + # with full host_list or current reachable_hosts. + if len(cmd_list) == len(self.host_list): + cmd_hosts = list(self.host_list) + elif len(cmd_list) == len(self.reachable_hosts): + cmd_hosts = list(self.reachable_hosts) + else: + raise ValueError( + "cmd_list length must match host_list length or reachable_hosts length " + f"(cmd_list={len(cmd_list)}, host_list={len(self.host_list)}, " + f"reachable_hosts={len(self.reachable_hosts)})" + ) - # Apply env_prefix to all commands (for logging compatibility) + # Keep raw commands for shard payloads (worker Pssh applies env_prefix once). + # Build expanded commands separately for logging compatibility. + raw_commands = list(cmd_list) if self.env_prefix: - expanded = [f"{self.env_prefix} ; {cmd}" for cmd in cmd_list] + expanded = [f"{self.env_prefix} ; {cmd}" for cmd in raw_commands] else: - expanded = list(cmd_list) - - # Only filter commands if some hosts are unreachable - if len(self.reachable_hosts) < len(self.host_list): - # Filter commands to match reachable_hosts order - filtered_commands = [] - for i, host in enumerate(self.host_list): - if host in self.reachable_hosts: - filtered_commands.append(expanded[i]) - else: - # All hosts are reachable, use expanded directly - filtered_commands = expanded + expanded = list(raw_commands) + + # Align commands to current reachable_hosts ordering for shard payloads. + command_by_host = dict(zip(cmd_hosts, raw_commands)) + filtered_commands = [command_by_host[h] for h in self.reachable_hosts if h in command_by_host] self.log.info("%s", filtered_commands) @@ -259,11 +286,13 @@ def exec_cmd_list(self, cmd_list, timeout=None, print_console=True): # Merge results, update state, and convert to cmd_output dict cmd_output = self._merge_shard_returns(shard_returns) - self._print_merged_outputs(cmd_output, cmd=None, cmd_list=expanded, print_console=print_console) + self._print_merged_outputs( + cmd_output, cmd=None, cmd_list=expanded, cmd_hosts=cmd_hosts, print_console=print_console + ) # Log per-host command execution if self.log: - for host, cmd in zip(self.host_list, expanded): + for host, cmd in zip(cmd_hosts, expanded): self.log.debug(f"Command on {host}: {cmd}") return cmd_output @@ -281,7 +310,7 @@ def scp_file(self, local_file, remote_file, recurse=False): def upload_file(self, local_file, remote_file, recurse=False): """Upload file with automatic sharding if needed.""" - if not self._use_process_sharding: + if self.pssh is not None: return self.pssh.upload_file(local_file, remote_file, recurse=recurse) self.log.info('SFTP upload %s -> %s on %s', local_file, remote_file, self.reachable_hosts) @@ -300,7 +329,7 @@ def upload_file(self, local_file, remote_file, recurse=False): def download_file(self, remote_file, local_file, recurse=False, suffix_separator='_'): """Download file with automatic sharding if needed.""" - if not self._use_process_sharding: + if self.pssh is not None: return self.pssh.download_file(remote_file, local_file, recurse=recurse, suffix_separator=suffix_separator) self.log.info('SFTP download %s -> %s from %s', remote_file, local_file, self.reachable_hosts) @@ -320,7 +349,7 @@ def download_file(self, remote_file, local_file, recurse=False, suffix_separator def reboot_connections(self): """Reboot connections with automatic sharding if needed.""" - if not self._use_process_sharding: + if self.pssh is not None: return self.pssh.reboot_connections() self.log.info('Rebooting Connections') @@ -330,9 +359,83 @@ def reboot_connections(self): shard_returns = self.sharder.execute_sharded(payloads) return self._merge_shard_returns(shard_returns, merge_unreachable=False) + def upload_file_list(self, node_path_map): + """Upload different files to different hosts with automatic sharding if needed.""" + if self.pssh is not None: + return self.pssh.upload_file_list(node_path_map) + + if not node_path_map: + return {} + + self.log.info(f"Uploading files to hosts with sharding: {len(node_path_map)} file mappings") + + # Shard the file operations across hosts + host_chunks = list(self.sharder.chunk_hosts(self.reachable_hosts)) + payloads = [] + + for shard_hosts in host_chunks: + # Create subset of node_path_map for this shard + shard_node_path_map = {h: node_path_map[h] for h in shard_hosts if h in node_path_map} + if shard_node_path_map: + # Worker host_list must match node_path_map keys for upload_file_list. + # Pssh.upload_file_list delegates to ParallelSSHClient.copy_file(copy_args=...), + # which requires one copy_args entry per worker host in order. + target_hosts = [h for h in shard_hosts if h in shard_node_path_map] + payloads.append( + self.sharder.create_payloads( + 'upload_file_list', + [target_hosts], + self._shard_init_kwargs(), + node_path_map=shard_node_path_map, + )[0] + ) + + if not payloads: + return {} + + shard_returns = self.sharder.execute_sharded(payloads) + + # Merge results from all shards + merged_results = {} + for shard in shard_returns: + result = shard.get('result', {}) + merged_results.update(result) + + return merged_results + + def prune_nodes(self, nodes_to_remove): + """ + Explicitly prune nodes from future operations. + + Works in both modes: + - sharded mode: updates wrapper host state + - non-sharded mode: delegates to Single-process Pssh (which rebuilds its client) + """ + if not nodes_to_remove: + return [] + + remove_set = {h for h in nodes_to_remove if h} + removed = [h for h in self.reachable_hosts if h in remove_set] + if not removed: + return [] + + if self.pssh is not None: + # Non-sharded mode: let single-process Pssh own prune + client rebuild. + removed = self.pssh.prune_nodes(removed) + self._sync_pssh_state() + return removed + + # Sharded mode: workers create per-call Pssh instances from payload host lists, so + # pruning only needs to update wrapper host state used for future shard construction. + self.reachable_hosts = [h for h in self.reachable_hosts if h not in remove_set] + for host in removed: + if host not in self.unreachable_hosts: + self.unreachable_hosts.append(host) + return removed + def destroy_clients(self): """Destroy clients - handle both sharded and non-sharded modes.""" - if self._use_process_sharding: + if self.pssh is None: self.log.info('Cleaning up sharded mode state ..') # In sharded mode, no persistent connections to destroy self.client = None diff --git a/cvs/lib/parallel/pssh.py b/cvs/lib/parallel/pssh.py index 26a582037..a93713847 100644 --- a/cvs/lib/parallel/pssh.py +++ b/cvs/lib/parallel/pssh.py @@ -35,6 +35,7 @@ def __init__( stop_on_errors=True, env_vars=None, process_output=True, + **ssh_client_kwargs, ): # Backward compatibility warning for log parameter if log is not None: @@ -57,6 +58,7 @@ def __init__( self.stop_on_errors = stop_on_errors self.process_output = process_output self.unreachable_hosts = [] + self.ssh_client_kwargs = ssh_client_kwargs self.env_prefix = build_env_prefix(env_vars) self.log.debug(f"Environ vars: {self.env_prefix}") @@ -64,10 +66,16 @@ def __init__( self.log.info("%s", self.reachable_hosts) self.log.info("%s", self.user) self.log.info("%s", self.pkey) - self.client = ParallelSSHClient(self.reachable_hosts, user=self.user, pkey=self.pkey, keepalive_seconds=30) + self.client = ParallelSSHClient( + self.reachable_hosts, user=self.user, pkey=self.pkey, keepalive_seconds=30, **self.ssh_client_kwargs + ) else: self.client = ParallelSSHClient( - self.reachable_hosts, user=self.user, password=self.password, keepalive_seconds=30 + self.reachable_hosts, + user=self.user, + password=self.password, + keepalive_seconds=30, + **self.ssh_client_kwargs, ) def check_connectivity(self, hosts): @@ -77,18 +85,60 @@ def check_connectivity(self, hosts): """ if not hosts: return [] + temp_ssh_client_kwargs = self.ssh_client_kwargs.copy() + temp_ssh_client_kwargs['timeout'] = 2 + temp_ssh_client_kwargs['num_retries'] = 0 temp_client = ParallelSSHClient( hosts, user=self.user, pkey=self.pkey if self.password is None else None, password=self.password, - num_retries=0, - timeout=2, + **temp_ssh_client_kwargs, ) output = temp_client.run_command('echo 1', stop_on_errors=False, read_timeout=2) unreachable = [item.host for item in output if item.exception] return unreachable + def prune_nodes(self, nodes_to_remove): + """ + Explicitly prune hosts from this Pssh instance and rebuild client. + + Args: + nodes_to_remove: Iterable of hostnames/IPs to remove. + + Returns: + list: Hosts actually removed from reachable_hosts. + """ + if not nodes_to_remove: + return [] + + remove_set = {h for h in nodes_to_remove if h} + removed = [h for h in self.reachable_hosts if h in remove_set] + if not removed: + return [] + + for host in removed: + self.log.warning(f"Host {host} is unreachable, pruning from reachable hosts list.") + + self.reachable_hosts = [h for h in self.reachable_hosts if h not in remove_set] + for host in removed: + if host not in self.unreachable_hosts: + self.unreachable_hosts.append(host) + + if self.password is None: + self.client = ParallelSSHClient( + self.reachable_hosts, user=self.user, pkey=self.pkey, keepalive_seconds=30, **self.ssh_client_kwargs + ) + else: + self.client = ParallelSSHClient( + self.reachable_hosts, + user=self.user, + password=self.password, + keepalive_seconds=30, + **self.ssh_client_kwargs, + ) + return removed + def prune_unreachable_hosts(self, output): """ Prune unreachable hosts from self.reachable_hosts if they have ConnectionError or Timeout exceptions and also fail connectivity check. @@ -98,27 +148,13 @@ def prune_unreachable_hosts(self, output): of potential unreachability, so we perform an additional connectivity check before pruning. This ensures that hosts are not permanently removed from the list for recoverable errors. """ - initial_unreachable_len = len(self.unreachable_hosts) failed_hosts = [ item.host for item in output if item.exception and isinstance(item.exception, (ConnectionError, Timeout, SessionError)) ] unreachable = self.check_connectivity(failed_hosts) - for host in unreachable: - self.log.warning(f"Host {host} is unreachable, pruning from reachable hosts list.") - self.unreachable_hosts.append(host) - self.reachable_hosts.remove(host) - if len(self.unreachable_hosts) > initial_unreachable_len: - # Recreate client with filtered reachable_hosts, only if hosts were actually pruned - if self.password is None: - self.client = ParallelSSHClient( - self.reachable_hosts, user=self.user, pkey=self.pkey, keepalive_seconds=30 - ) - else: - self.client = ParallelSSHClient( - self.reachable_hosts, user=self.user, password=self.password, keepalive_seconds=30 - ) + self.prune_nodes(unreachable) def inform_unreachability(self, cmd_output, include_exit_codes=False): """ @@ -370,6 +406,51 @@ def download_file(self, remote_file, local_file, recurse=False, suffix_separator ) from errors[0][1] return result + def upload_file_list(self, node_path_map): + """ + Upload different files to different hosts using SFTP. + + Args: + node_path_map: dict mapping {host: (local_path, remote_path)} + + Returns: + dict: {host: "host: SUCCESS" | "host: FAILED - ..."} + """ + if not node_path_map: + return {} + + # Build copy_args for each host + copy_args = [] + valid_hosts = [] + + for host in self.reachable_hosts: + if host in node_path_map: + local_path, remote_path = node_path_map[host] + copy_args.append({'local_file': local_path, 'remote_file': remote_path}) + valid_hosts.append(host) + + if not copy_args: + return {} + + self.log.info(f"Uploading {len(copy_args)} different files to {len(valid_hosts)} hosts") + + # Use copy_file with copy_args - returns greenlets + cmds = self.client.copy_file("%(local_file)s", "%(remote_file)s", copy_args=copy_args) + + # Wait for greenlets to complete (like upload_file does) + self.client.pool.join() + + # Process greenlet results + results = {} + for cmd, host in zip(cmds, valid_hosts): + try: + cmd.get() # This will raise if the greenlet failed + results[host] = f"{host}: SUCCESS" + except Exception as e: + results[host] = f"{host}: FAILED - {e}" + + return results + def reboot_connections(self): self.log.info('Rebooting Connections') self.client.run_command('reboot -f', stop_on_errors=self.stop_on_errors) diff --git a/cvs/lib/parallel/unittests/test_multiprocess_pssh.py b/cvs/lib/parallel/unittests/test_multiprocess_pssh.py index a6c251d4d..f66d15bec 100644 --- a/cvs/lib/parallel/unittests/test_multiprocess_pssh.py +++ b/cvs/lib/parallel/unittests/test_multiprocess_pssh.py @@ -29,7 +29,7 @@ def test_init_no_sharding_small_host_list(self): ) # For small lists, no sharder should be created (key difference!) self.assertFalse(hasattr(pssh, 'sharder')) - self.assertFalse(pssh._use_process_sharding) + self.assertIsNotNone(pssh.pssh) # But pssh instance should always exist self.assertTrue(hasattr(pssh, 'pssh')) @@ -51,7 +51,7 @@ def test_init_with_sharding_large_host_list(self, mock_sharder_class, mock_init_ ) # Verify sharder was created mock_sharder_class.assert_called_once_with(config) - self.assertTrue(pssh._use_process_sharding) + self.assertIsNone(pssh.pssh) # In sharded mode: pssh is None, sharder exists self.assertIsNone(pssh.pssh) self.assertTrue(hasattr(pssh, 'sharder')) @@ -196,7 +196,6 @@ def test_exec_cmd_list_with_unreachable_hosts(self): pssh.reachable_hosts = reachable_hosts_only # host2 is missing pssh.config = config pssh.sharder = mock_sharder - pssh._use_process_sharding = True # Additional attributes needed by _shard_init_kwargs pssh.user = "test" pssh.password = None @@ -210,10 +209,9 @@ def test_exec_cmd_list_with_unreachable_hosts(self): # Verify create_payloads was called and check the command mapping mock_sharder.create_payloads.assert_called() - # The cmd_list should be filtered to match reachable hosts - # Original: cmd_list=["cmd1", "cmd2", "cmd3", "cmd4"] for ["host1", "host2", "host3", "host4"] - # Filtered: should be ["cmd1", "cmd3", "cmd4"] for ["host1", "host3", "host4"] - # This validates that host3 gets cmd3 (not cmd2) and host4 gets cmd4 (not cmd3) + # The cmd_list should match reachable hosts exactly (proper usage) + # Passed: cmd_list=["cmd1", "cmd3", "cmd4"] for ["host1", "host3", "host4"] + # This validates proper 1:1 mapping between commands and reachable hosts self.assertEqual(result, {"host1": "result1", "host3": "result3", "host4": "result4"}) @@ -370,6 +368,76 @@ def test_upload_file_with_sharding(self): # upload_file should return empty dict (merged void results) self.assertEqual(result, {}) + def test_upload_file_list_with_subset_hosts_per_shard(self): + """Test upload_file_list shards only targeted hosts per worker payload.""" + host_list = ["host1", "host2", "host3", "host4"] + config = ParallelConfig(hosts_per_shard=2, max_workers_per_cpu=1) + node_path_map = { + "host2": ("/tmp/local2.sh", "/tmp/remote2.sh"), + "host3": ("/tmp/local3.sh", "/tmp/remote3.sh"), + } + + self.mock_execute_sharded.return_value = [ + {'result': {"host2": "host2: SUCCESS"}, 'reachable_hosts': ["host2"], 'unreachable_hosts': []}, + {'result': {"host3": "host3: SUCCESS"}, 'reachable_hosts': ["host3"], 'unreachable_hosts': []}, + ] + + pssh = MultiProcessPssh(self.mock_log, host_list, user="test", config=config) + result = pssh.upload_file_list(node_path_map) + + self.mock_execute_sharded.assert_called_once() + payloads = self.mock_execute_sharded.call_args[0][0] + self.assertEqual(len(payloads), 2) + + self.assertEqual(payloads[0]['operation'], 'upload_file_list') + self.assertEqual(payloads[0]['init']['host_list'], ["host2"]) + self.assertEqual(payloads[0]['node_path_map'], {"host2": ("/tmp/local2.sh", "/tmp/remote2.sh")}) + + self.assertEqual(payloads[1]['operation'], 'upload_file_list') + self.assertEqual(payloads[1]['init']['host_list'], ["host3"]) + self.assertEqual(payloads[1]['node_path_map'], {"host3": ("/tmp/local3.sh", "/tmp/remote3.sh")}) + + self.assertEqual(result, {"host2": "host2: SUCCESS", "host3": "host3: SUCCESS"}) + + def test_prune_nodes_sharded_mode_updates_wrapper_state(self): + """Test prune_nodes in sharded mode updates wrapper state directly.""" + with patch('cvs.lib.parallel.multiprocess_pssh.MultiProcessPssh._init_sharded'): + with patch('cvs.lib.parallel.multiprocess_pssh.PsshSharder'): + pssh = MultiProcessPssh(self.mock_log, ["host1", "host2", "host3"], user="test") + pssh.pssh = None + pssh.reachable_hosts = ["host1", "host2", "host3"] + pssh.host_list = ["host1", "host2", "host3"] + pssh.unreachable_hosts = [] + + removed = pssh.prune_nodes(["host2", "hostX"]) + + self.assertEqual(removed, ["host2"]) + self.assertEqual(pssh.reachable_hosts, ["host1", "host3"]) + self.assertEqual(pssh.host_list, ["host1", "host2", "host3"]) + self.assertEqual(pssh.unreachable_hosts, ["host2"]) + + def test_prune_nodes_non_sharded_delegates_to_single_process_pssh(self): + """Test prune_nodes in non-sharded mode delegates to single-process Pssh.""" + config = ParallelConfig(hosts_per_shard=32) + pssh = MultiProcessPssh(self.mock_log, ["host1", "host2"], user="test", config=config) + pssh.reachable_hosts = ["host1", "host2"] + pssh.host_list = ["host1", "host2"] + pssh.unreachable_hosts = [] + + pssh.pssh = MagicMock() + pssh.pssh.prune_nodes.return_value = ["host2"] + pssh.pssh.reachable_hosts = ["host1"] + pssh.pssh.host_list = ["host1"] + pssh.pssh.unreachable_hosts = ["host2"] + + removed = pssh.prune_nodes(["host2"]) + + pssh.pssh.prune_nodes.assert_called_once_with(["host2"]) + self.assertEqual(removed, ["host2"]) + self.assertEqual(pssh.reachable_hosts, ["host1"]) + self.assertEqual(pssh.host_list, ["host1", "host2"]) + self.assertEqual(pssh.unreachable_hosts, ["host2"]) + def test_download_file_with_sharding(self): """Test download_file with sharding uses sharder and merges results.""" config = ParallelConfig(hosts_per_shard=1, max_workers_per_cpu=1) diff --git a/cvs/lib/preflight/__init__.py b/cvs/lib/preflight/__init__.py new file mode 100644 index 000000000..e702aeb97 --- /dev/null +++ b/cvs/lib/preflight/__init__.py @@ -0,0 +1,7 @@ +# Preflight testing module +""" +Preflight testing components for cluster validation. + +This module provides various preflight checks to validate cluster health +before running workloads. +""" diff --git a/cvs/lib/preflight/base.py b/cvs/lib/preflight/base.py new file mode 100644 index 000000000..670dc46ee --- /dev/null +++ b/cvs/lib/preflight/base.py @@ -0,0 +1,124 @@ +""" +Base classes and utilities for preflight testing modules. +""" + +from abc import ABC, abstractmethod +from cvs.lib import globals + +log = globals.log + + +class PreflightCheck(ABC): + """ + Abstract base class for all preflight checks. + + Each preflight check should inherit from this class and implement the run() method. + This provides a consistent interface for all preflight operations. + """ + + def __init__(self, phdl, config_dict=None): + """ + Initialize the preflight check. + + Args: + phdl: Parallel SSH handle for cluster nodes + config_dict: Optional configuration dictionary + """ + self.phdl = phdl + self.config_dict = config_dict or {} + self.results = {} + + @abstractmethod + def run(self): + """ + Execute the preflight check. + + Returns: + dict: Results of the preflight check + """ + pass + + def get_results(self): + """Get the results of the last run.""" + return self.results + + def log_info(self, message): + """Log an info message.""" + log.info(f"[{self.__class__.__name__}] {message}") + + def log_error(self, message): + """Log an error message.""" + log.error(f"[{self.__class__.__name__}] {message}") + + def log_warning(self, message): + """Log a warning message.""" + log.warning(f"[{self.__class__.__name__}] {message}") + + @staticmethod + def partition_nodes_into_groups(node_list, group_size): + """ + Partition nodes into groups based on configurable group_size. + + Args: + node_list: List of all cluster nodes + group_size: Configured group size from preflight config + + Returns: + dict: {group_id: [node_list]} mapping + """ + import math + + groups = {} + num_groups = math.ceil(len(node_list) / group_size) + + for i in range(num_groups): + start_idx = i * group_size + end_idx = min(start_idx + group_size, len(node_list)) + groups[f"group_{i + 1}"] = node_list[start_idx:end_idx] + + return groups + + @staticmethod + def calculate_resource_requirements(group_size, num_interfaces=8): + """ + Calculate resource requirements for connectivity testing. + + Args: + group_size: Number of nodes per group + num_interfaces: Number of interfaces per node (default: 8) + + Returns: + dict: Resource requirement calculations + """ + intra_group_fds = (group_size - 1) * (num_interfaces**2) + inter_group_fds = group_size * (num_interfaces**2) + + return { + 'intra_group_fds_per_node': intra_group_fds, + 'inter_group_fds_per_node': inter_group_fds, + 'max_fds_per_node': inter_group_fds, + 'script_size_kb': (inter_group_fds * 85) // 1024, # ~85 chars per command + } + + @staticmethod + def find_host_group(host, groups): + """ + Find which group a host belongs to. + + Args: + host: Hostname to find + groups: Dictionary of group_id -> [nodes] + + Returns: + str: Group ID that contains the host, or None if not found + """ + for group_id, group_nodes in groups.items(): + if host in group_nodes: + return group_id + return None + + +# Module-level aliases for imports: cvs.lib.preflight.base.partition_nodes_into_groups +partition_nodes_into_groups = PreflightCheck.partition_nodes_into_groups +calculate_resource_requirements = PreflightCheck.calculate_resource_requirements +find_host_group = PreflightCheck.find_host_group diff --git a/cvs/lib/preflight/gid_consistency.py b/cvs/lib/preflight/gid_consistency.py new file mode 100644 index 000000000..b0e39c92a --- /dev/null +++ b/cvs/lib/preflight/gid_consistency.py @@ -0,0 +1,116 @@ +""" +GID Consistency Checking Module + +This module provides functionality for checking RDMA GID consistency across cluster nodes. +""" + +from cvs.lib.preflight.base import PreflightCheck + + +class GidConsistencyCheck(PreflightCheck): + """Check GID consistency across cluster RDMA interfaces.""" + + def __init__(self, phdl, gid_index="3", expected_interfaces=None, config_dict=None): + """ + Initialize GID consistency check. + + Args: + phdl: Parallel SSH handle for cluster nodes + gid_index: GID index to check (default: "3") + expected_interfaces: List of specific interfaces to check (if None, checks all) + config_dict: Optional configuration dictionary + """ + super().__init__(phdl, config_dict) + self.gid_index = gid_index + self.expected_interfaces = expected_interfaces + + def _build_gid_check_command(self): + """Build the remote shell snippet that enumerates GIDs per device.""" + if self.expected_interfaces: + interface_filter = " ".join([f"/sys/class/infiniband/{iface}" for iface in self.expected_interfaces]) + self.log_info( + f"Checking GID consistency for index {self.gid_index} on specific interfaces: {self.expected_interfaces}" + ) + return f""" + for dev in {interface_filter}; do + if [ -d "$dev" ]; then + dev_name=$(basename "$dev") + echo "DEVICE:$dev_name" + if [ -f "$dev/ports/1/gids/{self.gid_index}" ]; then + gid_value=$(cat "$dev/ports/1/gids/{self.gid_index}" 2>/dev/null) + if [ -n "$gid_value" ] && [ "$gid_value" != "0000:0000:0000:0000:0000:0000:0000:0000" ]; then + echo "GID_OK:$gid_value" + else + echo "GID_EMPTY:$gid_value" + fi + else + echo "GID_MISSING:No GID file" + fi + else + dev_name=$(basename "$dev") + echo "DEVICE:$dev_name" + echo "DEVICE_MISSING:Interface not found" + fi + done + """ + self.log_info(f"Checking GID consistency for index {self.gid_index} on all interfaces") + return f""" + for dev in /sys/class/infiniband/*/; do + if [ -d "$dev" ]; then + dev_name=$(basename "$dev") + echo "DEVICE:$dev_name" + if [ -f "$dev/ports/1/gids/{self.gid_index}" ]; then + gid_value=$(cat "$dev/ports/1/gids/{self.gid_index}" 2>/dev/null) + if [ -n "$gid_value" ] && [ "$gid_value" != "0000:0000:0000:0000:0000:0000:0000:0000" ]; then + echo "GID_OK:$gid_value" + else + echo "GID_EMPTY:$gid_value" + fi + else + echo "GID_MISSING:No GID file" + fi + fi + done + """ + + def _parse_gid_output_for_node(self, node, output): + """Parse remote command output into a per-node result dict.""" + node_result = {'status': 'PASS', 'interfaces': {}, 'errors': []} + current_device = None + for line in output.strip().split('\n'): + if line.startswith('DEVICE:'): + current_device = line.split(':', 1)[1] + node_result['interfaces'][current_device] = {} + elif line.startswith('GID_OK:'): + gid_value = line.split(':', 1)[1] + node_result['interfaces'][current_device] = {'status': 'OK', 'gid_value': gid_value} + elif line.startswith('GID_EMPTY:'): + gid_value = line.split(':', 1)[1] + node_result['interfaces'][current_device] = {'status': 'EMPTY', 'gid_value': gid_value} + node_result['status'] = 'FAIL' + node_result['errors'].append(f"GID index {self.gid_index} is empty on {current_device}") + elif line.startswith('GID_MISSING:'): + error_msg = line.split(':', 1)[1] + node_result['interfaces'][current_device] = {'status': 'MISSING', 'error': error_msg} + node_result['status'] = 'FAIL' + node_result['errors'].append(f"GID index {self.gid_index} missing on {current_device}: {error_msg}") + elif line.startswith('DEVICE_MISSING:'): + error_msg = line.split(':', 1)[1] + node_result['interfaces'][current_device] = {'status': 'DEVICE_MISSING', 'error': error_msg} + node_result['status'] = 'FAIL' + node_result['errors'].append(f"Interface {current_device} not found: {error_msg}") + return node_result + + def run(self): + """ + Execute GID consistency check across all cluster nodes. + + Returns: + dict: Results with per-node GID status + """ + cmd = self._build_gid_check_command() + self.results = {} + out_dict = self.phdl.exec(cmd) + for node, output in out_dict.items(): + self.results[node] = self._parse_gid_output_for_node(node, output) + return self.results diff --git a/cvs/lib/preflight/interface_consistency.py b/cvs/lib/preflight/interface_consistency.py new file mode 100644 index 000000000..91a25eb8d --- /dev/null +++ b/cvs/lib/preflight/interface_consistency.py @@ -0,0 +1,124 @@ +""" +Interface Consistency Checking Module + +This module provides functionality for checking RDMA interface consistency across cluster nodes. +""" + +from cvs.lib.preflight.base import PreflightCheck +from cvs.lib import linux_utils + + +class InterfaceConsistencyCheck(PreflightCheck): + """Check RDMA interface consistency across cluster nodes.""" + + def __init__(self, phdl, expected_interfaces=None, config_dict=None): + """ + Initialize interface consistency check. + + Args: + phdl: Parallel SSH handle for cluster nodes + expected_interfaces: List of expected interface names + config_dict: Optional configuration dictionary + """ + super().__init__(phdl, config_dict) + self.expected_interfaces = expected_interfaces or ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + + def _evaluate_node_interfaces(self, node, node_rdma_info): + """Build result dict for one node from RDMA NIC info.""" + found_interfaces = list(node_rdma_info.keys()) + + result = { + 'interfaces': [], + 'status': 'PASS', + 'errors': [], + 'expected_interfaces': self.expected_interfaces, + 'found_interfaces': found_interfaces, + 'missing_interfaces': [], + 'unexpected_interfaces': [], + 'inactive_interfaces': [], + 'down_interfaces': [], + } + + if not found_interfaces: + result['status'] = 'FAIL' + result['errors'].append("No RDMA interfaces found") + result['missing_interfaces'] = self.expected_interfaces.copy() + return result + + missing = [] + inactive_expected = [] + down_expected = [] + + for expected_iface in self.expected_interfaces: + if expected_iface not in found_interfaces: + missing.append(expected_iface) + else: + iface_info = node_rdma_info[expected_iface] + device_status = iface_info.get('device_status', 'UNKNOWN') + link_status = iface_info.get('link_status', 'UNKNOWN') + + if device_status != 'ACTIVE': + inactive_expected.append(f"{expected_iface} (state: {device_status})") + if link_status not in ['LINK_UP', 'LinkUp']: + down_expected.append(f"{expected_iface} (physical_state: {link_status})") + + result['missing_interfaces'] = missing + result['inactive_interfaces'] = inactive_expected + result['down_interfaces'] = down_expected + + unexpected = [] + for found_iface in found_interfaces: + if found_iface not in self.expected_interfaces: + unexpected.append(found_iface) + + result['unexpected_interfaces'] = unexpected + + for interface in found_interfaces: + iface_info = node_rdma_info.get(interface, {}) + device_status = iface_info.get('device_status', 'UNKNOWN') + link_status = iface_info.get('link_status', 'UNKNOWN') + + is_expected = interface in self.expected_interfaces + is_functional = device_status == 'ACTIVE' and link_status in ['LINK_UP', 'LinkUp'] + + result['interfaces'].append( + { + 'name': interface, + 'expected': is_expected, + 'device_status': device_status, + 'link_status': link_status, + 'functional': is_functional, + } + ) + + if missing: + result['status'] = 'FAIL' + result['errors'].append(f"Missing expected interfaces: {', '.join(missing)}") + + if inactive_expected: + result['status'] = 'FAIL' + result['errors'].append(f"Expected interfaces not active: {', '.join(inactive_expected)}") + + if down_expected: + result['status'] = 'FAIL' + result['errors'].append(f"Expected interfaces link down: {', '.join(down_expected)}") + + return result + + def run(self): + """ + Execute interface consistency check across all cluster nodes. + + Returns: + dict: Results with per-node interface status + """ + self.log_info(f"Checking RDMA interface presence (expected: {self.expected_interfaces})") + + rdma_dict = linux_utils.get_rdma_nic_dict(self.phdl) + + self.results = {} + for node in self.phdl.reachable_hosts: + node_rdma_info = rdma_dict.get(node, {}) + self.results[node] = self._evaluate_node_interfaces(node, node_rdma_info) + + return self.results diff --git a/cvs/lib/preflight/rdma_connectivity.py b/cvs/lib/preflight/rdma_connectivity.py new file mode 100644 index 000000000..7417d8d40 --- /dev/null +++ b/cvs/lib/preflight/rdma_connectivity.py @@ -0,0 +1,1460 @@ +''' +RDMA Connectivity Testing Module + +This module provides functions for testing RDMA connectivity across cluster nodes +using parallel group-based algorithms. +''' + +from cvs.lib.utils_lib import * +from cvs.lib.verify_lib import * +from cvs.lib import globals +from collections import defaultdict + +from cvs.lib.preflight.base import PreflightCheck, partition_nodes_into_groups +import re +import shlex +import time +# Import execute_round_with_script_coordination locally to avoid circular imports + +log = globals.log + + +def get_nested_config(config_dict, section, key, default): + """ + Get configuration value from nested structure. + + Args: + config_dict: Full configuration dictionary + section: Section name (e.g., 'debug', 'connectivity_check.rdma') + key: Parameter key within the section + default: Default value if not found + + Returns: + Configuration value or default + """ + if not config_dict: + return default + + # Handle nested sections like 'connectivity_check.rdma' + sections = section.split('.') + current = config_dict + + for sec in sections: + if isinstance(current, dict) and sec in current: + current = current[sec] + else: + return default + + if isinstance(current, dict) and key in current: + return current[key] + return default + + +class RdmaConnectivityCheck(PreflightCheck): + """Check RDMA connectivity across cluster nodes with multiple testing modes.""" + + def _scriptlet_enabled(self) -> bool: + """ + True when preflight config requests scriptlet debug (preserve scripts/logs and, + for connectivity, attach strace to generated server ibv_rc_pingpong lines). + + Accepts bool or common string forms from JSON ("true", "1", etc.). + """ + if not self.config_dict: + return False + v = get_nested_config(self.config_dict, 'debug', 'scriptlet', False) + if isinstance(v, str): + return v.strip().lower() in ('1', 'true', 'yes', 'on') + return bool(v) + + def _port_listen_retry_max(self) -> int: + """Max extra ScriptLet batches after initial run for PORT_LISTEN_FAILED (default 3).""" + cfg = self.config_dict or {} + raw = get_nested_config(cfg, 'connectivity_check.rdma', 'port_retry_max', 3) + try: + n = int(raw) + except (TypeError, ValueError): + n = 3 + return max(0, min(n, 10)) + + def _port_listen_retry_gap(self) -> int: + """Port offset step when remapping retry batches (reduces collision with ephemerals).""" + cfg = self.config_dict or {} + raw = get_nested_config(cfg, 'connectivity_check.rdma', 'port_retry_gap', 1000) + try: + g = int(raw) + except (TypeError, ValueError): + g = 1000 + return max(1, min(g, 65535)) + + @staticmethod + def _generate_test_pair_key(assignment): + """Stable result key for one directed iface test.""" + return f"{assignment['server_node']} <-> {assignment['client_node']} ({assignment['server_iface']}->{assignment['client_iface']})" + + @staticmethod + def _is_port_listen_failure(test_result: dict) -> bool: + if not isinstance(test_result, dict) or test_result.get('status') != 'FAIL': + return False + for line in test_result.get('error_details') or []: + if 'PORT_LISTEN_FAILED' in str(line): + return True + return False + + def _artifacts_root_dir(self) -> str: + cfg = self.config_dict or {} + root = get_nested_config(cfg, 'reporting', 'artifacts_root_dir', '/tmp/preflight') + return str(root).rstrip('/') + + def _remote_rdma_workspace_root(self) -> str: + """ + Remote tree root for ScriptLet logs and scripts. + + Lives under ``artifacts_root_dir`` so operators can point NFS-mounted preflight reports here. + """ + return f'{self._artifacts_root_dir()}/rdma_connectivity_workspace' + + def _begin_full_mesh_rdma_artifacts(self) -> None: + """ + Once per ``full_mesh`` run: ``rm -rf`` remote workspace and create a new session id. + + Clears only ``/rdma_connectivity_workspace``, not sibling HTML files. + """ + if getattr(self, '_full_mesh_rdma_artifact_wipe_done', False): + return + root = self._remote_rdma_workspace_root() + self._rdma_session_id = time.strftime('%Y%m%d_%H%M%S') + q = shlex.quote + self.phdl.exec(f"rm -rf {q(root)} && mkdir -p {q(root)}", timeout=180, print_console=False) + self._full_mesh_rdma_artifact_wipe_done = True + log.info( + "RDMA full_mesh: cleared remote workspace %s once (HTML in %s preserved); session %s — " + "per-round logs under %s//", + root, + self._artifacts_root_dir(), + self._rdma_session_id, + f"{root}/{self._rdma_session_id}", + ) + + def _artifact_workspace_dir(self, work_segment: str) -> str: + """Remote directory for one coordination round or inter-group wave.""" + sess = getattr(self, '_rdma_session_id', None) + if not sess: + self._rdma_session_id = time.strftime('%Y%m%d_%H%M%S') + sess = self._rdma_session_id + safe = re.sub(r'[^a-zA-Z0-9_.-]+', '_', work_segment.strip()) or 'round' + return f"{self._remote_rdma_workspace_root()}/{sess}/{safe}" + + def __init__( + self, + phdl, + node_list, + mode="basic", + port_range="10000-50000", + timeout=90, + expected_interfaces=None, + gid_index="3", + parallel_group_size=128, + config_dict=None, + ): + """ + Initialize RDMA connectivity check. + + Args: + phdl: Parallel SSH handle for cluster nodes + node_list: List of cluster nodes + mode: "basic", "full_mesh", or "skip" + port_range: Port range for ibv_rc_pingpong tests (e.g., "10000-50000") + timeout: Test timeout in seconds + expected_interfaces: List of RDMA interfaces + gid_index: GID index to use + parallel_group_size: Group size for parallel testing + config_dict: Optional configuration dictionary + """ + super().__init__(phdl, config_dict) + self.node_list = node_list + self.mode = mode + self.port_range = port_range + self.timeout = timeout + self.expected_interfaces = expected_interfaces + self.gid_index = gid_index + self.parallel_group_size = parallel_group_size + + def run(self): + """ + Execute RDMA connectivity testing with multiple mode support. + + Returns: + dict: Comprehensive connectivity test results + """ + log.info(f"Checking RDMA connectivity using ibv_rc_pingpong (mode: {self.mode}, GID index: {self.gid_index})") + + # Handle skip mode + if self.mode == "skip": + log.info("RDMA connectivity test skipped by configuration") + return { + 'mode': 'skip', + 'total_pairs': 0, + 'successful_pairs': 0, + 'failed_pairs': 0, + 'pair_results': {}, + 'node_status': {}, + 'skipped': True, + 'pruned_nodes_after_intra': [], + 'partition_groups': {}, + 'inter_groups': {}, + 'inter_group_mode': '', + 'inter_group_waves': [], + 'inter_group_wave_chunk': None, + } + + # Parse port range + port_start, port_end = map(int, self.port_range.split('-')) + + results = { + 'mode': self.mode, + 'total_pairs': 0, + 'successful_pairs': 0, + 'failed_pairs': 0, + 'pair_results': {}, + 'node_status': {}, + 'gid_index': self.gid_index, + 'port_range': self.port_range, + 'timeout': self.timeout, + 'pruned_nodes_after_intra': [], + 'partition_groups': {}, + 'inter_groups': {}, + 'inter_group_mode': '', + 'inter_group_waves': [], + 'inter_group_wave_chunk': None, + } + + # Initialize node status + for node in self.node_list: + results['node_status'][node] = { + 'server_tests': 0, + 'client_tests': 0, + 'successful_tests': 0, + 'failed_tests': 0, + } + + if self.mode == "full_mesh": + # Use new parallel group-based algorithm + all_results, pruned_after_intra, mesh_meta = self._execute_full_mesh(port_start) + results['pruned_nodes_after_intra'] = pruned_after_intra + results['partition_groups'] = mesh_meta.get('partition_groups', {}) + results['inter_groups'] = mesh_meta.get('inter_groups', {}) + results['inter_group_mode'] = mesh_meta.get('inter_group_mode', 'multi_wave') + results['inter_group_waves'] = mesh_meta.get('inter_group_waves', []) + results['inter_group_wave_chunk'] = mesh_meta.get('inter_group_wave_chunk') + + # Process results from parallel algorithm + self._process_test_results(all_results, results) + + else: + # Basic mode using ScriptLet execution + batch_results = self._execute_basic(port_start) + self._process_test_results(batch_results, results) + + return results + + def _generate_interface_test_pairs(self): + """ + Generate node pairs for connectivity testing based on self.mode. + + Returns: + list: List of tuples representing node pairs + """ + if self.mode == "skip": + return [] + elif self.mode == "basic": + # Adjacent pairs like current IB tests + pairs = [] + for i in range(0, len(self.node_list) - 1, 2): + if i + 1 < len(self.node_list): + pairs.append((self.node_list[i], self.node_list[i + 1])) + return pairs + + elif self.mode == "full_mesh": + # All possible pairs + pairs = [] + for i in range(len(self.node_list)): + for j in range(i + 1, len(self.node_list)): + pairs.append((self.node_list[i], self.node_list[j])) + return pairs + + else: + raise ValueError(f"Unknown connectivity mode: {self.mode}") + + def _process_test_results(self, test_results, results_container): + """Process test results: count successes/failures and update node statistics.""" + for pair_key, pair_result in test_results.items(): + results_container['pair_results'][pair_key] = pair_result + + if pair_result['status'] == 'PASS': + results_container['successful_pairs'] += 1 + else: + results_container['failed_pairs'] += 1 + + # Update node statistics + if '(' in pair_key: + base_pair = pair_key.split(' (')[0] + server_node, client_node = base_pair.split(' <-> ') + else: + server_node, client_node = pair_key.split(' <-> ') + + results_container['node_status'][server_node]['server_tests'] += 1 + results_container['node_status'][client_node]['client_tests'] += 1 + + if pair_result['status'] == 'PASS': + results_container['node_status'][server_node]['successful_tests'] += 1 + results_container['node_status'][client_node]['successful_tests'] += 1 + else: + results_container['node_status'][server_node]['failed_tests'] += 1 + results_container['node_status'][client_node]['failed_tests'] += 1 + + results_container['total_pairs'] = len(test_results) + + def _execute_basic(self, port_start): + """ + Execute basic mode connectivity testing. + + Args: + port_start: Starting port number + + Returns: + dict: Test results + """ + log.info(f"Starting {self.mode} mode connectivity testing") + + # Generate node pairs based on mode + pairs = self._generate_interface_test_pairs() + if not pairs: + return {} + + # Convert pairs to assignments + assignments = self._create_test_assignments(pairs, port_start) + + # Execute via ScriptLet infrastructure + log.info(f"Executing {len(assignments)} test assignments via ScriptLet") + results = self._execute_test_assignments(assignments, f"{self.mode}_mode", f"{self.mode}_single") + + log.info(f"Completed {self.mode} mode testing: {len(results)} results") + return results + + def _create_test_assignments(self, pairs, base_port): + """ + Convert node pairs to ScriptLet assignment format. + + Args: + pairs: List of (server_node, client_node) tuples + base_port: Starting port number for assignments + + Returns: + list: List of assignment dictionaries + """ + if self.expected_interfaces is None: + expected_interfaces = ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + else: + expected_interfaces = self.expected_interfaces + + assignments = [] + current_port = base_port + + # Convert each pair to bidirectional interface matrix assignments + for node_a, node_b in pairs: + # Test both directions: A->B and B->A (like full mesh) + for server_node, client_node in [(node_a, node_b), (node_b, node_a)]: + for server_iface in expected_interfaces: + for client_iface in expected_interfaces: + assignments.append( + { + 'server_node': server_node, + 'client_node': client_node, + 'server_iface': server_iface, + 'client_iface': client_iface, + 'port': current_port, + } + ) + current_port += 1 + + return assignments + + def _execute_full_mesh(self, port_start): + """ + Execute parallel full mesh connectivity using class attributes. + + Args: + port_start: Starting port number + + Returns: + dict: Full mesh test results + """ + log.info(f"Starting parallel full mesh connectivity testing with group size {self.parallel_group_size}") + self._begin_full_mesh_rdma_artifacts() + + # Partition nodes into groups (use only reachable nodes) + reachable_node_list = list(self.phdl.reachable_hosts) + groups = partition_nodes_into_groups(reachable_node_list, self.parallel_group_size) + num_groups = len(groups) + + log.info(f"Partitioned {len(reachable_node_list)} reachable nodes into {num_groups} groups") + for group_id, group_nodes in groups.items(): + log.info(f"{group_id}: {len(group_nodes)} nodes") + + intra_results = self._execute_intragroup(groups, port_start) + for v in intra_results.values(): + if isinstance(v, dict): + v['round'] = 'intra_group' + + pruned_records = [] + groups_inter = groups + cfg = self.config_dict or {} + if cfg.get('rdma_prune_intra_failed_nodes', True): + _pruned_set, pruned_records, groups_inter = self._apply_intra_prune(intra_results, groups) + if pruned_records: + log.warning( + f"Round 1 pruning: excluding {len(pruned_records)} node(s) from inter-group tests: " + f"{', '.join(p['node'] for p in pruned_records)}" + ) + else: + log.info( + "Round 1 pruning: no nodes excluded from inter-group — " + "no node met the peer-failure fraction threshold for its group." + ) + else: + log.info( + "Round 1 pruning: disabled (config rdma_prune_intra_failed_nodes=false); " + "inter-group uses the same node set as after partitioning." + ) + + all_results = dict(intra_results) + + intra_assignments = self._calculate_port_assignments(groups, 'intra_group', port_start) + if intra_assignments: + port_cursor = max(a['port'] for a in intra_assignments) + 1 + else: + port_cursor = port_start + + inter_results, inter_meta = self._execute_intergroup_waves(groups_inter, port_cursor, cfg) + all_results.update(inter_results) + + mesh_meta = { + 'partition_groups': {k: list(v) for k, v in groups.items()}, + 'inter_groups': {k: list(v) for k, v in groups_inter.items()}, + 'inter_group_mode': inter_meta.get('mode', 'multi_wave'), + 'inter_group_waves': inter_meta.get('waves', []), + 'inter_group_wave_chunk': inter_meta.get('wave_group_pairs_chunk'), + } + + log.info(f"Parallel full mesh connectivity testing completed: {len(all_results)} total tests") + return all_results, pruned_records, mesh_meta + + def _execute_intergroup_waves(self, groups_inter, port_start, cfg): + """ + Run inter-group tests either in one coordination round (single_shot) or in multiple + waves (multi_wave) by chunking ordered group-pairs to reduce peak load. + + Config (``cfg``): + rdma_inter_group_mode: ``single_shot`` | ``multi_wave`` (default ``multi_wave``). + inter_full_mesh_group_pairs_per_wave: max ordered group-pairs (Gi->Gj keys) + per wave when multi_wave. If omitted, empty, or ``auto``, uses **max(1, Ng−1)** + where Ng is the number of inter-group partition groups. + (Backward compatible aliases: ``inter_group_wave_pairs``.) + + Returns: + tuple: (merged_results_dict, meta_dict with keys ``mode``, ``waves``) + """ + full = self._generate_all_intergroup_pairs(groups_inter) + if not full: + return {}, {'mode': 'none', 'waves': [], 'wave_group_pairs_chunk': None} + + mode = (cfg.get('rdma_inter_group_mode') or 'multi_wave').lower() + if mode not in ('single_shot', 'multi_wave'): + log.warning(f"Unknown rdma_inter_group_mode {mode!r}; using multi_wave") + mode = 'multi_wave' + + if mode == 'single_shot': + log.info(f"=== Round 2: Inter-group testing (single shot: {len(full)} ordered group pairs) ===") + wave_res = self._execute_test_round_with_retries( + full, 'inter_group', port_start, work_segment='inter_group_wave01_single' + ) + for v in wave_res.values(): + if isinstance(v, dict): + v['round'] = 'inter_group' + v['inter_wave'] = 1 + v['inter_waves_total'] = 1 + waves_meta = [ + { + 'wave': 1, + 'group_pair_keys': list(full.keys()), + 'num_group_pairs': len(full), + } + ] + return wave_res, {'mode': 'single_shot', 'waves': waves_meta, 'wave_group_pairs_chunk': None} + + ng = len(groups_inter) + default_chunk = max(1, ng - 1) + raw = get_nested_config( + cfg, + 'connectivity_check.rdma', + 'inter_full_mesh_group_pairs_per_wave', + get_nested_config(cfg, 'connectivity_check.rdma', 'inter_group_wave_pairs', None), + ) + if raw is None or (isinstance(raw, str) and raw.strip().lower() in ('', 'auto')): + chunk_sz = default_chunk + else: + try: + chunk_sz = max(1, int(raw)) + except (TypeError, ValueError): + chunk_sz = default_chunk + + keys = list(full.keys()) + chunks = [keys[i : i + chunk_sz] for i in range(0, len(keys), chunk_sz)] + n_waves = len(chunks) + log.info( + f"=== Round 2: Inter-group testing (multi-wave: {n_waves} wave(s), " + f"up to {chunk_sz} ordered group-pair(s) per wave, {len(full)} total group-pairs) ===" + ) + + merged = {} + waves_meta = [] + port_cursor = port_start + + for wi, key_chunk in enumerate(chunks, start=1): + sub = {k: full[k] for k in key_chunk} + log.info( + f"Inter-group wave {wi}/{n_waves}: {len(sub)} ordered group-pair(s) " + f"({', '.join(key_chunk[:5])}{'...' if len(key_chunk) > 5 else ''})" + ) + wave_res = self._execute_test_round_with_retries( + sub, 'inter_group', port_cursor, work_segment=f'inter_group_wave_{wi:02d}' + ) + for v in wave_res.values(): + if isinstance(v, dict): + v['round'] = 'inter_group' + v['inter_wave'] = wi + v['inter_waves_total'] = n_waves + merged.update(wave_res) + sub_assign = self._calculate_port_assignments(sub, 'inter_group', port_cursor) + if sub_assign: + port_cursor = max(a['port'] for a in sub_assign) + 1 + waves_meta.append( + { + 'wave': wi, + 'group_pair_keys': list(key_chunk), + 'num_group_pairs': len(key_chunk), + } + ) + + return merged, {'mode': 'multi_wave', 'waves': waves_meta, 'wave_group_pairs_chunk': chunk_sz} + + def _get_peer_failure_threshold(self): + """ + Fraction in (0, 1]. Nodes with ``failed_peers / (n-1) >= threshold`` are pruned before inter-group. + + Config: ``prune_failure_threshold`` (default ``0.5``). Invalid or out-of-range values fall back to 0.5. + """ + cfg = self.config_dict or {} + raw = get_nested_config(cfg, 'connectivity_check.rdma', 'prune_failure_threshold', 0.5) + try: + t = float(raw) + except (TypeError, ValueError): + log.warning(f"Invalid prune_failure_threshold {raw!r}; using 0.5") + t = 0.5 + if t <= 0 or t > 1: + log.warning(f"prune_failure_threshold {t} must be in (0, 1]; using 0.5") + t = 0.5 + return t + + def _failed_peers_from_intra(self, intra_results, groups): + """ + For each node, set of distinct peers in the same partition group with at least one **FAIL** intra test. + """ + failed_peers = defaultdict(set) + for pr in intra_results.values(): + if not isinstance(pr, dict) or pr.get('status') != 'FAIL': + continue + sn = pr.get('server_node') + cn = pr.get('client_node') + if not sn or not cn or sn == cn: + continue + for _gid, nodes in groups.items(): + node_set = set(nodes) + if sn in node_set and cn in node_set: + failed_peers[sn].add(cn) + failed_peers[cn].add(sn) + break + return failed_peers + + def _apply_intra_prune(self, intra_results, groups): + """ + Remove nodes whose intra-group **peer failure fraction** is at or above the configured threshold. + + For a node in a group of ``n`` nodes, fraction = ``(peers with ≥1 FAIL vs that node) / (n-1)``. + A peer counts if **any** intra test between the two nodes failed (any interface / role combo). + + Config: + ``prune_failure_threshold`` (default ``0.5``): prune when fraction >= threshold. + + Returns: + tuple: (pruned_node_set, pruned_records, groups_for_inter) + """ + threshold = self._get_peer_failure_threshold() + log.info( + f"Round 1 pruning: peer-failure fraction threshold {threshold:.0%} " + f"(prune_failure_threshold); prune when fraction ≥ threshold per node." + ) + failed_peers = self._failed_peers_from_intra(intra_results, groups) + pruned_records = [] + pruned_set = set() + for gid, nodes in groups.items(): + if len(nodes) <= 1: + continue + denom = len(nodes) - 1 + for node in nodes: + cnt = len(failed_peers.get(node, set())) + fraction = cnt / denom + if fraction >= threshold: + pruned_set.add(node) + pruned_records.append( + { + 'node': node, + 'group_id': gid, + 'reason': ( + f'Intra-group peer failure fraction {fraction:.0%} ({cnt}/{denom} peers with ≥1 FAIL) ' + f'≥ threshold {threshold:.0%} (prune_failure_threshold).' + ), + } + ) + + new_groups = {} + for gid, nodes in groups.items(): + kept = [x for x in nodes if x not in pruned_set] + if kept: + new_groups[gid] = kept + + return pruned_set, pruned_records, new_groups + + def _execute_intragroup(self, groups, port_start): + """ + Execute intragroup connectivity testing using class attributes. + + Args: + groups: Dictionary of group_id -> node_list + port_start: Starting port number + + Returns: + dict: Intragroup test results + """ + log.info("=== Round 1: Intra-group parallel testing ===") + return self._execute_test_round_with_retries(groups, "intra_group", port_start, work_segment='intra_group') + + def _generate_all_intergroup_pairs(self, groups): + """ + Build every ordered pair of distinct groups (servers in Gi, clients in Gj). + + Required for full mesh: each directed node pair (u, v) with u in Gi and v in Gj + must appear for both role orders when i != j. + """ + group_ids = list(groups.keys()) + inter_group_tests = {} + for gi in group_ids: + for gj in group_ids: + if gi == gj: + continue + inter_group_tests[f"{gi}_to_{gj}"] = { + 'group1': groups[gi], + 'group2': groups[gj], + } + return inter_group_tests + + def _execute_intergroup(self, groups, port_start): + """ + Execute intergroup connectivity testing using class attributes. + + Args: + groups: Dictionary of group_id -> node_list + port_start: Starting port number + + Returns: + dict: Intergroup test results + """ + log.info("=== Round 2: Inter-group testing (single coordination, legacy helper) ===") + + inter_group_tests = self._generate_all_intergroup_pairs(groups) + if not inter_group_tests: + return {} + + log.info(f"Inter-group: {len(inter_group_tests)} ordered group pairs (full mesh between groups)") + + res = self._execute_test_round_with_retries( + inter_group_tests, "inter_group", port_start, work_segment='inter_group_legacy' + ) + for v in res.values(): + if isinstance(v, dict): + v['round'] = 'inter_group' + v['inter_wave'] = 1 + v['inter_waves_total'] = 1 + return res + + def _analyze_output(self, client_output, server_output): + """ + Analyze ibv_rc_pingpong output for success/failure. + + Args: + client_output: Output from client side + server_output: Output from server side + + Returns: + bool: True if successful, False otherwise + """ + if not client_output: + return False + + # Success patterns: bandwidth/latency measurements or successful connection info + success_patterns = [ + r'\d+\s+bytes\s+in\s+[\d.]+\s+seconds', # Bandwidth results + r'local\s+address:\s+LID.*GID.*remote\s+address:\s+LID.*GID', # Connection info + r'\d+\s+\d+\s+[\d.]+\s+[\d.]+', # Latency results table + ] + + for pattern in success_patterns: + if re.search(pattern, client_output, re.IGNORECASE | re.MULTILINE): + return True + + return False + + def _extract_errors(self, client_output, server_output): + """ + Extract error messages from ibv_rc_pingpong output. + + Args: + client_output: Output from client side + server_output: Output from server side + + Returns: + list: List of error messages found + """ + errors = [] + + def extract_connection_details(output, role): + """Extract connection-related error details.""" + details = [] + if "Failed to modify QP" in output: + if "to RTR" in output: + details.append(f"{role}: Queue Pair failed to transition to Ready-To-Receive (RTR)") + elif "to RTS" in output: + details.append(f"{role}: Queue Pair failed to transition to Ready-To-Send (RTS)") + + if "Couldn't connect to" in output or "Unable to Connect" in output: + details.append(f"{role}: Connection establishment failed") + + if "Failed status transport retry counter exceeded" in output: + details.append(f"{role}: Transport retry limit exceeded") + + if "parse WC failed" in output: + details.append(f"{role}: Work Completion parsing failed") + + return details + + # Check client output + if client_output: + client_errors = extract_connection_details(client_output, "Client") + errors.extend(client_errors) + + # Additional client-specific patterns + if not client_errors: + if "timeout" in client_output.lower(): + errors.append("Client: Connection timeout") + elif not client_output.strip(): + errors.append("Client: No output received") + + # Check server output + if server_output: + server_errors = extract_connection_details(server_output, "Server") + errors.extend(server_errors) + + # If we found specific errors, format them nicely + if not errors: + errors.append("Unknown error - ibv_rc_pingpong failed to establish connection") + + return errors + + def _generate_intergroup_pairs(self, groups, round_num): + """ + Generate inter-group pairs for a specific round using round-robin algorithm. + + Args: + groups: Dictionary of group_id -> [nodes] + round_num: Round number for pair generation + + Returns: + dict: Inter-group test pairs for this round + """ + group_ids = list(groups.keys()) + num_groups = len(groups) + inter_group_tests = {} + + for i, group1_id in enumerate(group_ids): + group2_idx = (i + round_num - 2) % num_groups + if group2_idx != i: # Don't test group with itself + group2_id = group_ids[group2_idx] + inter_group_tests[f"{group1_id}_to_{group2_id}"] = { + 'group1': groups[group1_id], + 'group2': groups[group2_id], + } + + return inter_group_tests + + def _calculate_port_assignments(self, test_groups, round_type, port_start): + """ + Calculate port assignments for all tests using class attributes. + + Args: + test_groups: Test group configuration + round_type: "intra_group" or "inter_group" + port_start: Starting port number + + Returns: + list: List of dicts with port assignments + """ + assignments = [] + if self.expected_interfaces is None: + expected_interfaces = ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + else: + expected_interfaces = self.expected_interfaces + + # Track port usage per server node (each node can reuse the same port range) + node_port_counters = {} + + if round_type == "intra_group": + for group_id, group_nodes in test_groups.items(): + for server_node in group_nodes: + # Initialize port counter for this server node + if server_node not in node_port_counters: + node_port_counters[server_node] = 0 + + for client_node in group_nodes: + if server_node != client_node: + for server_iface in expected_interfaces: + for client_iface in expected_interfaces: + # Assign sequential port for this specific server node + port = port_start + node_port_counters[server_node] + assignments.append( + { + 'server_node': server_node, + 'client_node': client_node, + 'server_iface': server_iface, + 'client_iface': client_iface, + 'port': port, + } + ) + # Increment port counter only for this server node + node_port_counters[server_node] += 1 + + elif round_type == "inter_group": + for test_name, test_config in test_groups.items(): + group1_nodes = test_config['group1'] + group2_nodes = test_config['group2'] + + for server_node in group1_nodes: + # Initialize port counter for this server node + if server_node not in node_port_counters: + node_port_counters[server_node] = 0 + + for client_node in group2_nodes: + for server_iface in expected_interfaces: + for client_iface in expected_interfaces: + # Assign sequential port for this specific server node + port = port_start + node_port_counters[server_node] + assignments.append( + { + 'server_node': server_node, + 'client_node': client_node, + 'server_iface': server_iface, + 'client_iface': client_iface, + 'port': port, + } + ) + # Increment port counter only for this server node + node_port_counters[server_node] += 1 + + return assignments + + def _execute_test_round_with_retries(self, test_groups, round_type, port_start, *, work_segment='round'): + """ + Execute testing round with script coordination using class attributes. + + Retries subsets that fail with ``PORT_LISTEN_FAILED`` up to ``port_retry_max`` times + (default 3), using new TCP ports between attempts. + + Args: + test_groups: Test group configuration + round_type: "intra_group" or "inter_group" + port_start: Starting port number + work_segment: Path segment for this round's remote work dir (under + ``artifacts_root_dir/rdma_connectivity_workspace//``). + + Returns: + dict: Test results for the round + """ + assignments = self._calculate_port_assignments(test_groups, round_type, port_start) + if not assignments: + return {} + + max_pl_retries = self._port_listen_retry_max() + gap = self._port_listen_retry_gap() + merged: dict = {} + next_port_cursor = max(a['port'] for a in assignments) + gap + + pending: list = [dict(a) for a in assignments] + attempt = 0 + + while pending: + seg = work_segment if attempt == 0 else f"{work_segment}_plretry{attempt}" + batch = self._execute_test_assignments(pending, round_type, seg) + merged.update(batch) + + pl_failed_keys = { + pk for pk, res in batch.items() if isinstance(res, dict) and self._is_port_listen_failure(res) + } + if not pl_failed_keys: + break + if attempt >= max_pl_retries: + log.warning( + "PORT_LISTEN_FAILED persists for %d pair(s) after %d retries; keeping last results.", + len(pl_failed_keys), + max_pl_retries, + ) + break + + pk_to_assignment = {self._generate_test_pair_key(a): a for a in pending} + pending = [] + for pk in sorted(pl_failed_keys): + if pk not in pk_to_assignment: + continue + base = dict(pk_to_assignment[pk]) + base['port'] = next_port_cursor + next_port_cursor += 1 + pending.append(base) + + log.info( + "RDMA %s: retry batch %d for %d pair(s) with PORT_LISTEN_FAILED (new ports %s..)", + round_type, + attempt + 1, + len(pending), + pending[0]['port'] if pending else 'n/a', + ) + attempt += 1 + + log.info( + "ScriptLet-based %s round completed with %d pair result(s) (%d PORT_LISTEN retry wave(s) used)", + round_type, + len(merged), + attempt, + ) + return merged + + def _execute_test_assignments(self, assignments, round_type, work_segment): + """ + Execute a complete RDMA connectivity test round for the given assignments. + + Orchestrates a three-phase testing cycle: + 1. Server Phase: Deploy and start ibv_rc_pingpong servers on assigned nodes + 2. Client Phase: Deploy and execute ibv_rc_pingpong clients to connect to servers + 3. Collection Phase: Gather and analyze test results from server/client logs + + Args: + assignments: List of test assignments, each containing server_node, client_node, + server_iface, client_iface, and port + round_type: Type of round being executed ("intra_group", "inter_group", etc.) + work_segment: Path segment for remote workspace directory + + Returns: + dict: Test results mapped by test pair keys + """ + from cvs.lib.scriptlet import ScriptLet + + scriptlet_debug = self._scriptlet_enabled() + workspace_dir = self._artifact_workspace_dir(work_segment) + + log.info( + "ScriptLet batch (%s): %d assignment(s), dir %s", + round_type, + len(assignments), + workspace_dir, + ) + if scriptlet_debug: + log.info( + "scriptlet_debug: strace logs under %s/strace_server__.log", + workspace_dir, + ) + + # Phase 1: Server scripts context (kept open through client phase) + log.info("Phase 1: Starting ibv_rc_pingpong servers") + with ScriptLet( + self.phdl, + debug=scriptlet_debug, + scriptlet_workspace=workspace_dir, + cleanup_on_init=True, + preserve_workspace_on_exit=True, + ) as server_scriptlet: + for host in self.phdl.reachable_hosts: + server_commands = self._generate_server_commands_for_host(host, assignments, workspace_dir) + script_content = self._build_server_script(host, server_commands) + + if script_content: + script_id = f"servers_{host}_{round_type}" + server_scriptlet.create_script(script_id, script_content) + + server_script_mapping = { + host: f"servers_{host}_{round_type}" + for host in self.phdl.reachable_hosts + if f"servers_{host}_{round_type}" in server_scriptlet.local_scripts + } + + if server_script_mapping: + server_scriptlet.copy_script_list(server_script_mapping) + + # Phase 2: Client scripts nested so server cleanup doesn't run before clients start. + log.info("Phase 2: Starting ibv_rc_pingpong clients") + with ScriptLet( + self.phdl, + debug=scriptlet_debug, + scriptlet_workspace=workspace_dir, + cleanup_on_init=False, # Don't clean workspace since servers are using it + preserve_workspace_on_exit=True, + ) as client_scriptlet: + for host in self.phdl.reachable_hosts: + client_commands = self._generate_client_commands_for_host(host, assignments, workspace_dir) + script_content = self._build_client_script(host, client_commands) + + if script_content: + script_id = f"clients_{host}_{round_type}" + client_scriptlet.create_script(script_id, script_content) + + client_script_mapping = { + host: f"clients_{host}_{round_type}" + for host in self.phdl.reachable_hosts + if f"clients_{host}_{round_type}" in client_scriptlet.local_scripts + } + + if client_script_mapping: + client_scriptlet.copy_script_list(client_script_mapping) + + if server_script_mapping: + log.info(f"Starting {len(server_script_mapping)} server scripts in parallel") + server_results = server_scriptlet.run_parallel_group(server_script_mapping, timeout=90) + + failed_servers = [] + for node, output in server_results.items(): + if "All servers started" not in output: + failed_servers.append(f"{node}: {output}") + + if failed_servers: + log.error(f"Server startup failed on {len(failed_servers)} nodes:") + for failure in failed_servers: + log.error(f" {failure}") + raise RuntimeError(f"Server startup failures: {failed_servers}") + + log.info(f"All servers started successfully on {len(server_script_mapping)} nodes") + + log.info(f"Starting {len(client_script_mapping)} client scripts in parallel") + client_execution_results = client_scriptlet.run_parallel_group( + client_script_mapping, timeout=self.timeout + ) + + failed_clients = [] + for node, output in client_execution_results.items(): + if "All clients completed" not in output: + failed_clients.append(f"{node}: {output}") + + if failed_clients: + log.warning(f"Client execution issues on {len(failed_clients)} nodes:") + for failure in failed_clients[:5]: + log.warning(f" {failure}") + + log.info(f"Client execution completed on {len(client_script_mapping)} nodes") + # Client context exits here; server context remains active until this block completes. + # Server context exits here (after client phase) and runs its cleanup. + + # Phase 3: Collection and analysis (keeping existing separate context) + log.info("Phase 3: Collecting and analyzing test results") + + test_metadata = [] + workspace_dir = workspace_dir + for assignment in assignments: + server_node = assignment['server_node'] + client_node = assignment['client_node'] + server_iface = assignment['server_iface'] + client_iface = assignment['client_iface'] + port = assignment['port'] + + test_metadata.append( + { + 'server_node': server_node, + 'client_node': client_node, + 'server_iface': server_iface, + 'client_iface': client_iface, + 'port': port, + 'client_log_path': ( + f"{workspace_dir}/client_{client_iface}_{port}_{server_node.replace('.', '_').replace(':', '_')}.log" + ), + 'server_log_path': ( + f"{workspace_dir}/server_{server_iface}_{port}_{client_node.replace('.', '_').replace(':', '_')}.log" + ), + } + ) + + return self._collect_results_with_scriptlet(test_metadata, workspace_dir) + + def _generate_server_commands_for_host(self, host, assignments, workspace_dir): + commands = [] + workspace_dir = workspace_dir + for assignment in assignments: + if assignment['server_node'] == host: + server_iface = assignment['server_iface'] + client_node = assignment['client_node'] + port = assignment['port'] + client_node_token = client_node.replace('.', '_').replace(':', '_') + srv_log = f"{workspace_dir}/server_{server_iface}_{port}_{client_node_token}.log" + if self._scriptlet_enabled(): + trace_log = f"{workspace_dir}/strace_server_{server_iface}_{port}_{client_node_token}.log" + cmd = ( + f"timeout 120 strace -f -tt " + f"-e trace=bind,socket,setsockopt,listen,accept " + f"-o {trace_log} " + f"ibv_rc_pingpong -d {server_iface} -g {self.gid_index} -p {port} " + f"> {srv_log} 2>&1 &" + ) + else: + cmd = ( + f"timeout 120 ibv_rc_pingpong -d {server_iface} -g {self.gid_index} -p {port} " + f"> {srv_log} 2>&1 &" + ) + commands.append(cmd) + return commands + + def _generate_client_commands_for_host(self, host, assignments, workspace_dir): + commands = [] + workspace_dir = workspace_dir + for assignment in assignments: + if assignment['client_node'] == host: + client_iface = assignment['client_iface'] + server_node = assignment['server_node'] + port = assignment['port'] + server_node_token = server_node.replace('.', '_').replace(':', '_') + cmd = ( + f"timeout 30 ibv_rc_pingpong -d {client_iface} -g {self.gid_index} -p {port} " + f"{server_node} > {workspace_dir}/client_{client_iface}_{port}_{server_node_token}.log 2>&1 &" + ) + commands.append(cmd) + return commands + + def _generate_ibv_server_commands(self, host, test_groups, round_type, port_start, workspace_dir="/tmp/preflight"): + """ + Generate server commands for a round using class attributes. + + Args: + host: Host to generate commands for + test_groups: Test group configuration + round_type: "intra_group" or "inter_group" + port_start: Starting port number + workspace_dir: Temporary directory for logs + + Returns: + list: Server commands for this host + """ + assignments = self._calculate_port_assignments(test_groups, round_type, port_start) + return self._generate_server_commands_for_host(host, assignments, workspace_dir) + + def _generate_ibv_client_commands(self, host, test_groups, round_type, port_start, workspace_dir="/tmp/preflight"): + """ + Generate client commands for a round using class attributes. + + Args: + host: Host to generate commands for + test_groups: Test group configuration + round_type: "intra_group" or "inter_group" + port_start: Starting port number + workspace_dir: Temporary directory for logs + + Returns: + list: Client commands for this host + """ + assignments = self._calculate_port_assignments(test_groups, round_type, port_start) + return self._generate_client_commands_for_host(host, assignments, workspace_dir) + + def _format_ibv_commands_for_display(self, server_iface, client_iface, server_node, port): + """ + Return the ibv_rc_pingpong command lines shown in reports (match ScriptLet / batch timeouts). + + Omits shell redirections so operators can copy-paste after ssh; logs use /tmp/preflight/ when run via scripts. + """ + server_cmd = f"timeout 120 ibv_rc_pingpong -d {server_iface} -g {self.gid_index} -p {port}" + client_cmd = f"timeout 30 ibv_rc_pingpong -d {client_iface} -g {self.gid_index} -p {port} {server_node}" + return server_cmd, client_cmd + + def _create_collection_script(self, test_metadata_for_node, workspace_dir): + """ + Create ibv result collection script. + + Args: + test_metadata_for_node: Test metadata for a specific node + workspace_dir: Remote directory where client/server logs for this round live + + Returns: + str: Collection script content + """ + script_lines = [ + "#!/bin/bash", + "# Optimized ibv_rc_pingpong result collection script", + "# Only reports failed tests in key=value format", + "", + "# Function to analyze ibv_rc_pingpong output for success/failure", + "analyze_ibv_output() {", + " local log_file=\"$1\"", + " local test_key=\"$2\"", + " ", + " if [[ ! -f \"$log_file\" ]]; then", + " echo \"${test_key}=LOG_MISSING\"", + " return", + " fi", + " ", + " local content", + " content=$(<\"$log_file\")", + " ", + " # Check for success patterns", + " shopt -s nocasematch", + " if [[ \"$content\" =~ [0-9]+[[:space:]]+bytes[[:space:]]+in[[:space:]]+.*[[:space:]]+seconds ]] || [[ \"$content\" =~ local[[:space:]]+address:.*GID.*remote[[:space:]]+address:.*GID ]]; then", + " # Success - don't report (minimal output)", + " shopt -u nocasematch", + " return", + " fi", + " ", + " # Check for specific failure patterns", + " if [[ \"$content\" =~ Failed[[:space:]]+to[[:space:]]+modify[[:space:]]+QP.*to[[:space:]]+RTR ]]; then", + " echo \"${test_key}=QP_RTR_FAILED\"", + " elif [[ \"$content\" =~ Failed[[:space:]]+to[[:space:]]+modify[[:space:]]+QP.*to[[:space:]]+RTS ]]; then", + " echo \"${test_key}=QP_RTS_FAILED\"", + " elif [[ \"$content\" =~ Couldn.*t[[:space:]]+connect[[:space:]]+to|Unable[[:space:]]+to[[:space:]]+Connect ]]; then", + " echo \"${test_key}=CONNECTION_FAILED\"", + " elif [[ \"$content\" =~ Failed[[:space:]]+status[[:space:]]+transport[[:space:]]+retry[[:space:]]+counter[[:space:]]+exceeded ]]; then", + " echo \"${test_key}=TRANSPORT_RETRY_EXCEEDED\"", + " elif [[ \"$content\" =~ parse[[:space:]]+WC[[:space:]]+failed ]]; then", + " echo \"${test_key}=WC_PARSE_FAILED\"", + " elif [[ \"$content\" =~ No[[:space:]]+space[[:space:]]+left[[:space:]]+on[[:space:]]+device ]]; then", + " echo \"${test_key}=NO_SPACE_LEFT\"", + " elif [[ \"$content\" =~ Couldn.*t[[:space:]]+listen|listen[[:space:]]+to[[:space:]]+port|Address[[:space:]]+already[[:space:]]+in[[:space:]]+use|bind:[[:space:]]+Address[[:space:]]+already[[:space:]]+in[[:space:]]+use ]]; then", + " echo \"${test_key}=PORT_LISTEN_FAILED\"", + " elif [[ -z \"$content\" ]]; then", + " echo \"${test_key}=EMPTY_LOG\"", + " else", + " echo \"${test_key}=UNKNOWN_FAILURE\"", + " fi", + " shopt -u nocasematch", + "}", + "", + "# Analyze test results", + ] + + # Add analysis calls for each test involving this node + for test in test_metadata_for_node: + # Create a compact test identifier + test_key = f"{test['server_node']}-{test['client_node']}-{test['server_iface']}-{test['client_iface']}-{test['port']}" + + # Check if this node has client logs to analyze + if 'client_log_path' in test and test.get('node_role') == 'client': + client_log = test['client_log_path'] + script_lines.append(f"analyze_ibv_output \"{client_log}\" \"CLIENT_{test_key}\"") + + # Check if this node has server logs to analyze + if 'server_log_path' in test and test.get('node_role') == 'server': + server_log = test['server_log_path'] + script_lines.append(f"analyze_ibv_output \"{server_log}\" \"SERVER_{test_key}\"") + + script_lines.extend( + [ + "", + f"# Logs and artifacts remain under {workspace_dir} on each node", + "", + "exit 0", + ] + ) + + return "\n".join(script_lines) + + def _collect_results_with_scriptlet(self, test_metadata, workspace_dir): + """ + Collect test results using scriptlet with class attributes. + + Args: + test_metadata: Test metadata list + workspace_dir: Remote directory for this coordination round (same as server/client logs) + + Returns: + dict: Collected test results + """ + from cvs.lib.scriptlet import ScriptLet + + log.info(f"Starting ScriptLet-based result collection for {len(test_metadata)} tests") + + # Phase 1: Group test metadata by node (both client and server roles) + node_tests = {} + + for test in test_metadata: + client_node = test['client_node'] + server_node = test['server_node'] + + # Add test to client node's list + if client_node not in node_tests: + node_tests[client_node] = [] + # Mark this test as involving this node as client + test_copy = test.copy() + test_copy['node_role'] = 'client' + node_tests[client_node].append(test_copy) + + # Add test to server node's list (if different from client) + if server_node != client_node: + if server_node not in node_tests: + node_tests[server_node] = [] + # Mark this test as involving this node as server + test_copy = test.copy() + test_copy['node_role'] = 'server' + node_tests[server_node].append(test_copy) + + log.info(f"Distributing result collection across {len(node_tests)} nodes") + + scriptlet_debug = self._scriptlet_enabled() + + # Phase 2: Generate and execute collection scripts in parallel + results = {} + + with ScriptLet( + self.phdl, + debug=scriptlet_debug, + scriptlet_workspace=workspace_dir, + cleanup_on_init=False, + preserve_workspace_on_exit=True, + ) as scriptlet: + collect_mapping = {} + for node, node_test_list in node_tests.items(): + if node in self.phdl.reachable_hosts: + script_id = f"collect_{node}" + script_content = self._create_collection_script(node_test_list, workspace_dir) + scriptlet.create_script(script_id, script_content) + collect_mapping[node] = script_id + + if collect_mapping: + scriptlet.copy_script_list(collect_mapping) + exec_results = scriptlet.run_parallel_group(collect_mapping, timeout=60) + for node, script_output in exec_results.items(): + if script_output: + for line in script_output.strip().split('\n'): + if '=' in line and line.strip(): + key, value = line.split('=', 1) + results[key] = {'status': 'FAIL', 'error': value} + + # Phase 3: Fill in successful tests (those not reported as failures) + for test in test_metadata: + test_key = f"{test['server_node']}-{test['client_node']}-{test['server_iface']}-{test['client_iface']}-{test['port']}" + client_key = f"CLIENT_{test_key}" + server_key = f"SERVER_{test_key}" + + # If neither client nor server reported failure, mark as success + if client_key not in results and server_key not in results: + pair_key = ( + f"{test['server_node']} <-> {test['client_node']} ({test['server_iface']}->{test['client_iface']})" + ) + server_cmd, client_cmd = self._format_ibv_commands_for_display( + test['server_iface'], + test['client_iface'], + test['server_node'], + test['port'], + ) + results[pair_key] = { + 'status': 'PASS', + 'server_node': test['server_node'], + 'client_node': test['client_node'], + 'server_iface': test['server_iface'], + 'client_iface': test['client_iface'], + 'port': test['port'], + 'server_cmd': server_cmd, + 'client_cmd': client_cmd, + } + else: + # Convert failure keys to pair format + pair_key = ( + f"{test['server_node']} <-> {test['client_node']} ({test['server_iface']}->{test['client_iface']})" + ) + if client_key in results or server_key in results: + error_details = [] + if client_key in results: + error_details.append(f"Client: {results[client_key]['error']}") + if server_key in results: + error_details.append(f"Server: {results[server_key]['error']}") + + server_cmd, client_cmd = self._format_ibv_commands_for_display( + test['server_iface'], + test['client_iface'], + test['server_node'], + test['port'], + ) + results[pair_key] = { + 'status': 'FAIL', + 'server_node': test['server_node'], + 'client_node': test['client_node'], + 'server_iface': test['server_iface'], + 'client_iface': test['client_iface'], + 'port': test['port'], + 'error_details': error_details, + 'server_cmd': server_cmd, + 'client_cmd': client_cmd, + } + + # Phase 2 stores raw log keys (CLIENT_*/SERVER_*); Phase 3 adds display pair_keys. + # Drop intermediates so callers only see "node <-> node (iface->iface)" keys. + for k in list(results.keys()): + if k.startswith(('CLIENT_', 'SERVER_')): + del results[k] + + return results + + def _build_server_script(self, host, server_commands): + """Build server startup script for ibv_rc_pingpong processes.""" + if not server_commands: + return None + + commands_block = "\n".join(server_commands) + return f"""#!/bin/bash +set -e # Exit on any error + +# Start all ibv_rc_pingpong servers in background +{commands_block} + +# Allow servers to initialize +sleep 1 + +echo "All servers started on {host}" +exit 0 +""" + + def _build_client_script(self, host, client_commands): + """Build client execution script for ibv_rc_pingpong processes.""" + if not client_commands: + return None + + commands_block = "\n".join(client_commands) + return f"""#!/bin/bash +set -e # Exit on any error + +# Run all ibv_rc_pingpong clients in parallel +{commands_block} + +# Wait for all client processes to complete +wait + +echo "All clients completed on {host}" +exit 0 +""" diff --git a/cvs/lib/preflight/report.py b/cvs/lib/preflight/report.py new file mode 100644 index 000000000..4c691bfc8 --- /dev/null +++ b/cvs/lib/preflight/report.py @@ -0,0 +1,1473 @@ +""" +Report Generation Module + +This module provides functionality for generating preflight test reports in various formats. +""" + +import csv +import html +import json +from pathlib import Path + +from cvs.lib.preflight.base import PreflightCheck +from cvs.lib import globals + +log = globals.log + + +def get_nested_config(config_dict, section, key, default): + """ + Get configuration value from nested structure. + + Args: + config_dict: Full configuration dictionary + section: Section name (e.g., 'reporting') + key: Parameter key within the section + default: Default value if not found + + Returns: + Configuration value or default + """ + if not config_dict: + return default + + # Handle nested sections like 'connectivity_check.rdma' + sections = section.split('.') + current = config_dict + + for sec in sections: + if isinstance(current, dict) and sec in current: + current = current[sec] + else: + return default + + if isinstance(current, dict) and key in current: + return current[key] + return default + + +def _config_flag_enabled(value, default=True): + """Normalize mixed bool/string config flags.""" + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in ('1', 'true', 'yes', 'on') + return bool(value) + + +class PreflightReportGenerator(PreflightCheck): + """Generate preflight test reports.""" + + def __init__(self, phdl, results, config_dict=None): + """ + Initialize report generator. + + Args: + phdl: Parallel SSH handle for cluster nodes + results: Preflight test results to generate reports from + config_dict: Optional configuration dictionary + """ + super().__init__(phdl, config_dict) + self.results = results + + def run(self): + """ + Generate all configured reports. + + Returns: + dict: Report generation results + """ + report_results = {} + + summary = self._generate_preflight_summary() + report_results['summary'] = summary + self.results['summary'] = summary + + if _config_flag_enabled(get_nested_config(self.config_dict, 'reporting', 'generate_html_report', 'true')): + html_path, rdma_csv_path = self._generate_html_report() + report_results['html_report'] = html_path + if rdma_csv_path: + report_results['rdma_pairs_csv'] = rdma_csv_path + + return report_results + + def _generate_preflight_summary(self): + """Build summary dict from ``self.results`` (preflight_results bundle).""" + gid_results = self.results.get('gid_consistency', {}) + connectivity_results = self.results.get('rdma_connectivity', {}) + rocm_results = self.results.get('rocm_versions', {}) + interface_results = self.results.get('interface_names', {}) + reachability_results = self.results.get('node_reachability') + ssh_connectivity_results = self.results.get('ssh_connectivity') + summary = { + 'overall_status': 'PASS', + 'checks': { + 'ssh_reachability': self._summarize_reachability_results(reachability_results), + 'gid_consistency': self._summarize_gid_results(gid_results), + 'rdma_connectivity': self._summarize_connectivity_results(connectivity_results), + 'rocm_versions': self._summarize_rocm_results(rocm_results), + 'interface_names': self._summarize_interface_results(interface_results), + }, + 'recommendations': [], + } + + if ssh_connectivity_results: + summary['checks']['ssh_connectivity'] = self._summarize_ssh_connectivity_results(ssh_connectivity_results) + + # Determine overall status (skipped tests don't affect overall status) + for check_name, check_summary in summary['checks'].items(): + if check_summary['status'] == 'FAIL': + summary['overall_status'] = 'FAIL' + + # Generate recommendations + if summary['checks']['gid_consistency']['status'] == 'FAIL': + summary['recommendations'].append( + "Fix GID configuration on RDMA interfaces before running performance tests" + ) + + if summary['checks']['rdma_connectivity']['status'] == 'FAIL': + summary['recommendations'].append("Address RDMA connectivity issues between node pairs") + elif summary['checks']['rdma_connectivity']['status'] == 'SKIPPED': + summary['recommendations'].append("Consider running RDMA connectivity tests for comprehensive validation") + + if summary['checks']['rocm_versions']['status'] == 'FAIL': + summary['recommendations'].append("Ensure consistent ROCm versions across all cluster nodes") + + if summary['checks']['interface_names']['status'] == 'FAIL': + summary['recommendations'].append("Standardize RDMA interface naming across cluster nodes") + + if summary['overall_status'] == 'PASS': + summary['recommendations'].append("All preflight checks passed - cluster is ready for performance testing") + + return summary + + def _generate_html_report(self, output_path=None): + """Write HTML report for preflight test results. + + Returns: + tuple: (html_path_str, rdma_pairs_csv_path_str_or_None) + """ + from datetime import datetime + + if output_path is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_dir = get_nested_config(self.config_dict, 'reporting', 'artifacts_root_dir', '/tmp/preflight') + Path(output_dir).mkdir(parents=True, exist_ok=True) + output_path = Path(output_dir) / f"preflight_report_{timestamp}.html" + else: + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + + html_content = self._generate_html_content() + + # Write HTML file + with open(output_path, 'w', encoding='utf-8') as f: + f.write(html_content) + + log.info(f"HTML report generated: {output_path}") + + rdma_csv_path = None + if _config_flag_enabled(get_nested_config(self.config_dict, 'reporting', 'generate_rdma_pairs_csv', 'true')): + rdma_csv_path = self._write_rdma_pairs_csv(output_path) + if rdma_csv_path: + log.info(f"RDMA pairs CSV generated: {rdma_csv_path}") + + return str(output_path), rdma_csv_path + + def _summarize_gid_results(self, gid_results): + """Summarize GID consistency check results.""" + total_interfaces = 0 + ok_interfaces = 0 + failed_nodes = [] + + for node, result in gid_results.items(): + for interface, interface_result in result['interfaces'].items(): + total_interfaces += 1 + if interface_result.get('status') == 'OK': + ok_interfaces += 1 + + if result['status'] == 'FAIL': + failed_nodes.append(node) + + return { + 'status': 'PASS' if not failed_nodes else 'FAIL', + 'total_interfaces': total_interfaces, + 'ok_interfaces': ok_interfaces, + 'failed_nodes': failed_nodes, + 'summary': f"{ok_interfaces}/{total_interfaces} interfaces have valid GID", + } + + def _summarize_connectivity_results(self, connectivity_results): + """Summarize RDMA connectivity check results.""" + # Handle skipped tests (either by configuration or due to failures) + if connectivity_results.get('skipped', False) or connectivity_results.get('status') == 'SKIPPED': + msg = connectivity_results.get('message', 'RDMA connectivity test skipped') + excl_if = connectivity_results.get('excluded_nodes_interface_check') or [] + excl_gid = connectivity_results.get('excluded_nodes_gid') or [] + extra = [] + if excl_if: + extra.append(f"{len(excl_if)} failed interface presence") + if excl_gid: + extra.append(f"{len(excl_gid)} failed GID consistency") + if extra: + msg += " (" + "; ".join(extra) + ")" + return { + 'status': 'SKIPPED', + 'total_pairs': 0, + 'successful_pairs': 0, + 'failed_pairs': 0, + 'mode': connectivity_results.get('mode', 'unknown'), + 'summary': msg, + } + + # Handle normal test results + if 'failed_pairs' not in connectivity_results: + # Fallback for malformed results + return { + 'status': 'ERROR', + 'total_pairs': 0, + 'successful_pairs': 0, + 'failed_pairs': 0, + 'mode': connectivity_results.get('mode', 'unknown'), + 'summary': 'RDMA connectivity test failed to complete properly', + } + + summary = f"{connectivity_results['successful_pairs']}/{connectivity_results['total_pairs']} pairs connected successfully" + pruned = connectivity_results.get('pruned_nodes_after_intra') or [] + if pruned: + summary += f"; {len(pruned)} node(s) excluded from inter-group after Round 1 (intra-group)" + excl_iface = connectivity_results.get('excluded_nodes_interface_check') or [] + if excl_iface: + summary += f"; {len(excl_iface)} node(s) excluded from RDMA (failed interface presence check before mesh)" + excl_gid = connectivity_results.get('excluded_nodes_gid') or [] + if excl_gid: + summary += f"; {len(excl_gid)} node(s) excluded from RDMA (failed GID consistency before mesh)" + + return { + 'status': 'PASS' if connectivity_results['failed_pairs'] == 0 else 'FAIL', + 'total_pairs': connectivity_results['total_pairs'], + 'successful_pairs': connectivity_results['successful_pairs'], + 'failed_pairs': connectivity_results['failed_pairs'], + 'mode': connectivity_results['mode'], + 'summary': summary, + } + + def _summarize_rocm_results(self, rocm_results): + """Summarize ROCm version check results.""" + total_nodes = len(rocm_results) + consistent_nodes = sum(1 for result in rocm_results.values() if result['status'] == 'PASS') + failed_nodes = [node for node, result in rocm_results.items() if result['status'] == 'FAIL'] + + return { + 'status': 'PASS' if consistent_nodes == total_nodes else 'FAIL', + 'total_nodes': total_nodes, + 'consistent_nodes': consistent_nodes, + 'failed_nodes': failed_nodes, + 'summary': f"{consistent_nodes}/{total_nodes} nodes have consistent ROCm version", + } + + def _summarize_interface_results(self, interface_results): + """Summarize interface name check results.""" + total_nodes = len(interface_results) + compliant_nodes = sum(1 for result in interface_results.values() if result['status'] == 'PASS') + failed_nodes = [node for node, result in interface_results.items() if result['status'] == 'FAIL'] + + return { + 'status': 'PASS' if compliant_nodes == total_nodes else 'FAIL', + 'total_nodes': total_nodes, + 'compliant_nodes': compliant_nodes, + 'failed_nodes': failed_nodes, + 'summary': f"{compliant_nodes}/{total_nodes} nodes have compliant interface names", + } + + def _summarize_reachability_results(self, reachability_results): + """Summarize SSH reachability check results.""" + if not reachability_results: + return { + 'status': 'UNKNOWN', + 'total_nodes': 0, + 'reachable_nodes': 0, + 'unreachable_nodes': [], + 'summary': 'SSH reachability test not performed', + } + + total_nodes = reachability_results.get('total_nodes', 0) + reachable_nodes = reachability_results.get('reachable_nodes', 0) + unreachable_nodes = reachability_results.get('unreachable_nodes', []) + + status = 'PASS' if len(unreachable_nodes) == 0 else 'WARNING' + + return { + 'status': status, + 'total_nodes': total_nodes, + 'reachable_nodes': reachable_nodes, + 'unreachable_nodes': unreachable_nodes, + 'summary': f"{reachable_nodes}/{total_nodes} nodes reachable", + } + + def _summarize_ssh_connectivity_results(self, ssh_connectivity_results): + """Summarize SSH full mesh connectivity check results.""" + if not ssh_connectivity_results: + return { + 'status': 'UNKNOWN', + 'total_pairs': 0, + 'successful_pairs': 0, + 'failed_pairs': 0, + 'summary': 'SSH connectivity test not performed', + } + + if ssh_connectivity_results.get('skipped', False): + return { + 'status': 'SKIPPED', + 'total_pairs': 0, + 'successful_pairs': 0, + 'failed_pairs': 0, + 'summary': 'SSH connectivity test skipped', + } + + total_pairs = ssh_connectivity_results.get('total_pairs', 0) + successful_pairs = ssh_connectivity_results.get('successful_pairs', 0) + failed_pairs = ssh_connectivity_results.get('failed_pairs', 0) + + # Determine status based on results + if 'error' in ssh_connectivity_results: + status = 'ERROR' + summary = f"SSH connectivity test failed: {ssh_connectivity_results['error']}" + elif failed_pairs == 0: + status = 'PASS' + summary = f"{successful_pairs}/{total_pairs} pairs connected successfully" + else: + status = 'FAIL' + success_rate = (successful_pairs / total_pairs * 100) if total_pairs > 0 else 0 + summary = f"{successful_pairs}/{total_pairs} pairs connected successfully ({success_rate:.1f}%)" + + return { + 'status': status, + 'total_pairs': total_pairs, + 'successful_pairs': successful_pairs, + 'failed_pairs': failed_pairs, + 'summary': summary, + } + + def _generate_html_content(self): + """Generate the complete HTML content for the preflight report.""" + from datetime import datetime + + results = self.results + config_dict = self.config_dict + summary = results.get('summary', {}) + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + html = f""" + + + + + + GPU Cluster Preflight Check Report + + + +
+
+

GPU Cluster Preflight Check Report

+
+

Generated: {timestamp}

+

Overall Status: {summary.get('overall_status', 'UNKNOWN')}

+
+
+ + {self._generate_executive_summary_html(summary)} + {self._generate_gid_consistency_html(results.get('gid_consistency', {}))} + {self._generate_connectivity_html(results.get('rdma_connectivity', {}))} + {self._generate_ssh_connectivity_html(results.get('ssh_connectivity', {}))} + {self._generate_rocm_versions_html(results.get('rocm_versions', {}))} + {self._generate_interface_names_html(results.get('interface_names', {}))} + {self._generate_configuration_html(config_dict)} + {self._generate_recommendations_html(summary.get('recommendations', []))} +
+ + + """ + return html + + def _get_html_styles(self): + """Return CSS styles for the HTML report.""" + return """ + body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + line-height: 1.6; + margin: 0; + padding: 0; + background-color: #f5f5f5; + } + .container { + max-width: 1200px; + margin: 0 auto; + padding: 20px; + background-color: white; + box-shadow: 0 0 10px rgba(0,0,0,0.1); + } + header { + border-bottom: 3px solid #007acc; + padding-bottom: 20px; + margin-bottom: 30px; + } + h1 { + color: #333; + margin: 0; + } + h2 { + color: #007acc; + border-bottom: 2px solid #e0e0e0; + padding-bottom: 10px; + } + h3 { + color: #555; + } + .report-meta { + margin-top: 10px; + color: #666; + } + .status-pass { + color: #28a745; + font-weight: bold; + } + .status-fail { + color: #dc3545; + font-weight: bold; + } + .status-skipped { + color: #6c757d; + font-weight: bold; + } + .error-summary { + color: #dc3545; + font-weight: bold; + margin-bottom: 15px; + padding: 10px; + background-color: #f8d7da; + border: 1px solid #f5c6cb; + border-radius: 4px; + } + .summary-table { + width: 100%; + border-collapse: collapse; + margin: 20px 0; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + } + .summary-table th { + background-color: #f8f9fa; + font-weight: bold; + padding: 15px 12px; + text-align: left; + border-bottom: 2px solid #dee2e6; + } + .summary-table td { + padding: 12px; + border-bottom: 1px solid #dee2e6; + } + .summary-table tr:hover { + background-color: #f8f9fa; + } + .check-name { + font-weight: 600; + color: #495057; + } + .status-cell { + text-align: center; + width: 120px; + } + .status-badge { + display: inline-block; + padding: 6px 12px; + border-radius: 20px; + font-size: 0.9em; + font-weight: 600; + white-space: nowrap; + } + .status-badge.status-pass { + background-color: #d4edda; + color: #155724; + border: 1px solid #c3e6cb; + } + .status-badge.status-fail { + background-color: #f8d7da; + color: #721c24; + border: 1px solid #f5c6cb; + } + .status-badge.status-skipped { + background-color: #e2e3e5; + color: #6c757d; + border: 1px solid #d6d8db; + } + .results-cell { + text-align: center; + font-weight: 600; + color: #495057; + width: 100px; + } + .details-cell { + color: #6c757d; + font-size: 0.95em; + } + .summary-row-pass { + border-left: 4px solid #28a745; + } + .summary-row-fail { + border-left: 4px solid #dc3545; + } + .summary-row-skipped { + border-left: 4px solid #6c757d; + } + table { + width: 100%; + border-collapse: collapse; + margin: 20px 0; + } + th, td { + border: 1px solid #ddd; + padding: 12px; + text-align: left; + } + th { + background-color: #f2f2f2; + font-weight: bold; + } + tr:nth-child(even) { + background-color: #f9f9f9; + } + .gid-cell { + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 0.9em; + word-break: break-all; + max-width: 200px; + } + .gid-cell small { + color: #666; + font-weight: normal; + } + code { + background-color: #f8f9fa; + border: 1px solid #e9ecef; + border-radius: 3px; + padding: 2px 6px; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 0.85em; + color: #495057; + word-break: break-all; + display: block; + margin: 2px 0; + } + .connectivity-matrix { + display: grid; + gap: 2px; + margin: 20px 0; + } + .matrix-cell { + padding: 8px; + text-align: center; + border: 1px solid #ddd; + font-size: 12px; + } + .matrix-cell.header { + background-color: #f2f2f2; + font-weight: bold; + } + .matrix-cell.pass { + background-color: #d4edda; + color: #155724; + } + .matrix-cell.fail { + background-color: #f8d7da; + color: #721c24; + } + .matrix-cell.not-tested { + background-color: #e2e3e5; + color: #6c757d; + } + .error-list { + background-color: #f8d7da; + border: 1px solid #f5c6cb; + border-radius: 4px; + padding: 15px; + margin: 10px 0; + } + .error-list ul { + margin: 0; + padding-left: 20px; + } + .error-list li { + color: #721c24; + } + .recommendations { + background-color: #d1ecf1; + border: 1px solid #bee5eb; + border-radius: 4px; + padding: 20px; + margin: 20px 0; + } + .recommendations h3 { + color: #0c5460; + margin-top: 0; + } + .recommendations ul { + margin: 0; + padding-left: 20px; + } + .config-section { + background-color: #f8f9fa; + border: 1px solid #dee2e6; + border-radius: 4px; + padding: 15px; + margin: 10px 0; + } + .config-section pre { + background-color: #e9ecef; + padding: 10px; + border-radius: 4px; + overflow-x: auto; + } + + /* Collapsible details styling */ + details { + border: 1px solid #ddd; + border-radius: 6px; + margin: 15px 0; + overflow: hidden; + } + details summary { + background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%); + padding: 12px 15px; + cursor: pointer; + font-weight: 600; + border-bottom: 1px solid #ddd; + outline: none; + transition: background-color 0.2s ease; + } + details summary:hover { + background: linear-gradient(135deg, #e9ecef 0%, #dee2e6 100%); + } + details[open] summary { + background: linear-gradient(135deg, #007acc 0%, #0056b3 100%); + color: white; + border-bottom-color: #0056b3; + } + details summary::marker { + font-size: 1.2em; + } + details .content { + padding: 15px; + background-color: #fafbfc; + } + details table { + margin-top: 0; + } + """ + + def _generate_executive_summary_html(self, summary): + """Generate summary section with table layout.""" + if not summary or 'checks' not in summary: + return "

Summary

No summary data available.

" + + html = """ +
+

Summary

+ + + + + + + + + + + """ + + for check_name, check_summary in summary['checks'].items(): + status = check_summary['status'] + if status == 'PASS': + status_class = 'pass' + status_icon = '✅' + elif status == 'FAIL': + status_class = 'fail' + status_icon = '❌' + else: # SKIPPED + status_class = 'skipped' + status_icon = '⏭️' + + display_name = check_name.replace('_', ' ').title() + + # Extract key metrics from summary for the Results column + summary_text = check_summary['summary'] + results_text = summary_text + + # Try to extract numbers for cleaner display + if '/' in summary_text: + # Extract fraction like "102/102" or "48/51" + import re + + fraction_match = re.search(r'(\d+/\d+)', summary_text) + if fraction_match: + results_text = fraction_match.group(1) + + html += f""" + + + + + + + """ + + html += """ + +
CheckStatusResultsDetails
{display_name} + + {status_icon} {status} + + {results_text}{summary_text}
+
+ """ + return html + + def _generate_gid_consistency_html(self, gid_results): + """Generate GID inconsistencies section - only show failed nodes.""" + if not gid_results: + return "" + + # Filter to only failed nodes + failed_nodes = {node: result for node, result in gid_results.items() if result['status'] == 'FAIL'} + + if not failed_nodes: + return "" # No failures, no section needed + + html = """ +
+

GID Inconsistencies

+

The following nodes have GID consistency issues:

+ + + + + + + + + + """ + + for node, result in failed_nodes.items(): + # Build details for failed interfaces only + failed_interfaces = [] + issues = [] + + for iface_name, iface_result in result.get('interfaces', {}).items(): + if iface_result.get('status') != 'OK': + failed_interfaces.append(iface_name) + status = iface_result.get('status', 'UNKNOWN') + error = iface_result.get('error', '') + issues.append(f"{iface_name}: {status} ({error})" if error else f"{iface_name}: {status}") + + # Add general errors + if result.get('errors'): + issues.extend(result['errors']) + + failed_ifaces_str = ', '.join(failed_interfaces) if failed_interfaces else 'Multiple' + issues_str = '; '.join(issues) + + html += f""" + + + + + + """ + + html += """ + +
NodeFailed InterfacesIssues
{node}{failed_ifaces_str}{issues_str}
+
+ """ + return html + + @staticmethod + def _rdma_pair_row_fields(pair_key, pair_result): + """Parse one RDMA pair result for HTML/CSV rows. Returns None if ``pair_result`` is not a dict.""" + if not isinstance(pair_result, dict): + return None + error_details = pair_result.get('error_details', []) + error_msg = 'Connection failed' + if error_details: + for err in error_details: + if '|' in err: + error_msg = err.split('|')[0].strip() + else: + error_msg = err + if '(' in pair_key: + base_pair = pair_key.split(' (')[0] + interface = pair_key.split('(')[1].rstrip(')') + else: + base_pair = pair_key + interface = pair_result.get('interface', 'default') + return { + 'base_pair': base_pair, + 'interface': interface, + 'server_cmd': pair_result.get('server_cmd', 'N/A'), + 'client_cmd': pair_result.get('client_cmd', 'N/A'), + 'server_node': pair_result.get('server_node', 'unknown'), + 'client_node': pair_result.get('client_node', 'unknown'), + 'error_msg': error_msg, + 'server_output': pair_result.get('server_output', ''), + 'client_output': pair_result.get('client_output', ''), + 'status': pair_result.get('status'), + } + + @staticmethod + def _rdma_pair_error_plaintext(fields, max_log_chars=4096): + """Build plain-text error column for CSV (mirrors HTML failure detail content without markup).""" + if fields.get('status') != 'FAIL': + return '' + parts = [f"Error: {fields['error_msg']}"] + so = '' if fields.get('server_output') is None else str(fields.get('server_output')) + co = '' if fields.get('client_output') is None else str(fields.get('client_output')) + if so.strip() and 'EMPTY_LOG' not in so: + parts.append(f"Server Log:\n{so.strip()[:max_log_chars]}") + elif 'EMPTY_LOG' in so: + parts.append('Server Log: EMPTY_LOG') + if co.strip() and 'EMPTY_LOG' not in co: + parts.append(f"Client Log:\n{co.strip()[:max_log_chars]}") + elif 'EMPTY_LOG' in co: + parts.append('Client Log: EMPTY_LOG') + return '\n'.join(parts) + + def _write_rdma_pairs_csv(self, html_path: Path): + """ + Write **failed** RDMA ``pair_results`` rows to a UTF-8 CSV next to the HTML report. + + If there are no failures, no file is written (returns None). + + Columns match the HTML failure table: node pair, interface, ssh-wrapped commands, error text. + """ + rc = self.results.get('rdma_connectivity') or {} + if rc.get('skipped') or not rc.get('pair_results'): + return None + pair_results = rc['pair_results'] + failed_rows = [] + for pair_key in sorted(pair_results.keys()): + pr = pair_results[pair_key] + fields = self._rdma_pair_row_fields(pair_key, pr) + if fields is None or fields.get('status') != 'FAIL': + continue + failed_rows.append( + [ + fields['base_pair'], + fields['interface'], + f'ssh {fields["server_node"]} "{fields["server_cmd"]}"', + f'ssh {fields["client_node"]} "{fields["client_cmd"]}"', + self._rdma_pair_error_plaintext(fields), + ] + ) + if not failed_rows: + return None + out_path = html_path.parent / f'{html_path.stem}_rdma_pairs.csv' + headers = [ + 'Failed Node Pair', + 'Interface', + 'Server Command', + 'Client Command', + 'Error Details', + ] + with open(out_path, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow(headers) + writer.writerows(failed_rows) + return str(out_path) + + def _generate_rdma_topology_html(self, connectivity_results): + """ + Partition groups, hosts per group, inter-group mode, and multi-wave schedule for full_mesh runs. + """ + partition = connectivity_results.get('partition_groups') or {} + if not partition: + return "" + + inter_groups = connectivity_results.get('inter_groups') or {} + mode = connectivity_results.get('inter_group_mode') or '' + waves = connectivity_results.get('inter_group_waves') or [] + + def _group_block(title, groups_map): + parts = [f"

{html.escape(title)}

", '
'] + for gid in sorted(groups_map.keys(), key=lambda x: (str(type(x)), str(x))): + hosts = groups_map[gid] + if not isinstance(hosts, (list, tuple)): + hosts = [hosts] + host_list = ', '.join(html.escape(str(h)) for h in hosts) + parts.append( + f"
{html.escape(str(gid))}
" + f"
{host_list}
" + ) + parts.append('
') + return '\n'.join(parts) + + blocks = [ + '
', + '

Topology (partition groups)

', + '

Reachable nodes were partitioned for intra-group (Round 1) and inter-group (Round 2) testing.

', + _group_block('Groups and hosts (initial partition)', partition), + ] + + if inter_groups and inter_groups != partition: + blocks.append(_group_block('Groups and hosts (inter-group phase, after prune)', inter_groups)) + + mode_label = { + 'single_shot': 'single shot (one coordination round for all ordered group-pairs)', + 'multi_wave': 'multi-wave (ordered group-pairs split across waves)', + 'none': 'no inter-group phase (single group or empty)', + }.get(mode, html.escape(str(mode))) + + blocks.append(f'

Inter-group mode: {mode_label}

') + + ng = len(inter_groups) if inter_groups else len(partition) + chunk = connectivity_results.get('inter_group_wave_chunk') + if chunk is None and ng >= 2: + chunk = max(1, ng - 1) + elif chunk is None: + chunk = 1 + if ng >= 2 and mode == 'multi_wave': + n_ordered = ng * (ng - 1) + n_waves_approx = (n_ordered + chunk - 1) // chunk + blocks.append( + '

' + 'Why this many waves? With G partition groups, inter-group ' + 'testing schedules G×(G−1) ordered directed group-pairs ' + '(every Gi→Gj with i≠j), not one wave per source group where one group is server for all others. ' + 'Multi-wave splits that ordered list into chunks of up to ' + f'inter_full_mesh_group_pairs_per_wave (resolved chunk size {chunk}; ' + 'default max(1, G−1) when unset or auto), so the wave count is ' + f'ceil(G×(G−1) / {chunk}). With G={ng}, that is {n_ordered} ordered pairs ' + f'→ {n_waves_approx} wave(s).' + '

' + ) + + if waves: + blocks.append('

Inter-group waves (ordered group-pair keys)

') + blocks.append('
    ') + total_w = len(waves) + for w in waves: + wi = w.get('wave', '?') + n_gp = w.get('num_group_pairs', 0) + keys = w.get('group_pair_keys') or [] + preview = ', '.join(html.escape(str(k)) for k in keys[:8]) + if len(keys) > 8: + preview += f' … (+{len(keys) - 8} more)' + blocks.append( + f'
  • Wave {wi}/{total_w} — ' + f'{n_gp} ordered group-pair(s)' + f'
    {preview}
    • ' + ) + blocks.append('
') + + blocks.append('
') + return '\n'.join(blocks) + + def _generate_pruned_nodes_after_intra_html(self, records): + """HTML block listing nodes removed after Round 1 before inter-group tests.""" + if not records: + return "" + rows = [] + for r in records: + rows.append( + f"{html.escape(str(r.get('node', '')))}" + f"{html.escape(str(r.get('group_id', '')))}" + f"{html.escape(str(r.get('reason', '')))}" + ) + return f""" +
+

Nodes excluded before Round 2 (inter-group)

+

These nodes met the configured intra-group peer failure fraction threshold (see prune_failure_threshold): enough distinct peers in the same partition group had at least one failing intra test versus that node. They were omitted from inter-group tests.

+ + + {''.join(rows)} +
NodeGroupReason
+
+ """ + + def _rdma_failure_detail_table_rows(self, pair_results, round_tag, max_detail_rows, inter_wave=None): + """ + Build … fragments for failed pairs. ``round_tag`` None = all failures. + ``inter_wave`` filters inter-group rows to one wave (legacy rows without ``inter_wave`` count as wave 1). + Returns (html_fragment, rows_emitted). + """ + row_fragments = [] + if not pair_results: + return '', 0 + + for pair_key, pair_result in pair_results.items(): + if not isinstance(pair_result, dict) or pair_result.get('status') != 'FAIL': + continue + if round_tag is not None and pair_result.get('round') != round_tag: + continue + if inter_wave is not None: + if pair_result.get('round') != 'inter_group': + continue + pw = pair_result.get('inter_wave') or 1 + if pw != inter_wave: + continue + if max_detail_rows > 0 and len(row_fragments) >= max_detail_rows: + break + + fields = self._rdma_pair_row_fields(pair_key, pair_result) + if fields is None: + continue + base_pair = fields['base_pair'] + interface = fields['interface'] + server_cmd = fields['server_cmd'] + client_cmd = fields['client_cmd'] + server_node = fields['server_node'] + client_node = fields['client_node'] + error_msg = fields['error_msg'] + + error_details_html = f"Error: {html.escape(str(error_msg))}
" + + server_output = fields.get('server_output', '') + if server_output and server_output.strip() and 'EMPTY_LOG' not in server_output: + so = html.escape(server_output.strip()[:4096]) + error_details_html += f"
Server Log:
{so}
" + elif 'EMPTY_LOG' in str(server_output): + error_details_html += "
Server Log: EMPTY_LOG" + + client_output = fields.get('client_output', '') + if client_output and client_output.strip() and 'EMPTY_LOG' not in client_output: + co = html.escape(client_output.strip()[:4096]) + error_details_html += f"
Client Log:
{co}
" + elif 'EMPTY_LOG' in str(client_output): + error_details_html += "
Client Log: EMPTY_LOG" + + row_fragments.append( + f""" + + {html.escape(base_pair)} + {html.escape(interface)} + ssh {html.escape(str(server_node))} "{html.escape(str(server_cmd))}" + ssh {html.escape(str(client_node))} "{html.escape(str(client_cmd))}" + {error_details_html} + + """ + ) + + return ''.join(row_fragments), len(row_fragments) + + def _generate_connectivity_html(self, connectivity_results): + """Generate RDMA connectivity section: topology, pruned nodes, and failure analysis.""" + if not connectivity_results: + return "" + + if connectivity_results.get('skipped', False): + ex_if = connectivity_results.get('excluded_nodes_interface_check') or [] + ex_gid = connectivity_results.get('excluded_nodes_gid') or [] + if ex_if or ex_gid: + msg = html.escape(str(connectivity_results.get('message', 'RDMA connectivity test skipped'))) + iface_items = ''.join(f'
  • {html.escape(str(n))}
    • ' for n in ex_if) + gid_items = ''.join(f'
  • {html.escape(str(n))}
    • ' for n in ex_gid) + iface_block = ( + f"""
    +

    Excluded (interface presence)

      {iface_items}
    """ + if ex_if + else "" + ) + gid_block = ( + f"""
    +

    Excluded (GID consistency)

      {gid_items}
    """ + if ex_gid + else "" + ) + return f""" +
    +

    RDMA connectivity

    +

    {msg}

    + {iface_block} + {gid_block} +
    + """ + return "" + + failed_pairs = connectivity_results.get('failed_pairs', 0) + pruned = connectivity_results.get('pruned_nodes_after_intra') or [] + pair_results = connectivity_results.get('pair_results') or {} + partition_groups = connectivity_results.get('partition_groups') or {} + + topology_html = self._generate_rdma_topology_html(connectivity_results) if partition_groups else "" + + excluded_iface = connectivity_results.get('excluded_nodes_interface_check') or [] + excluded_iface_html = '' + if excluded_iface: + items = ''.join(f'
  • {html.escape(str(n))}
    • ' for n in excluded_iface) + excluded_iface_html = f""" +
    +

    Nodes excluded before RDMA mesh (interface presence)

    +

    These nodes failed the interface presence check (missing, down, or inactive expected RDMA interfaces) and were not included in RDMA connectivity testing.

    +
      {items}
    +
    + """ + + excluded_gid = connectivity_results.get('excluded_nodes_gid') or [] + excluded_gid_html = '' + if excluded_gid: + gitems = ''.join(f'
  • {html.escape(str(n))}
    • ' for n in excluded_gid) + excluded_gid_html = f""" +
    +

    Nodes excluded before RDMA mesh (GID consistency)

    +

    These nodes failed GID consistency for the configured index and were not included in RDMA connectivity testing.

    +
      {gitems}
    +
    + """ + + excluded_panels_html = excluded_iface_html + excluded_gid_html + + if failed_pairs == 0 and not pruned and not topology_html and not excluded_panels_html: + return "" + + cfg = self.config_dict or {} + try: + max_detail_rows = int(cfg.get('rdma_report_max_detail_rows', 5000)) + except (TypeError, ValueError): + max_detail_rows = 5000 + + pruned_html = self._generate_pruned_nodes_after_intra_html(pruned) + + if failed_pairs == 0: + return f""" +
    +

    RDMA connectivity

    + {excluded_panels_html} + {topology_html} + {pruned_html} +
    + """ + + use_rounds = any( + isinstance(p, dict) and p.get('round') in ('intra_group', 'inter_group') for p in pair_results.values() + ) + + table_header = """ + + + + + + + + + + + + """ + + blocks = [] + + def append_round_block(title, rtag, inter_wave_idx=None): + if rtag == 'inter_group' and inter_wave_idx is not None: + n_fail = sum( + 1 + for p in pair_results.values() + if isinstance(p, dict) + and p.get('status') == 'FAIL' + and p.get('round') == 'inter_group' + and (p.get('inter_wave') or 1) == inter_wave_idx + ) + else: + n_fail = sum( + 1 + for p in pair_results.values() + if isinstance(p, dict) and p.get('status') == 'FAIL' and p.get('round') == rtag + ) + if n_fail == 0: + return + log_preview = n_fail if max_detail_rows <= 0 else min(n_fail, max_detail_rows) + iw = inter_wave_idx if rtag == 'inter_group' else None + detail_body, n_shown = self._rdma_failure_detail_table_rows( + pair_results, rtag, max_detail_rows, inter_wave=iw + ) + omitted = n_fail - n_shown if max_detail_rows > 0 and n_fail > n_shown else 0 + omit_note = ( + f"

    {omitted:,} additional failures in this round omitted (rdma_report_max_detail_rows).

    " + if omitted + else "" + ) + safe_title = html.escape(title) + blocks.append( + f""" +
    + {safe_title} ({n_fail} failed tests) +
    +

    Detailed rows (up to {log_preview} shown):

    + {omit_note} + {table_header} + {detail_body} +
    +
    Failed Node PairInterfaceServer CommandClient CommandError Details
    + + + """ + ) + + if use_rounds: + append_round_block('Round 1 — Intra-group failures', 'intra_group') + + inter_mode = connectivity_results.get('inter_group_mode') or '' + inter_waves = connectivity_results.get('inter_group_waves') or [] + split_inter = inter_mode == 'multi_wave' and len(inter_waves) > 1 + + if split_inter: + total_w = len(inter_waves) + for wmeta in inter_waves: + widx = wmeta.get('wave', 1) + n_gp = wmeta.get('num_group_pairs', 0) + title = f'Inter-group wave {widx}/{total_w} ({n_gp} ordered group-pairs in this wave)' + append_round_block(title, 'inter_group', inter_wave_idx=widx) + else: + append_round_block('Round 2 — Inter-group failures', 'inter_group', inter_wave_idx=None) + + rounds_body = '\n'.join(blocks) if blocks else '

    No per-round failure breakdown available.

    ' + else: + log_preview = failed_pairs if max_detail_rows <= 0 else min(failed_pairs, max_detail_rows) + detail_body, n_shown = self._rdma_failure_detail_table_rows(pair_results, None, max_detail_rows) + omitted = failed_pairs - n_shown if max_detail_rows > 0 and failed_pairs > n_shown else 0 + omit_note = ( + f"

    {omitted:,} additional failures omitted (rdma_report_max_detail_rows).

    " + if omitted + else "" + ) + rounds_body = f""" +
    + All failed connection tests ({failed_pairs} failed tests) +
    +

    Detailed rows (up to {log_preview} shown):

    + {omit_note} + {table_header} + {detail_body} + + +
    +
    + """ + + return f""" +
    +

    RDMA connectivity

    + {excluded_panels_html} + {topology_html} +

    Connection failures

    +

    Found {failed_pairs} failed connection(s) out of {connectivity_results.get('total_pairs', 0)} total pairs tested.

    + {pruned_html} + {rounds_body} +
    + """ + + def _generate_ssh_connectivity_html(self, ssh_results): + """Generate SSH connection failures section - only show failed pairs.""" + if not ssh_results: + return "" + + if ssh_results.get('skipped', False): + return "" # Skip section entirely if test was skipped + + # Check if there are any failures + failed_pairs = ssh_results.get('failed_pairs', 0) + if failed_pairs == 0: + return "" # No failures, no section needed + + html = f""" +
    +

    SSH Connection Failures

    +

    Found {failed_pairs} failed SSH connection(s) out of {ssh_results.get('total_pairs', 0)} total pairs tested:

    + + + + + + + + + + """ + + # Only show failed pairs + if ssh_results.get('pair_results'): + # Sort failed pairs for consistent display + failed_pairs_list = [] + for pair_key, status in ssh_results['pair_results'].items(): + if status.startswith('FAILED'): + # Parse pair key (format: "source → target") + if ' → ' in pair_key: + source_node, target_node = pair_key.split(' → ', 1) + error_msg = ( + status.replace('FAILED - ', '') if 'FAILED - ' in status else 'SSH connection failed' + ) + failed_pairs_list.append((source_node.strip(), target_node.strip(), error_msg)) + + # Sort by source node, then target node + failed_pairs_list.sort(key=lambda x: (x[0], x[1])) + + # Generate table rows + for source_node, target_node, error_msg in failed_pairs_list: + html += f""" + + + + + + """ + + html += """ + +
    Source NodeTarget NodeError Details
    {source_node}{target_node}{error_msg}
    +
    + """ + + return html + + def _generate_rocm_versions_html(self, rocm_results): + """Generate ROCm version inconsistencies section - only show failed nodes.""" + if not rocm_results: + return "" + + # Filter to only failed nodes + failed_nodes = {node: result for node, result in rocm_results.items() if result['status'] == 'FAIL'} + + if not failed_nodes: + return "" # No failures, no section needed + + html = """ +
    +

    ROCm Version Inconsistencies

    +

    The following nodes have ROCm version mismatches:

    + + + + + + + + + + + """ + + for node, result in failed_nodes.items(): + errors = ', '.join(result.get('errors', [])) if result.get('errors') else 'Version mismatch' + + html += f""" + + + + + + + """ + + html += """ + +
    NodeDetected VersionExpected VersionIssue
    {node}{result.get('detected_version', 'Unknown')}{result.get('expected_version', 'Unknown')}{errors}
    +
    + """ + return html + + def _generate_interface_names_html(self, interface_results): + """Generate RDMA interface inconsistencies section - only show failed nodes.""" + if not interface_results: + return "" + + # Filter to only failed nodes + failed_nodes = {node: result for node, result in interface_results.items() if result['status'] == 'FAIL'} + + if not failed_nodes: + return "" # No failures, no section needed + + html = """ +
    +

    RDMA Interface Inconsistencies

    +

    The following nodes have RDMA interface issues:

    + + + + + + + + + + + + """ + + for node, result in failed_nodes.items(): + missing_interfaces = ', '.join(result.get('missing_interfaces', [])) or 'None' + inactive_interfaces = ', '.join(result.get('inactive_interfaces', [])) or 'None' + down_interfaces = ', '.join(result.get('down_interfaces', [])) or 'None' + errors = ', '.join(result.get('errors', [])) if result.get('errors') else 'Interface issues detected' + + html += f""" + + + + + + + + """ + + html += """ + +
    NodeMissingInactiveDownIssues
    {node}{missing_interfaces}{inactive_interfaces}{down_interfaces}{errors}
    +
    + """ + return html + + def _generate_configuration_html(self, config_dict): + """Generate configuration section.""" + html = """ +
    +

    Test Configuration

    +
    + 
    +        """
+
+        # Filter out comment fields for cleaner display
+        clean_config = {k: v for k, v in config_dict.items() if not k.startswith('_comment')}
+        html += json.dumps(clean_config, indent=2)
+
+        html += """
+
    +
    +
    + """ + return html + + def _generate_recommendations_html(self, recommendations): + """Generate recommendations section.""" + if not recommendations: + return "" + + html = """ +
    +
    +

    Recommendations

    +
      + """ + + for recommendation in recommendations: + html += f"
    • {recommendation}
      • " + + html += """ +
    +
    +
    + """ + return html diff --git a/cvs/lib/preflight/unittests/__init__.py b/cvs/lib/preflight/unittests/__init__.py new file mode 100644 index 000000000..391ce8983 --- /dev/null +++ b/cvs/lib/preflight/unittests/__init__.py @@ -0,0 +1 @@ +# Preflight unittests module diff --git a/cvs/lib/preflight/unittests/test_rdma_connectivity.py b/cvs/lib/preflight/unittests/test_rdma_connectivity.py new file mode 100644 index 000000000..5b71e5ccc --- /dev/null +++ b/cvs/lib/preflight/unittests/test_rdma_connectivity.py @@ -0,0 +1,485 @@ +# cvs/lib/preflight/unittests/test_rdma_connectivity.py +"""Unit tests for RDMA preflight connectivity (class-based implementation).""" + +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +# Add cvs package root (four levels up: unittests -> preflight -> lib -> cvs) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) + +from cvs.lib.preflight.base import partition_nodes_into_groups +from cvs.lib.preflight.rdma_connectivity import RdmaConnectivityCheck + + +def _make_checker( + phdl=None, + node_list=None, + *, + expected_interfaces=None, + parallel_group_size=2, + **kwargs, +): + """Build RdmaConnectivityCheck with sensible test defaults.""" + phdl = phdl or MagicMock() + node_list = node_list or ['A', 'B', 'C', 'D'] + return RdmaConnectivityCheck( + phdl, + node_list, + mode=kwargs.get('mode', 'full_mesh'), + port_range=kwargs.get('port_range', '5000-6000'), + timeout=kwargs.get('timeout', 30), + expected_interfaces=expected_interfaces, + gid_index=kwargs.get('gid_index', '3'), + parallel_group_size=parallel_group_size, + config_dict=kwargs.get('config_dict', {}), + ) + + +class TestPartitionNodesIntoGroups(unittest.TestCase): + """Test node partitioning (PreflightCheck.partition_nodes_into_groups / base alias).""" + + def test_partition_4_nodes_group_size_2(self): + nodes = ['A', 'B', 'C', 'D'] + groups = partition_nodes_into_groups(nodes, group_size=2) + + self.assertEqual(len(groups), 2) + self.assertEqual(groups['group_1'], ['A', 'B']) + self.assertEqual(groups['group_2'], ['C', 'D']) + + def test_partition_4_nodes_group_size_4(self): + nodes = ['A', 'B', 'C', 'D'] + groups = partition_nodes_into_groups(nodes, group_size=4) + + self.assertEqual(len(groups), 1) + self.assertEqual(groups['group_1'], ['A', 'B', 'C', 'D']) + + def test_partition_6_nodes_group_size_4(self): + nodes = ['A', 'B', 'C', 'D', 'E', 'F'] + groups = partition_nodes_into_groups(nodes, group_size=4) + + self.assertEqual(len(groups), 2) + self.assertEqual(groups['group_1'], ['A', 'B', 'C', 'D']) + self.assertEqual(groups['group_2'], ['E', 'F']) + + def test_partition_5_nodes_group_size_2(self): + """Odd node count: last group is smaller (common with parallel_group_size=2).""" + nodes = ['A', 'B', 'C', 'D', 'E'] + groups = partition_nodes_into_groups(nodes, group_size=2) + + self.assertEqual(len(groups), 3) + self.assertEqual(groups['group_1'], ['A', 'B']) + self.assertEqual(groups['group_2'], ['C', 'D']) + self.assertEqual(groups['group_3'], ['E']) + + def test_partition_empty_list(self): + groups = partition_nodes_into_groups([], group_size=2) + self.assertEqual(len(groups), 0) + + def test_partition_single_node(self): + groups = partition_nodes_into_groups(['A'], group_size=2) + self.assertEqual(len(groups), 1) + self.assertEqual(groups['group_1'], ['A']) + + +class TestCalculatePortAssignments(unittest.TestCase): + """Test RdmaConnectivityCheck._calculate_port_assignments.""" + + def setUp(self): + self.interfaces = ['if1', 'if2', 'if3', 'if4', 'if5', 'if6', 'if7', 'if8'] + self.port_start = 5000 + + def test_intragroup_assignments_2_nodes(self): + checker = _make_checker(expected_interfaces=self.interfaces) + test_groups = {'group_1': ['A', 'B']} + + assignments = checker._calculate_port_assignments(test_groups, 'intra_group', self.port_start) + + self.assertEqual(len(assignments), 128) + + a_to_b = [a for a in assignments if a['server_node'] == 'A' and a['client_node'] == 'B'] + b_to_a = [a for a in assignments if a['server_node'] == 'B' and a['client_node'] == 'A'] + + self.assertEqual(len(a_to_b), 64) + self.assertEqual(len(b_to_a), 64) + + a_to_b_interfaces = [(a['server_iface'], a['client_iface']) for a in a_to_b] + expected_combinations = [(s_if, c_if) for s_if in self.interfaces for c_if in self.interfaces] + self.assertEqual(len(a_to_b_interfaces), len(expected_combinations)) + for combo in expected_combinations: + self.assertIn(combo, a_to_b_interfaces) + + def test_intragroup_assignments_4_nodes(self): + checker = _make_checker(expected_interfaces=self.interfaces) + test_groups = {'group_1': ['A', 'B', 'C', 'D']} + + assignments = checker._calculate_port_assignments(test_groups, 'intra_group', self.port_start) + + self.assertEqual(len(assignments), 768) + + self_connections = [a for a in assignments if a['server_node'] == a['client_node']] + self.assertEqual(len(self_connections), 0) + + node_pairs = {(a['server_node'], a['client_node']) for a in assignments} + self.assertEqual(len(node_pairs), 12) + + def test_intergroup_assignments(self): + checker = _make_checker(expected_interfaces=self.interfaces) + test_groups = {'group_1_to_group_2': {'group1': ['A', 'B'], 'group2': ['C', 'D']}} + + assignments = checker._calculate_port_assignments(test_groups, 'inter_group', self.port_start) + + self.assertEqual(len(assignments), 256) + + for assignment in assignments: + self.assertIn(assignment['server_node'], ['A', 'B']) + self.assertIn(assignment['client_node'], ['C', 'D']) + + def test_port_assignment_uniqueness_per_server(self): + checker = _make_checker(expected_interfaces=self.interfaces) + test_groups = {'group_1': ['A', 'B']} + + assignments = checker._calculate_port_assignments(test_groups, 'intra_group', self.port_start) + + a_assignments = [a for a in assignments if a['server_node'] == 'A'] + a_ports = [a['port'] for a in a_assignments] + + self.assertEqual(len(set(a_ports)), 64) + self.assertEqual(min(a_ports), self.port_start) + self.assertEqual(max(a_ports), self.port_start + 63) + + b_assignments = [a for a in assignments if a['server_node'] == 'B'] + b_ports = [a['port'] for a in b_assignments] + + self.assertEqual(len(set(b_ports)), 64) + self.assertEqual(min(b_ports), self.port_start) + self.assertEqual(max(b_ports), self.port_start + 63) + + +class TestConnectivityMathInvariant(unittest.TestCase): + """Total mocked tests should be invariant to parallel_group_size when mesh is full.""" + + def setUp(self): + self.nodes = ['A', 'B', 'C', 'D'] + self.interfaces = ['if1', 'if2', 'if3', 'if4', 'if5', 'if6', 'if7', 'if8'] + + @patch.object(RdmaConnectivityCheck, '_execute_test_round_with_retries') + def test_execute_full_mesh_group_size_invariant(self, mock_execute_round): + def make_side_effect(checker): + invocation = [0] + + def mock_execute_side_effect(test_groups, round_type, port_start, **kwargs): + assignments = checker._calculate_port_assignments(test_groups, round_type, port_start) + n = invocation[0] + invocation[0] += 1 + return {f'assignment_{n}_{i}': f'mock_result_{n}_{i}' for i in range(len(assignments))} + + return mock_execute_side_effect + + mock_phdl = MagicMock() + mock_phdl.reachable_hosts = self.nodes + + c2 = _make_checker( + phdl=mock_phdl, + node_list=self.nodes, + expected_interfaces=self.interfaces, + parallel_group_size=2, + ) + mock_execute_round.side_effect = make_side_effect(c2) + results_group2, _, _ = c2._execute_full_mesh(5000) + + c4 = _make_checker( + phdl=mock_phdl, + node_list=self.nodes, + expected_interfaces=self.interfaces, + parallel_group_size=4, + ) + mock_execute_round.side_effect = make_side_effect(c4) + results_group4, _, _ = c4._execute_full_mesh(5000) + + self.assertEqual(len(results_group2), len(results_group4)) + + expected_total = 4 * 3 * 8 * 8 + self.assertEqual(len(results_group2), expected_total) + self.assertEqual(len(results_group4), expected_total) + + @patch.object(RdmaConnectivityCheck, '_execute_test_round_with_retries') + def test_full_mesh_five_nodes_eight_ifaces_parallel_group_size_2(self, mock_execute_round): + """ + Full mesh total tests = N × (N − 1) × I²; parallel_group_size only batches work, not totals. + + With N=5, I=8: 5 × 4 × 64 = 1280. (If a live run shows 768 with five hosts, that is 4×3×8×8 — + typically only four reachable nodes participated in RDMA, or a stale report.) + """ + nodes = ['A', 'B', 'C', 'D', 'E'] + interfaces = ['if1', 'if2', 'if3', 'if4', 'if5', 'if6', 'if7', 'if8'] + n, i = len(nodes), len(interfaces) + expected_total = n * (n - 1) * i * i + + def make_side_effect(checker): + invocation = [0] + + def mock_execute_side_effect(test_groups, round_type, port_start, **kwargs): + assignments = checker._calculate_port_assignments(test_groups, round_type, port_start) + inv = invocation[0] + invocation[0] += 1 + return {f'assignment_{inv}_{j}': f'mock_{inv}_{j}' for j in range(len(assignments))} + + return mock_execute_side_effect + + mock_phdl = MagicMock() + mock_phdl.reachable_hosts = nodes + + checker = _make_checker( + phdl=mock_phdl, + node_list=nodes, + expected_interfaces=interfaces, + parallel_group_size=2, + ) + mock_execute_round.side_effect = make_side_effect(checker) + + results, _, _ = checker._execute_full_mesh(5000) + + self.assertEqual(len(results), expected_total) + self.assertEqual(expected_total, 1280) + + @patch.object(RdmaConnectivityCheck, '_execute_test_round_with_retries') + def test_execute_full_mesh_completeness(self, mock_execute_round): + generated_assignments = [] + + mock_phdl = MagicMock() + mock_phdl.reachable_hosts = self.nodes + + checker = _make_checker( + phdl=mock_phdl, + node_list=self.nodes, + expected_interfaces=self.interfaces, + parallel_group_size=2, + ) + + def capture_assignments(test_groups, round_type, port_start, **kwargs): + assignments = checker._calculate_port_assignments(test_groups, round_type, port_start) + generated_assignments.extend(assignments) + return {f'result_{i}': f'mock_{i}' for i in range(len(assignments))} + + mock_execute_round.side_effect = capture_assignments + + checker._execute_full_mesh(5000) + + connections = {(a['server_node'], a['client_node']) for a in generated_assignments} + + expected_connections = { + ('A', 'B'), + ('A', 'C'), + ('A', 'D'), + ('B', 'A'), + ('B', 'C'), + ('B', 'D'), + ('C', 'A'), + ('C', 'B'), + ('C', 'D'), + ('D', 'A'), + ('D', 'B'), + ('D', 'C'), + } + + self.assertEqual(connections, expected_connections, 'Algorithm should test all directed node pairs') + + +class TestAlgorithmBugDetection(unittest.TestCase): + """Inter-group port assignments for one round (same helper as production).""" + + def test_incomplete_connectivity_with_2_groups(self): + nodes = ['A', 'B', 'C', 'D'] + interfaces = ['if1', 'if2'] + + groups = partition_nodes_into_groups(nodes, group_size=2) + checker = _make_checker(expected_interfaces=interfaces, node_list=nodes) + # Round 1 is the first inter-group round used by _execute_intergroup (range(1, num_groups)) + inter_group_tests = checker._generate_intergroup_pairs(groups, round_num=1) + all_inter_assignments = checker._calculate_port_assignments(inter_group_tests, 'inter_group', 5000) + + # Two directed inter-group tests (g1→g2 and g2→g1) × 2×2 nodes × 2×2 iface pairs = 2×16 = 32 + self.assertEqual(len(all_inter_assignments), 32) + + +class TestGenerateIntergroupPairs(unittest.TestCase): + """RdmaConnectivityCheck._generate_intergroup_pairs round-robin behavior.""" + + def setUp(self): + self.groups_2 = {'group_1': ['A', 'B'], 'group_2': ['C', 'D']} + self.groups_3 = {'group_1': ['A', 'B'], 'group_2': ['C', 'D'], 'group_3': ['E', 'F']} + + def test_round_2_with_2_groups(self): + checker = _make_checker() + # For 2 groups, round_num=2 yields no pairs (group2_idx == i for both i); round 1 is used in practice. + pairs_empty = checker._generate_intergroup_pairs(self.groups_2, round_num=2) + self.assertEqual(pairs_empty, {}) + pairs_r1 = checker._generate_intergroup_pairs(self.groups_2, round_num=1) + self.assertTrue(pairs_r1) + + def test_all_rounds_with_2_groups(self): + checker = _make_checker() + all_pairs = {} + num_groups = len(self.groups_2) + + # Match _execute_intergroup: range(1, num_groups) → only round 1 when num_groups == 2 + for round_num in range(1, num_groups): + round_pairs = checker._generate_intergroup_pairs(self.groups_2, round_num) + all_pairs.update(round_pairs) + + connections = set() + for _test_name, test_config in all_pairs.items(): + group1_nodes = test_config['group1'] + group2_nodes = test_config['group2'] + for node1 in group1_nodes: + for node2 in group2_nodes: + connections.add((node1, node2)) + + expected_connections = { + ('A', 'C'), + ('A', 'D'), + ('B', 'C'), + ('B', 'D'), + ('C', 'A'), + ('C', 'B'), + ('D', 'A'), + ('D', 'B'), + } + + self.assertEqual( + connections, + expected_connections, + 'Round-robin algorithm should generate all inter-group directed pairs', + ) + + def test_round_generation_with_3_groups(self): + checker = _make_checker() + all_pairs = {} + num_groups = len(self.groups_3) + + for round_num in range(2, num_groups + 1): + round_pairs = checker._generate_intergroup_pairs(self.groups_3, round_num) + all_pairs.update(round_pairs) + + self.assertTrue(all_pairs) + + +class TestExecuteIntragroup(unittest.TestCase): + """RdmaConnectivityCheck._execute_intragroup delegates to coordination.""" + + @patch.object(RdmaConnectivityCheck, '_execute_test_round_with_retries') + def test_execute_intragroup(self, mock_execute_round): + mock_execute_round.return_value = {'test1': 'result1'} + + groups = {'group_1': ['A', 'B'], 'group_2': ['C', 'D']} + mock_phdl = MagicMock() + checker = _make_checker(phdl=mock_phdl, node_list=['A', 'B', 'C', 'D'], expected_interfaces=['if1', 'if2']) + + result = checker._execute_intragroup(groups, 5000) + + mock_execute_round.assert_called_once_with(groups, 'intra_group', 5000, work_segment='intra_group') + self.assertEqual(result, {'test1': 'result1'}) + + +class TestExecuteIntergroup(unittest.TestCase): + """RdmaConnectivityCheck._execute_intergroup runs all ordered group pairs in one coordination round.""" + + @patch.object(RdmaConnectivityCheck, '_execute_test_round_with_retries') + def test_execute_intergroup_with_2_groups(self, mock_execute_round): + mock_execute_round.return_value = {'test1': 'result1'} + + groups = {'group_1': ['A', 'B'], 'group_2': ['C', 'D']} + mock_phdl = MagicMock() + checker = _make_checker(phdl=mock_phdl, node_list=['A', 'B', 'C', 'D'], expected_interfaces=['if1', 'if2']) + + checker._execute_intergroup(groups, 5000) + + self.assertEqual(mock_execute_round.call_count, 1) + test_groups, round_type, port = mock_execute_round.call_args[0] + self.assertEqual(mock_execute_round.call_args.kwargs.get('work_segment'), 'inter_group_legacy') + self.assertEqual(round_type, 'inter_group') + self.assertEqual(port, 5000) + self.assertIn('group_1_to_group_2', test_groups) + self.assertIn('group_2_to_group_1', test_groups) + + +class TestIntraPeerPrune(unittest.TestCase): + """Round 1 prune: peer failure fraction ≥ threshold (default 50%).""" + + def test_prunes_nodes_at_half_threshold_three_node_group(self): + """One failed pair A–B: A and B each have 1/2 peers failed → prune; C has 0/2 → keep.""" + groups = {'group_1': ['A', 'B', 'C']} + intra_results = { + 't1': {'status': 'FAIL', 'server_node': 'A', 'client_node': 'B'}, + } + checker = _make_checker(config_dict={'connectivity_check': {'rdma': {'prune_failure_threshold': 0.5}}}) + pruned_set, records, new_g = checker._apply_intra_prune(intra_results, groups) + self.assertEqual(pruned_set, {'A', 'B'}) + self.assertEqual(new_g['group_1'], ['C']) + self.assertEqual(len(records), 2) + + def test_no_prune_when_below_threshold(self): + groups = {'group_1': ['A', 'B', 'C']} + intra_results = { + 't1': {'status': 'FAIL', 'server_node': 'A', 'client_node': 'B'}, + } + checker = _make_checker(config_dict={'connectivity_check': {'rdma': {'prune_failure_threshold': 0.6}}}) + pruned_set, _records, new_g = checker._apply_intra_prune(intra_results, groups) + self.assertEqual(pruned_set, set()) + self.assertEqual(new_g['group_1'], ['A', 'B', 'C']) + + +class TestGenerateServerCommandsScriptletDebug(unittest.TestCase): + """Server scriptlet lines: optional strace when scriptlet is set.""" + + def test_default_no_strace(self): + checker = _make_checker(expected_interfaces=['if1'], config_dict={}) + groups = {'group_1': ['A', 'B']} + cmds = checker._generate_ibv_server_commands('A', groups, 'intra_group', 5000, '/tmp/preflight') + self.assertTrue(cmds) + self.assertTrue(all('strace' not in c for c in cmds)) + + def test_scriptlet_debug_adds_strace(self): + checker = _make_checker(expected_interfaces=['if1'], config_dict={'debug': {'scriptlet': True}}) + groups = {'group_1': ['A', 'B']} + cmds = checker._generate_ibv_server_commands('A', groups, 'intra_group', 5000, '/tmp/preflight') + self.assertTrue(cmds) + for c in cmds: + with self.subTest(cmd=c[:80]): + self.assertIn('strace', c) + self.assertIn('-e trace=bind,socket,setsockopt,listen,accept', c) + self.assertIn('-o /tmp/preflight/strace_server_', c) + + def test_scriptlet_debug_accepts_string_true(self): + checker = _make_checker(expected_interfaces=['if1'], config_dict={'debug': {'scriptlet': 'true'}}) + groups = {'group_1': ['A', 'B']} + cmds = checker._generate_ibv_server_commands('A', groups, 'intra_group', 5000, '/tmp/preflight') + self.assertTrue(any('strace' in c for c in cmds)) + + +class TestEdgeCases(unittest.TestCase): + """Edge cases for port assignments.""" + + def setUp(self): + self.interfaces = ['if1', 'if2'] + self.port_start = 5000 + + def test_single_node_group(self): + checker = _make_checker(expected_interfaces=self.interfaces) + test_groups = {'group_1': ['A']} + + assignments = checker._calculate_port_assignments(test_groups, 'intra_group', self.port_start) + self.assertEqual(len(assignments), 0) + + def test_empty_interfaces(self): + checker = _make_checker(expected_interfaces=[]) + test_groups = {'group_1': ['A', 'B']} + + assignments = checker._calculate_port_assignments(test_groups, 'intra_group', self.port_start) + self.assertEqual(len(assignments), 0) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/cvs/lib/preflight/version_check.py b/cvs/lib/preflight/version_check.py new file mode 100644 index 000000000..95de33af8 --- /dev/null +++ b/cvs/lib/preflight/version_check.py @@ -0,0 +1,84 @@ +""" +ROCm Version Checking Module + +This module provides functionality for checking ROCm version consistency across cluster nodes. +""" + +import json + +from cvs.lib.preflight.base import PreflightCheck + + +class RocmVersionCheck(PreflightCheck): + """Check ROCm version consistency across cluster nodes.""" + + def __init__(self, phdl, expected_version, config_dict=None): + """ + Initialize ROCm version check. + + Args: + phdl: Parallel SSH handle for cluster nodes + expected_version: Expected ROCm version string (e.g., "6.2.0") + config_dict: Optional configuration dictionary + """ + super().__init__(phdl, config_dict) + self.expected_version = expected_version + + @staticmethod + def _extract_rocm_version(output): + """Parse ROCm version from amd-smi JSON output.""" + raw = output.strip() + if not raw or raw == 'NOT_FOUND': + return 'NOT_FOUND' + + try: + payload = json.loads(raw) + except json.JSONDecodeError: + return 'NOT_FOUND' + + version = None + if isinstance(payload, list) and payload: + version = payload[0].get('rocm_version') + elif isinstance(payload, dict): + version = payload.get('rocm_version') + + if isinstance(version, str) and version.strip(): + return version.strip() + return 'NOT_FOUND' + + def run(self): + """ + Execute ROCm version check across all cluster nodes. + + Returns: + dict: Results with per-node ROCm version status + """ + self.log_info(f"Checking ROCm version consistency (expected: {self.expected_version})") + + cmd = """ + if command -v amd-smi >/dev/null 2>&1; then + amd-smi version --json 2>/dev/null || echo 'NOT_FOUND' + else + echo 'NOT_FOUND' + fi + """ + self.results = {} + out_dict = self.phdl.exec(cmd) + + for node, output in out_dict.items(): + version = self._extract_rocm_version(output) + self.results[node] = { + 'detected_version': version, + 'expected_version': self.expected_version, + 'status': 'PASS' if version == self.expected_version else 'FAIL', + 'errors': [], + } + + if version == 'NOT_FOUND': + self.results[node]['errors'].append("ROCm version not found via amd-smi JSON output") + elif version != self.expected_version: + self.results[node]['errors'].append( + f"Version mismatch: expected {self.expected_version}, found {version}" + ) + + return self.results diff --git a/cvs/lib/scriptlet.py b/cvs/lib/scriptlet.py new file mode 100644 index 000000000..a86be9ec1 --- /dev/null +++ b/cvs/lib/scriptlet.py @@ -0,0 +1,401 @@ +""" +ScriptLet: Efficient script distribution and execution for cluster operations. + +This module provides the ScriptLet class for managing script lifecycle across +cluster nodes, optimizing SSH operations by batching script distribution and +execution. +""" + +import os +import tempfile +import logging +import shlex + +log = logging.getLogger(__name__) + + +class ScriptLet: + """ + A utility class for efficient script distribution and execution across cluster nodes. + + Key features: + - Batch script copying to minimize SSH overhead + - Parallel script execution across nodes + - Centralized script lifecycle management + - Optional debug mode for script preservation + - Minimal output collection for performance + + Responsibilities: + 1. Generate and manage script files locally + 2. Copy scripts to remote nodes efficiently + 3. Execute scripts and collect results in batch + 4. Handle cleanup of temporary files + 5. Support debugging through script preservation + """ + + def __init__( + self, + phdl, + debug=False, + cleanup_on_exit=True, + scriptlet_workspace="/tmp/preflight", + cleanup_on_init=False, + preserve_workspace_on_exit=False, + ): + """ + Initialize ScriptLet with a parallel SSH handle. + + Args: + phdl: Parallel SSH handle with upload_file_list (or copy_script_list) and exec_cmd_list methods + debug: If True, preserve scripts for debugging (don't auto-cleanup) + cleanup_on_exit: Whether to cleanup temp files on destruction + scriptlet_workspace: Remote directory for storing scripts and logs + cleanup_on_init: Whether to cleanup workspace on initialization + preserve_workspace_on_exit: If True, exit cleanup removes only ScriptLet ``.sh`` files, not + ``rm -rf`` of ``scriptlet_workspace`` (keeps co-located logs and other artifacts in that directory). + """ + self.phdl = phdl + self.debug = debug + self.cleanup_on_exit = cleanup_on_exit and not debug + self.workspace_dir = scriptlet_workspace + self.cleanup_on_init = bool(cleanup_on_init) + self.preserve_workspace_on_exit = bool(preserve_workspace_on_exit) + self.local_scripts = {} # {script_id: local_path} + self.remote_scripts = {} # {script_id: {"host": host, "path": remote_path}} + + # Clean up temp directory for fresh test environment if requested + # Directory will be recreated when needed by _ensure_workspace_directory() + if cleanup_on_init: + self._cleanup_workspace_directory(force=True) + + log.info( + f"ScriptLet initialized (debug={debug}, cleanup_on_exit={self.cleanup_on_exit}, workspace={self.workspace_dir})" + ) + + def create_script(self, script_id, script_content, remote_path=None): + """ + Create a local script file with given content. + + Args: + script_id: Unique identifier for this script + script_content: Shell script content + remote_path: Target path on remote nodes (auto-generated if None) + + Returns: + str: Local path of created script file + + Raises: + ValueError: If script_id already exists + """ + if script_id in self.local_scripts: + raise ValueError(f"Script ID '{script_id}' already exists") + + if remote_path is None: + remote_path = f"{self.workspace_dir}/scriptlet_{script_id}.sh" + + # Create temporary local script file + with tempfile.NamedTemporaryFile(mode='w', suffix='.sh', delete=False, prefix=f'scriptlet_{script_id}_') as f: + f.write(script_content) + local_path = f.name + + # Make script executable + os.chmod(local_path, 0o755) + + self.local_scripts[script_id] = local_path + self.remote_scripts[script_id] = {"host": None, "path": remote_path} # host will be set when copied + + log.debug(f"Created script '{script_id}': {local_path} -> {remote_path}") + return local_path + + def copy_script(self, script_id, target_nodes=None): + """ + Copy a single script to specified nodes or all reachable nodes. + + Args: + script_id: ID of script to copy + target_nodes: List of nodes to copy to (all reachable nodes if None) + + Returns: + Dict: {node: "SUCCESS/FAILED - details"} + + Raises: + ValueError: If script_id doesn't exist + """ + if script_id not in self.local_scripts: + raise ValueError(f"Script '{script_id}' not found. Create it first with create_script()") + + local_path = self.local_scripts[script_id] + remote_path = self.remote_scripts[script_id]["path"] + + if target_nodes is None: + target_nodes = self.phdl.reachable_hosts + + node_path_map = {node: (local_path, remote_path) for node in target_nodes} + + log.info(f"Copying script '{script_id}' to {len(target_nodes)} nodes") + results = self.phdl.upload_file_list(node_path_map) + + # Log copy results + successful = [r for r in results.values() if "SUCCESS" in r] + failed = [r for r in results.values() if "FAILED" in r] + log.info(f"Script copy completed: {len(successful)} successful, {len(failed)} failed") + + if failed and log.isEnabledFor(logging.WARNING): + for failure in failed[:3]: # Log first 3 failures + log.warning(f"Copy failed: {failure}") + + return results + + def copy_script_list(self, script_mapping): + """ + Copy different scripts to different nodes using phdl native parallel SCP (libssh2). + + Args: + script_mapping: {node: script_id} mapping + + Returns: + Dict: {node: "node: SUCCESS/FAILED - details"} + + Raises: + ValueError: If any script_id doesn't exist + """ + missing_scripts = [sid for sid in script_mapping.values() if sid not in self.local_scripts] + if missing_scripts: + raise ValueError(f"Scripts not found: {missing_scripts}") + + # Update remote_scripts with host information + for host, script_id in script_mapping.items(): + if script_id in self.remote_scripts: + self.remote_scripts[script_id]["host"] = host + + node_path_map = { + node: (self.local_scripts[sid], self.remote_scripts[sid]["path"]) for node, sid in script_mapping.items() + } + + # When cleanup_on_init=True, the workspace was just force-cleaned and must be recreated. + # For cleanup_on_init=False flows (e.g., subsequent phases sharing the same workspace), + # skip the extra mkdir round-trip to reduce overhead. + if self.cleanup_on_init: + self._ensure_workspace_directory(list(script_mapping.keys())) + + log.info(f"Copying {len(set(script_mapping.values()))} different scripts to {len(script_mapping)} nodes") + + results = self.phdl.upload_file_list(node_path_map) + + successful = [r for r in results.values() if "SUCCESS" in r] + failed = [r for r in results.values() if "FAILED" in r] + log.info(f"Script copy completed: {len(successful)} successful, {len(failed)} failed") + + if failed: + for failure in list(failed)[:3]: + log.warning(f"Copy failed: {failure}") + + return results + + def run_parallel_group(self, script_mapping, timeout=30, cleanup_after_run=False): + """ + Execute different scripts on different nodes in parallel using a single phdl.exec_cmd_list call. + + This is the efficient way to run scripts when each node needs to execute a different script, + avoiding the "Script not targeted for this node" messages and enabling true parallelization. + + Args: + script_mapping: {node: script_id} mapping + timeout: Command timeout in seconds + cleanup_after_run: Whether to cleanup scripts after execution (default: False, + let __exit__ handle cleanup for better efficiency) + + Returns: + Dict: {node: execution_output} + + Raises: + ValueError: If any script_id doesn't exist + """ + # Validate all script IDs exist + missing_scripts = [sid for sid in script_mapping.values() if sid not in self.local_scripts] + if missing_scripts: + raise ValueError(f"Scripts not found: {missing_scripts}") + + # Build command list in the same order as phdl.reachable_hosts to avoid KeyError + cmd_list = [] + executed_nodes = [] + + for node in self.phdl.reachable_hosts: + if node in script_mapping: + script_id = script_mapping[node] + remote_script_path = self.remote_scripts[script_id]["path"] + quoted_remote_script_path = shlex.quote(remote_script_path) + cmd_list.append(f"chmod +x {quoted_remote_script_path} && {quoted_remote_script_path}") + else: + # Node is reachable but no script for it - use dummy command + cmd_list.append("echo 'No script for this node'") + executed_nodes.append(node) + + log.info( + f"Executing {len([c for c in cmd_list if 'chmod' in c])} scripts across {len(executed_nodes)} reachable nodes" + ) + + # Execute all scripts in parallel using single phdl call + raw_results = self.phdl.exec_cmd_list(cmd_list, timeout=timeout, print_console=False) + + # Convert results back to node-keyed dictionary, only include nodes that had actual scripts + results = {} + for i, node in enumerate(executed_nodes): + if node in script_mapping and node in raw_results: + results[node] = raw_results[node] + + # Handle cleanup if requested + if cleanup_after_run and not self.debug: + self.cleanup_script_list(script_mapping) + + return results + + def cleanup_script_list(self, script_mapping): + """ + Clean up different scripts from different nodes in batch mode for efficiency. + + Args: + script_mapping: {node: script_id} mapping (same format as copy_script_list) + """ + if self.debug: + log.debug("Cleanup skipped due to debug mode") + return + + if not script_mapping: + return + + # Clean up local files (only once per unique script) + unique_scripts = set(script_mapping.values()) + for script_id in unique_scripts: + if script_id in self.local_scripts: + try: + os.unlink(self.local_scripts[script_id]) + del self.local_scripts[script_id] + except OSError as e: + log.warning(f"Failed to cleanup local script '{script_id}': {e}") + + # Build parallel cleanup commands for remote files + cleanup_commands = [] + for node in self.phdl.reachable_hosts: + if node in script_mapping: + script_id = script_mapping[node] + if script_id in self.remote_scripts: + remote_path = self.remote_scripts[script_id]["path"] + cleanup_commands.append(f"rm -f {shlex.quote(remote_path)} 2>/dev/null || true") + else: + cleanup_commands.append("true") # No-op for nodes without scripts + else: + cleanup_commands.append("true") # No-op for nodes not in mapping + + # Execute all cleanup commands in parallel + if cleanup_commands: + log.info(f"Cleaning up {len(unique_scripts)} scripts from {len(script_mapping)} nodes") + try: + self.phdl.exec_cmd_list(cleanup_commands, print_console=False) + except Exception as e: + log.warning(f"Failed to cleanup remote scripts: {e}") + + # Remove scripts from remote tracking + for script_id in unique_scripts: + if script_id in self.remote_scripts: + del self.remote_scripts[script_id] + + def cleanup(self): + """ + Clean up local and remote script files. + """ + if self.debug: + log.debug("Cleanup skipped due to debug mode") + return + + # Cleanup all scripts using batch method for efficiency + if self.remote_scripts or self.local_scripts: + # Use batch cleanup for remote scripts if any exist + if self.remote_scripts: + # Build script mapping from remote_scripts structure + script_mapping = {} + for script_id, script_info in self.remote_scripts.items(): + if script_info["host"] is not None: + script_mapping[script_info["host"]] = script_id + + if script_mapping: + log.debug( + f"Using batch cleanup for {len(script_mapping)} scripts across {len(script_mapping)} hosts" + ) + self.cleanup_script_list(script_mapping) + + # Clean any remaining local scripts that weren't cleaned by batch + for script_id in list(self.local_scripts.keys()): + if script_id in self.local_scripts: + try: + os.unlink(self.local_scripts[script_id]) + except OSError as e: + log.warning(f"Failed to cleanup local script '{script_id}': {e}") + del self.local_scripts[script_id] + + if not self.preserve_workspace_on_exit: + self._cleanup_workspace_directory() + + log.debug("Cleanup completed for all scripts") + + def _cleanup_workspace_directory(self, force=False): + """Clean up the entire workspace directory on remote nodes.""" + if self.debug and not force: + log.debug(f"Debug mode enabled - preserving workspace directory '{self.workspace_dir}' for inspection") + return + + cleanup_cmd = f"rm -rf {shlex.quote(self.workspace_dir)} 2>/dev/null || true" + try: + self.phdl.exec(cleanup_cmd, print_console=False) + log.info(f"Cleaned up workspace directory: {self.workspace_dir}") + except Exception as e: + log.warning(f"Failed to cleanup workspace directory '{self.workspace_dir}': {e}") + + def _ensure_workspace_directory(self, target_nodes): + """Ensure workspace directory exists on target nodes.""" + if not target_nodes: + return + + mkdir_cmd = f"mkdir -p {shlex.quote(self.workspace_dir)}" + try: + # Create directory on all target nodes using simple exec + self.phdl.exec(mkdir_cmd, timeout=10, print_console=False) + log.debug(f"Ensured workspace directory exists: {self.workspace_dir}") + except Exception as e: + log.warning(f"Failed to create workspace directory '{self.workspace_dir}': {e}") + + def list_scripts(self): + """ + List all managed scripts and their paths. + + Returns: + Dict: {script_id: {'local': local_path, 'remote': remote_path, 'host': host}} + """ + scripts_info = {} + + for script_id in self.local_scripts: + remote_info = self.remote_scripts.get(script_id, {"path": "Not set", "host": "Not assigned"}) + scripts_info[script_id] = { + 'local': self.local_scripts[script_id], + 'remote': remote_info["path"], + 'host': remote_info["host"], + } + return scripts_info + + def __del__(self): + """Cleanup on destruction if enabled.""" + if self.cleanup_on_exit: + try: + self.cleanup() + except Exception: + # Don't raise exceptions in destructor + pass + + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit with cleanup.""" + if not self.debug: + self.cleanup() diff --git a/cvs/lib/unittests/test_linux_utils.py b/cvs/lib/unittests/test_linux_utils.py index 5c6b5292b..efbee8220 100644 --- a/cvs/lib/unittests/test_linux_utils.py +++ b/cvs/lib/unittests/test_linux_utils.py @@ -20,7 +20,7 @@ def test_get_rdma_nic_dict_with_hyphenated_devices(self): result = linux_utils.get_rdma_nic_dict(mock_phdl) # Verify the function was called with correct command - mock_phdl.exec.assert_called_once_with('sudo rdma link') + mock_phdl.exec.assert_called_once_with('rdma link') # Verify all RDMA devices are parsed self.assertIn('node1', result) diff --git a/cvs/parsers/schemas.py b/cvs/parsers/schemas.py index c8dcb3e74..495f92f8f 100644 --- a/cvs/parsers/schemas.py +++ b/cvs/parsers/schemas.py @@ -749,15 +749,213 @@ def validate_path_not_placeholder(cls, v: str, info) -> str: return v +# ============================================================================= +# Preflight Check Configuration Schema +# ============================================================================= + + +class PreflightParallelismConfig(BaseModel): + """Legacy parallelism settings for preflight checks.""" + + model_config = ConfigDict(extra="allow") + + parallel_group_size: int = Field( + default=128, + ge=2, + le=512, + description=("Legacy alias for RDMA grouping. Prefer connectivity_check.rdma.nodes_per_full_mesh_group."), + ) + + +class PreflightDebugConfig(BaseModel): + """Debug and troubleshooting settings for preflight checks.""" + + model_config = ConfigDict(extra="allow") + + scriptlet: bool = Field( + default=False, + description=( + "Enable ScriptLet debug: preserve generated scripts/logs on remote nodes. " + "For RDMA connectivity, also wraps each ibv_rc_pingpong server in strace with " + "per-test traces under /tmp/preflight/strace_server__.log (expensive at scale)." + ), + ) + + +class PreflightNodeCheckConfig(BaseModel): + """Individual node validation settings.""" + + model_config = ConfigDict(extra="allow") + + gid_index: str = Field(default="3", description="GID index to check on all RDMA interfaces (typically 3 for RoCE)") + expected_rocm_version: str = Field(default="6.2.0", description="Expected ROCm version across all cluster nodes") + rdma_interfaces: List[str] = Field( + default_factory=lambda: ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"], + description="List of specific RDMA interface names to check and test", + ) + + +class PreflightRdmaConfig(BaseModel): + """RDMA connectivity testing settings.""" + + model_config = ConfigDict(extra="allow") + + connectivity_mode: str = Field(default="basic", description="RDMA connectivity testing: basic, full_mesh, or skip") + nodes_per_full_mesh_group: int = Field( + default=128, + ge=2, + le=512, + description=( + "Number of nodes in each full-mesh partition group (2-512). " + "Smaller groups use fewer resources per node but require more rounds." + ), + ) + parallel_group_size: int = Field( + default=128, + ge=2, + le=512, + description="Legacy alias for nodes_per_full_mesh_group.", + ) + ibv_test_timeout: int = Field( + default=90, + ge=1, + description="Timeout in seconds for RDMA connectivity tests using ibv_rc_pingpong", + ) + ibv_test_port_range: str = Field( + default="10000-50000", description="Port range for RDMA connectivity tests (format: start-end)" + ) + inter_full_mesh_group_pairs_per_wave: str = Field( + default="auto", description="Max ordered group-pairs per wave during inter-group testing ('auto' or integer)" + ) + inter_group_wave_pairs: str = Field( + default="auto", + description="Legacy alias for inter_full_mesh_group_pairs_per_wave.", + ) + prune_failure_threshold: float = Field( + default=0.5, + gt=0.0, + le=1.0, + description=( + "Round 1 (intra) prune before inter-group: prune nodes whose fraction of peers with ≥1 FAIL " + "intra test is ≥ this value (default 0.5). Peers counted per distinct other node in the same partition group." + ), + ) + port_retry_max: int = Field( + default=3, + ge=0, + le=10, + description=( + "After each ScriptLet wave (intra/inter), rerun only pairs whose logs show PORT_LISTEN_FAILED, " + "up to this many extra batches with new TCP ports (default 3)." + ), + ) + port_retry_gap: int = Field( + default=1000, + ge=1, + le=65535, + description=( + "When remapping ports for PORT_LISTEN_FAILED retries, start at (max port in batch) + this gap " + "to reduce overlap with ephemeral ports." + ), + ) + exclude_failed_interface_nodes: str = Field( + default="true", + description=( + "Legacy hint for reporting: preflight now prunes interface- and GID-failed nodes from the SSH " + "host list before RDMA; interface failures are not run in the mesh regardless of this flag." + ), + ) + + @field_validator('connectivity_mode') + @classmethod + def validate_connectivity_check(cls, v: str) -> str: + """Validate RDMA connectivity check setting.""" + valid_modes = ['basic', 'full_mesh', 'skip'] + if v not in valid_modes: + raise ValueError(f"connectivity_mode must be one of: {', '.join(valid_modes)}") + return v + + @field_validator('ibv_test_port_range') + @classmethod + def validate_port_range(cls, v: str) -> str: + """Validate port range format.""" + try: + start, end = map(int, v.split('-')) + if start >= end or start < 1024 or end > 65535: + raise ValueError("Invalid port range") + except (ValueError, AttributeError): + raise ValueError("ibv_test_port_range must be in format 'start-end' with valid port numbers") + return v + + +class PreflightConnectivityCheckConfig(BaseModel): + """Connectivity check settings by protocol.""" + + model_config = ConfigDict(extra="allow") + + rdma: PreflightRdmaConfig = Field(default_factory=PreflightRdmaConfig, description="RDMA connectivity settings") + + +class PreflightReportingConfig(BaseModel): + """Report generation and output settings.""" + + model_config = ConfigDict(extra="allow") + + generate_html_report: str = Field(default="true", description="Whether to generate HTML report") + artifacts_root_dir: str = Field( + default="/tmp/preflight", + description=( + "Root directory for preflight artifacts. HTML report output and RDMA full_mesh ScriptLet logs use " + "/rdma_connectivity_workspace/// on each node (NFS-friendly)." + ), + ) + generate_rdma_pairs_csv: str = Field( + default="true", + description="If true, write preflight_report_*_rdma_pairs.csv beside the HTML report (failed pairs only)", + ) + + +class PreflightConfigFile(BaseModel): + """ + Schema for preflight check configuration file. + + Uses nested structure organized by execution phase for better organization. + """ + + model_config = ConfigDict(extra="allow") # Allow comment fields + + parallelism: PreflightParallelismConfig = Field( + default_factory=PreflightParallelismConfig, description="Parallel execution settings" + ) + debug: PreflightDebugConfig = Field( + default_factory=PreflightDebugConfig, description="Debug and troubleshooting options" + ) + node_check: PreflightNodeCheckConfig = Field( + default_factory=PreflightNodeCheckConfig, description="Individual node validation settings" + ) + connectivity_check: PreflightConnectivityCheckConfig = Field( + default_factory=PreflightConnectivityCheckConfig, description="Inter-node connectivity tests" + ) + reporting: PreflightReportingConfig = Field( + default_factory=PreflightReportingConfig, description="Report generation and output settings" + ) + + def validate_config_file( config_path: Union[str, Path], config_type: str = "auto" -) -> Union[AortaBenchmarkConfigFile, ClusterConfigFile, PytorchXditWanConfigFile, PytorchXditFluxConfigFile]: +) -> Union[ + AortaBenchmarkConfigFile, + ClusterConfigFile, + PytorchXditWanConfigFile, + PytorchXditFluxConfigFile, + PreflightConfigFile, +]: """ Load and validate a configuration file. Args: config_path: Path to configuration file (YAML or JSON) - config_type: Type of config - "aorta", "cluster", "pytorch_xdit_wan", "pytorch_xdit_flux", or "auto" (detect from content) + config_type: Type of config - "aorta", "cluster", "pytorch_xdit_wan", "pytorch_xdit_flux", "preflight", or "auto" (detect from content) Returns: Validated Pydantic model @@ -788,6 +986,8 @@ def validate_config_file( if config_type == "auto": if "node_dict" in raw_config: config_type = "cluster" + elif "preflight" in raw_config: + config_type = "preflight" elif "aorta_path" in raw_config: config_type = "aorta" elif "config" in raw_config and "benchmark_params" in raw_config: @@ -807,13 +1007,19 @@ def validate_config_file( else: raise ValueError( f"Cannot auto-detect config type for {config_path}. " - f"Specify config_type='aorta', config_type='cluster', config_type='pytorch_xdit_wan', or config_type='pytorch_xdit_flux'" + f"Specify config_type='aorta', config_type='cluster', config_type='pytorch_xdit_wan', config_type='pytorch_xdit_flux', or config_type='preflight'" ) # Validate with appropriate schema try: if config_type == "cluster": return ClusterConfigFile.model_validate(raw_config) + elif config_type == "preflight": + # Extract preflight section for validation + if "preflight" in raw_config: + return PreflightConfigFile.model_validate(raw_config["preflight"]) + else: + raise ValueError("Preflight config must contain 'preflight' section") elif config_type == "aorta": return AortaBenchmarkConfigFile.model_validate(raw_config) elif config_type == "pytorch_xdit_wan": diff --git a/cvs/tests/preflight/README.md b/cvs/tests/preflight/README.md new file mode 100644 index 000000000..a1a4bde9b --- /dev/null +++ b/cvs/tests/preflight/README.md @@ -0,0 +1,334 @@ +# GPU Cluster Preflight Checks + +A comprehensive validation system for GPU clusters before running performance tests, training workloads, and inference tasks. + +## Overview + +The preflight checks system validates essential cluster health and configuration consistency across all nodes. It performs four critical validations: + +1. **GID Consistency** - Ensures RDMA interfaces have valid Global Identifier entries +2. **RDMA Connectivity** - Tests node-to-node RDMA communication using `rping` +3. **ROCm Version Consistency** - Verifies consistent ROCm versions across all nodes +4. **RDMA Interface Presence** - Validates that expected RDMA interfaces are present + +## Quick Start + +### Basic Usage + +```bash +# Run preflight checks with default configuration +cvs run preflight_checks \ + --cluster_file cluster.json \ + --config_file cvs/input/config_file/preflight/preflight_config.json +``` + +### With Custom HTML Report + +```bash +# Generate HTML report in specific location +cvs run preflight_checks \ + --cluster_file cluster.json \ + --config_file preflight_config.json \ + --html /path/to/preflight_report.html \ + --self-contained-html +``` + +## Test Modes + +### Basic Mode (Default) +- Tests adjacent node pairs (like current IB performance tests) +- Fast execution (~30 seconds for 8 nodes) +- 14.3% coverage for 8-node cluster (4 out of 28 possible pairs) +- Good for quick validation + +### Full Mesh Mode +- Tests all possible node pair combinations +- Comprehensive coverage (100% of all pairs) +- Longer execution (~5-10 minutes for 8 nodes) +- Uses batched approach to maximize parallelism +- Recommended for thorough validation + +### Sample Mode +- Tests random 20% of all possible node pairs +- Balanced speed vs coverage (~1-2 minutes for 8 nodes) +- Good for regular health checks + +## Configuration + +### Default Configuration File +Located at: `cvs/input/config_file/preflight/preflight_config.json` + +```json +{ + "preflight": { + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.2.0", + "rdma_interfaces": ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + }, + "connectivity_check": { + "rdma": { + "connectivity_mode": "basic", + "ibv_test_timeout": "10", + "ibv_test_port_range": "10000-50000" + } + }, + "reporting": { + "generate_html_report": "true", + "artifacts_root_dir": "/tmp/preflight" + } + } +} +``` + +### Key Parameters + +- **`connectivity_check.rdma.connectivity_mode`**: `"basic"`, `"full_mesh"`, or `"skip"` +- **`node_check.expected_rocm_version`**: ROCm version expected across all nodes +- **`node_check.rdma_interfaces`**: List of expected RDMA interface names +- **`connectivity_check.rdma.ibv_test_timeout`**: Timeout in seconds for ibv_rc_pingpong tests +- **`connectivity_check.rdma.ibv_test_port_range`**: Port range for parallel ibv_rc_pingpong tests + +## Output and Reporting + +### Console Output +``` +✅ GID Consistency: PASS (64/64 interfaces have GID index 3) +⚪ RDMA Connectivity: SKIPPED (Test skipped by configuration) +✅ ROCm Versions: PASS (All nodes running 6.2.0) +✅ Interface Names: PASS (All interfaces match rocep*s0 pattern) + +Overall Status: PASS - Cluster ready for performance testing (connectivity not tested) +``` + +### HTML Report +Comprehensive HTML report includes: +- Executive summary with pass/fail status +- Detailed per-node results for each check +- RDMA connectivity matrix showing all tested pairs +- Configuration details and recommendations +- Error details for failed checks + +## Integration with Performance Tests + +### Recommended Workflow + +```bash +# 1. Run preflight checks first +cvs run preflight_checks \ + --cluster_file cluster.json \ + --config_file preflight_config.json + +# 2. If preflight passes, run IB performance tests +cvs run ib_perf_bw_test \ + --cluster_file cluster.json \ + --config_file ib_config.json + +# 3. Run RCCL training tests +cvs run rccl_multinode_default_cvs \ + --cluster_file cluster.json \ + --config_file rccl_config.json + +# 4. Run inference workloads +cvs run pytorch_xdit_wan \ + --cluster_file cluster.json \ + --config_file inference_config.json +``` + +### Benefits of Preflight Validation + +- **Early Problem Detection**: Catch configuration issues before expensive performance tests +- **Time Savings**: Avoid running long performance tests on misconfigured clusters +- **Clear Diagnostics**: Detailed error reporting for quick issue resolution +- **Comprehensive Coverage**: Validates all critical cluster health aspects + +## Architecture + +### Test Execution Flow + +``` +1. Load and validate cluster + preflight configurations +2. Test SSH connectivity to all nodes +3. Run parallel preflight checks: + - GID consistency across all RDMA interfaces + - RDMA connectivity using rping (mode-dependent) + - ROCm version consistency via amd-smi + - Interface naming pattern compliance +4. Generate comprehensive summary and HTML report +5. Return overall PASS/FAIL status +``` + +### Parallel Execution + +- **SSH Operations**: Parallel execution across all cluster nodes +- **RDMA Connectivity**: Batched parallel testing to maximize throughput +- **Error Collection**: Continues all checks even if some fail (collect-all mode) + +## Troubleshooting + +### Common Issues + +#### GID Check Failures +```bash +# Check RDMA driver status +lsmod | grep rdma + +# Verify interface status +rdma link show + +# Check GID entries manually +cat /sys/class/infiniband/*/ports/1/gids/3 +``` + +#### RDMA Connectivity Failures +```bash +# Verify rping availability +which rping + +# Check firewall status +sudo ufw status + +# Test manual connectivity +rping -s -p 9000 # Server +rping -c -a -p 9000 # Client +``` + +#### ROCm Version Mismatches +```bash +# Check ROCm version on each node +amd-smi version + +# Verify consistent installation +ls -la /opt/rocm/ +``` + +#### Interface Name Inconsistencies +```bash +# List RDMA interfaces +ls /sys/class/infiniband/ + +# Check interface details +ibv_devinfo +``` + +### Performance Considerations + +| Mode | 8 Nodes | 16 Nodes | 32 Nodes | +|------|---------|----------|----------| +| Basic | ~30s | ~45s | ~60s | +| Full Mesh | ~5-10min | ~20-30min | ~2-3hrs | +| Skip | ~5s | ~5s | ~5s | + +### Network Requirements + +- **SSH Access**: Passwordless SSH to all cluster nodes +- **RDMA Interfaces**: Active InfiniBand or RoCE interfaces +- **Port Access**: ibv_test_port_range must not be blocked by firewalls +- **Privileges**: Some checks may require sudo access + +## Files and Structure + +``` +cvs/cvs/tests/preflight/ +├── __init__.py +├── preflight_checks.py # Main test module +└── README.md # This file + +cvs/cvs/lib/ +└── preflight_lib.py # Core preflight functions + +cvs/cvs/input/config_file/preflight/ +├── preflight_config.json # Default configuration +└── README_preflight_config.md # Configuration guide +``` + +## Advanced Usage + +### Custom Configuration + +```json +{ + "preflight": { + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.2.0", + "rdma_interfaces": ["mlx5_0", "mlx5_1"] + }, + "connectivity_check": { + "rdma": { + "connectivity_mode": "full_mesh", + "ibv_test_timeout": "15", + "ibv_test_port_range": "10000-10999" + } + }, + "reporting": { + "generate_html_report": "true", + "artifacts_root_dir": "/shared/preflight_reports" + } + } +} +``` + +### Scripted Validation + +```bash +#!/bin/bash +# Automated cluster validation script + +CLUSTER_FILE="production_cluster.json" +PREFLIGHT_CONFIG="production_preflight.json" + +echo "Running preflight checks..." +if cvs run preflight_checks \ + --cluster_file "$CLUSTER_FILE" \ + --config_file "$PREFLIGHT_CONFIG"; then + + echo "✅ Preflight checks passed - proceeding with performance tests" + + # Run performance tests + cvs run ib_perf_bw_test --cluster_file "$CLUSTER_FILE" --config_file ib_config.json + cvs run rccl_multinode_default_cvs --cluster_file "$CLUSTER_FILE" --config_file rccl_config.json + +else + echo "❌ Preflight checks failed - fix issues before running performance tests" + exit 1 +fi +``` + +### Large Cluster Considerations + +For clusters with 100+ nodes: +- Use `"basic"` mode for regular health checks +- Use `"full_mesh"` mode only for critical validation +- Consider hierarchical testing (rack-by-rack) +- Increase `ibv_test_timeout` for high-latency networks +- Monitor network bandwidth during full mesh tests + +## Contributing + +When adding new preflight checks: + +1. Add the check function to `preflight_lib.py` +2. Add the test function to `preflight_checks.py` +3. Update the HTML report generation +4. Add configuration parameters if needed +5. Update documentation + +### Example: Adding a New Check + +```python +# In preflight_lib.py +def check_gpu_health(phdl): + """Check GPU health across all nodes.""" + cmd = "rocm-smi --showtemp --showpower" + # Implementation here + return results + +# In preflight_checks.py +def test_gpu_health(phdl, config_dict): + """Test GPU health across cluster nodes.""" + results = preflight_lib.check_gpu_health(phdl) + # Validation and reporting here +``` + +This system provides a solid foundation for ensuring cluster health before running critical workloads. \ No newline at end of file diff --git a/cvs/tests/preflight/__init__.py b/cvs/tests/preflight/__init__.py new file mode 100644 index 000000000..67b881dfa --- /dev/null +++ b/cvs/tests/preflight/__init__.py @@ -0,0 +1,6 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. +''' diff --git a/cvs/tests/preflight/preflight_checks.py b/cvs/tests/preflight/preflight_checks.py new file mode 100644 index 000000000..317137223 --- /dev/null +++ b/cvs/tests/preflight/preflight_checks.py @@ -0,0 +1,665 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. +''' + +import pytest +import json + +# Import new modular preflight classes +from cvs.lib.preflight.gid_consistency import GidConsistencyCheck +from cvs.lib.preflight.version_check import RocmVersionCheck +from cvs.lib.preflight.interface_consistency import InterfaceConsistencyCheck + +# RdmaConnectivityCheck not used - using legacy function temporarily +from cvs.lib.preflight.report import PreflightReportGenerator +from cvs.lib.parallel.multiprocess_pssh import MultiProcessPssh as Pssh +from cvs.lib.parallel.config import ParallelConfig +from cvs.lib.utils_lib import * +from cvs.lib.verify_lib import * +from cvs.parsers.schemas import validate_config_file + +from cvs.lib import globals + +log = globals.log + + +def get_nested_config(config_dict, section, key, default): + """ + Get configuration value from nested structure. + + Args: + config_dict: Full configuration dictionary + section: Section name (e.g., 'node_check', 'connectivity_check.rdma') + key: Parameter key within the section + default: Default value if not found + + Returns: + Configuration value or default + """ + if not config_dict: + return default + + # Handle nested sections like 'connectivity_check.rdma' + sections = section.split('.') + current = config_dict + + for sec in sections: + if isinstance(current, dict) and sec in current: + current = current[sec] + else: + return default + + if isinstance(current, dict) and key in current: + return current[key] + return default + + +def _config_flag_enabled(value, default=True): + """Normalize mixed bool/string config flags.""" + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in ('1', 'true', 'yes', 'on') + return bool(value) + + +# Global results storage for HTML report generation +preflight_results = {} + + +def _prune_nodes_from_phdl(phdl, failed_nodes, reason): + """ + Remove ``failed_nodes`` from ``phdl.reachable_hosts`` and recreate the parallel client. + + Later preflight steps then only target hosts that passed the previous check. + """ + if not failed_nodes: + return + remove = {n for n in failed_nodes if n} + on_host = [h for h in phdl.reachable_hosts if h in remove] + if not on_host: + return + pruned = phdl.prune_nodes(on_host) + if not pruned: + return + log.info(f"{reason} Pruned {len(pruned)} node(s) from further preflight tests: {', '.join(sorted(pruned))}") + + +# Override update_test_result for preflight tests to be reporting-only +def preflight_update_test_result(): + """ + Preflight-specific test result handler that reports issues but doesn't fail tests. + This allows preflight to be a comprehensive reporting tool rather than a pass/fail test. + """ + if len(globals.error_list) > 0: + log.info(f"Preflight detected {len(globals.error_list)} issues (see detailed logs above)") + # Clear the error list to prevent test failure + globals.error_list.clear() + # Always pass - preflight is for reporting, not failing + + +@pytest.fixture(scope="module") +def cluster_file(pytestconfig): + """ + Return the path to the cluster configuration JSON file passed via pytest CLI. + + Expects: + - pytest to be invoked with: --cluster_file + + Args: + pytestconfig: Built-in pytest config object used to access CLI options. + + Returns: + str: Filesystem path to the cluster configuration file. + """ + return pytestconfig.getoption("cluster_file") + + +@pytest.fixture(scope="module") +def config_file(pytestconfig): + """ + Return the path to the test configuration JSON file passed via pytest CLI. + + Expects: + - pytest to be invoked with: --config_file + + Args: + pytestconfig: Built-in pytest config object used to access CLI options. + + Returns: + str: Filesystem path to the test configuration file. + """ + return pytestconfig.getoption("config_file") + + +@pytest.fixture(scope="module") +def cluster_dict(cluster_file): + """ + Load and validate cluster configuration from JSON file. + + Args: + cluster_file (str): Path to cluster configuration file. + + Returns: + dict: Validated cluster configuration dictionary. + """ + cluster_config = validate_config_file(cluster_file, config_type="cluster") + cluster_dict = cluster_config.model_dump() + + # Resolve path placeholders + cluster_dict = resolve_cluster_config_placeholders(cluster_dict) + log.info(f"Loaded cluster configuration with {len(cluster_dict['node_dict'])} nodes") + + return cluster_dict + + +@pytest.fixture(scope="module") +def config_dict(config_file, cluster_dict): + """ + Load and validate test configuration from JSON file. + + Args: + config_file (str): Path to test configuration file. + cluster_dict (dict): Cluster configuration for placeholder resolution. + + Returns: + dict: Validated test configuration dictionary. + """ + with open(config_file) as json_file: + config_dict_t = json.load(json_file) + + if 'preflight' not in config_dict_t: + raise ValueError("Configuration file must contain 'preflight' section") + + config_dict = config_dict_t['preflight'] + + # Resolve path placeholders + config_dict = resolve_test_config_placeholders(config_dict, cluster_dict) + log.info("Loaded preflight configuration") + log.info(config_dict) + + return config_dict + + +@pytest.fixture(scope="module") +def phdl(cluster_dict, config_dict): + """ + Build and return a parallel SSH handle (Pssh) for all cluster nodes. + + Args: + cluster_dict (dict): Cluster metadata fixture containing: + - node_dict: dict of node_name -> node_details + - username: SSH username + - priv_key_file: path to SSH private key + + Returns: + Pssh: Handle configured for all nodes (for broadcast/parallel operations). + """ + node_list = list(cluster_dict['node_dict'].keys()) + env_vars = cluster_dict.get("env_vars") + log.info(f"Creating parallel SSH handle for {len(node_list)} nodes") + + # Create config with RDMA full-mesh partition group size + # (fallbacks: legacy rdma.parallel_group_size, then preflight.parallelism.parallel_group_size). + hosts_per_shard = get_nested_config( + config_dict, + 'connectivity_check.rdma', + 'nodes_per_full_mesh_group', + get_nested_config( + config_dict, + 'connectivity_check.rdma', + 'parallel_group_size', + get_nested_config(config_dict, 'parallelism', 'parallel_group_size', 32), + ), + ) + config = ParallelConfig(hosts_per_shard=hosts_per_shard) + + phdl = Pssh( + log, + node_list, + user=cluster_dict['username'], + pkey=cluster_dict['priv_key_file'], + env_vars=env_vars, + stop_on_errors=False, + config=config, + timeout=60, + num_retries=2, + retry_delay=2, + ) + return phdl + + +@pytest.fixture(scope="module") +def shdl(cluster_dict): + """ + Build and return a parallel SSH handle (Pssh) for the head node only. + + Args: + cluster_dict (dict): Cluster metadata fixture (see phdl docstring). + + Returns: + Pssh: Handle configured for head node only (for single-node operations). + """ + head_node = cluster_dict['head_node_dict']['mgmt_ip'] + env_vars = cluster_dict.get("env_vars") + log.info(f"Creating single SSH handle for head node: {head_node}") + + shdl = Pssh( + log, + [head_node], + user=cluster_dict['username'], + pkey=cluster_dict['priv_key_file'], + env_vars=env_vars, + stop_on_errors=False, + ) + return shdl + + +def test_node_reachability(phdl): + """ + Test basic SSH connectivity to all cluster nodes. + + Logs unreachable nodes but continues with reachable ones. + This allows preflight tests to run on available nodes. + """ + # Clear any previous errors for preflight reporting mode + globals.error_list.clear() + + log.info("Testing node reachability via SSH") + + # Simple connectivity test + cmd = "echo 'SSH_OK'" + out_dict = phdl.exec(cmd, timeout=60) # Generous timeout for multiprocessing coordination + + failed_nodes = [] + reachable_nodes = [] + + for node, output in out_dict.items(): + if 'SSH_OK' not in output: + failed_nodes.append(node) + if 'ABORT: Host Unreachable Error' in output: + log.warning(f"Node {node} is unreachable (will be excluded from further tests)") + else: + log.error(f"Node {node} failed connectivity test: {output.strip()}") + else: + reachable_nodes.append(node) + + if failed_nodes: + log.warning(f"Unreachable nodes ({len(failed_nodes)}): {', '.join(failed_nodes)}") + log.info(f"Continuing preflight tests with {len(reachable_nodes)} reachable nodes") + + log.info(f"Node reachability: {len(reachable_nodes)}/{len(out_dict)} nodes reachable") + + # Store reachability results for summary + global preflight_results + preflight_results['node_reachability'] = { + 'total_nodes': len(out_dict), + 'reachable_nodes': len(reachable_nodes), + 'unreachable_nodes': failed_nodes, + 'status': 'PASS' if len(failed_nodes) == 0 else 'WARNING', + } + + # Drop all nodes that did not return SSH_OK from phdl (explicit prune; not only SSH client exceptions) + _prune_nodes_from_phdl(phdl, failed_nodes, "Reachability:") + + preflight_update_test_result() + + +def test_rocm_version_consistency(phdl, config_dict): + """ + Test ROCm version consistency across all cluster nodes. + + Verifies that all nodes are running the expected ROCm version + as specified in the configuration. + + Nodes that fail this check are **not** removed from ``phdl`` so the next test + (RDMA interface consistency) still runs on the full reachability-passed set. + """ + global preflight_results + + expected_version = get_nested_config(config_dict, 'node_check', 'expected_rocm_version', '6.2.0') + log.info(f"Testing ROCm version consistency (expected: {expected_version})") + + version_checker = RocmVersionCheck(phdl, expected_version, config_dict) + results = version_checker.run() + preflight_results['rocm_versions'] = results + + # Analyze results and report failures + failed_nodes = [] + version_summary = {} + + for node, result in results.items(): + detected_version = result['detected_version'] + if detected_version in version_summary: + version_summary[detected_version].append(node) + else: + version_summary[detected_version] = [node] + + if result['status'] == 'FAIL': + failed_nodes.append(node) + for error in result['errors']: + log.error(f"Node {node}: {error}") + + if failed_nodes: + log.warning(f"ROCm version inconsistencies on {len(failed_nodes)} nodes: {', '.join(failed_nodes)}") + else: + log.info("ROCm version consistency check: All reachable nodes passed") + + log.info( + f"ROCm version results: {len(results) - len(failed_nodes)}/{len(results)} nodes have expected version {expected_version}" + ) + # Intentionally do not prune ROCm failures from phdl (see docstring). + preflight_update_test_result() + + +def test_interface_name_consistency(phdl, config_dict): + """ + Test RDMA interface presence and consistency across all cluster nodes. + + Verifies that the expected RDMA interfaces are present on all nodes + as specified in the configuration. + + Nodes that fail are removed from ``phdl`` before the GID consistency check. + """ + global preflight_results + + expected_interfaces = get_nested_config( + config_dict, 'node_check', 'rdma_interfaces', ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + ) + log.info(f"Testing interface presence (expected: {expected_interfaces})") + + interface_checker = InterfaceConsistencyCheck(phdl, expected_interfaces, config_dict) + results = interface_checker.run() + preflight_results['interface_names'] = results + + # Analyze results and report failures + failed_nodes = [] + total_interfaces = 0 + compliant_interfaces = 0 + + for node, result in results.items(): + if result['status'] == 'FAIL': + failed_nodes.append(node) + for error in result['errors']: + log.error(f"Node {node}: {error}") + + for interface in result['interfaces']: + total_interfaces += 1 + if interface['expected'] and interface.get('functional', True): + compliant_interfaces += 1 + + if failed_nodes: + log.warning(f"Interface naming inconsistencies on {len(failed_nodes)} nodes: {', '.join(failed_nodes)}") + else: + log.info("Interface naming consistency check: All reachable nodes passed") + + log.info( + f"Interface presence results: {compliant_interfaces}/{total_interfaces} interfaces are expected interfaces" + ) + + _prune_nodes_from_phdl(phdl, failed_nodes, "Interface consistency:") + preflight_update_test_result() + + +def test_gid_consistency(phdl, config_dict): + """ + Test GID consistency across specified RDMA interfaces in the cluster. + + Verifies that the specified GID index exists and is valid on the + specified RDMA interfaces across all cluster nodes. + + Nodes that fail are removed from ``phdl`` before RDMA connectivity testing. + """ + global preflight_results + + gid_index = get_nested_config(config_dict, 'node_check', 'gid_index', '3') + expected_interfaces = get_nested_config( + config_dict, 'node_check', 'rdma_interfaces', ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + ) + log.info(f"Testing GID consistency for index {gid_index} on interfaces: {expected_interfaces}") + + gid_checker = GidConsistencyCheck(phdl, gid_index, expected_interfaces, config_dict) + results = gid_checker.run() + preflight_results['gid_consistency'] = results + + # Analyze results and report failures + failed_nodes = [] + total_interfaces = 0 + ok_interfaces = 0 + + for node, result in results.items(): + if result['status'] == 'FAIL': + failed_nodes.append(node) + for error in result['errors']: + log.error(f"Node {node}: {error}") + + for interface, interface_result in result['interfaces'].items(): + total_interfaces += 1 + if interface_result.get('status') == 'OK': + ok_interfaces += 1 + + if failed_nodes: + log.warning(f"GID consistency issues on {len(failed_nodes)} nodes: {', '.join(failed_nodes)}") + else: + log.info("GID consistency check: All nodes passed") + + log.info(f"GID consistency results: {ok_interfaces}/{total_interfaces} interfaces have valid GID index {gid_index}") + + _prune_nodes_from_phdl(phdl, failed_nodes, "GID consistency:") + preflight_update_test_result() + + +def test_rdma_connectivity(phdl, cluster_dict, config_dict): + """ + Test RDMA connectivity between cluster nodes using ibv_rc_pingpong. + + Uses direct IB verbs (same as RCCL) for more accurate connectivity testing + that can detect issues that rping might miss. + + Tests connectivity based on the specified mode (basic, full_mesh, or skip) + and reports any connection failures. + + ``phdl`` excludes nodes that failed reachability, interface consistency, or GID + consistency; those steps prune before the next. ROCm version mismatches are reported + but **not** pruned. Results may include ``excluded_nodes_interface_check`` and + ``excluded_nodes_gid`` for the report (hosts already removed from ``phdl``). + """ + global preflight_results + + # Host list matches prior-step pruning (reachability, interface, GID); not full cluster_dict. + node_list = list(phdl.reachable_hosts) + + iface_results = preflight_results.get('interface_names') or {} + excluded_nodes_interface_check = sorted(n for n, r in iface_results.items() if r.get('status') == 'FAIL') + + gid_results = preflight_results.get('gid_consistency') or {} + excluded_nodes_gid = sorted(n for n, r in gid_results.items() if r.get('status') == 'FAIL') + + log.info( + f"RDMA connectivity: {len(node_list)} host(s) on phdl after reachability / interface / GID pruning " + f"(ROCm mismatches are not pruned)." + ) + + mode = get_nested_config(config_dict, 'connectivity_check.rdma', 'connectivity_mode', 'basic') + port_range = get_nested_config(config_dict, 'connectivity_check.rdma', 'ibv_test_port_range', '10000-50000') + timeout = int(get_nested_config(config_dict, 'connectivity_check.rdma', 'ibv_test_timeout', 90)) + expected_interfaces = get_nested_config( + config_dict, 'node_check', 'rdma_interfaces', ["rocep28s0", "rocep62s0", "rocep79s0", "rocep96s0"] + ) + gid_index = get_nested_config(config_dict, 'node_check', 'gid_index', '3') + parallel_group_size = get_nested_config( + config_dict, + 'connectivity_check.rdma', + 'nodes_per_full_mesh_group', + get_nested_config( + config_dict, + 'connectivity_check.rdma', + 'parallel_group_size', + get_nested_config(config_dict, 'parallelism', 'parallel_group_size', 128), + ), + ) + + log.info( + f"Testing RDMA connectivity using parallel algorithm (mode: {mode}, group_size: {parallel_group_size}, timeout: {timeout}s, interfaces: {expected_interfaces}, GID: {gid_index})" + ) + + if mode != 'skip' and len(phdl.reachable_hosts) < 2: + log.warning( + 'RDMA connectivity skipped: fewer than 2 hosts remain after reachability / interface / GID pruning.' + ) + skip_results = { + 'mode': mode, + 'skipped': True, + 'message': 'Too few nodes for RDMA after reachability, interface, and GID pruning', + 'total_pairs': 0, + 'successful_pairs': 0, + 'failed_pairs': 0, + 'pair_results': {}, + 'node_status': {}, + 'excluded_nodes_interface_check': excluded_nodes_interface_check, + 'excluded_nodes_gid': excluded_nodes_gid, + } + preflight_results['rdma_connectivity'] = skip_results + preflight_update_test_result() + return + + # Use the new modular RDMA connectivity check (supports all modes) + from cvs.lib.preflight.rdma_connectivity import RdmaConnectivityCheck + + rdma_checker = RdmaConnectivityCheck( + phdl, node_list, mode, port_range, timeout, expected_interfaces, gid_index, parallel_group_size, config_dict + ) + results = rdma_checker.run() + if excluded_nodes_interface_check: + results['excluded_nodes_interface_check'] = excluded_nodes_interface_check + if excluded_nodes_gid: + results['excluded_nodes_gid'] = excluded_nodes_gid + preflight_results['rdma_connectivity'] = results + + # Handle skipped test + if results.get('skipped', False): + log.info("RDMA connectivity test skipped by configuration") + preflight_update_test_result() + return + + # Analyze results and report failures + if results['failed_pairs'] > 0: + failed_pairs = [] + for pair_key, pair_result in results['pair_results'].items(): + if pair_result['status'] == 'FAIL': + failed_pairs.append(pair_key) + for error in pair_result['error_details']: + log.error(f"Pair {pair_key}: {error}") + + log.warning( + f"RDMA connectivity issues: {results['failed_pairs']} failed pairs out of {results['total_pairs']} total" + ) + for pair in failed_pairs[:5]: # Log first 5 failed pairs + log.warning(f"Failed pair: {pair}") + else: + log.info("RDMA connectivity check: All tested pairs connected successfully") + + log.info( + f"RDMA connectivity results: {results['successful_pairs']}/{results['total_pairs']} pairs connected successfully" + ) + preflight_update_test_result() + + +def test_generate_preflight_report(phdl, config_dict, request): + """ + Generate comprehensive preflight check report. + + Creates a summary of all preflight check results and generates + an HTML report for easy review. + """ + global preflight_results + + log.info("Generating preflight check report") + + # Ensure we have results from all checks + required_checks = ['gid_consistency', 'rocm_versions', 'interface_names', 'rdma_connectivity'] + missing_checks = [check for check in required_checks if check not in preflight_results] + + if missing_checks: + log.error(f"Missing results for checks: {', '.join(missing_checks)}") + # Create empty results for missing checks to allow report generation + for check in missing_checks: + preflight_results[check] = {'status': 'SKIPPED', 'message': 'Check was skipped due to earlier failures'} + + # Generate comprehensive summary using new report generator + report_generator = PreflightReportGenerator(phdl, preflight_results, config_dict) + report_results = report_generator.run() + summary = report_results['summary'] + + preflight_results['summary'] = summary + + # Log summary to console + log.info("=== PREFLIGHT CHECK SUMMARY ===") + for check_name, check_summary in summary['checks'].items(): + status_icon = "✅" if check_summary['status'] == 'PASS' else "❌" + log.info( + f"{status_icon} {check_name.replace('_', ' ').title()}: {check_summary['status']} - {check_summary['summary']}" + ) + + log.info(f"\nOverall Status: {summary['overall_status']}") + + if summary['recommendations']: + log.info("\nRecommendations:") + for i, recommendation in enumerate(summary['recommendations'], 1): + log.info(f"{i}. {recommendation}") + + # Generate HTML report + html_report_path = None + try: + if _config_flag_enabled(get_nested_config(config_dict, 'reporting', 'generate_html_report', 'true')): + # HTML report is generated as part of the report generator run() above + html_report_path = report_results.get('html_report') + log.info(f"HTML report generated: {html_report_path}") + rdma_csv = report_results.get('rdma_pairs_csv') + if rdma_csv: + log.info(f"RDMA pairs CSV generated: {rdma_csv}") + else: + log.info("HTML report generation disabled in configuration") + except Exception as e: + log.warning(f"Failed to generate HTML report: {e}") + + # Add HTML report to main test report bundle + if html_report_path and hasattr(request.config, '_html_report_manager'): + try: + copied_path = request.config._html_report_manager.add_html_to_report( + html_report_path, link_name="Preflight Checks Report", request=request + ) + + rdma_csv = report_results.get('rdma_pairs_csv') + if rdma_csv: + try: + csv_copied = request.config._html_report_manager.add_html_to_report( + rdma_csv, link_name="RDMA failed pairs (CSV)", request=request + ) + if csv_copied: + log.info(f'RDMA failed pairs CSV added to report bundle: {csv_copied}') + except Exception as e: + log.warning(f"Failed to add RDMA CSV to report bundle: {e}") + + if copied_path: + log.info(f'Preflight report saved and added to report bundle: {copied_path}') + else: + log.info( + f'Preflight report is saved under {html_report_path}, please copy it to your web server under /var/www/html folder to view' + ) + except Exception as e: + log.warning(f"Failed to add preflight report to bundle: {e}") + log.info(f"Preflight report available at: {html_report_path}") + + # Report overall status but don't fail the test + if summary['overall_status'] == 'FAIL': + log.warning("One or more preflight checks detected issues - see detailed results above") + log.info("Preflight report generated successfully - review HTML report for detailed analysis") + else: + log.info("All preflight checks passed - cluster is ready for performance testing") + preflight_update_test_result()