diff --git a/cvs/monitors/cluster-mon/backend/app/api/cluster.py b/cvs/monitors/cluster-mon/backend/app/api/cluster.py index 6d3364c94..399856f34 100644 --- a/cvs/monitors/cluster-mon/backend/app/api/cluster.py +++ b/cvs/monitors/cluster-mon/backend/app/api/cluster.py @@ -179,7 +179,9 @@ async def get_cluster_status() -> Dict[str, Any]: if isinstance(node_data, dict) and "error" not in node_data: for gpu_id, gpu_metrics in node_data.items(): if isinstance(gpu_metrics, dict): - temp = gpu_metrics.get("Temperature (Sensor junction) (C)") or gpu_metrics.get("Temperature (Sensor edge) (C)") + temp = gpu_metrics.get("Temperature (Sensor junction) (C)") or gpu_metrics.get( + "Temperature (Sensor edge) (C)" + ) if temp: total_temp += float(temp) temp_count += 1 @@ -242,11 +244,7 @@ async def get_cluster_health() -> Dict[str, Any]: # Create alerts for unhealthy nodes if node_status == "unhealthy": for issue in issues: - health_data["alerts"].append({ - "severity": "warning", - "node": node, - "message": issue - }) + health_data["alerts"].append({"severity": "warning", "node": node, "message": issue}) # Check unreachable nodes for node in app_state.ssh_manager.unreachable_hosts: @@ -254,10 +252,6 @@ async def get_cluster_health() -> Dict[str, Any]: "status": "unreachable", "issues": ["Node is unreachable via SSH"], } - health_data["alerts"].append({ - "severity": "critical", - "node": node, - "message": f"Node {node} is unreachable" - }) + health_data["alerts"].append({"severity": "critical", "node": node, "message": f"Node {node} is unreachable"}) return health_data diff --git a/cvs/monitors/cluster-mon/backend/app/api/config.py b/cvs/monitors/cluster-mon/backend/app/api/config.py index bd671863e..f1a93e4db 100644 --- a/cvs/monitors/cluster-mon/backend/app/api/config.py +++ b/cvs/monitors/cluster-mon/backend/app/api/config.py @@ -7,8 +7,6 @@ from typing import Dict, Any, List, Optional import yaml import os -import signal -import asyncio from pathlib import Path router = APIRouter() @@ -75,6 +73,7 @@ async def update_configuration(config: ClusterConfigUpdate) -> Dict[str, Any]: Saves nodes to config/nodes.txt and updates cluster.yaml """ import logging + logger = logging.getLogger(__name__) logger.info("=" * 80) @@ -148,6 +147,7 @@ async def update_configuration(config: ClusterConfigUpdate) -> Dict[str, Any]: # SECURITY: Store password in memory only, never persist to disk from app.main import app_state + if config.auth_method == "password" and config.password: app_state.ssh_password = config.password else: @@ -213,7 +213,9 @@ async def update_configuration(config: ClusterConfigUpdate) -> Dict[str, Any]: logger.info("=" * 80) password_note = "" - if config.auth_method == "password" or (config.use_jump_host and config.jump_host and config.jump_host.auth_method == "password"): + if config.auth_method == "password" or ( + config.use_jump_host and config.jump_host and config.jump_host.auth_method == "password" + ): password_note = " Passwords are stored in memory only." return { diff --git a/cvs/monitors/cluster-mon/backend/app/api/logs.py b/cvs/monitors/cluster-mon/backend/app/api/logs.py index 4f8168653..10d7146df 100644 --- a/cvs/monitors/cluster-mon/backend/app/api/logs.py +++ b/cvs/monitors/cluster-mon/backend/app/api/logs.py @@ -29,7 +29,4 @@ async def get_dmesg_errors() -> Dict[str, Any]: return logs_data except Exception as e: - raise HTTPException( - status_code=500, - detail=f"Failed to collect logs: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Failed to collect logs: {str(e)}") diff --git a/cvs/monitors/cluster-mon/backend/app/api/nodes.py b/cvs/monitors/cluster-mon/backend/app/api/nodes.py index 134ec8470..fcd24b581 100644 --- a/cvs/monitors/cluster-mon/backend/app/api/nodes.py +++ b/cvs/monitors/cluster-mon/backend/app/api/nodes.py @@ -84,9 +84,9 @@ async def list_nodes() -> List[Dict[str, Any]]: util = gpu_entry.get("GPU use (%)", 0) total_util += float(util) if util else 0 - node_info["avg_gpu_util"] = round( - total_util / node_info["gpu_count"], 2 - ) if node_info["gpu_count"] > 0 else 0 + node_info["avg_gpu_util"] = ( + round(total_util / node_info["gpu_count"], 2) if node_info["gpu_count"] > 0 else 0 + ) # Handle dict format (old rocm-smi format) elif isinstance(node_util, dict) and 'error' not in node_util: @@ -97,9 +97,9 @@ async def list_nodes() -> List[Dict[str, Any]]: util = gpu_metrics.get("GPU use (%)", 0) total_util += float(util) if util else 0 - node_info["avg_gpu_util"] = round( - total_util / node_info["gpu_count"], 2 - ) if node_info["gpu_count"] > 0 else 0 + node_info["avg_gpu_util"] = ( + round(total_util / node_info["gpu_count"], 2) if node_info["gpu_count"] > 0 else 0 + ) if node in temp_data: node_temp = temp_data[node] @@ -114,7 +114,9 @@ async def list_nodes() -> List[Dict[str, Any]]: temp_data_field = gpu_entry.get("temperature", {}) if isinstance(temp_data_field, dict): # Try different sensor names - temp = temp_data_field.get("hotspot", temp_data_field.get("edge", temp_data_field.get("junction", 0))) + temp = temp_data_field.get( + "hotspot", temp_data_field.get("edge", temp_data_field.get("junction", 0)) + ) if isinstance(temp, dict): temp = temp.get("value", 0) else: @@ -131,9 +133,12 @@ async def list_nodes() -> List[Dict[str, Any]]: for gpu_temp in node_temp.values(): if isinstance(gpu_temp, dict): # Try junction (hotspot) first, then edge, then memory - temp = gpu_temp.get("Temperature (Sensor junction) (C)", - gpu_temp.get("Temperature (Sensor edge) (C)", - gpu_temp.get("Temperature (Sensor memory) (C)", 0))) + temp = gpu_temp.get( + "Temperature (Sensor junction) (C)", + gpu_temp.get( + "Temperature (Sensor edge) (C)", gpu_temp.get("Temperature (Sensor memory) (C)", 0) + ), + ) if temp: try: total_temp += float(temp) @@ -228,9 +233,12 @@ async def get_node_details(node_id: str) -> Dict[str, Any]: # Temperature - try junction (hotspot) first, then edge, then memory if gpu_id in temp_data and isinstance(temp_data[gpu_id], dict): - temp = temp_data[gpu_id].get("Temperature (Sensor junction) (C)", - temp_data[gpu_id].get("Temperature (Sensor edge) (C)", - temp_data[gpu_id].get("Temperature (Sensor memory) (C)", 0))) + temp = temp_data[gpu_id].get( + "Temperature (Sensor junction) (C)", + temp_data[gpu_id].get( + "Temperature (Sensor edge) (C)", temp_data[gpu_id].get("Temperature (Sensor memory) (C)", 0) + ), + ) try: gpu_info["temperature_c"] = float(temp) if temp else 0.0 except (ValueError, TypeError): diff --git a/cvs/monitors/cluster-mon/backend/app/api/packages.py b/cvs/monitors/cluster-mon/backend/app/api/packages.py index f7b221088..a14eb9d39 100644 --- a/cvs/monitors/cluster-mon/backend/app/api/packages.py +++ b/cvs/monitors/cluster-mon/backend/app/api/packages.py @@ -38,12 +38,10 @@ def install_package(request: PackageInstallRequest) -> Dict[str, Any]: # Get the appropriate installer if request.package.lower() == 'lldp': from app.installers.lldp_installer import LLDPInstaller + installer = LLDPInstaller(ssh_manager) else: - raise HTTPException( - status_code=400, - detail=f"Unsupported package: {request.package}. Supported: lldp" - ) + raise HTTPException(status_code=400, detail=f"Unsupported package: {request.package}. Supported: lldp") # Run installation (synchronous) result = installer.install_package() @@ -51,17 +49,14 @@ def install_package(request: PackageInstallRequest) -> Dict[str, Any]: return { "success": True, "message": f"Installation complete: {result['successful']} successful, {result['failed']} failed", - **result + **result, } except HTTPException: raise except Exception as e: logger.error(f"Package installation failed: {str(e)}", exc_info=True) - raise HTTPException( - status_code=500, - detail=f"Package installation failed: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Package installation failed: {str(e)}") @router.get("/status/{package}") @@ -88,7 +83,7 @@ def get_package_status(package: str) -> Dict[str, Any]: "installed_nodes": [], "not_installed_nodes": [], "status_by_node": {}, - "note": "SSH not configured yet. Save configuration first." + "note": "SSH not configured yet. Save configuration first.", } logger.info(f"Checking package status: {package}") @@ -96,12 +91,10 @@ def get_package_status(package: str) -> Dict[str, Any]: # Get the appropriate installer if package.lower() == 'lldp': from app.installers.lldp_installer import LLDPInstaller + installer = LLDPInstaller(ssh_manager) else: - raise HTTPException( - status_code=400, - detail=f"Unsupported package: {package}. Supported: lldp" - ) + raise HTTPException(status_code=400, detail=f"Unsupported package: {package}. Supported: lldp") # Check installation status (synchronous) installed_map = installer.check_installation() @@ -116,17 +109,14 @@ def get_package_status(package: str) -> Dict[str, Any]: "not_installed_count": len(not_installed_nodes), "installed_nodes": installed_nodes, "not_installed_nodes": not_installed_nodes, - "status_by_node": installed_map + "status_by_node": installed_map, } except HTTPException: raise except Exception as e: logger.error(f"Failed to check package status: {str(e)}", exc_info=True) - raise HTTPException( - status_code=500, - detail=f"Failed to check package status: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Failed to check package status: {str(e)}") @router.get("/list") @@ -141,7 +131,7 @@ async def list_supported_packages() -> Dict[str, Any]: "name": "LLDP Daemon", "description": "Link Layer Discovery Protocol daemon for network topology discovery", "package_name": "lldpd", - "check_command": "lldpcli" + "check_command": "lldpcli", }, # Add more packages here as we implement them # { diff --git a/cvs/monitors/cluster-mon/backend/app/api/restart.py b/cvs/monitors/cluster-mon/backend/app/api/restart.py index 1363051d7..6151289e4 100644 --- a/cvs/monitors/cluster-mon/backend/app/api/restart.py +++ b/cvs/monitors/cluster-mon/backend/app/api/restart.py @@ -19,6 +19,7 @@ async def restart_backend() -> Dict[str, Any]: This triggers a graceful shutdown and restart by executing the backend in a new process. """ + async def do_restart(): await asyncio.sleep(1) # Restart using os.execv to replace current process diff --git a/cvs/monitors/cluster-mon/backend/app/api/ssh_keys.py b/cvs/monitors/cluster-mon/backend/app/api/ssh_keys.py index f700964db..84fc7bd5f 100644 --- a/cvs/monitors/cluster-mon/backend/app/api/ssh_keys.py +++ b/cvs/monitors/cluster-mon/backend/app/api/ssh_keys.py @@ -4,7 +4,6 @@ from fastapi import APIRouter, UploadFile, File, HTTPException from typing import Dict, Any -import os from pathlib import Path import logging @@ -26,10 +25,7 @@ async def upload_ssh_key(file: UploadFile = File(...)) -> Dict[str, Any]: # Only allow common SSH key filenames for security allowed_names = ["id_rsa", "id_ed25519", "id_ecdsa", "cluster_id_ed25519", "known_hosts", "config"] if file.filename not in allowed_names: - raise HTTPException( - status_code=400, - detail=f"Invalid key filename. Allowed: {', '.join(allowed_names)}" - ) + raise HTTPException(status_code=400, detail=f"Invalid key filename. Allowed: {', '.join(allowed_names)}") # Create .ssh directory if it doesn't exist ssh_dir = Path("/root/.ssh") @@ -37,6 +33,7 @@ async def upload_ssh_key(file: UploadFile = File(...)) -> Dict[str, Any]: # Ensure directory has correct ownership import os + try: os.chown(ssh_dir, 0, 0) # root:root ssh_dir.chmod(0o700) @@ -52,6 +49,7 @@ async def upload_ssh_key(file: UploadFile = File(...)) -> Dict[str, Any]: # Set proper permissions and ownership import os + if file.filename in ["known_hosts", "config"]: key_path.chmod(0o644) else: @@ -76,7 +74,7 @@ async def upload_ssh_key(file: UploadFile = File(...)) -> Dict[str, Any]: "message": f"SSH key '{file.filename}' uploaded successfully", "filename": file.filename, "size": len(content), - "path": str(key_path) + "path": str(key_path), } except Exception as e: @@ -93,25 +91,21 @@ async def list_ssh_keys() -> Dict[str, Any]: ssh_dir = Path("/root/.ssh") if not ssh_dir.exists(): - return { - "keys": [], - "message": "No SSH keys directory found" - } + return {"keys": [], "message": "No SSH keys directory found"} keys = [] for key_file in ssh_dir.iterdir(): if key_file.is_file(): stat = key_file.stat() - keys.append({ - "filename": key_file.name, - "size": stat.st_size, - "permissions": oct(stat.st_mode)[-3:], - }) + keys.append( + { + "filename": key_file.name, + "size": stat.st_size, + "permissions": oct(stat.st_mode)[-3:], + } + ) - return { - "keys": keys, - "count": len(keys) - } + return {"keys": keys, "count": len(keys)} except Exception as e: logger.error(f"Failed to list SSH keys: {e}") @@ -137,10 +131,7 @@ async def delete_ssh_key(filename: str) -> Dict[str, Any]: key_path.unlink() logger.info(f"SSH key deleted: {filename}") - return { - "success": True, - "message": f"SSH key '{filename}' deleted successfully" - } + return {"success": True, "message": f"SSH key '{filename}' deleted successfully"} except HTTPException: raise diff --git a/cvs/monitors/cluster-mon/backend/app/collectors/gpu_collector.py b/cvs/monitors/cluster-mon/backend/app/collectors/gpu_collector.py index 26334e745..eb52f5882 100644 --- a/cvs/monitors/cluster-mon/backend/app/collectors/gpu_collector.py +++ b/cvs/monitors/cluster-mon/backend/app/collectors/gpu_collector.py @@ -5,7 +5,7 @@ import json import logging -from typing import Dict, List, Any, Optional +from typing import Dict, Any from datetime import datetime logger = logging.getLogger(__name__) @@ -26,8 +26,8 @@ def parse_json_output(output_dict: Dict[str, str]) -> Dict[str, Any]: Dictionary mapping host -> parsed JSON data """ parsed = {} - #logger.info('#========== before parse ===========#') - #logger.info(output_dict) + # logger.info('#========== before parse ===========#') + # logger.info(output_dict) for host, output in output_dict.items(): try: if output and not output.startswith("ERROR") and not output.startswith("ABORT"): @@ -39,7 +39,7 @@ def parse_json_output(output_dict: Dict[str, str]) -> Dict[str, Any]: # Find the start of JSON (first '[' or '{') json_start = min( (output.find('[') if '[' in output else len(output)), - (output.find('{') if '{' in output else len(output)) + (output.find('{') if '{' in output else len(output)), ) if json_start < len(output): output = output[json_start:].strip() @@ -60,8 +60,8 @@ def parse_json_output(output_dict: Dict[str, str]) -> Dict[str, Any]: logger.error(f"Output was: {output[:500]}") parsed[host] = {"error": f"JSON parse error: {str(e)}"} - #logger.info('#=========== parsed value ===============#') - #logger.info(parsed) + # logger.info('#=========== parsed value ===============#') + # logger.info(parsed) return parsed async def collect_gpu_utilization(self, ssh_manager) -> Dict[str, Any]: @@ -292,7 +292,7 @@ async def collect_pcie_info(self, ssh_manager) -> Dict[str, Any]: for i, line in enumerate(lines): if bdf in line: # Look for LnkSta in next ~50 lines (within same device section) - for j in range(i+1, min(i+50, len(lines))): + for j in range(i + 1, min(i + 50, len(lines))): if 'LnkSta:' in lines[j]: width_match = re.search(r'Width (x\d+)', lines[j]) speed_match = re.search(r'Speed ([0-9.]+GT/s)', lines[j]) @@ -432,7 +432,7 @@ def _parse_utilization_from_amd_smi(self, amd_smi_data: Dict) -> Dict: util_data[node][gpu_id] = { 'GPU use (%)': str(gfx_val), 'GFX Activity': str(gfx_val), - 'UMC Activity': str(umc_val) + 'UMC Activity': str(umc_val), } else: util_data[node] = data @@ -504,9 +504,12 @@ def _parse_temperature_from_amd_smi(self, amd_smi_data: Dict) -> Dict: mem_val = mem_obj.get('value', 0) if isinstance(mem_obj, dict) else 0 # Handle "N/A" strings - if edge_obj == "N/A": edge_val = 0 - if hotspot_obj == "N/A": hotspot_val = 0 - if mem_obj == "N/A": mem_val = 0 + if edge_obj == "N/A": + edge_val = 0 + if hotspot_obj == "N/A": + hotspot_val = 0 + if mem_obj == "N/A": + mem_val = 0 temp_data[node][gpu_id] = { 'Temperature (Sensor edge) (C)': str(edge_val), @@ -550,42 +553,42 @@ def _parse_pcie_metrics_from_amd_smi(self, amd_smi_data: Dict) -> Dict: if gpu_list: pcie_info[node] = {} for gpu in gpu_list: - gpu_id = f"card{gpu.get('gpu', 0)}" - pcie_data = gpu.get('pcie', {}) - - if pcie_data: - # Flatten nested values for frontend display - width = pcie_data.get('width', '-') - if width != '-' and width != 'N/A': - width = f"x{width}" - - # Flatten speed object - speed_obj = pcie_data.get('speed', {}) - if isinstance(speed_obj, dict): - speed_val = speed_obj.get('value', '-') - speed_unit = speed_obj.get('unit', 'GT/s') - speed = f"{speed_val} {speed_unit}" if speed_val != '-' else '-' - else: - speed = str(speed_obj) if speed_obj and speed_obj != 'N/A' else '-' - - # Flatten bandwidth object - bw_obj = pcie_data.get('bandwidth', {}) - if isinstance(bw_obj, dict): - bw_val = bw_obj.get('value', '-') - bw_unit = bw_obj.get('unit', 'Mb/s') - bandwidth = f"{bw_val} {bw_unit}" if bw_val != '-' else '-' - else: - bandwidth = str(bw_obj) if bw_obj and bw_obj != 'N/A' else '-' - - pcie_info[node][gpu_id] = { - 'width': width, - 'speed': speed, - 'bandwidth': bandwidth, - 'replay_count': pcie_data.get('replay_count', 0), - 'l0_to_recovery_count': pcie_data.get('l0_to_recovery_count', 0), - 'nak_sent_count': pcie_data.get('nak_sent_count', 0), - 'nak_received_count': pcie_data.get('nak_received_count', 0), - } + gpu_id = f"card{gpu.get('gpu', 0)}" + pcie_data = gpu.get('pcie', {}) + + if pcie_data: + # Flatten nested values for frontend display + width = pcie_data.get('width', '-') + if width != '-' and width != 'N/A': + width = f"x{width}" + + # Flatten speed object + speed_obj = pcie_data.get('speed', {}) + if isinstance(speed_obj, dict): + speed_val = speed_obj.get('value', '-') + speed_unit = speed_obj.get('unit', 'GT/s') + speed = f"{speed_val} {speed_unit}" if speed_val != '-' else '-' + else: + speed = str(speed_obj) if speed_obj and speed_obj != 'N/A' else '-' + + # Flatten bandwidth object + bw_obj = pcie_data.get('bandwidth', {}) + if isinstance(bw_obj, dict): + bw_val = bw_obj.get('value', '-') + bw_unit = bw_obj.get('unit', 'Mb/s') + bandwidth = f"{bw_val} {bw_unit}" if bw_val != '-' else '-' + else: + bandwidth = str(bw_obj) if bw_obj and bw_obj != 'N/A' else '-' + + pcie_info[node][gpu_id] = { + 'width': width, + 'speed': speed, + 'bandwidth': bandwidth, + 'replay_count': pcie_data.get('replay_count', 0), + 'l0_to_recovery_count': pcie_data.get('l0_to_recovery_count', 0), + 'nak_sent_count': pcie_data.get('nak_sent_count', 0), + 'nak_received_count': pcie_data.get('nak_received_count', 0), + } elif isinstance(data, list): # Direct list format - also flatten @@ -806,9 +809,12 @@ def normalize_metrics(self, raw_metrics: Dict[str, Any]) -> Dict[str, Any]: # Extract temperature (prefer junction/hotspot, fallback to edge or mem) if gpu_id in node_temp: # Try junction (hotspot) first, then edge, then mem - temp_val = node_temp[gpu_id].get("Temperature (Sensor junction) (C)", - node_temp[gpu_id].get("Temperature (Sensor edge) (C)", - node_temp[gpu_id].get("Temperature (Sensor memory) (C)", 0))) + temp_val = node_temp[gpu_id].get( + "Temperature (Sensor junction) (C)", + node_temp[gpu_id].get( + "Temperature (Sensor edge) (C)", node_temp[gpu_id].get("Temperature (Sensor memory) (C)", 0) + ), + ) try: gpu_metrics["temperature_c"] = float(temp_val) if temp_val and temp_val != 'N/A' else 0.0 except (ValueError, TypeError): diff --git a/cvs/monitors/cluster-mon/backend/app/collectors/gpu_software_collector.py b/cvs/monitors/cluster-mon/backend/app/collectors/gpu_software_collector.py index 4201bc2bc..867c48866 100644 --- a/cvs/monitors/cluster-mon/backend/app/collectors/gpu_software_collector.py +++ b/cvs/monitors/cluster-mon/backend/app/collectors/gpu_software_collector.py @@ -42,8 +42,10 @@ async def collect_rocm_version(self, ssh_manager) -> Dict[str, Any]: logger.info("Collecting ROCM version") # Get actual ROCm version from file - import asyncio - rocm_ver_output = await ssh_manager.exec_async("cat /opt/rocm*/.info/version 2>/dev/null | head -1 || echo 'N/A'") + + rocm_ver_output = await ssh_manager.exec_async( + "cat /opt/rocm*/.info/version 2>/dev/null | head -1 || echo 'N/A'" + ) driver_output = await ssh_manager.exec_async("rocm-smi --showdriverversion 2>/dev/null || echo 'Not available'") rocm_version = {} diff --git a/cvs/monitors/cluster-mon/backend/app/collectors/nic_advanced_collector.py b/cvs/monitors/cluster-mon/backend/app/collectors/nic_advanced_collector.py index 56ddd0669..502159ca1 100644 --- a/cvs/monitors/cluster-mon/backend/app/collectors/nic_advanced_collector.py +++ b/cvs/monitors/cluster-mon/backend/app/collectors/nic_advanced_collector.py @@ -6,7 +6,7 @@ import re import json import logging -from typing import Dict, Any, List +from typing import Dict, Any from datetime import datetime logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ async def collect_nic_pcie_info(self, ssh_manager) -> Dict[str, Any]: logger.info("Collecting NIC PCIe information (optimized)") # Get ALL NIC PCIe info in one command per node - #cmd = "sudo lspci -vvv 2>/dev/null | grep -A 30 -i 'ethernet\\|network' | grep -E '^[0-9a-f]{2}:|Ethernet|Network|LnkCap:|LnkSta:'" + # cmd = "sudo lspci -vvv 2>/dev/null | grep -A 30 -i 'ethernet\\|network' | grep -E '^[0-9a-f]{2}:|Ethernet|Network|LnkCap:|LnkSta:'" cmd = "sudo lspci -vvv 2>/dev/null | egrep -A 30 -i 'ethernet\\|network' | egrep '^[0-9a-f]{2}:|Ethernet|Network|LnkCap:|LnkSta:'" result = await ssh_manager.exec_async(cmd) @@ -129,7 +129,9 @@ async def collect_nic_pcie_info(self, ssh_manager) -> Dict[str, Any]: 'link_width_current': link_sta_width, } - logger.info(f"NIC PCIe collection complete: {len(pcie_info)} nodes, {sum(len(v) for v in pcie_info.values())} devices total") + logger.info( + f"NIC PCIe collection complete: {len(pcie_info)} nodes, {sum(len(v) for v in pcie_info.values())} devices total" + ) return pcie_info async def collect_congestion_info(self, ssh_manager) -> Dict[str, Any]: diff --git a/cvs/monitors/cluster-mon/backend/app/collectors/nic_collector.py b/cvs/monitors/cluster-mon/backend/app/collectors/nic_collector.py index fcdb021a3..2b85d8250 100644 --- a/cvs/monitors/cluster-mon/backend/app/collectors/nic_collector.py +++ b/cvs/monitors/cluster-mon/backend/app/collectors/nic_collector.py @@ -122,7 +122,9 @@ async def collect_rdma_stats(self, ssh_manager) -> Dict[str, Any]: if key not in metadata_keys and isinstance(value, (int, float)): rdma_stats[node][dev_key][key] = value - logger.info(f"Node {node}: Device {dev_key} - parsed {len(rdma_stats[node][dev_key])} stats") + logger.info( + f"Node {node}: Device {dev_key} - parsed {len(rdma_stats[node][dev_key])} stats" + ) except json.JSONDecodeError as e: logger.warning(f"Failed to parse RDMA stats JSON for {node}: {e}") @@ -430,7 +432,6 @@ async def collect_all_metrics(self, ssh_manager) -> Dict[str, Any]: "lldp": {...} } """ - import asyncio logger.info("Collecting all NIC metrics") diff --git a/cvs/monitors/cluster-mon/backend/app/collectors/nic_devlink_collector.py b/cvs/monitors/cluster-mon/backend/app/collectors/nic_devlink_collector.py index b8567ec56..0ea4662f0 100644 --- a/cvs/monitors/cluster-mon/backend/app/collectors/nic_devlink_collector.py +++ b/cvs/monitors/cluster-mon/backend/app/collectors/nic_devlink_collector.py @@ -80,7 +80,7 @@ async def collect_devlink_info(self, ssh_manager) -> Dict[str, Any]: versions = dev_data.get("versions", {}) fixed = versions.get("fixed", {}) running = versions.get("running", {}) - stored = versions.get("stored", {}) + versions.get("stored", {}) # Determine vendor based on driver if driver == "bnxt_en": @@ -97,12 +97,7 @@ async def collect_devlink_info(self, ssh_manager) -> Dict[str, Any]: # Normalize firmware versions across vendors # Broadcom Thor2 and NVIDIA CX7 use "fw" field # AMD AINIC uses specific fields like "fw.a35_fip_a" - fw_version = ( - running.get("fw") or - running.get("fw.version") or - running.get("fw.a35_fip_a") or - "-" - ) + fw_version = running.get("fw") or running.get("fw.version") or running.get("fw.a35_fip_a") or "-" fw_psid = fixed.get("fw.psid") or "-" fw_mgmt = running.get("fw.mgmt") or "-" diff --git a/cvs/monitors/cluster-mon/backend/app/collectors/nic_software_collector.py b/cvs/monitors/cluster-mon/backend/app/collectors/nic_software_collector.py index 667de6de2..b738659da 100644 --- a/cvs/monitors/cluster-mon/backend/app/collectors/nic_software_collector.py +++ b/cvs/monitors/cluster-mon/backend/app/collectors/nic_software_collector.py @@ -221,10 +221,7 @@ async def collect_pci_device_info(self, ssh_manager) -> Dict[str, Any]: # Parse PCI address and device info match = re.match(r"([\da-f:\.]+)\s+(.+)", line, re.I) if match: - devices.append({ - "pci_address": match.group(1), - "description": match.group(2).strip() - }) + devices.append({"pci_address": match.group(1), "description": match.group(2).strip()}) pci_info[host] = {"devices": devices} diff --git a/cvs/monitors/cluster-mon/backend/app/core/config.py b/cvs/monitors/cluster-mon/backend/app/core/config.py index 1e947c093..3a9df112b 100644 --- a/cvs/monitors/cluster-mon/backend/app/core/config.py +++ b/cvs/monitors/cluster-mon/backend/app/core/config.py @@ -86,11 +86,13 @@ class Settings(BaseSettings): # API api_prefix: str = "/api" - cors_origins: List[str] = Field(default_factory=lambda: [ - "http://localhost:3000", - "http://localhost:5173", - # Configured via environment or simple_config.py - ]) + cors_origins: List[str] = Field( + default_factory=lambda: [ + "http://localhost:3000", + "http://localhost:5173", + # Configured via environment or simple_config.py + ] + ) class Config: env_file = ".env" @@ -168,5 +170,6 @@ def load_from_yaml(cls, yaml_file: str) -> "Settings": except Exception as e: print(f"Error loading YAML config: {e}") import traceback + traceback.print_exc() settings = Settings() diff --git a/cvs/monitors/cluster-mon/backend/app/core/cvs_parallel_ssh_reliable.py b/cvs/monitors/cluster-mon/backend/app/core/cvs_parallel_ssh_reliable.py index 1c1c42b7c..0f24f723a 100644 --- a/cvs/monitors/cluster-mon/backend/app/core/cvs_parallel_ssh_reliable.py +++ b/cvs/monitors/cluster-mon/backend/app/core/cvs_parallel_ssh_reliable.py @@ -31,8 +31,19 @@ class Pssh: """ def __init__( - self, log, host_list, user=None, password=None, pkey='id_rsa', host_key_check=False, stop_on_errors=True, - timeout=30, proxy_host=None, proxy_user=None, proxy_password=None, proxy_pkey=None + self, + log, + host_list, + user=None, + password=None, + pkey='id_rsa', + host_key_check=False, + stop_on_errors=True, + timeout=30, + proxy_host=None, + proxy_user=None, + proxy_password=None, + proxy_pkey=None, ): self.log = log self.host_list = host_list @@ -47,10 +58,7 @@ def __init__( self.timeout = timeout # Build client parameters - client_params = { - 'user': self.user, - 'keepalive_seconds': 30 - } + client_params = {'user': self.user, 'keepalive_seconds': 30} # Add authentication if self.password is None: @@ -63,7 +71,7 @@ def __init__( # Add jump host/proxy if configured if proxy_host: - logger.info(f"Configuring jump host proxy:") + logger.info("Configuring jump host proxy:") logger.info(f" Proxy host: {proxy_host}") logger.info(f" Proxy user: {proxy_user}") logger.info(f" Proxy password: {'***SET***' if proxy_password else 'NOT SET'}") @@ -77,7 +85,7 @@ def __init__( elif proxy_pkey: client_params['proxy_pkey'] = proxy_pkey - logger.info(f"Creating ParallelSSHClient with params:") + logger.info("Creating ParallelSSHClient with params:") logger.info(f" Hosts: {self.reachable_hosts}") logger.info(f" User: {client_params.get('user')}") logger.info(f" Password: {'***SET***' if client_params.get('password') else 'NOT SET'}") @@ -86,7 +94,7 @@ def __init__( logger.info(f" Proxy password: {'***SET***' if client_params.get('proxy_password') else 'NOT SET'}") self.client = ParallelSSHClient(self.reachable_hosts, **client_params) - logger.info(f"✅ ParallelSSHClient created successfully") + logger.info("✅ ParallelSSHClient created successfully") def check_connectivity(self, hosts): """ @@ -341,8 +349,6 @@ def scp(src, dst, srcusername, srcpassword, dstusername=None, dstpassword=None): time.sleep(1) ssh.exec_command('sshpass -p %s scp %s %s@%s:%s' % (dstpassword, srcfile, dstusername, dstip, dstfile)) - async def exec_async(self, cmd, timeout=None, print_console=True): '''Async wrapper - just calls exec() directly''' return self.exec(cmd, timeout, print_console) - diff --git a/cvs/monitors/cluster-mon/backend/app/core/jump_host_pssh.py b/cvs/monitors/cluster-mon/backend/app/core/jump_host_pssh.py index 72f4d79b1..d4f000874 100644 --- a/cvs/monitors/cluster-mon/backend/app/core/jump_host_pssh.py +++ b/cvs/monitors/cluster-mon/backend/app/core/jump_host_pssh.py @@ -4,7 +4,6 @@ """ import paramiko -from pssh.clients import ParallelSSHClient from typing import List, Optional, Dict import logging @@ -61,7 +60,9 @@ def _connect_to_jump_host(self): """Connect to jump host using paramiko.""" logger.info(f"Connecting to jump host: {self.jump_host}") logger.info(f" Jump user: {self.jump_user}") - logger.info(f" Jump password: {'***SET*** (length={len(self.jump_password)})' if self.jump_password else 'NOT SET'}") + logger.info( + f" Jump password: {'***SET*** (length={len(self.jump_password)})' if self.jump_password else 'NOT SET'}" + ) logger.info(f" Jump pkey: {self.jump_pkey if self.jump_pkey else 'NOT SET'}") self.jump_client = paramiko.SSHClient() @@ -70,8 +71,10 @@ def _connect_to_jump_host(self): try: if self.jump_password: logger.info(f"Attempting password authentication to {self.jump_host}...") - logger.info(f" Password value check: {self.jump_password[:3]}*** (showing first 3 chars for verification)") - logger.info(f"Using password authentication for jump host") + logger.info( + f" Password value check: {self.jump_password[:3]}*** (showing first 3 chars for verification)" + ) + logger.info("Using password authentication for jump host") self.jump_client.connect( hostname=self.jump_host, username=self.jump_user, @@ -111,8 +114,8 @@ def _create_parallel_client(self): logger.info(f" Target user: {self.target_user}") logger.info(f" Target pkey (on jump host): {self.target_pkey}") logger.info(f" Max parallel: {self.max_parallel}") - logger.info(f" Method: Execute SSH commands on jump host using key file there") - logger.info(f"✅ Ready to execute commands via jump host") + logger.info(" Method: Execute SSH commands on jump host using key file there") + logger.info("✅ Ready to execute commands via jump host") def _execute_on_node(self, node: str, cmd: str, timeout: Optional[int] = None) -> str: """Execute command on a single node via jump host.""" @@ -160,8 +163,7 @@ def exec(self, cmd: str, timeout: Optional[int] = None, print_console: bool = Tr with ThreadPoolExecutor(max_workers=self.max_parallel) as executor: # Submit all tasks future_to_node = { - executor.submit(self._execute_on_node, node, cmd, timeout): node - for node in self.target_hosts + executor.submit(self._execute_on_node, node, cmd, timeout): node for node in self.target_hosts } # Collect results as they complete diff --git a/cvs/monitors/cluster-mon/backend/app/core/simple_config.py b/cvs/monitors/cluster-mon/backend/app/core/simple_config.py index e5bba7c35..1b14e8945 100644 --- a/cvs/monitors/cluster-mon/backend/app/core/simple_config.py +++ b/cvs/monitors/cluster-mon/backend/app/core/simple_config.py @@ -5,7 +5,7 @@ import yaml from pathlib import Path -from typing import List, Optional, Dict, Any +from typing import List, Optional class SimpleConfig: @@ -14,7 +14,6 @@ class SimpleConfig: def __init__(self, yaml_path: str = None): # Auto-detect config path for both dev and Docker if yaml_path is None: - import os # Try Docker path first docker_path = Path("/app/config/cluster.yaml") if docker_path.exists(): @@ -49,6 +48,7 @@ def load_nodes_from_file(self) -> List[str]: """Load node IPs from nodes file.""" # Try multiple possible paths import os + possible_paths = [ Path("/app/config/nodes.txt"), # Docker path (first priority) Path("../config/nodes.txt"), # Development path @@ -70,6 +70,7 @@ def load_nodes_from_file(self) -> List[str]: @property def ssh_username(self) -> str: import os + default_user = os.getenv("USER", "root") return self.config_data.get("ssh", {}).get("username", default_user) @@ -78,6 +79,7 @@ def ssh_password(self) -> Optional[str]: # SECURITY: Password is stored in memory only (app_state), never in YAML try: from app.main import app_state + return app_state.ssh_password except: return None @@ -104,6 +106,7 @@ def jump_host(self) -> Optional[str]: @property def jump_host_username(self) -> str: import os + default_user = os.getenv("USER", "root") return self.config_data.get("ssh", {}).get("jump_host", {}).get("username", default_user) @@ -113,6 +116,7 @@ def jump_host_password(self) -> Optional[str]: # However, for testing/development, we also check YAML try: from app.main import app_state + if app_state.jump_host_password: return app_state.jump_host_password except: @@ -131,17 +135,20 @@ def jump_host_key_file(self) -> str: def node_username_via_jumphost(self) -> str: """Username for cluster nodes when using jump host.""" import os + default_user = os.getenv("USER", "root") # First check for node_username_via_jumphost at ssh level, then check jump_host.node_username - return self.config_data.get("ssh", {}).get("node_username_via_jumphost") or \ - self.config_data.get("ssh", {}).get("jump_host", {}).get("node_username", default_user) + return self.config_data.get("ssh", {}).get("node_username_via_jumphost") or self.config_data.get("ssh", {}).get( + "jump_host", {} + ).get("node_username", default_user) @property def node_key_file_on_jumphost(self) -> str: """Path to private key ON JUMP HOST for accessing cluster nodes.""" # First check for node_key_file_on_jumphost at ssh level, then check jump_host.node_key_file - return self.config_data.get("ssh", {}).get("node_key_file_on_jumphost") or \ - self.config_data.get("ssh", {}).get("jump_host", {}).get("node_key_file", "~/.ssh/id_rsa") + return self.config_data.get("ssh", {}).get("node_key_file_on_jumphost") or self.config_data.get("ssh", {}).get( + "jump_host", {} + ).get("node_key_file", "~/.ssh/id_rsa") # Polling Configuration @property @@ -169,6 +176,7 @@ def gpu_util_threshold(self) -> float: @property def cors_origins(self) -> List[str]: import os + # Allow all origins in Docker, or specific origins from environment variable cors_env = os.getenv("CORS_ORIGINS", "*") if cors_env == "*": @@ -225,12 +233,16 @@ def polling(self): class PollingConfig: def __init__(self, parent): import os + # Allow environment variable override self.interval = int(os.getenv('POLLING__INTERVAL', parent.polling_interval)) self.batch_size = parent.polling_batch_size self.stagger_delay = parent.polling_stagger_delay - self.failure_threshold = int(os.getenv('POLLING__FAILURE_THRESHOLD', - parent.config_data.get('polling', {}).get('failure_threshold', 5))) + self.failure_threshold = int( + os.getenv( + 'POLLING__FAILURE_THRESHOLD', parent.config_data.get('polling', {}).get('failure_threshold', 5) + ) + ) return PollingConfig(self) diff --git a/cvs/monitors/cluster-mon/backend/app/installers/base_installer.py b/cvs/monitors/cluster-mon/backend/app/installers/base_installer.py index d9f9d4dc8..418063c38 100644 --- a/cvs/monitors/cluster-mon/backend/app/installers/base_installer.py +++ b/cvs/monitors/cluster-mon/backend/app/installers/base_installer.py @@ -3,7 +3,7 @@ """ import logging -from typing import Dict, List, Any, Optional +from typing import Dict, Any, Optional from abc import ABC, abstractmethod logger = logging.getLogger(__name__) @@ -170,29 +170,26 @@ def install_package(self) -> Dict[str, Any]: # Create temporary SSH manager for just these nodes from app.core.parallel_ssh import ParallelSSHManager + temp_manager = ParallelSSHManager( host_list=os_nodes, user=self.ssh_manager.pssh.user, password=getattr(self.ssh_manager.pssh, 'password', None), pkey=getattr(self.ssh_manager.pssh, 'pkey', '~/.ssh/id_rsa'), timeout=300, # 5 minute timeout for installation - stop_on_errors=False + stop_on_errors=False, ) results = temp_manager.exec(install_cmd, timeout=300) for node in os_nodes: if node in results: - installation_results[node] = { - "success": True, - "os_type": os_type, - "output": results[node] - } + installation_results[node] = {"success": True, "os_type": os_type, "output": results[node]} else: installation_results[node] = { "success": False, "os_type": os_type, - "error": "Installation command failed or timed out" + "error": "Installation command failed or timed out", } # Add already installed nodes to results @@ -201,7 +198,7 @@ def install_package(self) -> Dict[str, Any]: "success": True, "os_type": os_map.get(node, 'unknown'), "already_installed": True, - "message": "Package already installed" + "message": "Package already installed", } # Add unsupported OS nodes to results @@ -209,7 +206,7 @@ def install_package(self) -> Dict[str, Any]: installation_results[item["node"]] = { "success": False, "os_type": item["os"], - "error": f"Unsupported OS: {item['os']}" + "error": f"Unsupported OS: {item['os']}", } # Summary @@ -225,5 +222,5 @@ def install_package(self) -> Dict[str, Any]: "failed": failed, "already_installed": len(already_installed), "unsupported_os": len(unsupported_os), - "results": installation_results + "results": installation_results, } diff --git a/cvs/monitors/cluster-mon/backend/app/main.py b/cvs/monitors/cluster-mon/backend/app/main.py index d83d25a89..558032bf7 100644 --- a/cvs/monitors/cluster-mon/backend/app/main.py +++ b/cvs/monitors/cluster-mon/backend/app/main.py @@ -9,7 +9,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles -from typing import List +from typing import List, Union, Optional import os from pathlib import Path @@ -32,16 +32,12 @@ backupCount=3, # Keep 3 backup files (backend.log.1, backend.log.2, backend.log.3) ) rotating_handler.setLevel(LOG_LEVEL) -rotating_handler.setFormatter( - logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") -) +rotating_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) # Also keep console output console_handler = logging.StreamHandler() console_handler.setLevel(LOG_LEVEL) -console_handler.setFormatter( - logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") -) +console_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) # Configure root logger logging.basicConfig( @@ -64,7 +60,7 @@ class AppState: """Global application state.""" def __init__(self): - self.ssh_manager: ParallelSSHManager = None + self.ssh_manager: Optional[Union[Pssh, JumpHostPssh]] = None self.gpu_collector: GPUMetricsCollector = None self.nic_collector: NICMetricsCollector = None self.latest_metrics: dict = {} @@ -132,17 +128,14 @@ async def reload_configuration(): # 4. Reload configuration from files logger.info("Reloading configuration from cluster.yaml and nodes.txt...") from app.core.simple_config import SimpleConfig + new_config = SimpleConfig() # 5. Load new nodes nodes = new_config.load_nodes_from_file() if not nodes: logger.warning("No nodes found in configuration after reload") - return { - "success": False, - "error": "No nodes configured in nodes.txt", - "nodes_count": 0 - } + return {"success": False, "error": "No nodes configured in nodes.txt", "nodes_count": 0} logger.info(f"Loaded {len(nodes)} nodes from configuration") @@ -152,7 +145,11 @@ async def reload_configuration(): if not using_jump_password and not using_direct_password: # Using key-based auth - verify key exists - key_file_path = new_config.ssh.jump_host.key_file if (new_config.ssh.jump_host.enabled and new_config.ssh.jump_host.host) else new_config.ssh.key_file + key_file_path = ( + new_config.ssh.jump_host.key_file + if (new_config.ssh.jump_host.enabled and new_config.ssh.jump_host.host) + else new_config.ssh.key_file + ) key_file_expanded = os.path.expanduser(key_file_path) if key_file_path.startswith("~") else key_file_path logger.info(f"Checking for SSH key (key-based auth): {key_file_expanded}") @@ -163,25 +160,26 @@ async def reload_configuration(): "success": False, "error": f"SSH key file not found: {key_file_path}. Please upload SSH keys via the Configuration UI.", "nodes_count": len(nodes), - "requires_key_upload": True + "requires_key_upload": True, } else: logger.info(f"✅ SSH key file found: {key_file_expanded}") # List the key file to verify import subprocess + try: result = subprocess.run(['ls', '-l', key_file_expanded], capture_output=True, text=True) logger.info(f"Key file details: {result.stdout.strip()}") except: pass else: - logger.info(f"✅ Using password authentication - no key file check needed") + logger.info("✅ Using password authentication - no key file check needed") # 7. Reinitialize SSH manager with new configuration try: if new_config.ssh.jump_host.enabled and new_config.ssh.jump_host.host: num_nodes = len(nodes) - max_parallel = min(num_nodes, 5) + min(num_nodes, 5) logger.info(f"Reinitializing with jump host: {new_config.ssh.jump_host.host}") logger.info(f"Jump Host Username: {new_config.ssh.jump_host.username}") @@ -200,7 +198,7 @@ async def reload_configuration(): max_parallel=min(len(nodes), 5), # Limit to 5 to avoid exhausting paramiko channels (conservative) timeout=new_config.ssh.timeout, ) - logger.info(f"JumpHostPssh initialized successfully") + logger.info("JumpHostPssh initialized successfully") else: logger.info("Reinitializing with direct SSH (no jump host)") logger.info(f"Username: {new_config.ssh.username}") @@ -218,11 +216,7 @@ async def reload_configuration(): logger.info("Direct SSH manager reinitialized") except Exception as e: logger.error(f"Failed to reinitialize SSH manager: {e}") - return { - "success": False, - "error": f"Failed to initialize SSH manager: {str(e)}", - "nodes_count": len(nodes) - } + return {"success": False, "error": f"Failed to initialize SSH manager: {str(e)}", "nodes_count": len(nodes)} # 7. Restart metrics collection if app_state.ssh_manager and nodes: @@ -236,16 +230,12 @@ async def reload_configuration(): "success": True, "message": "Configuration reloaded successfully", "nodes_count": len(nodes), - "jump_host_enabled": new_config.ssh.jump_host.enabled + "jump_host_enabled": new_config.ssh.jump_host.enabled, } except Exception as e: logger.error(f"Error during configuration reload: {e}", exc_info=True) - return { - "success": False, - "error": str(e), - "nodes_count": 0 - } + return {"success": False, "error": str(e), "nodes_count": 0} def update_node_status(node: str, is_error: bool, error_type: str = 'unreachable'): @@ -272,7 +262,9 @@ def update_node_status(node: str, is_error: bool, error_type: str = 'unreachable # Only change status after consecutive failures exceed threshold if app_state.node_failure_count[node] >= failure_threshold: app_state.node_health_status[node] = error_type - logger.warning(f"Node {node} marked as {error_type} after {app_state.node_failure_count[node]} consecutive failures") + logger.warning( + f"Node {node} marked as {error_type} after {app_state.node_failure_count[node]} consecutive failures" + ) else: # Success - reset counter and mark healthy if app_state.node_failure_count[node] > 0: @@ -535,5 +527,5 @@ async def root(): "status": "running", "nodes": len(settings.nodes) if settings.nodes else 0, "collecting": app_state.is_collecting, - "note": "Frontend not built. Run 'cd frontend && npm run build' to build the UI." + "note": "Frontend not built. Run 'cd frontend && npm run build' to build the UI.", }