Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions cvs/monitors/cluster-mon/backend/app/api/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,9 @@ async def get_cluster_status() -> Dict[str, Any]:
if isinstance(node_data, dict) and "error" not in node_data:
for gpu_id, gpu_metrics in node_data.items():
if isinstance(gpu_metrics, dict):
temp = gpu_metrics.get("Temperature (Sensor junction) (C)") or gpu_metrics.get("Temperature (Sensor edge) (C)")
temp = gpu_metrics.get("Temperature (Sensor junction) (C)") or gpu_metrics.get(
"Temperature (Sensor edge) (C)"
)
if temp:
total_temp += float(temp)
temp_count += 1
Expand Down Expand Up @@ -242,22 +244,14 @@ async def get_cluster_health() -> Dict[str, Any]:
# Create alerts for unhealthy nodes
if node_status == "unhealthy":
for issue in issues:
health_data["alerts"].append({
"severity": "warning",
"node": node,
"message": issue
})
health_data["alerts"].append({"severity": "warning", "node": node, "message": issue})

# Check unreachable nodes
for node in app_state.ssh_manager.unreachable_hosts:
health_data["nodes"][node] = {
"status": "unreachable",
"issues": ["Node is unreachable via SSH"],
}
health_data["alerts"].append({
"severity": "critical",
"node": node,
"message": f"Node {node} is unreachable"
})
health_data["alerts"].append({"severity": "critical", "node": node, "message": f"Node {node} is unreachable"})

return health_data
8 changes: 5 additions & 3 deletions cvs/monitors/cluster-mon/backend/app/api/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
from typing import Dict, Any, List, Optional
import yaml
import os
import signal
import asyncio
from pathlib import Path

router = APIRouter()
Expand Down Expand Up @@ -75,6 +73,7 @@ async def update_configuration(config: ClusterConfigUpdate) -> Dict[str, Any]:
Saves nodes to config/nodes.txt and updates cluster.yaml
"""
import logging

logger = logging.getLogger(__name__)

logger.info("=" * 80)
Expand Down Expand Up @@ -148,6 +147,7 @@ async def update_configuration(config: ClusterConfigUpdate) -> Dict[str, Any]:

# SECURITY: Store password in memory only, never persist to disk
from app.main import app_state

if config.auth_method == "password" and config.password:
app_state.ssh_password = config.password
else:
Expand Down Expand Up @@ -213,7 +213,9 @@ async def update_configuration(config: ClusterConfigUpdate) -> Dict[str, Any]:
logger.info("=" * 80)

password_note = ""
if config.auth_method == "password" or (config.use_jump_host and config.jump_host and config.jump_host.auth_method == "password"):
if config.auth_method == "password" or (
config.use_jump_host and config.jump_host and config.jump_host.auth_method == "password"
):
password_note = " Passwords are stored in memory only."

return {
Expand Down
5 changes: 1 addition & 4 deletions cvs/monitors/cluster-mon/backend/app/api/logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,4 @@ async def get_dmesg_errors() -> Dict[str, Any]:
return logs_data

except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to collect logs: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Failed to collect logs: {str(e)}")
34 changes: 21 additions & 13 deletions cvs/monitors/cluster-mon/backend/app/api/nodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,9 @@ async def list_nodes() -> List[Dict[str, Any]]:
util = gpu_entry.get("GPU use (%)", 0)
total_util += float(util) if util else 0

node_info["avg_gpu_util"] = round(
total_util / node_info["gpu_count"], 2
) if node_info["gpu_count"] > 0 else 0
node_info["avg_gpu_util"] = (
round(total_util / node_info["gpu_count"], 2) if node_info["gpu_count"] > 0 else 0
)

# Handle dict format (old rocm-smi format)
elif isinstance(node_util, dict) and 'error' not in node_util:
Expand All @@ -97,9 +97,9 @@ async def list_nodes() -> List[Dict[str, Any]]:
util = gpu_metrics.get("GPU use (%)", 0)
total_util += float(util) if util else 0

node_info["avg_gpu_util"] = round(
total_util / node_info["gpu_count"], 2
) if node_info["gpu_count"] > 0 else 0
node_info["avg_gpu_util"] = (
round(total_util / node_info["gpu_count"], 2) if node_info["gpu_count"] > 0 else 0
)

if node in temp_data:
node_temp = temp_data[node]
Expand All @@ -114,7 +114,9 @@ async def list_nodes() -> List[Dict[str, Any]]:
temp_data_field = gpu_entry.get("temperature", {})
if isinstance(temp_data_field, dict):
# Try different sensor names
temp = temp_data_field.get("hotspot", temp_data_field.get("edge", temp_data_field.get("junction", 0)))
temp = temp_data_field.get(
"hotspot", temp_data_field.get("edge", temp_data_field.get("junction", 0))
)
if isinstance(temp, dict):
temp = temp.get("value", 0)
else:
Expand All @@ -131,9 +133,12 @@ async def list_nodes() -> List[Dict[str, Any]]:
for gpu_temp in node_temp.values():
if isinstance(gpu_temp, dict):
# Try junction (hotspot) first, then edge, then memory
temp = gpu_temp.get("Temperature (Sensor junction) (C)",
gpu_temp.get("Temperature (Sensor edge) (C)",
gpu_temp.get("Temperature (Sensor memory) (C)", 0)))
temp = gpu_temp.get(
"Temperature (Sensor junction) (C)",
gpu_temp.get(
"Temperature (Sensor edge) (C)", gpu_temp.get("Temperature (Sensor memory) (C)", 0)
),
)
if temp:
try:
total_temp += float(temp)
Expand Down Expand Up @@ -228,9 +233,12 @@ async def get_node_details(node_id: str) -> Dict[str, Any]:

# Temperature - try junction (hotspot) first, then edge, then memory
if gpu_id in temp_data and isinstance(temp_data[gpu_id], dict):
temp = temp_data[gpu_id].get("Temperature (Sensor junction) (C)",
temp_data[gpu_id].get("Temperature (Sensor edge) (C)",
temp_data[gpu_id].get("Temperature (Sensor memory) (C)", 0)))
temp = temp_data[gpu_id].get(
"Temperature (Sensor junction) (C)",
temp_data[gpu_id].get(
"Temperature (Sensor edge) (C)", temp_data[gpu_id].get("Temperature (Sensor memory) (C)", 0)
),
)
try:
gpu_info["temperature_c"] = float(temp) if temp else 0.0
except (ValueError, TypeError):
Expand Down
30 changes: 10 additions & 20 deletions cvs/monitors/cluster-mon/backend/app/api/packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,30 +38,25 @@ def install_package(request: PackageInstallRequest) -> Dict[str, Any]:
# Get the appropriate installer
if request.package.lower() == 'lldp':
from app.installers.lldp_installer import LLDPInstaller

installer = LLDPInstaller(ssh_manager)
else:
raise HTTPException(
status_code=400,
detail=f"Unsupported package: {request.package}. Supported: lldp"
)
raise HTTPException(status_code=400, detail=f"Unsupported package: {request.package}. Supported: lldp")

# Run installation (synchronous)
result = installer.install_package()

return {
"success": True,
"message": f"Installation complete: {result['successful']} successful, {result['failed']} failed",
**result
**result,
}

except HTTPException:
raise
except Exception as e:
logger.error(f"Package installation failed: {str(e)}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Package installation failed: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Package installation failed: {str(e)}")


@router.get("/status/{package}")
Expand All @@ -88,20 +83,18 @@ def get_package_status(package: str) -> Dict[str, Any]:
"installed_nodes": [],
"not_installed_nodes": [],
"status_by_node": {},
"note": "SSH not configured yet. Save configuration first."
"note": "SSH not configured yet. Save configuration first.",
}

logger.info(f"Checking package status: {package}")

# Get the appropriate installer
if package.lower() == 'lldp':
from app.installers.lldp_installer import LLDPInstaller

installer = LLDPInstaller(ssh_manager)
else:
raise HTTPException(
status_code=400,
detail=f"Unsupported package: {package}. Supported: lldp"
)
raise HTTPException(status_code=400, detail=f"Unsupported package: {package}. Supported: lldp")

# Check installation status (synchronous)
installed_map = installer.check_installation()
Expand All @@ -116,17 +109,14 @@ def get_package_status(package: str) -> Dict[str, Any]:
"not_installed_count": len(not_installed_nodes),
"installed_nodes": installed_nodes,
"not_installed_nodes": not_installed_nodes,
"status_by_node": installed_map
"status_by_node": installed_map,
}

except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to check package status: {str(e)}", exc_info=True)
raise HTTPException(
status_code=500,
detail=f"Failed to check package status: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Failed to check package status: {str(e)}")


@router.get("/list")
Expand All @@ -141,7 +131,7 @@ async def list_supported_packages() -> Dict[str, Any]:
"name": "LLDP Daemon",
"description": "Link Layer Discovery Protocol daemon for network topology discovery",
"package_name": "lldpd",
"check_command": "lldpcli"
"check_command": "lldpcli",
},
# Add more packages here as we implement them
# {
Expand Down
1 change: 1 addition & 0 deletions cvs/monitors/cluster-mon/backend/app/api/restart.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ async def restart_backend() -> Dict[str, Any]:
This triggers a graceful shutdown and restart by executing
the backend in a new process.
"""

async def do_restart():
await asyncio.sleep(1)
# Restart using os.execv to replace current process
Expand Down
37 changes: 14 additions & 23 deletions cvs/monitors/cluster-mon/backend/app/api/ssh_keys.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

from fastapi import APIRouter, UploadFile, File, HTTPException
from typing import Dict, Any
import os
from pathlib import Path
import logging

Expand All @@ -26,17 +25,15 @@ async def upload_ssh_key(file: UploadFile = File(...)) -> Dict[str, Any]:
# Only allow common SSH key filenames for security
allowed_names = ["id_rsa", "id_ed25519", "id_ecdsa", "cluster_id_ed25519", "known_hosts", "config"]
if file.filename not in allowed_names:
raise HTTPException(
status_code=400,
detail=f"Invalid key filename. Allowed: {', '.join(allowed_names)}"
)
raise HTTPException(status_code=400, detail=f"Invalid key filename. Allowed: {', '.join(allowed_names)}")

# Create .ssh directory if it doesn't exist
ssh_dir = Path("/root/.ssh")
ssh_dir.mkdir(mode=0o700, exist_ok=True)

# Ensure directory has correct ownership
import os

try:
os.chown(ssh_dir, 0, 0) # root:root
ssh_dir.chmod(0o700)
Expand All @@ -52,6 +49,7 @@ async def upload_ssh_key(file: UploadFile = File(...)) -> Dict[str, Any]:

# Set proper permissions and ownership
import os

if file.filename in ["known_hosts", "config"]:
key_path.chmod(0o644)
else:
Expand All @@ -76,7 +74,7 @@ async def upload_ssh_key(file: UploadFile = File(...)) -> Dict[str, Any]:
"message": f"SSH key '{file.filename}' uploaded successfully",
"filename": file.filename,
"size": len(content),
"path": str(key_path)
"path": str(key_path),
}

except Exception as e:
Expand All @@ -93,25 +91,21 @@ async def list_ssh_keys() -> Dict[str, Any]:
ssh_dir = Path("/root/.ssh")

if not ssh_dir.exists():
return {
"keys": [],
"message": "No SSH keys directory found"
}
return {"keys": [], "message": "No SSH keys directory found"}

keys = []
for key_file in ssh_dir.iterdir():
if key_file.is_file():
stat = key_file.stat()
keys.append({
"filename": key_file.name,
"size": stat.st_size,
"permissions": oct(stat.st_mode)[-3:],
})
keys.append(
{
"filename": key_file.name,
"size": stat.st_size,
"permissions": oct(stat.st_mode)[-3:],
}
)

return {
"keys": keys,
"count": len(keys)
}
return {"keys": keys, "count": len(keys)}

except Exception as e:
logger.error(f"Failed to list SSH keys: {e}")
Expand All @@ -137,10 +131,7 @@ async def delete_ssh_key(filename: str) -> Dict[str, Any]:
key_path.unlink()
logger.info(f"SSH key deleted: {filename}")

return {
"success": True,
"message": f"SSH key '{filename}' deleted successfully"
}
return {"success": True, "message": f"SSH key '{filename}' deleted successfully"}

except HTTPException:
raise
Expand Down
Loading
Loading