diff --git a/src/cpp/server/backends/llamacpp_server.cpp b/src/cpp/server/backends/llamacpp_server.cpp index a7652516fe..67ddb39fc2 100644 --- a/src/cpp/server/backends/llamacpp_server.cpp +++ b/src/cpp/server/backends/llamacpp_server.cpp @@ -525,6 +525,23 @@ void LlamaCppServer::load(const std::string& model_name, << std::endl; } } + +#ifdef __linux__ + // On NVIDIA Optimus/PRIME laptops in On-Demand mode the dGPU is only + // activated for applications that opt in via __NV_PRIME_RENDER_OFFLOAD. + // Without this, CUDA reports "no CUDA-capable device is detected" even + // though the kernel module is loaded and /proc/driver/nvidia/gpus exists. + // Setting the variable is harmless on non-Optimus (single-GPU) systems. + const char* existing_prime = std::getenv("__NV_PRIME_RENDER_OFFLOAD"); + if (!existing_prime || existing_prime[0] == '\0') { + env_vars.push_back({"__NV_PRIME_RENDER_OFFLOAD", "1"}); + const char* existing_provider = std::getenv("__NV_PRIME_RENDER_OFFLOAD_PROVIDER"); + if (!existing_provider || existing_provider[0] == '\0') { + env_vars.push_back({"__NV_PRIME_RENDER_OFFLOAD_PROVIDER", "NVIDIA-G0"}); + } + LOG(INFO, "LlamaCpp") << "Setting __NV_PRIME_RENDER_OFFLOAD=1 for PRIME Offload compatibility" << std::endl; + } +#endif } #ifdef __APPLE__ diff --git a/src/cpp/server/model_manager.cpp b/src/cpp/server/model_manager.cpp index 04c1403efa..4bdc181609 100644 --- a/src/cpp/server/model_manager.cpp +++ b/src/cpp/server/model_manager.cpp @@ -2123,6 +2123,29 @@ std::map ModelManager::filter_models_by_backend( if (largest_mem_pool_gb > 0.0) { LOG(INFO, "ModelManager") << " - Largest memory pool: " << std::fixed << std::setprecision(1) << largest_mem_pool_gb << std::endl; } + if (system_info.contains("devices") && system_info["devices"].contains("nvidia_gpu")) { + const auto& nvidia_gpus = system_info["devices"]["nvidia_gpu"]; + if (nvidia_gpus.is_array()) { + for (const auto& gpu : nvidia_gpus) { + if (gpu.value("available", false)) { + std::string name = gpu.value("name", "unknown"); + std::string family = gpu.value("family", ""); + std::string cc = gpu.value("compute_capability", ""); + std::string suffix; + if (!cc.empty()) suffix = " (compute " + cc + ", " + (family.empty() ? "unsupported arch" : family) + ")"; + else if (!family.empty()) suffix = " (" + family + ")"; + else suffix = " (arch unknown -- nvidia-smi may have failed)"; + LOG(INFO, "ModelManager") << " - NVIDIA GPU: " << name << suffix << std::endl; + } else if (gpu.contains("error")) { + LOG(INFO, "ModelManager") << " - NVIDIA GPU: detection error: " << gpu["error"].get() << std::endl; + } + } + if (system_info["devices"].contains("nvidia_gpu_error")) { + LOG(INFO, "ModelManager") << " - NVIDIA GPU: detection error: " + << system_info["devices"]["nvidia_gpu_error"].get() << std::endl; + } + } + } debug_printed = true; } diff --git a/src/cpp/server/system_info.cpp b/src/cpp/server/system_info.cpp index e234acf6d1..b0c75a3912 100644 --- a/src/cpp/server/system_info.cpp +++ b/src/cpp/server/system_info.cpp @@ -1769,7 +1769,9 @@ std::string identify_cuda_arch_from_name(const std::string& device_name) { // Compact table: {sm_XX, {substrings that identify the architecture}}. // Listed highest-to-lowest; first match wins. static const std::vector>> TABLE = { - {"sm_120", {"rtx 50", "rtx50", "5090", "5080", "5070", "5060"}}, + {"sm_120", {"blackwell", "rtx 50", "rtx50", "5090", "5080", "5070", "5060", + "rtx pro 6000", "rtx pro 5000", "rtx pro 4500", "rtx pro 4000", + "rtx pro 3500", "rtx pro 3000", "rtx pro 2000", "rtx pro 1000"}}, {"sm_100", {"b100", "b200"}}, {"sm_90", {"h100", "h200"}}, {"sm_89", {"rtx 40", "rtx40", "4090", "4080", "4070", "4060", "l40", " l4"}}, @@ -2209,6 +2211,12 @@ std::string SystemInfo::get_cuda_visible_devices_for_arch(const std::string& arc // CUDA runtime ordinals and nvidia-smi/NVML indices can differ on mixed systems. // Passing a numeric nvidia-smi index can therefore accidentally expose the wrong GPU // (e.g. selecting sm_120 but making an sm_89 RTX 4090 visible as CUDA0). + // + // Only restrict CUDA_VISIBLE_DEVICES when there are mixed architectures that need + // hiding. If every available NVIDIA GPU matches the target arch, returning empty + // string lets the CUDA runtime enumerate them all without UUID filtering — which + // avoids driver-level UUID mismatches seen on some single-arch systems (e.g. RTX 50 + // Blackwell where UUID-based filtering can cause "no CUDA-capable device detected"). std::vector devices_to_expose; if (arch.empty()) { return ""; @@ -2225,6 +2233,7 @@ std::string SystemInfo::get_cuda_visible_devices_for_arch(const std::string& arc return ""; } + bool has_other_arch = false; int ordinal = 0; for (const auto& gpu : devices["nvidia_gpu"]) { if (!gpu.value("available", false)) { @@ -2241,9 +2250,16 @@ std::string SystemInfo::get_cuda_visible_devices_for_arch(const std::string& arc // than UUIDs because numeric CUDA ordinals can differ from nvidia-smi indices. devices_to_expose.push_back(std::to_string(ordinal)); } + } else { + has_other_arch = true; } ordinal++; } + + // No mixed architectures — no need to restrict CUDA_VISIBLE_DEVICES. + if (!has_other_arch) { + return ""; + } } catch (...) { devices_to_expose.clear(); } @@ -3033,6 +3049,52 @@ std::vector LinuxSystemInfo::get_nvidia_gpu_devices() { return gpus; } + // Secondary: /proc/driver/nvidia/gpus/*/information — readable whenever the + // nvidia kernel module is loaded, even when the GPU is in Optimus power-save + // mode and nvidia-smi fails. Provides the full model name and GPU UUID, which + // is enough for identify_cuda_arch_from_name() to determine the sm_XX family. + // (No compute_capability here; family is resolved from the name.) + { + fs::path gpus_dir = "/proc/driver/nvidia/gpus"; + std::error_code ec; + if (fs::exists(gpus_dir, ec) && fs::is_directory(gpus_dir, ec)) { + LOG(WARNING, "SystemInfo") << "nvidia-smi detection failed; falling back to /proc/driver/nvidia/gpus" << std::endl; + std::string driver_version = get_nvidia_driver_version(); + double vram = get_nvidia_vram(); + for (const auto& entry : fs::directory_iterator(gpus_dir, ec)) { + fs::path info_path = entry.path() / "information"; + std::ifstream info_file(info_path); + if (!info_file.is_open()) continue; + + std::string model; + std::string uuid; + std::string line; + while (std::getline(info_file, line)) { + if (line.rfind("Model:", 0) == 0) { + model = line.substr(6); + size_t s = model.find_first_not_of(" \t"); + if (s != std::string::npos) model = model.substr(s); + } else if (line.rfind("GPU UUID:", 0) == 0) { + uuid = line.substr(9); + size_t s = uuid.find_first_not_of(" \t"); + if (s != std::string::npos) uuid = uuid.substr(s); + } + } + + if (!model.empty()) { + GPUInfo gpu; + gpu.name = model; + gpu.uuid = uuid; + gpu.available = true; + gpu.driver_version = driver_version; + if (vram > 0.0) gpu.vram_gb = vram; + gpus.push_back(gpu); + } + } + if (!gpus.empty()) return gpus; + } + } + // Fallback: lspci (for systems where nvidia-smi is unavailable) FILE* pipe = popen("lspci 2>/dev/null | grep -iE 'vga|3d|display'", "r"); if (!pipe) {