Add diagnostic logging to chase sd-cli.exe ESRGAN upscale crash on Windows ROCm

src/cpp/server/backends/sd_server.cpp

@@ -1,3 +1,17 @@
+// On Windows, set up header guards BEFORE any other includes
+#ifdef _WIN32
+#ifndef _WINSOCKAPI_
+#define _WINSOCKAPI_
+#endif
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <winsock2.h>
+#include <windows.h>
+#ifdef ERROR
+#undef ERROR
+#endif
+#endif
+
 #include "lemon/backends/sd_server.h"
 #include "lemon/backends/backend_utils.h"
 #include "lemon/backend_manager.h"
@@ -26,6 +40,100 @@ namespace backends {
 static const char* ROCM_STABLE_RUNTIME_DIR = "rocm-stable-runtime";
 
 namespace {
+
+#ifdef _WIN32
+// Walk the PE import table of `exe_path` and log any DLL that cannot be
+// resolved. Called on STATUS_DLL_NOT_FOUND (0xC0000135) to surface the
+// exact missing dependency without requiring ProcMon on the runner.
+static void log_missing_dll_imports_win32(const std::string& exe_path) {
+    LOG(INFO, "SDServer") << "DLL check: scanning imports of " << exe_path << std::endl;
+
+    // Map the image with sections at virtual addresses; no DLL resolution,
+    // no code executed.
+    HMODULE hmod = LoadLibraryExA(
+        exe_path.c_str(), nullptr,
+        LOAD_LIBRARY_AS_IMAGE_RESOURCE | LOAD_LIBRARY_AS_DATAFILE);
+    if (!hmod) {
+        LOG(WARNING, "SDServer") << "DLL check: LoadLibraryExA failed for "
+            << exe_path << " err=" << GetLastError() << std::endl;
+        return;
+    }
+
+    // LOAD_LIBRARY_AS_IMAGE_RESOURCE sets bit 1 of the handle; clear it to
+    // get the true module base suitable for pointer arithmetic.
+    auto base = reinterpret_cast<const BYTE*>(
+        reinterpret_cast<uintptr_t>(hmod) & ~static_cast<uintptr_t>(3));
+
+    auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
+    if (dos->e_magic != IMAGE_DOS_SIGNATURE) {
+        LOG(WARNING, "SDServer") << "DLL check: invalid DOS signature in "
+            << exe_path << std::endl;
+        FreeLibrary(hmod);
+        return;
+    }
+
+    auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
+    if (nt->Signature != IMAGE_NT_SIGNATURE) {
+        LOG(WARNING, "SDServer") << "DLL check: invalid NT signature in "
+            << exe_path << std::endl;
+        FreeLibrary(hmod);
+        return;
+    }
+
+    DWORD import_rva = nt->OptionalHeader.DataDirectory[
+        IMAGE_DIRECTORY_ENTRY_IMPORT].VirtualAddress;
+    if (import_rva == 0) {
+        LOG(INFO, "SDServer") << "DLL check: no import directory in "
+            << exe_path << std::endl;
+        FreeLibrary(hmod);
+        return;
+    }
+
+    // Directory of the exe so we can check co-located (bundled) DLLs first.
+    std::string exe_dir;
+    auto sep = exe_path.find_last_of("\\/");
+    if (sep != std::string::npos) exe_dir = exe_path.substr(0, sep);
+
+    auto* imp = reinterpret_cast<const IMAGE_IMPORT_DESCRIPTOR*>(
+        base + import_rva);
+    for (; imp->Name != 0; ++imp) {
+        const char* dll_name = reinterpret_cast<const char*>(base + imp->Name);
+
+        // Basic sanity: ensure dll_name is a non-empty printable string
+        // (guards against corrupted PE headers in diagnostic code)
+        if (!dll_name || dll_name[0] == '\0') continue;
+        constexpr size_t MAX_DLL_NAME = 260;
+        bool valid = true;
+        for (size_t i = 0; i < MAX_DLL_NAME; ++i) {
+            if (dll_name[i] == '\0') break;
+            if (i == MAX_DLL_NAME - 1) { valid = false; break; }
+        }
+        if (!valid) continue;
+
+        // 1) Check beside sd-cli.exe (bundled ROCm DLLs live here)
+        std::string local_path = (exe_dir.empty() ? std::string(".") : exe_dir) + "\\" + dll_name;
+        if (GetFileAttributesA(local_path.c_str()) != INVALID_FILE_ATTRIBUTES) {
+            continue; // found locally — ok
+        }
+
+        // 2) Let Windows resolve via PATH / known-DLLs / system32
+        HMODULE htest = LoadLibraryExA(
+            dll_name, nullptr, LOAD_LIBRARY_AS_DATAFILE);
+        if (htest) {
+            FreeLibrary(htest);
+            continue; // resolvable — ok
+        }
+
+        DWORD err = GetLastError();
+        LOG(WARNING, "SDServer") << "DLL check MISSING: " << dll_name
+            << " (not in " << (exe_dir.empty() ? "." : exe_dir)
+            << ", err=" << err << ")" << std::endl;
+    }
+
+    FreeLibrary(hmod);
+}
+#endif // _WIN32
+
 bool is_rocm_backend(const std::string& backend) {
     return backend == "rocm" || backend == "rocm-stable" || backend == "rocm-preview";
 }
@@ -556,6 +664,31 @@ std::string SDServer::upscale_via_cli(
         "-o", output_path.string()
     };
 
+    // Diagnostic: log all launch parameters before starting sd-cli
+    LOG(INFO, "SDServer") << "sd-cli upscale launch:"
+        << " cli=" << cli_exe_path
+        << " model=" << upscale_model_path
+        << " input=" << input_path.string()
+        << " output=" << output_path.string()
+        << " input_bytes=" << raw.size()
+        << std::endl;
+
+    // Log reconstructed command line for easy copy/paste reproduction
+    {
+        std::string cmdline = "\"" + cli_exe_path + "\"";
+        for (const auto& arg : cli_args) {
+            cmdline += " \"" + arg + "\"";
+        }
+        LOG(INFO, "SDServer") << "sd-cli cmdline: " << cmdline << std::endl;
+    }
+
+    // Log env vars passed to subprocess (PATH/LD_LIBRARY_PATH for backend)
+    if (!env_vars.empty()) {
+        for (const auto& kv : env_vars) {
+            LOG(INFO, "SDServer") << "sd-cli env: " << kv.first << "=" << kv.second << std::endl;
+        }
+    }
+
     // inherit_output = true so subprocess stderr/stdout is visible in server
     // logs for debugging failed upscale operations
     auto proc = ProcessManager::start_process(
@@ -573,9 +706,26 @@ std::string SDServer::upscale_via_cli(
         LOG(INFO, "SDServer") << "ESRGAN upscale complete ("
             << raw.size() << " -> " << upscaled_data.size() << " bytes)" << std::endl;
     } else {
-        LOG(WARNING, "SDServer") << "ESRGAN upscale failed (exit code: "
-            << exit_code << ", model: " << upscale_model_path
-            << ", cli: " << cli_exe_path << ")" << std::endl;
+        std::error_code ec;
+        bool out_exists = fs::exists(output_path, ec);
+        uintmax_t out_size = out_exists ? fs::file_size(output_path, ec) : 0;
+        LOG(WARNING, "SDServer") << "ESRGAN upscale failed"
+            << " exit_code=" << exit_code
+            << " (0x" << std::hex << static_cast<uint32_t>(exit_code) << std::dec << ")"
+            << " out_exists=" << (out_exists ? "true" : "false")
+            << " out_size=" << out_size
+            << " model=" << upscale_model_path
+            << " cli=" << cli_exe_path
+            << std::endl;
+#ifdef _WIN32
+        // STATUS_DLL_NOT_FOUND — scan the import table to name the culprit DLL
+        if (static_cast<uint32_t>(exit_code) == 0xC0000135u) {
+            LOG(WARNING, "SDServer")
+                << "0xC0000135 = STATUS_DLL_NOT_FOUND;"
+                << " scanning sd-cli imports for missing DLLs..." << std::endl;
+            log_missing_dll_imports_win32(cli_exe_path);
+        }
+#endif
     }
 
     return result;

src/cpp/server/server.cpp

@@ -2582,6 +2582,11 @@ void Server::handle_image_upscale(const httplib::Request& req, httplib::Response
             b64_image, upscale_model_path, cli_exe.string(), env_vars);
 
         if (upscaled.empty()) {
+            LOG(WARNING, "Server") << "ESRGAN upscale returned empty result"
+                << " backend=" << resolved_backend
+                << " cli=" << cli_exe.string()
+                << " model=" << upscale_model_path
+                << std::endl;
             res.status = 500;
             nlohmann::json error = {{"error", {
                 {"message", "ESRGAN upscale failed"},

src/cpp/server/utils/process_manager.cpp

@@ -608,6 +608,12 @@ int ProcessManager::get_exit_code(ProcessHandle handle) {
         return -1;  // Still running
     }
 
+    if (exit_code != 0) {
+        LOG(WARNING, "ProcessManager") << "Process exited with non-zero code: "
+            << static_cast<int>(exit_code)
+            << " (0x" << std::hex << exit_code << std::dec << ")"
+            << std::endl;
+    }
     return static_cast<int>(exit_code);
 #else
     if (handle.pid <= 0) {
@@ -644,6 +650,12 @@ int ProcessManager::wait_for_exit(ProcessHandle handle, int timeout_seconds) {
 
     DWORD exit_code;
     GetExitCodeProcess(handle.handle, &exit_code);
+    if (exit_code != 0) {
+        LOG(WARNING, "ProcessManager") << "Process exited with non-zero code: "
+            << static_cast<int>(exit_code)
+            << " (0x" << std::hex << exit_code << std::dec << ")"
+            << std::endl;
+    }
     return exit_code;
 #else
     if (handle.pid <= 0) {