From 25166fe400a1d0446b6c2d4f440825cc9bbaec78 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Apr 2026 16:25:12 +0000 Subject: [PATCH 1/2] Initial plan From b4081297ee47d95e4b086d6c92893c26c765e5c7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Apr 2026 16:30:04 +0000 Subject: [PATCH 2/2] Add ProcMon DLL diagnostics for Stable Diffusion ROCm CI tests Agent-Logs-Url: https://github.com/lemonade-sdk/lemonade/sessions/04a006e7-b2cc-4146-a2d2-45b75658f8f7 Co-authored-by: Geramy <264964+Geramy@users.noreply.github.com> --- .../cpp_server_build_test_release.yml | 15 +++ ci/procmon-dll-diagnose.ps1 | 91 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 ci/procmon-dll-diagnose.ps1 diff --git a/.github/workflows/cpp_server_build_test_release.yml b/.github/workflows/cpp_server_build_test_release.yml index 2dbe38564..8ca2cf620 100644 --- a/.github/workflows/cpp_server_build_test_release.yml +++ b/.github/workflows/cpp_server_build_test_release.yml @@ -851,6 +851,21 @@ jobs: } } + - name: Diagnose sd-cli missing DLLs (ProcMon) + if: always() && matrix.script == 'server_sd.py' && contains(matrix.backends, 'rocm') + shell: PowerShell + run: .\ci\procmon-dll-diagnose.ps1 + + - name: Upload ProcMon diagnostics artifacts + if: always() && matrix.script == 'server_sd.py' && contains(matrix.backends, 'rocm') + uses: actions/upload-artifact@v4 + with: + name: sd-cli-procmon-${{ matrix.name }} + path: | + ${{ runner.temp }}\sd-cli.pml + ${{ runner.temp }}\sd-cli.csv + if-no-files-found: warn + - name: Capture and upload server logs if: always() uses: ./.github/actions/capture-server-logs diff --git a/ci/procmon-dll-diagnose.ps1 b/ci/procmon-dll-diagnose.ps1 new file mode 100644 index 000000000..fc41a39c4 --- /dev/null +++ b/ci/procmon-dll-diagnose.ps1 @@ -0,0 +1,91 @@ +# ProcMon-based diagnostic script for missing DLL failures on Windows. +# Downloads Sysinternals Process Monitor, captures a short trace while running +# sd-cli.exe, then filters the exported CSV for NAME NOT FOUND / PATH NOT FOUND +# entries on .dll files so CI logs show exactly which DLLs are missing. +# +# Usage: .\ci\procmon-dll-diagnose.ps1 +# Expected env vars: RUNNER_TEMP, LEMONADE_CACHE_DIR (or run from repo root) + +$ErrorActionPreference = "Continue" + +# Delay constants (seconds) – tuned for CI runner startup/shutdown latency +$PROCMON_START_WAIT = 3 # time for ProcMon to initialize its kernel driver +$POST_RUN_WAIT = 2 # drain remaining events after sd-cli exits +$PROCMON_STOP_WAIT = 3 # time for ProcMon to flush and close the backing file +$CSV_EXPORT_WAIT = 5 # time for ProcMon to convert .pml to .csv + +$sdCliRelPath = ".\ci-cache\bin\sd-cpp\rocm-preview\sd-cli.exe" + +if (-not (Test-Path $sdCliRelPath)) { + Write-Host "sd-cli.exe not found at $sdCliRelPath - skipping ProcMon diagnostics" + exit 0 +} + +$sdCli = (Resolve-Path $sdCliRelPath).Path +Write-Host "sd-cli.exe: $sdCli" + +# Download and extract ProcMon +$procmonZip = Join-Path $env:RUNNER_TEMP "Procmon.zip" +$procmonDir = Join-Path $env:RUNNER_TEMP "Procmon" + +New-Item -ItemType Directory -Force -Path $procmonDir | Out-Null + +Write-Host "Downloading Process Monitor..." +Invoke-WebRequest -Uri "https://download.sysinternals.com/files/ProcessMonitor.zip" ` + -OutFile $procmonZip -UseBasicParsing + +Expand-Archive -Path $procmonZip -DestinationPath $procmonDir -Force + +$procmon = Join-Path $procmonDir "Procmon64.exe" +$pml = Join-Path $env:RUNNER_TEMP "sd-cli.pml" +$csv = Join-Path $env:RUNNER_TEMP "sd-cli.csv" + +# Start ProcMon capture in the background; capture the process so we can verify it launched +Write-Host "Starting ProcMon capture..." +$pmProc = Start-Process -FilePath $procmon ` + -ArgumentList "/AcceptEula", "/Quiet", "/Minimized", "/BackingFile", $pml ` + -PassThru +if (-not $pmProc -or $pmProc.HasExited) { + Write-Host "WARNING: ProcMon failed to start – diagnostics will be unavailable" +} +Start-Sleep -Seconds $PROCMON_START_WAIT + +# Run sd-cli.exe once to reproduce the loader failure and capture its DLL look-ups +Write-Host "Running sd-cli.exe to capture DLL load activity..." +& $sdCli -M upscale --help +$sdExitCode = $LASTEXITCODE +Write-Host "sd-cli.exe exit code: $sdExitCode" + +Start-Sleep -Seconds $POST_RUN_WAIT + +# Stop ProcMon (saves .pml automatically) +Write-Host "Stopping ProcMon..." +& $procmon /Quiet /Terminate +Start-Sleep -Seconds $PROCMON_STOP_WAIT + +# Export the capture to CSV and verify the file was created +Write-Host "Exporting ProcMon log to CSV..." +& $procmon /OpenLog $pml /SaveAs $csv /AcceptEula +Start-Sleep -Seconds $CSV_EXPORT_WAIT + +if (-not (Test-Path $csv)) { + Write-Host "WARNING: CSV export not found at $csv – ProcMon export may have failed" + exit $sdExitCode +} + +# Filter for missing DLL entries that sd-cli.exe triggered +Write-Host "" +Write-Host "=== Likely missing DLLs (NAME NOT FOUND / PATH NOT FOUND for .dll files) ===" +$hits = Get-Content $csv | + Where-Object { $_ -match "(?i)sd-cli\.exe" } | + Where-Object { $_ -match "(?i)\.dll" } | + Where-Object { $_ -match "NAME NOT FOUND|PATH NOT FOUND" } + +if ($hits) { + $hits | Select-Object -First 200 | ForEach-Object { Write-Host $_ } +} else { + Write-Host "(no NAME NOT FOUND / PATH NOT FOUND .dll entries found for sd-cli.exe)" +} + +# Exit with sd-cli.exe's exit code so the CI step fails when sd-cli.exe is broken +exit $sdExitCode