diff --git a/.github/actions/windows-msvc-env/action.yml b/.github/actions/windows-msvc-env/action.yml new file mode 100644 index 000000000..a105ca429 --- /dev/null +++ b/.github/actions/windows-msvc-env/action.yml @@ -0,0 +1,109 @@ +name: Set up Windows MSVC / clang-cl environment +description: > + Import the Visual Studio x64 developer environment (vcvarsall) into the job so + cl/clang-cl/ninja and the MSVC headers and libraries are available to later + steps. Exposes the resolved C++ compiler path and the MSVC toolset version for + use in cache keys. + +inputs: + compiler: + description: Compiler to set up - "msvc" (cl) or "clang-cl". + required: true + +outputs: + toolset: + description: MSVC toolset version (stable identifier for cache keys). + value: ${{ steps.setup.outputs.toolset }} + cxx-path: + description: Full path to the resolved C++ compiler. + value: ${{ steps.setup.outputs.cxx-path }} + +runs: + using: composite + steps: + - id: setup + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + if (-not (Test-Path $vswhere)) { throw "vswhere not found at $vswhere" } + $vsPath = & $vswhere -latest -products * -property installationPath + if (-not $vsPath) { throw "No Visual Studio installation found" } + Write-Host "Visual Studio: $vsPath" + + $vcvarsall = "$vsPath\VC\Auxiliary\Build\vcvarsall.bat" + if (-not (Test-Path $vcvarsall)) { throw "vcvarsall.bat not found at $vcvarsall" } + + # Snapshot the environment, run vcvarsall x64, then diff to capture its changes. + $before = @{} + Get-ChildItem env: | ForEach-Object { $before[$_.Name] = $_.Value } + + $tmp = [System.IO.Path]::GetTempFileName() + cmd /c "`"$vcvarsall`" x64 && set > `"$tmp`"" + if ($LASTEXITCODE -ne 0) { throw "vcvarsall.bat failed with exit code $LASTEXITCODE" } + $after = @{} + Get-Content $tmp | ForEach-Object { + if ($_ -match '^([^=]+)=(.*)$') { $after[$matches[1]] = $matches[2] } + } + Remove-Item $tmp + + # clang-cl is shipped with the VS LLVM component but is not added by vcvarsall. + $clangDir = $null + if ('${{ inputs.compiler }}' -eq 'clang-cl') { + $candidates = @( + "$vsPath\VC\Tools\Llvm\x64\bin\clang-cl.exe", + "$vsPath\VC\Tools\Llvm\bin\clang-cl.exe" + ) + foreach ($c in $candidates) { if (Test-Path $c) { $clangDir = Split-Path $c; break } } + if (-not $clangDir) { throw "clang-cl.exe not found under $vsPath\VC\Tools\Llvm" } + } + + # Apply to the current process so the compiler resolves within this step. + foreach ($name in $after.Keys) { + [System.Environment]::SetEnvironmentVariable($name, $after[$name], 'Process') + } + if ($clangDir) { $env:PATH = "$clangDir;$env:PATH" } + + # Persist non-PATH changes to GITHUB_ENV and PATH additions to GITHUB_PATH. + foreach ($name in $after.Keys) { + if ($name -ieq 'Path') { continue } + if ($before[$name] -ne $after[$name]) { + "$name=$($after[$name])" >> $env:GITHUB_ENV + } + } + $beforePath = @($before['Path'] -split ';') + $newEntries = @($after['Path'] -split ';') | Where-Object { $_ -and ($beforePath -notcontains $_) } + if ($clangDir) { $clangDir >> $env:GITHUB_PATH } + foreach ($p in $newEntries) { $p >> $env:GITHUB_PATH } + + # Resolve the compiler and a stable toolset version for cache keys. + if ('${{ inputs.compiler }}' -eq 'clang-cl') { + $cxx = (Get-Command clang-cl.exe -ErrorAction Stop).Source + } else { + $cxx = (Get-Command cl.exe -ErrorAction Stop).Source + } + Write-Host "C++ compiler: $cxx" + if ('${{ inputs.compiler }}' -eq 'clang-cl') { & $cxx --version 2>&1 | Write-Host } + + $toolsetFile = "$vsPath\VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt" + $toolset = (Get-Content $toolsetFile -ErrorAction Stop | Select-Object -First 1).Trim() + Write-Host "MSVC toolset: $toolset" + + "toolset=$toolset" >> $env:GITHUB_OUTPUT + "cxx-path=$cxx" >> $env:GITHUB_OUTPUT + + # Bootstrap vcpkg if not pre-installed. + $vcpkgRoot = if ($env:VCPKG_INSTALLATION_ROOT) { $env:VCPKG_INSTALLATION_ROOT } else { 'C:\vcpkg' } + if (Test-Path "$vcpkgRoot\vcpkg.exe") { + Write-Host "vcpkg already available at $vcpkgRoot" + } else { + Write-Host "vcpkg not found — bootstrapping into $vcpkgRoot" + git clone https://github.com/microsoft/vcpkg.git $vcpkgRoot --depth 1 + & "$vcpkgRoot\bootstrap-vcpkg.bat" -disableMetrics + if ($LASTEXITCODE -ne 0) { throw "vcpkg bootstrap failed ($LASTEXITCODE)" } + "VCPKG_INSTALLATION_ROOT=$vcpkgRoot" >> $env:GITHUB_ENV + } + # vcvarsall sets VCPKG_ROOT to VS's bundled copy; override it to match our + # standalone installation so vcpkg.exe doesn't warn about the mismatch. + "VCPKG_ROOT=$vcpkgRoot" >> $env:GITHUB_ENV diff --git a/.github/workflows/build-and-test-windows.yaml b/.github/workflows/build-and-test-windows.yaml new file mode 100644 index 000000000..9e45e1692 --- /dev/null +++ b/.github/workflows/build-and-test-windows.yaml @@ -0,0 +1,254 @@ +name: Build and Test (Windows) + +on: + push: + branches: + - main + pull_request: + branches: + - '**' + merge_group: + workflow_dispatch: + inputs: + run_slow_tests: + description: Run slow tests + required: false + default: false + type: boolean + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + VCPKG_TRIPLET: x64-windows-static-md + QDK_UARCH: x86-64-v3 + # Bump to force a rebuild of the cached dependencies. + DEPS_CACHE_VERSION: v1 + +jobs: + # ---------------------------------------------------------------------------- + # One job per compiler (msvc, clang-cl). On a cache miss the slow FetchContent + # dependencies (libint2/gauxc/ecpint/blaspp/lapackpp) and vcpkg packages are + # built and cached; on a hit they are restored and the build continues + # straight to compiling qdk. All steps run on the same runner so workspace + # paths are consistent and no CMakeCache juggling is needed. + # To force a dep cache rebuild, bump DEPS_CACHE_VERSION in the env block. + # ---------------------------------------------------------------------------- + build-test: + name: ${{ matrix.compiler }} - Build & test + runs-on: [self-hosted, 1ES.Pool=1es-gh-Dads_v5-pool-wus2, 1ES.ImageOverride=windows-2025-1espt, 'JobId=qdk_chemistry-win-${{ strategy.job-index }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}'] + timeout-minutes: 480 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + compiler: [msvc, clang-cl] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Set up ${{ matrix.compiler }} environment + id: env + uses: ./.github/actions/windows-msvc-env + with: + compiler: ${{ matrix.compiler }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Set memory-aware build parallelism + shell: pwsh + id: env2 + run: | + # qdk TUs that include libint2 engine.impl.h peak at several GB under MSVC; + # cap concurrency by available RAM (~4 GB/job) to avoid C1060 (compiler out + # of heap space). Scales up automatically on a larger runner. + $cpu = [int]$env:NUMBER_OF_PROCESSORS + $ramGB = [math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB) + $jobs = [math]::Min($cpu, [math]::Max(1, [math]::Round($ramGB / 4))) + Write-Host "CPUs=$cpu RAM=${ramGB}GB -> CMAKE_BUILD_PARALLEL_LEVEL=$jobs" + "CMAKE_BUILD_PARALLEL_LEVEL=$jobs" >> $env:GITHUB_ENV + # Include a short hash of the workspace path in the cache key so that + # runners with different workspace base directories (e.g. D:\a\ vs + # D:\a\_work\) each get their own cache entry and never hit a + # CMakeCache.txt path mismatch on restore. + $wsPath = "${{ github.workspace }}".ToLower().Replace('\', '/') + $wsBytes = [System.Text.Encoding]::UTF8.GetBytes($wsPath) + $wsHash = ([System.Security.Cryptography.SHA256]::Create().ComputeHash($wsBytes) | + ForEach-Object { $_.ToString('x2') }) -join '' + "wshash=$($wsHash.Substring(0, 12))" >> $env:GITHUB_OUTPUT + + - name: Restore dependency cache + id: cache + uses: actions/cache/restore@v5 + with: + path: | + vcpkg_installed + cpp/build-${{ matrix.compiler }} + key: >- + windows-deps-${{ matrix.compiler }}-${{ env.VCPKG_TRIPLET }}-vs${{ steps.env.outputs.toolset }}-${{ env.DEPS_CACHE_VERSION }}-ws${{ steps.env2.outputs.wshash }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json', 'vcpkg-overlay/**', 'cpp/manifest/qdk-chemistry/cgmanifest.json', 'external/macis/manifest/cgmanifest.json', 'cpp/cmake/third_party.cmake', 'cpp/cmake/patches/**', 'cpp/cmake/modules/DependencyManager.cmake', 'external/macis/src/lobpcgxx/CMakeLists.txt', '.pipelines/toolchains/windows.cmake') }} + + - name: Validate cached build directory + id: cache-validate + if: steps.cache.outputs.cache-hit == 'true' + shell: pwsh + run: | + # If the cached CMakeCache.txt was written on a runner with a different + # workspace base path (e.g. D:\a\ vs D:\a\_work\) the cached build.ninja + # files will reference stale absolute paths and cmake/ninja will fail. + # Detect this by checking whether the current workspace path appears in + # CMakeCache.txt (case-insensitive, forward-slash-normalised) and, if not, + # wipe the build directory so the configure + build steps run from scratch. + $buildDir = "cpp/build-${{ matrix.compiler }}" + $cacheFile = "$buildDir/CMakeCache.txt" + if (-not (Test-Path $cacheFile)) { + Write-Host "No CMakeCache.txt in restored cache — treating as cache miss" + "cache-valid=false" >> $env:GITHUB_OUTPUT + exit 0 + } + $ws = "${{ github.workspace }}".Replace('\', '/').ToLower() + $content = (Get-Content $cacheFile -Raw).ToLower().Replace('\', '/') + if ($content -notmatch [regex]::Escape($ws)) { + Write-Host "Workspace path mismatch in cached CMakeCache.txt (expected: $ws)" + Remove-Item -Recurse -Force $buildDir -ErrorAction SilentlyContinue + "cache-valid=false" >> $env:GITHUB_OUTPUT + } else { + Write-Host "Cached build directory paths match current workspace" + "cache-valid=true" >> $env:GITHUB_OUTPUT + } + + - name: Install vcpkg dependencies + if: steps.cache.outputs.cache-hit != 'true' || steps.cache-validate.outputs.cache-valid == 'false' + shell: pwsh + run: | + $vcpkg = $env:VCPKG_INSTALLATION_ROOT + if (-not $vcpkg) { $vcpkg = 'C:\vcpkg' } + & "$vcpkg\vcpkg.exe" install ` + --triplet ${{ env.VCPKG_TRIPLET }} ` + --x-manifest-root="${{ github.workspace }}" ` + --x-install-root="${{ github.workspace }}\vcpkg_installed" ` + --overlay-ports="${{ github.workspace }}\vcpkg-overlay\ports" + if ($LASTEXITCODE -ne 0) { throw "vcpkg install failed ($LASTEXITCODE)" } + + - name: Configure CMake + if: steps.cache.outputs.cache-hit != 'true' || steps.cache-validate.outputs.cache-valid == 'false' + shell: pwsh + run: | + $vcpkg = $env:VCPKG_INSTALLATION_ROOT + if (-not $vcpkg) { $vcpkg = 'C:\vcpkg' } + cmake -S cpp -B "cpp/build-${{ matrix.compiler }}" -GNinja ` + -DQDK_UARCH=${{ env.QDK_UARCH }} ` + -DQDK_CHEMISTRY_ENABLE_COVERAGE=OFF ` + -DQDK_CHEMISTRY_ENABLE_MPI=OFF ` + -DMACIS_ENABLE_TESTS=OFF ` + -DBUILD_SHARED_LIBS=OFF ` + -DBUILD_TESTING=ON ` + -DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=PRE_TEST ` + -DVCPKG_APPLOCAL_DEPS=OFF ` + -DCMAKE_BUILD_TYPE=Release ` + -DCMAKE_C_COMPILER="${{ steps.env.outputs.cxx-path }}" ` + -DCMAKE_CXX_COMPILER="${{ steps.env.outputs.cxx-path }}" ` + -DCMAKE_INSTALL_PREFIX="${{ github.workspace }}\install-${{ matrix.compiler }}" ` + -DCMAKE_TOOLCHAIN_FILE="$vcpkg\scripts\buildsystems\vcpkg.cmake" ` + -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE="${{ github.workspace }}\.pipelines\toolchains\windows.cmake" ` + -DVCPKG_TARGET_TRIPLET=${{ env.VCPKG_TRIPLET }} ` + -DVCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" ` + -DFETCHCONTENT_QUIET=OFF + if ($LASTEXITCODE -ne 0) { throw "CMake configure failed ($LASTEXITCODE)" } + + - name: Build C++ dependencies + if: steps.cache.outputs.cache-hit != 'true' || steps.cache-validate.outputs.cache-valid == 'false' + shell: pwsh + run: | + # libint2_cxx is a CMake INTERFACE target (no Ninja phony); libint2_obj is + # the actual OBJECT library that compiles all libint2 sources. + cmake --build "cpp/build-${{ matrix.compiler }}" --target libint2_obj ecpint gauxc blaspp lapackpp + if ($LASTEXITCODE -ne 0) { throw "Dependency build failed ($LASTEXITCODE)" } + + - name: Save dependency cache + if: steps.cache.outputs.cache-hit != 'true' || steps.cache-validate.outputs.cache-valid == 'false' + uses: actions/cache/save@v5 + with: + path: | + vcpkg_installed + cpp/build-${{ matrix.compiler }} + key: >- + windows-deps-${{ matrix.compiler }}-${{ env.VCPKG_TRIPLET }}-vs${{ steps.env.outputs.toolset }}-${{ env.DEPS_CACHE_VERSION }}-ws${{ steps.env2.outputs.wshash }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json', 'vcpkg-overlay/**', 'cpp/manifest/qdk-chemistry/cgmanifest.json', 'external/macis/manifest/cgmanifest.json', 'cpp/cmake/third_party.cmake', 'cpp/cmake/patches/**', 'cpp/cmake/modules/DependencyManager.cmake', 'external/macis/src/lobpcgxx/CMakeLists.txt', '.pipelines/toolchains/windows.cmake') }} + + - name: Build C++ library + shell: pwsh + run: | + # Dep targets are already up-to-date (either just built or restored from + # cache); Ninja skips them and only compiles qdk + test sources. + # Honors CMAKE_BUILD_PARALLEL_LEVEL set above (memory-aware). + cmake --build "cpp/build-${{ matrix.compiler }}" + if ($LASTEXITCODE -ne 0) { throw "CMake build failed ($LASTEXITCODE)" } + + - name: Run C++ tests + shell: pwsh + env: + OMP_NUM_THREADS: '2' + run: | + Push-Location "cpp/build-${{ matrix.compiler }}" + # Exclude MACIS_SERIAL_TEST (as on Linux) and libint2's own unit tests: + # libint2/unit/build is a compile-at-test-time meta-test of the vendored + # libint2 dependency that exceeds the ctest timeout under MSVC. qdk's own + # tests still run. + ctest --output-on-failure --verbose --timeout 400 ` + --output-junit ctest_results.xml ` + -E "MACIS_SERIAL_TEST|libint2/unit" + $code = $LASTEXITCODE + Pop-Location + if ($code -ne 0) { throw "ctest failed ($code)" } + + - name: Install C++ library + shell: pwsh + run: | + cmake --install "cpp/build-${{ matrix.compiler }}" + if ($LASTEXITCODE -ne 0) { throw "CMake install failed ($LASTEXITCODE)" } + + - name: Build & install Python package + shell: pwsh + run: | + $vcpkg = $env:VCPKG_INSTALLATION_ROOT + if (-not $vcpkg) { $vcpkg = 'C:\vcpkg' } + $prefix = "${{ github.workspace }}\vcpkg_installed\${{ env.VCPKG_TRIPLET }};${{ github.workspace }}\install-${{ matrix.compiler }}" + Push-Location python + # plugins (pyscf) do not build on Windows; install the test extra only. + # QDK_ALLOW_DEPENDENCY_FETCH=OFF: fail fast if the installed qdk is not + # reused (avoids a silent multi-hour rebuild of the C++ stack from source). + python -m pip install -v ".[test]" ` + --config-settings=cmake.args="-GNinja" ` + --config-settings=cmake.define.CMAKE_PREFIX_PATH="$prefix" ` + --config-settings=cmake.define.QDK_ALLOW_DEPENDENCY_FETCH=OFF ` + --config-settings=cmake.define.CMAKE_C_COMPILER="${{ steps.env.outputs.cxx-path }}" ` + --config-settings=cmake.define.CMAKE_CXX_COMPILER="${{ steps.env.outputs.cxx-path }}" ` + --config-settings=cmake.define.CMAKE_TOOLCHAIN_FILE="$vcpkg\scripts\buildsystems\vcpkg.cmake" ` + --config-settings=cmake.define.VCPKG_CHAINLOAD_TOOLCHAIN_FILE="${{ github.workspace }}\.pipelines\toolchains\windows.cmake" ` + --config-settings=cmake.define.VCPKG_TARGET_TRIPLET="${{ env.VCPKG_TRIPLET }}" ` + --config-settings=cmake.define.VCPKG_INSTALLED_DIR="${{ github.workspace }}\vcpkg_installed" + if ($LASTEXITCODE -ne 0) { Pop-Location; throw "Python package install failed ($LASTEXITCODE)" } + python -c "import qdk_chemistry; print('qdk_chemistry version:', qdk_chemistry.__version__)" + if ($LASTEXITCODE -ne 0) { Pop-Location; throw "Python import check failed ($LASTEXITCODE)" } + Pop-Location + + - name: Run Python tests + shell: pwsh + env: + OMP_NUM_THREADS: '2' + QDK_CHEMISTRY_RUN_SLOW_TESTS: ${{ inputs.run_slow_tests && '1' || '0' }} + run: | + Push-Location python + pytest -v --tb=short + $code = $LASTEXITCODE + Pop-Location + if ($code -ne 0) { throw "pytest failed ($code)" } diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index c128ae226..715c40938 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -15,6 +15,11 @@ on: required: false default: false type: boolean + rebuild_dep_cache: + description: Rebuild dependency cache from scratch (ignore existing cache) + required: false + default: false + type: boolean concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -219,7 +224,8 @@ jobs: - name: Cache C++ dependency install # Checks if manifest files or build script have changed to rebuild dependencies id: cache-cpp-deps - uses: actions/cache@v5 + if: ${{ !inputs.rebuild_dep_cache }} + uses: actions/cache/restore@v5 with: path: | ${{ env.CPP_DEPS_PREFIX }} @@ -244,6 +250,16 @@ jobs: ${{ github.workspace }}/cpp/manifest/qdk-chemistry/cgmanifest.json \ ${{ github.workspace }}/external/macis/manifest/cgmanifest.json + - name: Save C++ dependency cache + if: steps.cache-cpp-deps.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + with: + path: | + ${{ env.CPP_DEPS_PREFIX }} + key: >- + cpp-deps-${{ runner.os }}-${{ matrix.uarch }}- + ${{ hashFiles('cpp/manifest/qdk-chemistry/cgmanifest.json', 'external/macis/manifest/cgmanifest.json', '.devcontainer/scripts/install_cpp_dependencies.sh') }} + - name: Configure C++ RelWithDebInfo/Coverage build run: | if [ "${{ matrix.os-name }}" == "macOS" ]; then diff --git a/.gitignore b/.gitignore index 4e8546ada..eac2279cf 100644 --- a/.gitignore +++ b/.gitignore @@ -239,3 +239,6 @@ python/VERSION # Claude .claude + +# Copilot agent +agency.toml diff --git a/.pipelines/pip-scripts/bootstrap-conda.ps1 b/.pipelines/pip-scripts/bootstrap-conda.ps1 new file mode 100644 index 000000000..801706084 --- /dev/null +++ b/.pipelines/pip-scripts/bootstrap-conda.ps1 @@ -0,0 +1,113 @@ +<# +.SYNOPSIS + Bootstrap ms-ensureconda and create a named conda environment. + +.DESCRIPTION + PowerShell equivalent of bootstrap-conda.sh for Windows 1ES builds. + + This script is meant to be *called* (not dot-sourced) from other scripts + or YAML powershell steps. It writes all diagnostic output via Write-Host + and emits exactly one line to stdout: the resolved path to conda.exe. + Callers capture it with: + + $condaExe = & "$PSScriptRoot\bootstrap-conda.ps1" -EnvName buildenv -PythonVersion 3.11 + + Required env var: + SYSTEM_ACCESSTOKEN Pipeline access token, mapped from $(System.AccessToken). + Used by the azure_artifacts_conda_auth plugin that is + pre-registered in ms-ensureconda's conda distribution. + + Rationale: ms-ensureconda is the Microsoft-approved conda bootstrapper for CI + builds. See: + https://eng.ms/docs/more/languages-at-microsoft/python/articles/anaconda/install + All network access goes through the Azure Artifacts feed (1ES CFSClean blocks + public conda channels). The azure_artifacts_conda_auth plugin reads + ARTIFACTS_CONDA_TOKEN; we set it to SYSTEM_ACCESSTOKEN before every conda call. +#> +param( + # Name of the conda environment to create (e.g. "buildenv" or "testenv"). + [Parameter(Mandatory)] [string]$EnvName, + # Python version for the new environment (e.g. "3.11"). + [Parameter(Mandatory)] [string]$PythonVersion +) +$ErrorActionPreference = 'Stop' + +if (-not $env:SYSTEM_ACCESSTOKEN) { + throw "bootstrap-conda.ps1: SYSTEM_ACCESSTOKEN must be set before calling this script." +} + +$ENSURECONDA_PKG = 'ms-ensureconda==2026.6.1' +$MAX_ATTEMPTS = 3 +$RETRY_DELAY_SEC = 30 +$CONDA_FEED_ROOT = 'https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/Conda/repo' + +function _Bootstrap { + param([string]$EnvName, [string]$PythonVersion) + + Write-Host "Installing $ENSURECONDA_PKG and bootstrapping conda..." + + # Use a throwaway venv so we don't fight PEP 668 (externally-managed Python). + # Clean any stale venv first (idempotent on retries / self-hosted agents). + $bootstrapVenv = Join-Path $env:TEMP "qdk-bootstrap-venv-$EnvName" + Remove-Item -Recurse -Force $bootstrapVenv -ErrorAction SilentlyContinue + python -m venv $bootstrapVenv + if ($LASTEXITCODE -ne 0) { throw "Bootstrap venv creation failed ($LASTEXITCODE)" } + $venvPy = Join-Path $bootstrapVenv 'Scripts\python.exe' + + & $venvPy -m pip install --quiet $ENSURECONDA_PKG 2>&1 | Out-Host + if ($LASTEXITCODE -ne 0) { throw "ms-ensureconda install failed ($LASTEXITCODE)" } + + # ms-ensureconda --envfile writes a shell-sourceable KEY=VALUE file with + # CONDA_EXE, CONDA_BASH_HOOK, etc. We parse CONDA_EXE from it. + $condaEnvFile = Join-Path $env:TEMP "qdk-ensureconda-$EnvName.env" + $env:ARTIFACTS_CONDA_TOKEN = $env:SYSTEM_ACCESSTOKEN + & $venvPy -m ensureconda --envfile $condaEnvFile 2>&1 | Out-Host + if ($LASTEXITCODE -ne 0) { throw "ensureconda failed ($LASTEXITCODE)" } + + $condaExe = Get-Content $condaEnvFile | + Where-Object { $_ -match "^(?:export\s+)?CONDA_EXE=" } | + Select-Object -First 1 | + ForEach-Object { ($_ -split '=', 2)[1].Trim().Trim("'`"") } + if (-not $condaExe -or -not (Test-Path $condaExe)) { + throw "CONDA_EXE not found or path does not exist: '$condaExe'" + } + Write-Host "conda: $condaExe ($( & $condaExe --version 2>&1 ))" + + # Remove any pre-existing env (idempotent on self-hosted agents and retries). + & $condaExe env remove -y -n $EnvName 2>&1 | Out-Null + $LASTEXITCODE = 0 # env remove exits 1 when env doesn't exist; ignore + + # Public channels (conda.anaconda.org, repo.anaconda.com) are blocked under + # 1ES network isolation (CFSClean). Force all installs through the Azure + # Artifacts feed, which proxies `main` and `conda-forge` as named subpaths. + # The `main` channel carries python + pip; `conda-forge` is a fallback. + $env:ARTIFACTS_CONDA_TOKEN = $env:SYSTEM_ACCESSTOKEN + & $condaExe create --override-channels ` + --channel "$CONDA_FEED_ROOT/main" ` + --channel "$CONDA_FEED_ROOT/conda-forge" ` + --yes --quiet --name $EnvName "python=$PythonVersion" pip 2>&1 | Out-Host + if ($LASTEXITCODE -ne 0) { throw "conda create '$EnvName' failed ($LASTEXITCODE)" } + Write-Host "Conda env '$EnvName' created with Python $PythonVersion." + + # Return the resolved conda exe path (captured by caller). + return $condaExe +} + +$attempt = 1 +while ($true) { + try { + $condaExe = _Bootstrap -EnvName $EnvName -PythonVersion $PythonVersion + break + } catch { + if ($attempt -ge $MAX_ATTEMPTS) { + Write-Error "bootstrap-conda.ps1: failed after $MAX_ATTEMPTS attempts: $_" + exit 1 + } + Write-Warning "Bootstrap attempt $attempt/$MAX_ATTEMPTS failed: $_ Retrying in ${RETRY_DELAY_SEC}s..." + Start-Sleep -Seconds $RETRY_DELAY_SEC + $attempt++ + } +} + +# Emit only the conda exe path to stdout; callers capture this line. +Write-Output $condaExe diff --git a/.pipelines/pip-scripts/build-pip-wheels-windows-deps.ps1 b/.pipelines/pip-scripts/build-pip-wheels-windows-deps.ps1 new file mode 100644 index 000000000..d61386c43 --- /dev/null +++ b/.pipelines/pip-scripts/build-pip-wheels-windows-deps.ps1 @@ -0,0 +1,92 @@ +<# +.SYNOPSIS + Build C++ dependencies (vcpkg + FetchContent targets) for the Windows wheel pipeline. + +.DESCRIPTION + Invoked only on a dependency cache miss. Installs vcpkg packages, runs the CMake + configure pass, and builds the slow FetchContent targets (libint2, ecpint, gauxc, + blaspp, lapackpp). The build directory is subsequently cached by the calling + pipeline job; the full qdk build then starts from a warm build tree. + + Prerequisites (set by the YAML template before this script runs): + - INCLUDE, LIB, PATH already contain MSVC entries + (applied via ##vso[task.setvariable] / ##vso[task.prependpath]). + - CMAKE_BUILD_PARALLEL_LEVEL is set if caller wants a specific level + (otherwise computed here from CPU count and available RAM). +#> +param( + [Parameter(Mandatory)] [string]$SrcDir, + [Parameter(Mandatory)] [string]$ClPath, + [string]$March = 'x86-64-v3', + [string]$BuildType = 'Release', + [string]$VcpkgRoot +) +$ErrorActionPreference = 'Stop' + +# Fall back to well-known vcpkg location on MMS images. +if (-not $VcpkgRoot) { + $VcpkgRoot = if ($env:VCPKG_INSTALLATION_ROOT) { $env:VCPKG_INSTALLATION_ROOT } else { 'C:\vcpkg' } +} +if (-not (Test-Path "$VcpkgRoot\vcpkg.exe")) { throw "vcpkg.exe not found under '$VcpkgRoot'" } + +$buildDir = "$SrcDir\cpp\build-msvc" + +# Cap Ninja parallelism by available RAM (~4 GB/job for MSVC TUs pulling in +# libint2 headers). On the HB120 runner this typically allows all 120 cores. +if (-not $env:CMAKE_BUILD_PARALLEL_LEVEL) { + $cpu = [int]$env:NUMBER_OF_PROCESSORS + $ramGB = [math]::Floor((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB) + $jobs = [math]::Min($cpu, [math]::Max(1, [math]::Floor($ramGB / 4))) + Write-Host "CPUs=$cpu RAM=${ramGB} GB -> CMAKE_BUILD_PARALLEL_LEVEL=$jobs" + $env:CMAKE_BUILD_PARALLEL_LEVEL = $jobs +} + +# ─── vcpkg install ──────────────────────────────────────────────────────────── +# Route all vcpkg source downloads through the Terrapin internal mirror to +# avoid hitting external hosts (gitlab.com etc.) that are blocked by the +# 1ES CFSClean network isolation policy. +# See: https://eng.ms/docs/.../vcpkg (Step 4: Use Terrapin for Asset Caching) +# Do NOT set x-block-origin: github.com is accessible from the runner, but +# gitlab.com (used by eigen3) is blocked; Terrapin is the preferred source and +# falls back to the authoritative URL only on a miss. +$env:X_VCPKG_ASSET_SOURCES = "x-azurl,https://vcpkg.storage.devpackages.microsoft.io/artifacts/" + +Write-Host "=== vcpkg install ===" +& "$VcpkgRoot\vcpkg.exe" install ` + --triplet x64-windows-static-md ` + --x-manifest-root="$SrcDir" ` + --x-install-root="$SrcDir\vcpkg_installed" ` + --overlay-ports="$SrcDir\vcpkg-overlay\ports" +if ($LASTEXITCODE -ne 0) { throw "vcpkg install failed ($LASTEXITCODE)" } + +# ─── CMake configure (deps pass) ───────────────────────────────────────────── +Write-Host "=== CMake configure ===" +$cmakeArgs = @( + '-S', "$SrcDir\cpp", + '-B', $buildDir, + '-GNinja', + "-DQDK_UARCH=$March", + '-DQDK_CHEMISTRY_ENABLE_COVERAGE=OFF', + '-DQDK_CHEMISTRY_ENABLE_MPI=OFF', + '-DMACIS_ENABLE_TESTS=ON', + '-DBUILD_SHARED_LIBS=OFF', + '-DBUILD_TESTING=ON', + "-DCMAKE_BUILD_TYPE=$BuildType", + "-DCMAKE_C_COMPILER=$ClPath", + "-DCMAKE_CXX_COMPILER=$ClPath", + "-DCMAKE_INSTALL_PREFIX=$SrcDir\install-msvc", + "-DCMAKE_TOOLCHAIN_FILE=$VcpkgRoot\scripts\buildsystems\vcpkg.cmake", + "-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$SrcDir\.pipelines\toolchains\windows.cmake", + '-DVCPKG_TARGET_TRIPLET=x64-windows-static-md', + "-DVCPKG_INSTALLED_DIR=$SrcDir\vcpkg_installed", + '-DFETCHCONTENT_QUIET=OFF' +) +cmake @cmakeArgs +if ($LASTEXITCODE -ne 0) { throw "CMake configure failed ($LASTEXITCODE)" } + +# ─── Build slow FetchContent dependency targets ─────────────────────────────── +# Ninja reuses any previously-built artefacts from a restored partial cache, so +# this step is incremental when a stale cache is present. +Write-Host "=== Building C++ dependencies ===" +cmake --build $buildDir --target libint2 ecpint gauxc blaspp lapackpp +if ($LASTEXITCODE -ne 0) { throw "Dependency build failed ($LASTEXITCODE)" } diff --git a/.pipelines/pip-scripts/build-pip-wheels-windows.ps1 b/.pipelines/pip-scripts/build-pip-wheels-windows.ps1 new file mode 100644 index 000000000..426bfde13 --- /dev/null +++ b/.pipelines/pip-scripts/build-pip-wheels-windows.ps1 @@ -0,0 +1,216 @@ +<# +.SYNOPSIS + Build the full qdk C++ library, run C++ tests, and produce a Python wheel + using a conda-isolated build environment. + +.DESCRIPTION + This script is always invoked (unlike the deps script which is skipped on + cache hit). It: + 1. Rebuilds the full qdk + macis library on top of the cached dep tree. + 2. Runs the C++ test suite (ctest); results are written to an XML file + that the calling YAML template publishes with PublishTestResults@2. + 3. Installs the C++ library under install-msvc/. + 4. Bootstraps a conda environment via ms-ensureconda (the Microsoft-approved + conda bootstrapper). Public channels are blocked in 1ES CFSClean; all + packages are fetched from the Azure Artifacts Conda/PyPI feed. + 5. Installs build tooling (pip, build, scikit-build-core, etc.) and + generates a Component Governance PipReport. + 6. Rewrites relative README links to absolute GitHub URLs (equivalent to + prepare-readme.sh). + 7. Builds the distribution wheel with scikit-build-core. No wheel repair + is needed: x64-windows-static-md statically links all vcpkg deps. + 8. Copies the wheel to python/repaired_wheelhouse/. + + Prerequisites (set by the YAML template before this script runs): + - INCLUDE, LIB, PATH already contain MSVC entries. + - CMAKE_BUILD_PARALLEL_LEVEL is set (or computed here). + - SYSTEM_ACCESSTOKEN is in the environment (mapped by the YAML step via + env: SYSTEM_ACCESSTOKEN: $(System.AccessToken)). + - PIP_INDEX_URL is set by PipAuthenticate@1 at job level. +#> +param( + [Parameter(Mandatory)] [string]$SrcDir, + [Parameter(Mandatory)] [string]$ClPath, + [string]$March = 'x86-64-v3', + [string]$BuildType = 'Release', + [string]$PythonVersion = '3.11', + [string]$DevTag = 'None', + [string]$VcpkgRoot +) +$ErrorActionPreference = 'Stop' + +if (-not $VcpkgRoot) { + $VcpkgRoot = if ($env:VCPKG_INSTALLATION_ROOT) { $env:VCPKG_INSTALLATION_ROOT } else { 'C:\vcpkg' } +} + +$buildDir = "$SrcDir\cpp\build-msvc" +$installDir = "$SrcDir\install-msvc" + +if (-not $env:CMAKE_BUILD_PARALLEL_LEVEL) { + $cpu = [int]$env:NUMBER_OF_PROCESSORS + $ramGB = [math]::Floor((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB) + $jobs = [math]::Min($cpu, [math]::Max(1, [math]::Floor($ramGB / 4))) + Write-Host "CPUs=$cpu RAM=${ramGB} GB -> CMAKE_BUILD_PARALLEL_LEVEL=$jobs" + $env:CMAKE_BUILD_PARALLEL_LEVEL = $jobs +} + +# ─── CMake reconfigure + full build ────────────────────────────────────────── +# Re-configure the same build dir to refresh any source-path references, then +# build all qdk + macis targets. Dep .libs from the cache (or the deps step) +# are already present — Ninja skips them. +Write-Host "=== CMake configure (full build) ===" +$cmakeArgs = @( + '-S', "$SrcDir\cpp", + '-B', $buildDir, + '-GNinja', + "-DQDK_UARCH=$March", + '-DQDK_CHEMISTRY_ENABLE_COVERAGE=OFF', + '-DQDK_CHEMISTRY_ENABLE_MPI=OFF', + '-DMACIS_ENABLE_TESTS=ON', + '-DBUILD_SHARED_LIBS=OFF', + '-DBUILD_TESTING=ON', + '-DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=PRE_TEST', + '-DVCPKG_APPLOCAL_DEPS=OFF', + "-DCMAKE_BUILD_TYPE=$BuildType", + "-DCMAKE_C_COMPILER=$ClPath", + "-DCMAKE_CXX_COMPILER=$ClPath", + "-DCMAKE_INSTALL_PREFIX=$installDir", + "-DCMAKE_TOOLCHAIN_FILE=$VcpkgRoot\scripts\buildsystems\vcpkg.cmake", + "-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$SrcDir\.pipelines\toolchains\windows.cmake", + '-DVCPKG_TARGET_TRIPLET=x64-windows-static-md', + "-DVCPKG_INSTALLED_DIR=$SrcDir\vcpkg_installed", + '-DFETCHCONTENT_QUIET=OFF' +) +cmake @cmakeArgs +if ($LASTEXITCODE -ne 0) { throw "CMake configure failed ($LASTEXITCODE)" } + +Write-Host "=== CMake build ===" +cmake --build $buildDir +if ($LASTEXITCODE -ne 0) { throw "CMake build failed ($LASTEXITCODE)" } + +# ─── C++ tests ─────────────────────────────────────────────────────────────── +# Exclude MACIS_SERIAL_TEST and libint2/unit (compile-at-test-time meta-test) +# which can exceed the ctest timeout under MSVC. +# Save the ctest exit code; throw AFTER the wheel has been built so the +# PublishTestResults@2 task in the YAML always has something to publish. +Write-Host "=== ctest ===" +Push-Location $buildDir +ctest --output-on-failure --verbose --timeout 400 ` + --output-junit ctest_results.xml ` + -E "MACIS_SERIAL_TEST|libint2/unit" +$ctestCode = $LASTEXITCODE +Pop-Location +if ($ctestCode -ne 0) { + Write-Warning "ctest returned $ctestCode — continuing to build wheel, then will throw." +} + +# ─── Install C++ library ───────────────────────────────────────────────────── +Write-Host "=== cmake --install ===" +cmake --install $buildDir +if ($LASTEXITCODE -ne 0) { throw "CMake install failed ($LASTEXITCODE)" } + +# ─── Conda bootstrap ───────────────────────────────────────────────────────── +Write-Host "=== Conda bootstrap ===" +$condaExe = & "$PSScriptRoot\bootstrap-conda.ps1" -EnvName buildenv -PythonVersion $PythonVersion +if ($LASTEXITCODE -ne 0) { throw "Conda bootstrap failed ($LASTEXITCODE)" } + +# ─── Install Python build tooling ──────────────────────────────────────────── +Write-Host "=== pip install build tooling ===" +& $condaExe run -n buildenv python -m pip install --upgrade pip +if ($LASTEXITCODE -ne 0) { throw "pip upgrade failed" } +& $condaExe run -n buildenv python -m pip install -r "$SrcDir\.pipelines\requirements.txt" +if ($LASTEXITCODE -ne 0) { throw "pip install requirements failed" } + +# ─── Component Governance PipReport ────────────────────────────────────────── +# Snapshot the buildenv and feed it to pip install --report so Component +# Governance's PipReportDetector sees every package. +$manifestDir = "$SrcDir\python\build\build-manifest" +New-Item -ItemType Directory -Force -Path $manifestDir | Out-Null +$reqs = & $condaExe run -n buildenv python -m pip list --format=freeze +if ($LASTEXITCODE -ne 0) { throw "pip list failed ($LASTEXITCODE)" } +$reqs | Set-Content -Encoding utf8 "$manifestDir\requirements.txt" +$reqs | ForEach-Object { Write-Host $_ } +& $condaExe run -n buildenv python -m pip install ` + --dry-run --ignore-installed --quiet ` + --report "$manifestDir\component-detection-pip-report.json" ` + -r "$manifestDir\requirements.txt" +# PipReport failures are non-fatal (Component Governance is best-effort). +$LASTEXITCODE = 0 + +# ─── Prepare README (equivalent to prepare-readme.sh) ──────────────────────── +Write-Host "=== Prepare README ===" +try { + $ghBlob = 'https://github.com/microsoft/qdk-chemistry/blob/main' + $ghTree = 'https://github.com/microsoft/qdk-chemistry/tree/main' + $lines = Get-Content -Encoding utf8 "$SrcDir\README.md" + + # Apply substitutions line by line (sed -E equivalent). + $out = [System.Collections.Generic.List[string]]::new() + $deleting = $false + foreach ($line in $lines) { + # Range delete: ## Project Structure ... closing ``` + if ($line -match '^## Project Structure$') { $deleting = $true; continue } + if ($deleting) { + if ($line -match '^```$') { $deleting = $false } + continue + } + $line = $line -replace '\]\(\./([^)]+)\)', "]($ghBlob/`$1)" + $line = $line -replace '\]\(([A-Z][A-Z_]*\.(md|txt))\)', "]($ghBlob/`$1)" + $line = $line -replace '`examples/`', "[``examples/``]($ghTree/examples)" + $line = $line -replace '`cpp/include/`', "[``cpp/include/``]($ghTree/cpp/include)" + $out.Add($line) + } + $out | Set-Content -Encoding utf8 "$SrcDir\python\README.md" + Write-Host "README prepared." +} catch { + Write-Warning "prepare-readme failed: $_ (continuing)" +} + +# ─── Build Python wheel ─────────────────────────────────────────────────────── +# scikit-build-core picks up the pre-built C++ library via CMAKE_PREFIX_PATH so +# FetchContent is never triggered. QDK_ALLOW_DEPENDENCY_FETCH=OFF enforces this. +Write-Host "=== python -m build --wheel ===" +$prefix = "$SrcDir\vcpkg_installed\x64-windows-static-md;$installDir" +$buildArgs = @( + 'run', '-n', 'buildenv', + 'python', '-m', 'build', '--wheel', + "-C=build-dir=build/{wheel_tag}", + "-C=cmake.args=-GNinja", + "-C=cmake.define.QDK_UARCH=$March", + '-C=cmake.define.BUILD_SHARED_LIBS=OFF', + '-C=cmake.define.QDK_CHEMISTRY_ENABLE_MPI=OFF', + '-C=cmake.define.QDK_ENABLE_OPENMP=OFF', + '-C=cmake.define.QDK_CHEMISTRY_ENABLE_COVERAGE=OFF', + '-C=cmake.define.BUILD_TESTING=OFF', + "-C=cmake.define.QDK_ALLOW_DEPENDENCY_FETCH=OFF", + "-C=cmake.define.CMAKE_C_COMPILER=$ClPath", + "-C=cmake.define.CMAKE_CXX_COMPILER=$ClPath", + "-C=cmake.define.CMAKE_TOOLCHAIN_FILE=$VcpkgRoot\scripts\buildsystems\vcpkg.cmake", + "-C=cmake.define.VCPKG_CHAINLOAD_TOOLCHAIN_FILE=$SrcDir\.pipelines\toolchains\windows.cmake", + '-C=cmake.define.VCPKG_TARGET_TRIPLET=x64-windows-static-md', + "-C=cmake.define.VCPKG_INSTALLED_DIR=$SrcDir\vcpkg_installed", + "-C=cmake.define.CMAKE_PREFIX_PATH=$prefix" +) +Push-Location "$SrcDir\python" +& $condaExe @buildArgs +$wheelCode = $LASTEXITCODE +Pop-Location +if ($wheelCode -ne 0) { throw "python -m build --wheel failed ($wheelCode)" } + +# ─── Copy wheel to repaired_wheelhouse ─────────────────────────────────────── +# No wheel repair needed: x64-windows-static-md statically links all vcpkg +# deps; the only dynamic runtime deps (MSVCP140.dll, VCRUNTIME140.dll, UCRT) +# are always present on modern Windows. +$distDir = "$SrcDir\python\dist" +$outputDir = "$SrcDir\python\repaired_wheelhouse" +New-Item -ItemType Directory -Force -Path $outputDir | Out-Null +$wheels = Get-ChildItem "$distDir\qdk_chemistry*.whl" +if ($wheels.Count -ne 1) { + throw "Expected exactly 1 wheel in dist/, found $($wheels.Count): $($wheels.Name -join ', ')" +} +Copy-Item $wheels[0].FullName $outputDir +Write-Host "Wheel : $($wheels[0].Name)" +Write-Host "Output: $outputDir" + +# Deferred ctest failure: publish results step has already run by this point. +if ($ctestCode -ne 0) { throw "ctest failed ($ctestCode)" } diff --git a/.pipelines/python-wheels.yaml b/.pipelines/python-wheels.yaml index a8b063bf2..6a368657e 100644 --- a/.pipelines/python-wheels.yaml +++ b/.pipelines/python-wheels.yaml @@ -41,6 +41,11 @@ parameters: arch: aarch64 march: armv8-a imageName: ACES_VM_SharedPool_Tahoe + - name: windows_x86_64 + os: windows + arch: x86_64 + march: x86-64-v3 + imageName: windows-2025-1espt displayName: Target platforms - name: pythonVersions type: stringList @@ -93,7 +98,7 @@ extends: binaryLanguages: cpp psscriptanalyzer: enabled: false - justificationForDisabling: Powershell is not used in this repository. + justificationForDisabling: PSScriptAnalyzer not yet configured for the new Windows pip-scripts. break: false armory: enabled: false @@ -127,6 +132,12 @@ extends: - ImageOverride -equals ${{ target.imageName }} os: ${{ target.os }} hostArchitecture: Arm64 + ${{ elseif eq(target.os, 'windows') }}: + pool: + name: $(CPU_POOL_X86) + demands: + - ImageOverride -equals ${{ target.imageName }} + os: windows ${{ else }}: pool: name: $(CPU_POOL_X86) @@ -158,7 +169,7 @@ extends: onlyAddExtraIndex: false # Build - Linux x86_64 - - ${{ if eq(target.arch, 'x86_64') }}: + - ${{ if and(eq(target.arch, 'x86_64'), eq(target.os, 'linux')) }}: - template: .pipelines/templates/build-pip-wheels.yml@self parameters: march: ${{ target.march }} @@ -171,7 +182,7 @@ extends: devTag: ${{ parameters.devTag }} # Test - Linux x86_64 - - ${{ if eq(target.arch, 'x86_64') }}: + - ${{ if and(eq(target.arch, 'x86_64'), eq(target.os, 'linux')) }}: - template: .pipelines/templates/test-pip-wheels.yml@self parameters: agentBuildDirectory: $(Agent.BuildDirectory) @@ -233,11 +244,32 @@ extends: # the wheel is still built and published. condition: and(succeeded(), ne(variables['pythonVersion'], '3.10')) + # Build - Windows x86_64 + - ${{ if eq(target.os, 'windows') }}: + - template: .pipelines/templates/build-pip-wheels-windows.yml@self + parameters: + march: ${{ target.march }} + agentBuildDirectory: $(Agent.BuildDirectory) + pythonVersion: $(pythonVersion) + devTag: ${{ parameters.devTag }} + + # Test - Windows x86_64 + - ${{ if eq(target.os, 'windows') }}: + - template: .pipelines/templates/test-pip-wheels-windows.yml@self + parameters: + agentBuildDirectory: $(Agent.BuildDirectory) + pythonVersion: $(pythonVersion) + runSlowTests: ${{ parameters.runSlowTests }} + # Python 3.10 tests temporarily disabled (qre/cirq cannot resolve on 3.10); + # the wheel is still built and published. + condition: and(succeeded(), ne(variables['pythonVersion'], '3.10')) + # SBoM Manifest Generator Task requires we transfer ownership of the wheel directory - # back to the original base image user - - script: | - sudo chown -R $(id -u):$(id -g) $(System.DefaultWorkingDirectory)/python/repaired_wheelhouse - displayName: Fix wheel directory ownership + # back to the original base image user (Linux/macOS only; not needed on Windows). + - ${{ if ne(target.os, 'windows') }}: + - script: | + sudo chown -R $(id -u):$(id -g) $(System.DefaultWorkingDirectory)/python/repaired_wheelhouse + displayName: Fix wheel directory ownership - task: TwineAuthenticate@1 inputs: @@ -341,6 +373,21 @@ extends: - input: pipelineArtifact artifactName: macos-aarch64-wheels-python3.14 targetPath: $(System.DefaultWorkingDirectory)/artifacts/macos-aarch64-wheels-python3.14 + - input: pipelineArtifact + artifactName: windows-x86_64-wheels-python3.10 + targetPath: $(System.DefaultWorkingDirectory)/artifacts/windows-x86_64-wheels-python3.10 + - input: pipelineArtifact + artifactName: windows-x86_64-wheels-python3.11 + targetPath: $(System.DefaultWorkingDirectory)/artifacts/windows-x86_64-wheels-python3.11 + - input: pipelineArtifact + artifactName: windows-x86_64-wheels-python3.12 + targetPath: $(System.DefaultWorkingDirectory)/artifacts/windows-x86_64-wheels-python3.12 + - input: pipelineArtifact + artifactName: windows-x86_64-wheels-python3.13 + targetPath: $(System.DefaultWorkingDirectory)/artifacts/windows-x86_64-wheels-python3.13 + - input: pipelineArtifact + artifactName: windows-x86_64-wheels-python3.14 + targetPath: $(System.DefaultWorkingDirectory)/artifacts/windows-x86_64-wheels-python3.14 steps: - script: | mkdir -p $(System.DefaultWorkingDirectory)/target/wheels diff --git a/.pipelines/templates/build-pip-wheels-windows.yml b/.pipelines/templates/build-pip-wheels-windows.yml new file mode 100644 index 000000000..e40fe5cbe --- /dev/null +++ b/.pipelines/templates/build-pip-wheels-windows.yml @@ -0,0 +1,189 @@ +parameters: +- name: march + type: string + default: x86-64-v3 +- name: buildType + type: string + default: Release +- name: agentBuildDirectory + type: string + default: $(Agent.BuildDirectory) +- name: pythonVersion + type: string + default: '3.11' +- name: devTag + type: string + default: 'None' + +steps: +- checkout: self + persistCredentials: true + path: qdk-chemistry + fetchDepth: 1 + retryCountOnTaskFailure: 3 + +- pwsh: | + python "$(System.DefaultWorkingDirectory)\.pipelines\pip-scripts\versioning.py" ` + --version-file "$(System.DefaultWorkingDirectory)\VERSION" ` + --dev-tag ${{ parameters.devTag }} + Get-Content "$(System.DefaultWorkingDirectory)\VERSION" + displayName: Validate/Set Development Version for Wheels + condition: ne('${{ parameters.devTag }}', 'None') + +# Import the VS x64 developer environment (INCLUDE, LIB, PATH, etc.) so that +# cl.exe and the MSVC linker are available to all subsequent steps. +# Also resolves the MSVC toolset version for the dependency cache key. +- pwsh: | + $ErrorActionPreference = 'Stop' + + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $vsPath = & $vswhere -latest -products * -property installationPath + if (-not $vsPath) { throw "No Visual Studio installation found" } + Write-Host "Visual Studio: $vsPath" + + $vcvarsall = "$vsPath\VC\Auxiliary\Build\vcvarsall.bat" + if (-not (Test-Path $vcvarsall)) { throw "vcvarsall.bat not found at $vcvarsall" } + + # Snapshot env, run vcvarsall x64, diff to capture changes. + $before = @{} + Get-ChildItem env: | ForEach-Object { $before[$_.Name] = $_.Value } + $tmp = [System.IO.Path]::GetTempFileName() + cmd /c "`"$vcvarsall`" x64 && set > `"$tmp`"" + if ($LASTEXITCODE -ne 0) { throw "vcvarsall.bat failed ($LASTEXITCODE)" } + $after = @{} + Get-Content $tmp | ForEach-Object { + if ($_ -match '^([^=]+)=(.*)$') { $after[$matches[1]] = $matches[2] } + } + Remove-Item $tmp + + # Apply to current process so Get-Command can resolve cl.exe. + foreach ($name in $after.Keys) { + [System.Environment]::SetEnvironmentVariable($name, $after[$name], 'Process') + } + + # Locate cl.exe — vcvarsall x64 puts it on PATH. + # PS5-compatible null check (no ?. operator available in powershell.exe 5.1). + $clCmd = Get-Command cl.exe -ErrorAction SilentlyContinue + $cl = if ($clCmd) { $clCmd.Source } else { $null } + if (-not $cl) { throw "cl.exe not found on PATH after vcvarsall x64" } + Write-Host "cl.exe: $cl" + & $cl 2>&1 | Select-Object -First 2 | Write-Host + + # Export non-PATH changes to subsequent ADO steps. + foreach ($name in $after.Keys) { + if ($name -ieq 'Path') { continue } + if ($before[$name] -ne $after[$name]) { + Write-Host "##vso[task.setvariable variable=$name]$($after[$name])" + } + } + # Prepend new PATH entries from vcvarsall. + $beforePath = @($before['Path'] -split ';') + $newEntries = @($after['Path'] -split ';') | Where-Object { $_ -and ($beforePath -notcontains $_) } + $newEntries | ForEach-Object { Write-Host "##vso[task.prependpath]$_" } + + # MSVC toolset version: stable identifier for cache key. + $toolsetFile = "$vsPath\VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt" + $toolset = (Get-Content $toolsetFile -ErrorAction Stop | Select-Object -First 1).Trim() + Write-Host "MSVC toolset: $toolset" + Write-Host "##vso[task.setvariable variable=MSVC_TOOLSET]$toolset" + Write-Host "##vso[task.setvariable variable=CL_PATH]$cl" + + # Bootstrap vcpkg if not pre-installed. + $vcpkgRoot = if ($env:VCPKG_INSTALLATION_ROOT) { $env:VCPKG_INSTALLATION_ROOT } else { 'C:\vcpkg' } + if (Test-Path "$vcpkgRoot\vcpkg.exe") { + Write-Host "vcpkg already available at $vcpkgRoot" + } else { + Write-Host "vcpkg not found — bootstrapping into $vcpkgRoot" + git clone https://github.com/microsoft/vcpkg.git $vcpkgRoot --depth 1 + & "$vcpkgRoot\bootstrap-vcpkg.bat" -disableMetrics + if ($LASTEXITCODE -ne 0) { throw "vcpkg bootstrap failed ($LASTEXITCODE)" } + Write-Host "##vso[task.setvariable variable=VCPKG_INSTALLATION_ROOT]$vcpkgRoot" + } + # vcvarsall sets VCPKG_ROOT to VS's bundled copy; override it to match our + # standalone installation so vcpkg.exe doesn't warn about the mismatch. + Write-Host "##vso[task.setvariable variable=VCPKG_ROOT]$vcpkgRoot" + displayName: Set up MSVC environment + +# Hash all files that influence how the C++ dependencies are built. +# The result is used in the Cache@2 key so the cache is invalidated on any +# dep-affecting change without listing every file individually in the key. +- pwsh: | + $root = "$(System.DefaultWorkingDirectory)" + $files = @( + "$root\vcpkg.json", + "$root\vcpkg-configuration.json", + "$root\cpp\manifest\qdk-chemistry\cgmanifest.json", + "$root\external\macis\manifest\cgmanifest.json", + "$root\cpp\cmake\third_party.cmake", + "$root\cpp\cmake\modules\DependencyManager.cmake", + "$root\external\macis\src\lobpcgxx\CMakeLists.txt", + "$root\.pipelines\toolchains\windows.cmake" + ) + if (Test-Path "$root\vcpkg-overlay") { + $files += Get-ChildItem "$root\vcpkg-overlay" -Recurse -File | + Select-Object -ExpandProperty FullName + } + if (Test-Path "$root\cpp\cmake\patches") { + $files += Get-ChildItem "$root\cpp\cmake\patches" -Recurse -File | + Select-Object -ExpandProperty FullName + } + $hashes = $files | Where-Object { Test-Path $_ } | Sort-Object | + ForEach-Object { (Get-FileHash $_ -Algorithm SHA256).Hash } + $combined = $hashes -join "," + $hashBytes = [System.Security.Cryptography.SHA256]::Create().ComputeHash( + [System.Text.Encoding]::UTF8.GetBytes($combined)) + $hashStr = ($hashBytes | ForEach-Object { $_.ToString("x2") }) -join "" + Write-Host "Dependency files hash: $hashStr" + Write-Host "##vso[task.setvariable variable=DEPS_FILES_HASH]$hashStr" + displayName: Compute dependency cache key + +# Restore the dependency build dirs separately to avoid a known Windows issue +# with Cache@2 multi-line `path`: the task passes both lines as a single tar +# argument, causing "could not chdir" at save time. Two tasks work reliably. +- task: Cache@2 + displayName: Restore / save vcpkg dependency cache + inputs: + key: '"windows-vcpkg-x64-windows-static-md-v1" | "$(MSVC_TOOLSET)" | "$(DEPS_FILES_HASH)"' + path: $(System.DefaultWorkingDirectory)/vcpkg_installed + cacheHitVar: VCPKG_CACHE_RESTORED + +- task: Cache@2 + displayName: Restore / save C++ build cache + inputs: + key: '"windows-deps-msvc-x64-windows-static-md-v1" | "$(MSVC_TOOLSET)" | "$(DEPS_FILES_HASH)"' + path: $(System.DefaultWorkingDirectory)/cpp/build-msvc + cacheHitVar: DEPS_CACHE_RESTORED + +# Build vcpkg packages, configure CMake, and build the slow FetchContent targets +# (libint2, ecpint, gauxc, blaspp, lapackpp). Skipped entirely on cache hit. +- pwsh: | + & "$(System.DefaultWorkingDirectory)\.pipelines\pip-scripts\build-pip-wheels-windows-deps.ps1" ` + -SrcDir "$(System.DefaultWorkingDirectory)" ` + -March "${{ parameters.march }}" ` + -BuildType "${{ parameters.buildType }}" ` + -ClPath "$(CL_PATH)" + displayName: Build C++ dependencies (libint2 / gauxc / ecpint / blaspp / lapackpp) + condition: or(ne(variables['DEPS_CACHE_RESTORED'], 'true'), ne(variables['VCPKG_CACHE_RESTORED'], 'true')) + +# Full qdk build, C++ tests, conda environment setup, and Python wheel build. +# SYSTEM_ACCESSTOKEN is required by bootstrap-conda for the Azure Artifacts feed. +- pwsh: | + & "$(System.DefaultWorkingDirectory)\.pipelines\pip-scripts\build-pip-wheels-windows.ps1" ` + -SrcDir "$(System.DefaultWorkingDirectory)" ` + -March "${{ parameters.march }}" ` + -BuildType "${{ parameters.buildType }}" ` + -ClPath "$(CL_PATH)" ` + -PythonVersion "${{ parameters.pythonVersion }}" ` + -DevTag "${{ parameters.devTag }}" + displayName: Build C++ library, run tests, and build Python wheel + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + OMP_NUM_THREADS: '2' + +- task: PublishTestResults@2 + displayName: Publish C++ test results + inputs: + testResultsFormat: JUnit + testResultsFiles: '$(System.DefaultWorkingDirectory)/cpp/build-msvc/ctest_results.xml' + testRunTitle: 'C++ tests - Windows x86_64 Python ${{ parameters.pythonVersion }}' + condition: always() diff --git a/.pipelines/templates/test-pip-wheels-windows.yml b/.pipelines/templates/test-pip-wheels-windows.yml new file mode 100644 index 000000000..50d604158 --- /dev/null +++ b/.pipelines/templates/test-pip-wheels-windows.yml @@ -0,0 +1,79 @@ +parameters: +- name: agentBuildDirectory + type: string + default: $(Agent.BuildDirectory) +- name: pythonVersion + type: string + default: '3.11' +- name: runSlowTests + type: boolean + default: true +- name: condition + type: string + default: succeeded() + +steps: +# Bootstrap conda and create an isolated test environment, matching the +# approach used for Linux/macOS via bootstrap-conda.sh. +# SYSTEM_ACCESSTOKEN (mapped via env: below) authenticates the conda install +# against the Azure Artifacts feed (public channels blocked in 1ES CFSClean). +- pwsh: | + $ErrorActionPreference = 'Stop' + $scriptPath = "$(System.DefaultWorkingDirectory)\.pipelines\pip-scripts\bootstrap-conda.ps1" + $condaExe = & $scriptPath -EnvName testenv -PythonVersion '${{ parameters.pythonVersion }}' + if ($LASTEXITCODE -ne 0) { throw "Conda bootstrap failed ($LASTEXITCODE)" } + # Export conda exe path for subsequent steps. + Write-Host "##vso[task.setvariable variable=CONDA_EXE]$condaExe" + displayName: Set up conda test environment (Python ${{ parameters.pythonVersion }}) + condition: ${{ parameters.condition }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +# Install the wheel together with its test extra dependencies. +- pwsh: | + $ErrorActionPreference = 'Stop' + $wheels = Get-ChildItem "$(System.DefaultWorkingDirectory)\python\repaired_wheelhouse\qdk_chemistry*.whl" + if ($wheels.Count -ne 1) { + throw "Expected exactly 1 wheel, found $($wheels.Count): $($wheels.Name -join ', ')" + } + $wheel = $wheels[0].FullName + Write-Host "Installing: $wheel" + & "$(CONDA_EXE)" run -n testenv python -m pip install --upgrade pip + if ($LASTEXITCODE -ne 0) { throw "pip upgrade failed" } + & "$(CONDA_EXE)" run -n testenv python -m pip install "$wheel[test]" + if ($LASTEXITCODE -ne 0) { throw "pip install wheel[test] failed ($LASTEXITCODE)" } + displayName: Install wheel with test dependencies + condition: ${{ parameters.condition }} + +# Snapshot the testenv and generate a pip install --report for Component +# Governance's PipReportDetector (mirrors test-pip-wheels.sh lines 95-102). +- pwsh: | + $manifestDir = "$(System.DefaultWorkingDirectory)\python\build\test-manifest" + New-Item -ItemType Directory -Force -Path $manifestDir | Out-Null + $reqs = & "$(CONDA_EXE)" run -n testenv python -m pip list --format=freeze --exclude qdk_chemistry + if ($LASTEXITCODE -ne 0) { throw "pip list failed ($LASTEXITCODE)" } + $reqs | Set-Content -Encoding utf8 "$manifestDir\requirements.txt" + $reqs | ForEach-Object { Write-Host $_ } + & "$(CONDA_EXE)" run -n testenv python -m pip install ` + --dry-run --ignore-installed --quiet ` + --report "$manifestDir\component-detection-pip-report.json" ` + -r "$manifestDir\requirements.txt" + displayName: Generate Component Governance PipReport + condition: ${{ parameters.condition }} + continueOnError: true + +- pwsh: | + $ErrorActionPreference = 'Stop' + # Set env vars in the current process; conda run inherits them. + # conda run --env VAR=VALUE was added only in newer conda versions and is + # not universally available on 1ES runners. + $env:QSHARP_PYTHON_TELEMETRY = 'false' + $env:QDK_CHEMISTRY_RUN_SLOW_TESTS = '${{ parameters.runSlowTests }}' + $env:OMP_NUM_THREADS = '2' + Push-Location "$(System.DefaultWorkingDirectory)\python" + & "$(CONDA_EXE)" run -n testenv python -m pytest -v tests/ + $code = $LASTEXITCODE + Pop-Location + if ($code -ne 0) { throw "pytest failed ($code)" } + displayName: Run Python tests + condition: ${{ parameters.condition }} diff --git a/cpp/cmake/third_party.cmake b/cpp/cmake/third_party.cmake index e64379918..cc1266128 100644 --- a/cpp/cmake/third_party.cmake +++ b/cpp/cmake/third_party.cmake @@ -86,6 +86,13 @@ if(MSVC AND TARGET eritest-libint2) endif() endif() +# MSVC's /O2 optimizer is pathologically slow on libint2's large CMake Unity +# translation units (hours vs minutes for clang-cl). Disable Unity for libint2 on +# MSVC so the small generated TUs compile quickly and parallelize; clang-cl keeps it. +if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND TARGET libint2_obj) + set_target_properties(libint2_obj PROPERTIES UNITY_BUILD OFF) +endif() + # ecpint for ECP-related integral evaluation set(LIBECPINT_BUILD_TESTS OFF CACHE BOOL "Enable ECPINT Tests" FORCE) set(LIBECPINT_USE_PUGIXML OFF CACHE BOOL "Use pugixml for ECPINT" FORCE)