From 3101a68d702264ee1beae3c0ce10900cdede846c Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 18 Nov 2025 14:16:18 -0800 Subject: [PATCH 01/40] Revert revert libcurl https://github.com/microsoft/vcpkg-tool/pull/1856 --- .github/workflows/build.yaml | 5 + CMakeLists.txt | 3 + NOTICE.txt | 29 + .../end-to-end-tests-dir/asset-caching.ps1 | 60 +- azure-pipelines/signing.yml | 12 +- azure-pipelines/vcpkg-alpine/Dockerfile | 2 +- azure-pipelines/vcpkg-arm64/Dockerfile | 14 +- azure-pipelines/vcpkg-linux/Dockerfile | 14 +- cgmanifest.json | 13 + cmake/FindLibCURL.cmake | 93 ++ include/vcpkg/base/contractual-constants.h | 5 + include/vcpkg/base/curl.h | 69 + include/vcpkg/base/downloads.h | 26 +- include/vcpkg/base/files.h | 2 + include/vcpkg/base/message-data.inc.h | 45 +- include/vcpkg/metrics.h | 5 +- locales/messages.json | 25 +- src/vcpkg-test/downloads.cpp | 97 +- src/vcpkg-test/metrics.cpp | 36 + src/vcpkg.cpp | 1 + src/vcpkg/base/curl.cpp | 123 ++ src/vcpkg/base/downloads.cpp | 1372 ++++------------- src/vcpkg/base/files.cpp | 29 + src/vcpkg/binarycaching.cpp | 13 +- src/vcpkg/commands.cpp | 2 +- src/vcpkg/commands.z-check-tools-sha.cpp | 5 +- src/vcpkg/commands.z-upload-metrics.cpp | 20 +- src/vcpkg/metrics.cpp | 202 ++- 28 files changed, 922 insertions(+), 1400 deletions(-) create mode 100644 cmake/FindLibCURL.cmake create mode 100644 include/vcpkg/base/curl.h create mode 100644 src/vcpkg/base/curl.cpp diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7524e3d264..1596506fd2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -36,6 +36,11 @@ jobs: uses: github/codeql-action/init@v4 with: languages: javascript-typescript, c-cpp + - name: Install system libcurl + if: matrix.preset == 'linux-arm64-ci' || matrix.preset == 'linux-ci' + run: | + sudo apt update + sudo apt install -y libcurl4-openssl-dev - name: Configure and Build if: matrix.preset != 'windows-ci' run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d52ddf01a..11c2501111 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,6 +195,7 @@ set(TEST_SCRIPT_ASSET_CACHE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/test-script find_package(fmt REQUIRED) find_package(CMakeRC REQUIRED) +find_package(LibCURL REQUIRED) # === Target: locale-resources === @@ -229,6 +230,8 @@ target_compile_definitions(vcpkglib PUBLIC _FILE_OFFSET_BITS=64 ) +target_link_libraries(vcpkglib PUBLIC CURL::libcurl) + if(VCPKG_STANDALONE_BUNDLE_SHA) target_compile_definitions(vcpkglib PUBLIC "VCPKG_STANDALONE_BUNDLE_SHA=${VCPKG_STANDALONE_BUNDLE_SHA}" diff --git a/NOTICE.txt b/NOTICE.txt index 7b5c798535..3d26b1b4b5 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -71,6 +71,35 @@ SOFTWARE. ========================================= END OF CMakeRC NOTICES, INFORMATION, AND LICENSE +curl + +%% curl NOTICES, INFORMATION, AND LICENSE BEGIN HERE +========================================= +COPYRIGHT AND PERMISSION NOTICE + +Copyright (C) Daniel Stenberg, , and many +contributors, see the THANKS file. + +All rights reserved. + +Permission to use, copy, modify, and distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright +notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization of the copyright holder. +========================================= +END OF curl NOTICES, INFORMATION, AND LICENSE + The following third party software is incorporated into vcpkg-artifacts: --------------------------------------------------------- diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index ef7e956088..91ba717751 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -48,7 +48,8 @@ Throw-IfNotFailed $expected = @( "A suitable version of cmake was not found \(required v[0-9.]+\)\.", "Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-z]+", -"error: curl: \(37\) Couldn't open file [^\n]+", +"error: curl operation failed with error code 37 \((Couldn't|Could not) read a file:\/\/ file\)\.", +"error: Not a transient network error, won't retry download from file://$assetCacheRegex/[0-9a-z]+" "error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", @@ -100,7 +101,8 @@ if (-not ($actual -match $expected)) { Refresh-TestRoot $expected = @( "^Downloading https://localhost:1234/foobar\.html -> example3\.html", -"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"error: curl operation failed with error code 7 \((Couldn't|Could not) connect to server\)\.", +"error: Not a transient network error, won't retry download from https://localhost:1234/foobar\.html", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https://address:port``\.", @@ -123,9 +125,11 @@ if (-not ($actual -match $expected)) { Refresh-TestRoot $expected = @( "^Downloading example3\.html, trying https://localhost:1234/foobar\.html", +"error: curl operation failed with error code 7 \((Couldn't|Could not) connect to server\)\.", +"error: Not a transient network error, won't retry download from https://localhost:1234/foobar\.html", "Trying https://localhost:1235/baz\.html", -"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", -"error: curl: \(7\) Failed to connect to localhost port 1235( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"error: curl operation failed with error code 7 \((Couldn't|Could not) connect to server\)\.", +"error: Not a transient network error, won't retry download from https://localhost:1235/baz\.html", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https://address:port``\.", @@ -177,34 +181,12 @@ if (-not ($actual -match $expected)) { } # ... also with multiple authoritative URLs -if ($IsWindows) { - # WinHTTP - Refresh-TestRoot - $expected = @( - "^Downloading example3\.html, trying https://nonexistent\.example\.com", - "warning: Download https://nonexistent\.example\.com failed -- retrying after 1000ms", - "warning: Download https://nonexistent\.example\.com failed -- retrying after 2000ms", - "warning: Download https://nonexistent\.example\.com failed -- retrying after 4000ms", - "Trying https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt", - "Successfully downloaded example3\.html", - "$" - ) -join "`n" - - $actual = Run-VcpkgAndCaptureOutput @commonArgs x-download "$TestDownloadsRoot/example3.html" --sha512 65077997890f66f6041bb3284bb7b88e27631411ccbc253201ca4e00c4bcc58c0d77edffda4975498797cc10772c7fd68fbeb13cc4ac493a3471a9d49e5b6f24 --url https://nonexistent.example.com --url https://raw.githubusercontent.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE.txt - Throw-IfFailed - if (-not ($actual -match $expected)) { - throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" - } -} - -# Force curl with --header Refresh-TestRoot $expected = @( "^Downloading example3\.html, trying https://nonexistent\.example\.com", -"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in 1 seconds?\. 3 retries left\.", -"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in \d+ seconds?\. 2 retries left\.", -"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in \d+ seconds?\. 1 (retries|retry) left\.", -"Trying https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt", +"error: curl operation failed with error code 6 \((Couldn't|Could not) resolve (hostname|host name)\)\.", +"error: Not a transient network error, won't retry download from https://nonexistent\.example\.com", +"Trying https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt" "Successfully downloaded example3\.html", "$" ) -join "`n" @@ -212,7 +194,7 @@ $expected = @( $actual = Run-VcpkgAndCaptureOutput @commonArgs x-download "$TestDownloadsRoot/example3.html" --sha512 65077997890f66f6041bb3284bb7b88e27631411ccbc253201ca4e00c4bcc58c0d77edffda4975498797cc10772c7fd68fbeb13cc4ac493a3471a9d49e5b6f24 --url https://nonexistent.example.com --url https://raw.githubusercontent.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE.txt --header "Cache-Control: no-cache" Throw-IfFailed if (-not ($actual -match $expected)) { - throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" + throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed), headers (cache-control)" } # azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a) @@ -236,8 +218,10 @@ Refresh-TestRoot $expected = @( "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", "Asset cache miss; trying authoritative source https://localhost:1234/foobar\.html", -"error: curl: \(37\) Couldn't open file [^\n]+", -"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"error: curl operation failed with error code 37 \((Couldn't|Could not) read a file:// file\)\.", +"error: Not a transient network error, won't retry download from file://$assetCacheRegex/[0-9a-z]+", +"error: curl operation failed with error code 7 \((Couldn't|Could not) connect to server\)\.", +"error: Not a transient network error, won't retry download from https://localhost:1234/foobar\.html", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https://address:port``\.", @@ -296,7 +280,11 @@ if (-not ($actual -match $expected)) { $expected = @( "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", "Asset cache miss; trying authoritative source https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt", -"error: curl: \(37\) Couldn't open file [^\n]+", +"error: curl operation failed with error code 37 \((Couldn't|Could not) read a file:// file\)\.", +"error: Not a transient network error, won't retry download from file://$assetCacheRegex/[0-9a-z]+", +"[^\n]+example3\.html\.\d+\.part: error: download from https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt had an unexpected hash", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", +"note: Actual : 65077997890f66f6041bb3284bb7b88e27631411ccbc253201ca4e00c4bcc58c0d77edffda4975498797cc10772c7fd68fbeb13cc4ac493a3471a9d49e5b6f24", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https://address:port``\.", @@ -306,9 +294,6 @@ $expected = @( "The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", "3\. Your proxy's remote server is out of service\.", "If you believe this is not a temporary download server failure and vcpkg needs to be changed to download this file from a different location, please submit an issue to https://github\.com/Microsoft/vcpkg/issues", -"[^\n]+example3\.html\.\d+\.part: error: download from https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt had an unexpected hash", -"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", -"note: Actual : 65077997890f66f6041bb3284bb7b88e27631411ccbc253201ca4e00c4bcc58c0d77edffda4975498797cc10772c7fd68fbeb13cc4ac493a3471a9d49e5b6f24", "$" ) -join "`n" $actual = Run-VcpkgAndCaptureOutput @commonArgs x-download "$TestDownloadsRoot/example3.html" --sha512 d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b --url https://raw.githubusercontent.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE.txt "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite" @@ -358,7 +343,8 @@ if (-not ($actual -match $expected)) { Refresh-TestRoot $expected = @( "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", -"error: curl: \(37\) Couldn't open file [^\n]+", +"error: curl operation failed with error code 37 \((Couldn't|Could not) read a file:// file\)\.", +"error: Not a transient network error, won't retry download from file://$assetCacheRegex/[0-9a-z]+", "error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt", "note: or https://alternate\.example\.com", "note: If you are using a proxy, please ensure your proxy settings are correct\.", diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 17c080709d..0b7bf12519 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -188,7 +188,7 @@ extends: inputs: failOnStderr: true script: | - cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=10.13 -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$VCPKG_BASE_VERSION" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$VCPKG_STANDALONE_BUNDLE_SHA" "-DVCPKG_ARTIFACTS_SHA=$VCPKG_ARTIFACTS_SHA" -B "$(Build.BinariesDirectory)/build" 2>&1 + cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=10.13 -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_CURL_URL=$(curl-tarball-url)" "-DVCPKG_BASE_VERSION=$VCPKG_BASE_VERSION" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$VCPKG_STANDALONE_BUNDLE_SHA" "-DVCPKG_ARTIFACTS_SHA=$VCPKG_ARTIFACTS_SHA" -B "$(Build.BinariesDirectory)/build" 2>&1 make -j 8 -C "$(Build.BinariesDirectory)/build" zip -j "$(Build.ArtifactStagingDirectory)/vcpkg-macos.zip" "$(Build.BinariesDirectory)/build/vcpkg" - job: glibc_build @@ -226,7 +226,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-amd64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-linux/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-amd64:2025-11-17 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-linux/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_CURL_URL=$(curl-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-glibc" - job: muslc_build displayName: 'muslc (Alpine) Build' @@ -263,7 +263,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_CXX_FLAGS=\"-static -s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16.1 sh -c "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_CURL_URL=$(curl-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-muslc" - job: glibc_arm64_build displayName: 'glibc Arm64 Build' @@ -301,7 +301,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-arm64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-arm64/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-arm64:2025-11-17 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-arm64/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_CURL_URL=$(curl-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-glibc-arm64" - job: windows_and_sign displayName: 'Build Windows binaries and Sign' @@ -371,7 +371,7 @@ extends: script: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 cmake.exe --version - cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\amd64" 2>&1 + cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_CURL_URL=$(curl-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\amd64" 2>&1 ninja.exe -C "$(Build.BinariesDirectory)\amd64" - task: CmdLine@2 displayName: "Build vcpkg arm64 with CMake" @@ -380,7 +380,7 @@ extends: script: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=arm64 -host_arch=amd64 cmake.exe --version - cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_PDB_SUFFIX="-arm64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\arm64" 2>&1 + cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_PDB_SUFFIX="-arm64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_CURL_URL=$(curl-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\arm64" 2>&1 ninja.exe -C "$(Build.BinariesDirectory)\arm64" - task: NuGetToolInstaller@1 inputs: diff --git a/azure-pipelines/vcpkg-alpine/Dockerfile b/azure-pipelines/vcpkg-alpine/Dockerfile index e7e85974ee..07bdffc627 100644 --- a/azure-pipelines/vcpkg-alpine/Dockerfile +++ b/azure-pipelines/vcpkg-alpine/Dockerfile @@ -1,4 +1,4 @@ # The authoritative version of this file is in https://devdiv.visualstudio.com/DevDiv/_git/vcpkg-pme-utils FROM alpine:3.16 -RUN apk add alpine-sdk cmake ninja git curl tar gzip zip && apk upgrade +RUN apk add alpine-sdk cmake ninja git curl tar gzip zip curl-dev && apk upgrade diff --git a/azure-pipelines/vcpkg-arm64/Dockerfile b/azure-pipelines/vcpkg-arm64/Dockerfile index 77622dd235..484ccbc61c 100644 --- a/azure-pipelines/vcpkg-arm64/Dockerfile +++ b/azure-pipelines/vcpkg-arm64/Dockerfile @@ -51,13 +51,13 @@ RUN chroot /crossrootfs/arm64 apt-get clean && \ # Repeated runs of apt-get install workaround 'hash sum mismatch' errors # (This is also why these are a separate cache layer) -RUN chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" && \ +RUN chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev && \ chroot /crossrootfs/arm64 symlinks -cr /usr && \ chroot /crossrootfs/arm64 apt-get clean diff --git a/azure-pipelines/vcpkg-linux/Dockerfile b/azure-pipelines/vcpkg-linux/Dockerfile index 92b848f396..9ef4c79ecf 100644 --- a/azure-pipelines/vcpkg-linux/Dockerfile +++ b/azure-pipelines/vcpkg-linux/Dockerfile @@ -50,13 +50,13 @@ RUN chroot /crossrootfs/x64 apt-get clean && \ # Repeated runs of apt-get install workaround 'hash sum mismatch' errors # (This is also why these are a separate cache layer) -RUN chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" && \ +RUN chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev && \ chroot /crossrootfs/x64 symlinks -cr /usr && \ chroot /crossrootfs/x64 apt-get clean diff --git a/cgmanifest.json b/cgmanifest.json index 19dcc509fe..1244e27517 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -37,6 +37,19 @@ }, "DevelopmentDependency": false, "DependencyRoots": [] + }, + { + "Component": { + "Type": "other", + "other": { + "name": "curl", + "version": "8.17.0", + "downloadUrl": "https://github.com/curl/curl/archive/refs/tags/curl-8_17_0.tar.gz", + "hash": "88ab4b7aac12b26a6ad32fb0e1a9675288a45894438cb031102ef5d4ab6b33c2bc99cae0c70b71bdfa12eb49762827e2490555114c5eb4a6876b95e1f2a4eb74" + } + }, + "DevelopmentDependency": false, + "DependencyRoots": [] } ] } diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake new file mode 100644 index 0000000000..ddd346c349 --- /dev/null +++ b/cmake/FindLibCURL.cmake @@ -0,0 +1,93 @@ +if (WIN32) + option(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL "Use an external version of the libcurl library" OFF) +else() + option(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL "Use an external version of the libcurl library" ON) +endif() + +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() + +if (VCPKG_DEPENDENCY_EXTERNAL_LIBCURL) + find_package(CURL REQUIRED) + return() +endif() + +# This option exists to allow the URI to be replaced with a Microsoft-internal URI in official +# builds which have restricted internet access; see azure-pipelines/signing.yml +# Note that the SHA512 is the same, so vcpkg-tool contributors need not be concerned that we built +# with different content. +if(NOT VCPKG_LIBCURL_URL) + set(VCPKG_LIBCURL_URL "https://github.com/curl/curl/releases/download/curl-8_17_0/curl-8.17.0.tar.gz") +endif() + +include(FetchContent) +FetchContent_Declare( + LibCURL + URL "${VCPKG_LIBCURL_URL}" + URL_HASH "SHA512=88ab4b7aac12b26a6ad32fb0e1a9675288a45894438cb031102ef5d4ab6b33c2bc99cae0c70b71bdfa12eb49762827e2490555114c5eb4a6876b95e1f2a4eb74" +) + +if(NOT LibCURL_FIND_REQUIRED) + message(FATAL_ERROR "LibCURL must be REQUIRED") +endif() + +# This is in function() so no need to backup the variables +function(get_libcurl) + set(BUILD_CURL_EXE OFF) + set(BUILD_EXAMPLES OFF) + set(BUILD_LIBCURL_DOCS OFF) + set(BUILD_MISC_DOCS OFF) + set(BUILD_SHARED_LIBS OFF) + set(BUILD_TESTING OFF) + set(CURL_ENABLE_EXPORT_TARGET OFF) + set(CURL_USE_LIBSSH2 OFF) + set(CURL_USE_LIBPSL OFF) + if (WIN32) + set(CURL_USE_SCHANNEL ON) + endif() + set(ENABLE_CURL_MANUAL OFF) + set(ENABLE_UNICODE ON) + set(PICKY_COMPILER OFF) + set(USE_NGHTTP2 OFF) + set(USE_LIBIDN2 OFF) + set(CMAKE_DISABLE_FIND_PACKAGE_Perl ON) + set(CMAKE_DISABLE_FIND_PACKAGE_ZLIB ON) + set(CMAKE_DISABLE_FIND_PACKAGE_LibPSL ON) + set(CMAKE_DISABLE_FIND_PACKAGE_LibSSH2 ON) + set(CMAKE_DISABLE_FIND_PACKAGE_Brotli ON) + set(CMAKE_DISABLE_FIND_PACKAGE_Zstd ON) + set(CMAKE_DISABLE_FIND_PACKAGE_NGHTTP2 ON) + set(CMAKE_DISABLE_FIND_PACKAGE_Libidn2 ON) + if(MSVC) + string(APPEND CMAKE_C_FLAGS " /wd6101") + string(APPEND CMAKE_C_FLAGS " /wd6011") + string(APPEND CMAKE_C_FLAGS " /wd6054") + string(APPEND CMAKE_C_FLAGS " /wd6287") + string(APPEND CMAKE_C_FLAGS " /wd6323") + string(APPEND CMAKE_C_FLAGS " /wd6385") + string(APPEND CMAKE_C_FLAGS " /wd6387") + string(APPEND CMAKE_C_FLAGS " /wd28182") + string(APPEND CMAKE_C_FLAGS " /wd28251") + string(APPEND CMAKE_C_FLAGS " /wd28301") + else() + string(APPEND CMAKE_C_FLAGS " -Wno-error") + endif() + FetchContent_MakeAvailable(LibCURL) +endfunction() + +get_libcurl() + +if(NOT TARGET CURL::libcurl) + if(TARGET libcurl_static) + add_library(CURL::libcurl ALIAS libcurl_static) + target_compile_definitions(libcurl_static INTERFACE CURL_STATICLIB) + elseif(TARGET libcurl) + add_library(CURL::libcurl ALIAS libcurl) + if(NOT BUILD_SHARED_LIBS) + target_compile_definitions(libcurl INTERFACE CURL_STATICLIB) + endif() + else() + message(FATAL_ERROR "After FetchContent_MakeAvailable(LibCURL) no suitable curl target (libcurl or libcurl_static) was found.") + endif() +endif() diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h index f00ef4f10b..2703c055ef 100644 --- a/include/vcpkg/base/contractual-constants.h +++ b/include/vcpkg/base/contractual-constants.h @@ -589,4 +589,9 @@ namespace vcpkg inline constexpr StringLiteral StatusInstalled = "installed"; inline constexpr StringLiteral StatusNotInstalled = "not-installed"; inline constexpr StringLiteral StatusPurge = "purge"; + + // App Insights JSON response fields + inline constexpr StringLiteral AppInsightsResponseItemsReceived = "itemsReceived"; + inline constexpr StringLiteral AppInsightsResponseItemsAccepted = "itemsAccepted"; + inline constexpr StringLiteral AppInsightsResponseErrors = "errors"; } diff --git a/include/vcpkg/base/curl.h b/include/vcpkg/base/curl.h new file mode 100644 index 0000000000..61acc9d377 --- /dev/null +++ b/include/vcpkg/base/curl.h @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include + +#include + +VCPKG_MSVC_WARNING(push) +// note: disable warning triggered by curl headers +// ws2tcpip.h(968): warning C6101: Returning uninitialized memory '*Mtu': A successful path through the function does +// not set the named _Out_ parameter. +VCPKG_MSVC_WARNING(disable : 6101) +#include +#include +VCPKG_MSVC_WARNING(pop) + +namespace vcpkg +{ + CURLcode get_curl_global_init_status() noexcept; + + struct CurlEasyHandle + { + CurlEasyHandle(); + CurlEasyHandle(CurlEasyHandle&& other) noexcept; + CurlEasyHandle& operator=(CurlEasyHandle&& other) noexcept; + ~CurlEasyHandle(); + + CURL* get(); + + private: + CURL* m_ptr = nullptr; + }; + + struct CurlMultiHandle + { + CurlMultiHandle(); + CurlMultiHandle(CurlMultiHandle&& other) noexcept; + CurlMultiHandle& operator=(CurlMultiHandle&& other) noexcept; + ~CurlMultiHandle(); + + // Adds an easy handle to the multi handle but doesn't take ownership of it. + // Makes sure that the easy handle is removed from the multi handle on cleanup. + void add_easy_handle(CurlEasyHandle& easy_handle); + + CURLM* get(); + + private: + CURLM* m_ptr = nullptr; + std::vector m_easy_handles; + }; + + struct CurlHeaders + { + CurlHeaders() = default; + CurlHeaders(View headers); + CurlHeaders(CurlHeaders&& other) noexcept; + CurlHeaders& operator=(CurlHeaders&& other) noexcept; + ~CurlHeaders(); + + curl_slist* get() const; + + private: + curl_slist* m_headers = nullptr; + }; + + constexpr char vcpkg_curl_user_agent[] = + "vcpkg/" VCPKG_BASE_VERSION_AS_STRING "-" VCPKG_VERSION_AS_STRING " (curl)"; +} diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 068ab463b3..50300cfd04 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -37,20 +37,9 @@ namespace vcpkg View azure_blob_headers(); - // Parses a curl output line for curl invoked with - // -w "PREFIX%{http_code} %{exitcode} %{errormsg}" - // with specific handling for curl version < 7.75.0 which does not understand %{exitcode} %{errormsg} - // If the line is malformed for any reason, no entry to http_codes is added. - // Returns: true if the new version of curl's output with exitcode and errormsg was parsed; otherwise, false. - bool parse_curl_status_line(DiagnosticContext& context, - std::vector& http_codes, - StringLiteral prefix, - StringView this_line); - std::vector download_files_no_cache(DiagnosticContext& context, View> url_pairs, - View headers, - View secrets); + View headers); bool submit_github_dependency_graph_snapshot(DiagnosticContext& context, const Optional& maybe_github_server_url, @@ -58,19 +47,9 @@ namespace vcpkg const std::string& github_repository, const Json::Object& snapshot); - Optional invoke_http_request(DiagnosticContext& context, - StringLiteral method, - View headers, - StringView url, - View secrets, - StringView data = {}); - std::string format_url_query(StringView base_url, View query_params); - std::vector url_heads(DiagnosticContext& context, - View urls, - View headers, - View secrets); + std::vector url_heads(DiagnosticContext& context, View urls, View headers); struct AssetCachingSettings { @@ -107,7 +86,6 @@ namespace vcpkg bool store_to_asset_cache(DiagnosticContext& context, StringView raw_url, const SanitizedUrl& sanitized_url, - StringLiteral method, View headers, const Path& file); diff --git a/include/vcpkg/base/files.h b/include/vcpkg/base/files.h index d9b08a46fc..546e0c2975 100644 --- a/include/vcpkg/base/files.h +++ b/include/vcpkg/base/files.h @@ -103,6 +103,8 @@ namespace vcpkg // reads any remaining chunks of the file; used to implement read_to_end void read_to_end_suffix( std::string& output, std::error_code& ec, char* buffer, size_t buffer_size, size_t last_read); + uint64_t size(LineInfo li) const; + uint64_t size(std::error_code& ec) const; }; struct WriteFilePointer : FilePointer diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index fbdd6932e6..565554ed96 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -988,22 +988,16 @@ DECLARE_MESSAGE(CreationFailed, (msg::path), "", "Creating {path} failed.") DECLARE_MESSAGE(CurlFailedGeneric, (msg::exit_code), "curl is the name of a program, see curl.se.", - "curl operation failed with error code {exit_code}.") + "curl operation failed with error code {exit_code}") +DECLARE_MESSAGE(CurlDownloadTimeout, (), "", "Download timed out.") +DECLARE_MESSAGE(CurlFailedHttpResponse, + (msg::exit_code), + "curl is the name of a program, see curl.se.", + "curl operation failed with HTTP response code {exit_code}.") DECLARE_MESSAGE(CurlFailedToPut, - (msg::exit_code, msg::url), - "curl is the name of a program, see curl.se", - "curl failed to put file to {url} with exit code {exit_code}.") -DECLARE_MESSAGE(CurlFailedToPutHttp, - (msg::exit_code, msg::url, msg::value), + (msg::url, msg::value), "curl is the name of a program, see curl.se. {value} is an HTTP status code", - "curl failed to put file to {url} with exit code {exit_code} and http code {value}.") -DECLARE_MESSAGE( - CurlFailedToReturnExpectedNumberOfExitCodes, - (msg::exit_code, msg::command_line), - "", - "curl failed to return the expected number of exit codes; this can happen if something terminates curl " - "before it has finished. curl exited with {exit_code} which is normally the result code for the last operation, " - "but may be the result of a crash. The command line was {command_line}, and all output is below:") + "curl failed to PUT file to {url} with response code {value}.") DECLARE_MESSAGE(CurrentCommitBaseline, (msg::commit_sha), "", @@ -1085,10 +1079,6 @@ DECLARE_MESSAGE( (msg::sha), "", "failing download because the expected SHA512 was all zeros, please change the expected SHA512 to: {sha}") -DECLARE_MESSAGE(DownloadFailedRetrying, - (msg::value, msg::url), - "{value} is a number of milliseconds", - "Download {url} failed -- retrying after {value}ms") DECLARE_MESSAGE(DownloadFailedStatusCode, (msg::url, msg::value), "{value} is an HTTP status code", @@ -1125,6 +1115,18 @@ DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundle, (msg::version), "", "Downloadi DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundleLatest, (), "", "Downloading latest standalone bundle.") DECLARE_MESSAGE(DownloadingTools, (msg::count), "", "Downloading {count} tools") DECLARE_MESSAGE(DownloadOrUrl, (msg::url), "", "or {url}") +DECLARE_MESSAGE(DownloadTransientErrorRetry, + (msg::count, msg::value), + "{value} is the maximum number of attempts to download a file", + "Attempt {count} of {value}, retrying download.") +DECLARE_MESSAGE(DownloadTransientErrorRetriesExhausted, + (msg::url), + "", + "Reached maximum number of attempts, won't retry download from {url}.") +DECLARE_MESSAGE(DownloadNotTransientErrorWontRetry, + (msg::url), + "", + "Not a transient network error, won't retry download from {url}") DECLARE_MESSAGE(DownloadTryingAuthoritativeSource, (msg::url), "", "Trying {url}") DECLARE_MESSAGE(DownloadRootsDir, (msg::env_var), "", "Downloads directory (default: {env_var})") DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}") @@ -1132,10 +1134,6 @@ DECLARE_MESSAGE(DownloadSuccesfulUploading, (msg::path, msg::url), "", "Successfully downloaded {path}, storing to {url}") -DECLARE_MESSAGE(DownloadWinHttpError, - (msg::system_api, msg::exit_code, msg::url), - "", - "{url}: {system_api} failed with exit code {exit_code}.") DECLARE_MESSAGE(DuplicateDependencyOverride, (msg::package_name), "", "{package_name} already has an override") DECLARE_MESSAGE(DuplicatedKeyInObj, (msg::value), @@ -2600,7 +2598,7 @@ DECLARE_MESSAGE( "the license is not installed to ${{CURRENT_PACKAGES_DIR}}/share/${{PORT}}/copyright . This can be fixed by adding " "a call to vcpkg_install_copyright. To suppress this message, add set(VCPKG_POLICY_SKIP_COPYRIGHT_CHECK enabled)") DECLARE_MESSAGE(PortBugMissingLicenseFixIt, - (msg ::value), + (msg::value), "{value} is a CMake function call for the user to paste into their file, for example: " "vcpkg_install_copyright(FILE_LIST ${{SOURCE_PATH}}/COPYING ${{SOURCE_PATH}}/LICENSE.txt)", "Consider adding: {value}") @@ -3160,7 +3158,6 @@ DECLARE_MESSAGE(VcpkgUsage, "[]s, or --s should be preserved. @response_file should be localized to be consistent with the message " "named 'ResponseFileCode'.", "usage: vcpkg [--switches] [--options=values] [arguments] @response_file") -DECLARE_MESSAGE(InvalidUri, (msg::value), "{value} is the URI we attempted to parse.", "unable to parse uri: {value}") DECLARE_MESSAGE(VcpkgInVsPrompt, (msg::value, msg::triplet), "'{value}' is a VS prompt", diff --git a/include/vcpkg/metrics.h b/include/vcpkg/metrics.h index 07d750439d..e81ea0c95b 100644 --- a/include/vcpkg/metrics.h +++ b/include/vcpkg/metrics.h @@ -198,7 +198,6 @@ namespace vcpkg extern std::atomic g_should_send_metrics; void flush_global_metrics(const Filesystem&); -#if defined(_WIN32) - void winhttp_upload_metrics(StringView payload); -#endif // ^^^ _WIN32 + bool curl_upload_metrics(const std::string& payload); + bool parse_metrics_response(StringView response_body); } diff --git a/locales/messages.json b/locales/messages.json index c58fe8a1b7..54488e7124 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -570,14 +570,13 @@ "CreatingZipArchive": "Creating zip archive...", "CreationFailed": "Creating {path} failed.", "_CreationFailed.comment": "An example of {path} is /foo/bar.", - "CurlFailedGeneric": "curl operation failed with error code {exit_code}.", + "CurlDownloadTimeout": "Download timed out.", + "CurlFailedGeneric": "curl operation failed with error code {exit_code}", "_CurlFailedGeneric.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.", - "CurlFailedToPut": "curl failed to put file to {url} with exit code {exit_code}.", - "_CurlFailedToPut.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", - "CurlFailedToPutHttp": "curl failed to put file to {url} with exit code {exit_code} and http code {value}.", - "_CurlFailedToPutHttp.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", - "CurlFailedToReturnExpectedNumberOfExitCodes": "curl failed to return the expected number of exit codes; this can happen if something terminates curl before it has finished. curl exited with {exit_code} which is normally the result code for the last operation, but may be the result of a crash. The command line was {command_line}, and all output is below:", - "_CurlFailedToReturnExpectedNumberOfExitCodes.comment": "An example of {exit_code} is 127. An example of {command_line} is vcpkg install zlib.", + "CurlFailedHttpResponse": "curl operation failed with HTTP response code {exit_code}.", + "_CurlFailedHttpResponse.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.", + "CurlFailedToPut": "curl failed to PUT file to {url} with response code {value}.", + "_CurlFailedToPut.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.", "CurrentCommitBaseline": "You can use the current commit as a baseline, which is:\n\t\"builtin-baseline\": \"{commit_sha}\"", "_CurrentCommitBaseline.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.", "CycleDetectedDuring": "cycle detected during {spec}:", @@ -625,10 +624,10 @@ "DownloadFailedHashMismatchZero": "failing download because the expected SHA512 was all zeros, please change the expected SHA512 to: {sha}", "_DownloadFailedHashMismatchZero.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", "DownloadFailedProxySettings": "If you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https://address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See: https://github.com/microsoft/vcpkg-tool/pull/77\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is out of service.\nIf you believe this is not a temporary download server failure and vcpkg needs to be changed to download this file from a different location, please submit an issue to https://github.com/Microsoft/vcpkg/issues", - "DownloadFailedRetrying": "Download {url} failed -- retrying after {value}ms", - "_DownloadFailedRetrying.comment": "{value} is a number of milliseconds An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadFailedStatusCode": "{url}: failed: status code {value}", "_DownloadFailedStatusCode.comment": "{value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadNotTransientErrorWontRetry": "Not a transient network error, won't retry download from {url}", + "_DownloadNotTransientErrorWontRetry.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadOrUrl": "or {url}", "_DownloadOrUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadRootsDir": "Downloads directory (default: {env_var})", @@ -637,10 +636,12 @@ "_DownloadSuccesful.comment": "An example of {path} is /foo/bar.", "DownloadSuccesfulUploading": "Successfully downloaded {path}, storing to {url}", "_DownloadSuccesfulUploading.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadTransientErrorRetriesExhausted": "Reached maximum number of attempts, won't retry download from {url}.", + "_DownloadTransientErrorRetriesExhausted.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadTransientErrorRetry": "Attempt {count} of {value}, retrying download.", + "_DownloadTransientErrorRetry.comment": "{value} is the maximum number of attempts to download a file An example of {count} is 42.", "DownloadTryingAuthoritativeSource": "Trying {url}", "_DownloadTryingAuthoritativeSource.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", - "DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}.", - "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadedSources": "Downloaded sources for {spec}", "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.", "DownloadingAssetShaToFile": "Downloading asset cache entry {sha} -> {path}", @@ -1104,8 +1105,6 @@ "InvalidToolVersion": "Invalid tool version; expected a string containing a substring of between 1 and 3 numbers separated by dots.", "InvalidTriplet": "Invalid triplet: {triplet}", "_InvalidTriplet.comment": "An example of {triplet} is x64-windows.", - "InvalidUri": "unable to parse uri: {value}", - "_InvalidUri.comment": "{value} is the URI we attempted to parse.", "InvalidValueHashAdditionalFiles": "Variable VCPKG_HASH_ADDITIONAL_FILES contains invalid file path: '{path}'. The value must be an absolute path to an existent file.", "_InvalidValueHashAdditionalFiles.comment": "An example of {path} is /foo/bar.", "InvalidValuePostPortfileIncludes": "Variable VCPKG_POST_PORTFILE_INCLUDES contains invalid file path: '{path}'. The value must be an absolute path to an existent cmake file.", diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp index 3df1949a8e..1f601505b2 100644 --- a/src/vcpkg-test/downloads.cpp +++ b/src/vcpkg-test/downloads.cpp @@ -120,92 +120,25 @@ TEST_CASE ("parse_split_url_view", "[downloads]") } } -TEST_CASE ("parse_curl_status_line", "[downloads]") -{ - std::vector http_codes; - StringLiteral malformed_examples[] = { - "asdfasdf", // wrong prefix - "curl: unknown --write-out variable: 'exitcode'", // wrong prefixes, and also what old curl does - "curl: unknown --write-out variable: 'errormsg'", - "prefix", // missing spaces - "prefix42", // missing spaces - "prefix42 2", // missing space - "prefix42 2a", // non numeric exitcode - }; - - FullyBufferedDiagnosticContext bdc; - for (auto&& malformed : malformed_examples) - { - REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", malformed)); - REQUIRE(http_codes.empty()); - REQUIRE(bdc.empty()); - } - - // old curl output - REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix200 ")); - REQUIRE(http_codes == std::vector{200}); - REQUIRE(bdc.empty()); - http_codes.clear(); - - REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix404 ")); - REQUIRE(http_codes == std::vector{404}); - REQUIRE(bdc.empty()); - http_codes.clear(); - - REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix0 ")); // a failure, but we don't know that yet - REQUIRE(http_codes == std::vector{0}); - REQUIRE(bdc.empty()); - http_codes.clear(); - - // current curl output - REQUIRE(parse_curl_status_line(bdc, http_codes, "prefix", "prefix200 0 ")); - REQUIRE(http_codes == std::vector{200}); - REQUIRE(bdc.empty()); - http_codes.clear(); - - REQUIRE(parse_curl_status_line( - bdc, - http_codes, - "prefix", - "prefix0 60 schannel: SNI or certificate check failed: SEC_E_WRONG_PRINCIPAL (0x80090322) " - "- The target principal name is incorrect.")); - REQUIRE(http_codes == std::vector{0}); - REQUIRE(bdc.to_string() == - "error: curl operation failed with error code 60. schannel: SNI or certificate check failed: " - "SEC_E_WRONG_PRINCIPAL (0x80090322) - The target principal name is incorrect."); -} - TEST_CASE ("download_files", "[downloads]") { auto const dst = Test::base_temporary_directory() / "download_files"; - auto const url = [&](std::string l) -> auto { return std::pair(l, dst); }; + real_filesystem.create_directories(dst, VCPKG_LINE_INFO); + + static const std::vector> test_downloads{ + {"unknown://localhost:9/secret", dst / "test1"}, + {"http://localhost:9/not-exists/secret", dst / "test2"}, + }; FullyBufferedDiagnosticContext bdc; std::vector headers; - std::vector secrets; - auto results = download_files_no_cache( - bdc, - std::vector{url("unknown://localhost:9/secret"), url("http://localhost:9/not-exists/secret")}, - headers, - secrets); - REQUIRE(results == std::vector{0, 0}); - auto all_errors = bdc.to_string(); - if (all_errors == "error: curl operation failed with error code 7.") - { - // old curl, this is OK! - } - else - { - // new curl - REQUIRE_THAT( - all_errors, - Catch::Matches("error: curl operation failed with error code 1\\. Protocol \"unknown\" not supported( or " - "disabled in libcurl)?\n" - "error: curl operation failed with error code 7\\. ((Failed to connect to localhost port 9 " - "after [0-9]+ ms: ((Could not|Couldn't) connect to server|Connection refused))" - "|(getsockname\\(\\) failed with errno 22: Invalid argument))", - Catch::CaseSensitive::Yes)); - } + auto results = download_files_no_cache(bdc, test_downloads, headers); + REQUIRE(results == std::vector{-1, -1}); + auto all_errors = Strings::split(bdc.to_string(), '\n'); + REQUIRE(all_errors[0] == "error: curl operation failed with error code 1 (Unsupported protocol)."); + // Old versions of libcurl use "Couldn't" on error messages + REQUIRE((all_errors[1] == "error: curl operation failed with error code 7 (Could not connect to server)." || + all_errors[1] == "error: curl operation failed with error code 7 (Couldn't connect to server).")); } TEST_CASE ("try_parse_curl_max5_size", "[downloads]") @@ -374,7 +307,7 @@ TEST_CASE ("azblob", "[.][azblob]") FullyBufferedDiagnosticContext diagnostics{}; auto plain_put_success = store_to_asset_cache( - diagnostics, plain_put_url, SanitizedUrl{url, {}}, "PUT", azure_blob_headers(), data_filepath); + diagnostics, plain_put_url, SanitizedUrl{url, {}}, azure_blob_headers(), data_filepath); INFO(diagnostics.to_string()); CHECK(plain_put_success); } @@ -393,7 +326,7 @@ TEST_CASE ("azblob", "[.][azblob]") { FullyBufferedDiagnosticContext diagnostics{}; - auto results = download_files_no_cache(diagnostics, url_pairs, azure_blob_headers(), {}); + auto results = download_files_no_cache(diagnostics, url_pairs, azure_blob_headers()); INFO(diagnostics.to_string()); CHECK(results == std::vector{200, 200}); } diff --git a/src/vcpkg-test/metrics.cpp b/src/vcpkg-test/metrics.cpp index dbd24d08a2..e069ae4826 100644 --- a/src/vcpkg-test/metrics.cpp +++ b/src/vcpkg-test/metrics.cpp @@ -256,3 +256,39 @@ TEST_CASE ("payload smoke test", "[metrics]") )json"; REQUIRE(expected == actual); } + +TEST_CASE ("parse metrics response", "[metrics]") +{ + const std::string response = R"json( +{ + "itemsReceived": 1, + "itemsAccepted": 1, + "errors": [] +} +)json"; + auto parsed = parse_metrics_response(response); + CHECK(parsed); + + const std::string response_with_errors = R"json( +{ + "itemsReceived": 2, + "itemsAccepted": 1, + "errors": [ + { + "message": "Invalid payload" + } + ] +} +)json"; + auto parsed_with_errors = parse_metrics_response(response_with_errors); + CHECK(!parsed_with_errors); + + const std::string response_with_errors2 = R"json( +{ + "itemsReceived": 2, + "errors": [] +} +)json"; + auto parsed_with_errors2 = parse_metrics_response(response_with_errors2); + CHECK(!parsed_with_errors2); +} diff --git a/src/vcpkg.cpp b/src/vcpkg.cpp index 0dc64e15ea..3b3b18dda8 100644 --- a/src/vcpkg.cpp +++ b/src/vcpkg.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp new file mode 100644 index 0000000000..98305363c7 --- /dev/null +++ b/src/vcpkg/base/curl.cpp @@ -0,0 +1,123 @@ +#include +#include + +namespace +{ + struct CurlGlobalInit + { + CurlGlobalInit() : init_status(curl_global_init(CURL_GLOBAL_DEFAULT)) { } + ~CurlGlobalInit() { curl_global_cleanup(); } + + CurlGlobalInit(const CurlGlobalInit&) = delete; + CurlGlobalInit(CurlGlobalInit&&) = delete; + CurlGlobalInit& operator=(const CurlGlobalInit&) = delete; + CurlGlobalInit& operator=(CurlGlobalInit&&) = delete; + + CURLcode get_init_status() const { return init_status; } + + private: + CURLcode init_status; + }; +} + +namespace vcpkg +{ + CURLcode get_curl_global_init_status() noexcept + { + static CurlGlobalInit g_curl_global_init; + return g_curl_global_init.get_init_status(); + } + + CurlEasyHandle::CurlEasyHandle() { get_curl_global_init_status(); } + CurlEasyHandle::CurlEasyHandle(CurlEasyHandle&& other) noexcept : m_ptr(std::exchange(other.m_ptr, nullptr)) { } + CurlEasyHandle& CurlEasyHandle::operator=(CurlEasyHandle&& other) noexcept + { + m_ptr = std::exchange(other.m_ptr, nullptr); + return *this; + } + CurlEasyHandle::~CurlEasyHandle() + { + if (m_ptr) + { + curl_easy_cleanup(m_ptr); + } + } + CURL* CurlEasyHandle::get() + { + if (!m_ptr) + { + m_ptr = curl_easy_init(); + if (!m_ptr) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + } + return m_ptr; + } + + CurlMultiHandle::CurlMultiHandle() { get_curl_global_init_status(); } + CurlMultiHandle::CurlMultiHandle(CurlMultiHandle&& other) noexcept + : m_ptr(std::exchange(other.m_ptr, nullptr)), m_easy_handles(std::move(other.m_easy_handles)) + { + } + CurlMultiHandle& CurlMultiHandle::operator=(CurlMultiHandle&& other) noexcept + { + m_ptr = std::exchange(other.m_ptr, nullptr); + m_easy_handles = std::move(other.m_easy_handles); + return *this; + } + CurlMultiHandle::~CurlMultiHandle() + { + for (auto* easy_handle : m_easy_handles) + { + curl_multi_remove_handle(m_ptr, easy_handle); + } + + if (m_ptr) + { + curl_multi_cleanup(m_ptr); + } + } + void CurlMultiHandle::add_easy_handle(CurlEasyHandle& easy_handle) + { + auto* handle = easy_handle.get(); + if (curl_multi_add_handle(this->get(), handle) == CURLM_OK) + { + m_easy_handles.push_back(handle); + } + } + CURLM* CurlMultiHandle::get() + { + if (!m_ptr) + { + m_ptr = curl_multi_init(); + if (!m_ptr) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + } + return m_ptr; + } + + CurlHeaders::CurlHeaders(View headers) + { + for (const auto& header : headers) + { + m_headers = curl_slist_append(m_headers, header.c_str()); + } + } + CurlHeaders::CurlHeaders(CurlHeaders&& other) noexcept : m_headers(std::exchange(other.m_headers, nullptr)) { } + CurlHeaders& CurlHeaders::operator=(CurlHeaders&& other) noexcept + { + m_headers = std::exchange(other.m_headers, nullptr); + return *this; + } + CurlHeaders::~CurlHeaders() + { + if (m_headers) + { + curl_slist_free_all(m_headers); + } + } + curl_slist* CurlHeaders::get() const { return m_headers; } +} diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 4019e87cae..0515082bb7 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -15,24 +16,23 @@ #include #include -#include - #include using namespace vcpkg; namespace { - constexpr StringLiteral vcpkg_curl_user_agent_header = - "User-Agent: vcpkg/" VCPKG_BASE_VERSION_AS_STRING "-" VCPKG_VERSION_AS_STRING " (curl)"; - - void add_curl_headers(Command& cmd, View headers) + void set_common_curl_easy_options(CurlEasyHandle& easy_handle, StringView url, const CurlHeaders& request_headers) { - cmd.string_arg("-H").string_arg(vcpkg_curl_user_agent_header); - for (auto&& header : headers) - { - cmd.string_arg("-H").string_arg(header); - } + auto* curl = easy_handle.get(); + curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + curl_easy_setopt(curl, CURLOPT_URL, url_encode_spaces(url).c_str()); + curl_easy_setopt(curl, + CURLOPT_FOLLOWLOCATION, + 2L); // Follow redirects, change request method based on HTTP response code. + // https://curl.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html#CURLFOLLOWOBEYCODE + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + curl_easy_setopt(curl, CURLOPT_HEADEROPT, CURLHEADER_SEPARATE); // don't send headers to proxy CONNECT } } @@ -44,547 +44,6 @@ namespace vcpkg replace_secrets(m_sanitized_url, secrets); } -#if defined(_WIN32) - struct FormatMessageHLocalAlloc - { - LPWSTR buffer = nullptr; - - ~FormatMessageHLocalAlloc() - { - if (buffer) - { - LocalFree(buffer); - } - } - }; - - static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, - const SanitizedUrl& sanitized_url, - DWORD last_error) - { - const HMODULE winhttp_module = GetModuleHandleW(L"winhttp.dll"); - FormatMessageHLocalAlloc alloc; - DWORD tchars_excluding_terminating_null = 0; - if (winhttp_module) - { - tchars_excluding_terminating_null = - FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_HMODULE, - winhttp_module, - last_error, - 0, - reinterpret_cast(&alloc.buffer), - 0, - nullptr); - } - - auto result = msg::format( - msgDownloadWinHttpError, msg::system_api = api_name, msg::exit_code = last_error, msg::url = sanitized_url); - if (tchars_excluding_terminating_null && alloc.buffer) - { - while (tchars_excluding_terminating_null != 0 && - (alloc.buffer[tchars_excluding_terminating_null - 1] == L'\r' || - alloc.buffer[tchars_excluding_terminating_null - 1] == L'\n')) - { - --tchars_excluding_terminating_null; - } - - tchars_excluding_terminating_null = static_cast( - std::remove(alloc.buffer, alloc.buffer + tchars_excluding_terminating_null, L'\r') - alloc.buffer); - result.append_raw(' ').append_raw(Strings::to_utf8(alloc.buffer, tchars_excluding_terminating_null)); - } - - return result; - } - - static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, const SanitizedUrl& sanitized_url) - { - return format_winhttp_last_error_message(api_name, sanitized_url, GetLastError()); - } - - static void maybe_emit_winhttp_progress(MessageSink& machine_readable_progress, - const Optional& maybe_content_length, - std::chrono::steady_clock::time_point& last_write, - unsigned long long total_downloaded_size) - { - if (const auto content_length = maybe_content_length.get()) - { - const auto now = std::chrono::steady_clock::now(); - if ((now - last_write) >= std::chrono::milliseconds(100)) - { - const double percent = - (static_cast(total_downloaded_size) / static_cast(*content_length)) * 100; - machine_readable_progress.println(LocalizedString::from_raw(fmt::format("{:.2f}%", percent))); - last_write = now; - } - } - } - - struct WinHttpHandle - { - WinHttpHandle() = default; - WinHttpHandle(const WinHttpHandle&) = delete; - WinHttpHandle& operator=(const WinHttpHandle&) = delete; - - void require_null_handle() const - { - if (h) - { - Checks::unreachable(VCPKG_LINE_INFO, "WinHTTP handle type confusion"); - } - } - - void require_created_handle() const - { - if (!h) - { - Checks::unreachable(VCPKG_LINE_INFO, "WinHTTP handle not created"); - } - } - - bool Connect(DiagnosticContext& context, - const WinHttpHandle& session, - StringView hostname, - INTERNET_PORT port, - const SanitizedUrl& sanitized_url) - { - require_null_handle(); - session.require_created_handle(); - h = WinHttpConnect(session.h, Strings::to_utf16(hostname).c_str(), port, 0); - if (h) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpConnect", sanitized_url)); - return false; - } - - bool Open(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - _In_opt_z_ LPCWSTR pszAgentW, - _In_ DWORD dwAccessType, - _In_opt_z_ LPCWSTR pszProxyW, - _In_opt_z_ LPCWSTR pszProxyBypassW, - _In_ DWORD dwFlags) - { - require_null_handle(); - h = WinHttpOpen(pszAgentW, dwAccessType, pszProxyW, pszProxyBypassW, dwFlags); - if (h) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpOpen", sanitized_url)); - return false; - } - - bool OpenRequest(DiagnosticContext& context, - const WinHttpHandle& hConnect, - const SanitizedUrl& sanitized_url, - IN LPCWSTR pwszVerb, - StringView path_query_fragment, - IN LPCWSTR pwszVersion, - IN LPCWSTR pwszReferrer OPTIONAL, - IN LPCWSTR FAR* ppwszAcceptTypes OPTIONAL, - IN DWORD dwFlags) - { - require_null_handle(); - h = WinHttpOpenRequest(hConnect.h, - pwszVerb, - Strings::to_utf16(path_query_fragment).c_str(), - pwszVersion, - pwszReferrer, - ppwszAcceptTypes, - dwFlags); - if (h) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpOpenRequest", sanitized_url)); - return false; - } - - bool SendRequest(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - _In_reads_opt_(dwHeadersLength) LPCWSTR lpszHeaders, - IN DWORD dwHeadersLength, - _In_reads_bytes_opt_(dwOptionalLength) LPVOID lpOptional, - IN DWORD dwOptionalLength, - IN DWORD dwTotalLength, - IN DWORD_PTR dwContext) const - { - require_created_handle(); - if (WinHttpSendRequest( - h, lpszHeaders, dwHeadersLength, lpOptional, dwOptionalLength, dwTotalLength, dwContext)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpSendRequest", sanitized_url)); - return false; - } - - bool ReceiveResponse(DiagnosticContext& context, const SanitizedUrl& url) - { - require_created_handle(); - if (WinHttpReceiveResponse(h, NULL)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpReceiveResponse", url)); - return false; - } - - bool SetTimeouts(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - int nResolveTimeout, - int nConnectTimeout, - int nSendTimeout, - int nReceiveTimeout) const - { - require_created_handle(); - if (WinHttpSetTimeouts(h, nResolveTimeout, nConnectTimeout, nSendTimeout, nReceiveTimeout)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpSetTimeouts", sanitized_url)); - return false; - } - - bool SetOption(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - DWORD dwOption, - LPVOID lpBuffer, - DWORD dwBufferLength) const - { - require_created_handle(); - if (WinHttpSetOption(h, dwOption, lpBuffer, dwBufferLength)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpSetOption", sanitized_url)); - return false; - } - - DWORD QueryHeaders(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - DWORD dwInfoLevel, - LPWSTR pwszName, - LPVOID lpBuffer, - LPDWORD lpdwBufferLength, - LPDWORD lpdwIndex) const - { - require_created_handle(); - if (WinHttpQueryHeaders(h, dwInfoLevel, pwszName, lpBuffer, lpdwBufferLength, lpdwIndex)) - { - return 0; - } - - DWORD last_error = GetLastError(); - context.report_error(format_winhttp_last_error_message("WinHttpQueryHeaders", sanitized_url, last_error)); - return last_error; - } - - bool ReadData(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - LPVOID buffer, - DWORD dwNumberOfBytesToRead, - DWORD* numberOfBytesRead) - { - require_created_handle(); - if (WinHttpReadData(h, buffer, dwNumberOfBytesToRead, numberOfBytesRead)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpReadData", sanitized_url)); - return false; - } - - ~WinHttpHandle() - { - if (h) - { - // intentionally ignore failures - (void)WinHttpCloseHandle(h); - } - } - - private: - HINTERNET h{}; - }; - - enum class WinHttpTrialResult - { - failed, - succeeded, - retry - }; - - struct WinHttpSession - { - bool open(DiagnosticContext& context, const SanitizedUrl& sanitized_url) - { - if (!m_hSession.Open(context, - sanitized_url, - L"vcpkg/1.0", - WINHTTP_ACCESS_TYPE_NO_PROXY, - WINHTTP_NO_PROXY_NAME, - WINHTTP_NO_PROXY_BYPASS, - 0)) - { - return false; - } - - // Increase default timeouts to help connections behind proxies - // WinHttpSetTimeouts(HINTERNET hInternet, int nResolveTimeout, int nConnectTimeout, int nSendTimeout, int - // nReceiveTimeout); - if (!m_hSession.SetTimeouts(context, sanitized_url, 0, 120000, 120000, 120000)) - { - return false; - } - - // If the environment variable HTTPS_PROXY is set - // use that variable as proxy. This situation might exist when user is in a company network - // with restricted network/proxy settings - auto maybe_https_proxy_env = get_environment_variable(EnvironmentVariableHttpsProxy); - if (auto p_https_proxy = maybe_https_proxy_env.get()) - { - StringView p_https_proxy_view = *p_https_proxy; - if (p_https_proxy_view.size() != 0 && p_https_proxy_view.back() == '/') - { - // remove trailing slash - p_https_proxy_view = p_https_proxy_view.substr(0, p_https_proxy_view.size() - 1); - } - - std::wstring env_proxy_settings = Strings::to_utf16(p_https_proxy_view); - WINHTTP_PROXY_INFO proxy; - proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; - proxy.lpszProxy = env_proxy_settings.data(); - - // Try to get bypass list from environment variable - auto maybe_no_proxy_env = get_environment_variable(EnvironmentVariableNoProxy); - std::wstring env_noproxy_settings; - if (auto p_no_proxy = maybe_no_proxy_env.get()) - { - env_noproxy_settings = Strings::to_utf16(*p_no_proxy); - proxy.lpszProxyBypass = env_noproxy_settings.data(); - } - else - { - proxy.lpszProxyBypass = nullptr; - } - - if (!m_hSession.SetOption(context, sanitized_url, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy))) - { - return false; - } - } - // IE Proxy fallback, this works on Windows 10 - else - { - // We do not use WPAD anymore - // Directly read IE Proxy setting - auto ieProxy = get_windows_ie_proxy_server(); - if (ieProxy.has_value()) - { - WINHTTP_PROXY_INFO proxy; - proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; - proxy.lpszProxy = ieProxy.get()->server.data(); - proxy.lpszProxyBypass = ieProxy.get()->bypass.data(); - if (!m_hSession.SetOption(context, sanitized_url, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy))) - { - return false; - } - } - } - - // Use Windows 10 defaults on Windows 7 - DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | - WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2); - if (!m_hSession.SetOption(context, - sanitized_url, - WINHTTP_OPTION_SECURE_PROTOCOLS, - &secure_protocols, - sizeof(secure_protocols))) - { - return false; - } - - // Many open source mirrors such as https://download.gnome.org/ will redirect to http mirrors. - // `curl.exe -L` does follow https -> http redirection. - // Additionally, vcpkg hash checks the resulting archive. - DWORD redirect_policy(WINHTTP_OPTION_REDIRECT_POLICY_ALWAYS); - if (!m_hSession.SetOption( - context, sanitized_url, WINHTTP_OPTION_REDIRECT_POLICY, &redirect_policy, sizeof(redirect_policy))) - { - return false; - } - - return true; - } - - WinHttpHandle m_hSession; - }; - - struct WinHttpConnection - { - bool connect(DiagnosticContext& context, - const WinHttpSession& hSession, - StringView hostname, - INTERNET_PORT port, - const SanitizedUrl& sanitized_url) - { - // Specify an HTTP server. - return m_hConnect.Connect(context, hSession.m_hSession, hostname, port, sanitized_url); - } - - WinHttpHandle m_hConnect; - }; - - struct WinHttpRequest - { - bool open(DiagnosticContext& context, - const WinHttpConnection& hConnect, - StringView path_query_fragment, - const SanitizedUrl& sanitized_url, - bool https, - const wchar_t* method = L"GET") - { - if (!m_hRequest.OpenRequest(context, - hConnect.m_hConnect, - sanitized_url, - method, - path_query_fragment, - nullptr, - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, - https ? WINHTTP_FLAG_SECURE : 0)) - { - return false; - } - - // Send a request. - if (!m_hRequest.SendRequest( - context, sanitized_url, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0)) - { - return false; - } - - // End the request. - if (!m_hRequest.ReceiveResponse(context, sanitized_url)) - { - return false; - } - - return true; - } - - Optional query_status(DiagnosticContext& context, const SanitizedUrl& sanitized_url) const - { - DWORD status_code; - DWORD size = sizeof(status_code); - DWORD last_error = m_hRequest.QueryHeaders(context, - sanitized_url, - WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, - WINHTTP_HEADER_NAME_BY_INDEX, - &status_code, - &size, - WINHTTP_NO_HEADER_INDEX); - if (last_error) - { - return nullopt; - } - - return status_code; - } - - bool query_content_length(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - Optional& result) const - { - static constexpr DWORD buff_characters = 21; // 18446744073709551615 - wchar_t buff[buff_characters]; - DWORD size = sizeof(buff); - AttemptDiagnosticContext adc{context}; - DWORD last_error = m_hRequest.QueryHeaders(adc, - sanitized_url, - WINHTTP_QUERY_CONTENT_LENGTH, - WINHTTP_HEADER_NAME_BY_INDEX, - buff, - &size, - WINHTTP_NO_HEADER_INDEX); - if (!last_error) - { - adc.commit(); - result = Strings::strto(Strings::to_utf8(buff, size >> 1)); - return true; - } - - if (last_error == ERROR_WINHTTP_HEADER_NOT_FOUND) - { - adc.handle(); - return true; - } - - adc.commit(); - return false; - } - - WinHttpTrialResult write_response_body(DiagnosticContext& context, - MessageSink& machine_readable_progress, - const SanitizedUrl& sanitized_url, - const WriteFilePointer& file) - { - static constexpr DWORD buff_size = 65535; - std::unique_ptr buff{new char[buff_size]}; - Optional maybe_content_length; - auto last_write = std::chrono::steady_clock::now(); - if (!query_content_length(context, sanitized_url, maybe_content_length)) - { - return WinHttpTrialResult::retry; - } - - unsigned long long total_downloaded_size = 0; - for (;;) - { - DWORD this_read; - if (!m_hRequest.ReadData(context, sanitized_url, buff.get(), buff_size, &this_read)) - { - return WinHttpTrialResult::retry; - } - - if (this_read == 0) - { - return WinHttpTrialResult::succeeded; - } - - do - { - const auto this_write = static_cast(file.write(buff.get(), 1, this_read)); - if (this_write == 0) - { - context.report_error(format_filesystem_call_error( - std::error_code{errno, std::generic_category()}, "fwrite", {file.path()})); - return WinHttpTrialResult::failed; - } - - maybe_emit_winhttp_progress( - machine_readable_progress, maybe_content_length, last_write, total_downloaded_size); - this_read -= this_write; - total_downloaded_size += this_write; - } while (this_read > 0); - } - } - - WinHttpHandle m_hRequest; - }; -#endif - Optional parse_split_url_view(StringView raw_url) { auto sep = std::find(raw_url.begin(), raw_url.end(), ':'); @@ -672,107 +131,164 @@ namespace vcpkg return true; } - static std::vector curl_bulk_operation(DiagnosticContext& context, - View operation_args, - StringLiteral prefixArgs, - View headers, - View secrets) + static size_t write_file_callback(void* contents, size_t size, size_t nmemb, void* param) + { + if (!param) return 0; + return static_cast(param)->write(contents, size, nmemb); + } + + static size_t progress_callback( + void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { -#define GUID_MARKER "5ec47b8e-6776-4d70-b9b3-ac2a57bc0a1c" - static constexpr StringLiteral guid_marker = GUID_MARKER; - Command prefix_cmd{"curl"}; - if (!prefixArgs.empty()) + (void)ultotal; + (void)ulnow; + auto machine_readable_progress = static_cast(clientp); + if (dltotal && machine_readable_progress) { - prefix_cmd.raw_arg(prefixArgs); + double percentage = static_cast(dlnow) / static_cast(dltotal) * 100.0; + machine_readable_progress->println(LocalizedString::from_raw(fmt::format("{:.2f}%", percentage))); } + return 0; + } - prefix_cmd.string_arg("--retry").string_arg("3").string_arg("-L").string_arg("-sS").string_arg("-w").string_arg( - GUID_MARKER "%{http_code} %{exitcode} %{errormsg}\\n"); -#undef GUID_MARKER + static std::vector libcurl_bulk_operation(DiagnosticContext& context, + View urls, + View outputs, + View headers) + { + if (!outputs.empty() && outputs.size() != urls.size()) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + + std::vector return_codes(urls.size(), -1); + + CurlHeaders request_headers(headers); - std::vector ret; - ret.reserve(operation_args.size()); - add_curl_headers(prefix_cmd, headers); - while (ret.size() != operation_args.size()) + std::vector write_pointers; + write_pointers.reserve(urls.size()); + + std::vector easy_handles; + easy_handles.resize(urls.size()); + + CurlMultiHandle multi_handle; + for (size_t request_index = 0; request_index < urls.size(); ++request_index) { - // there's an edge case that we aren't handling here where not even one operation fits with the configured - // headers but this seems unlikely + const auto& url = urls[request_index]; + auto& easy_handle = easy_handles[request_index]; + auto* curl = easy_handle.get(); + + set_common_curl_easy_options(easy_handle, url, request_headers); + if (outputs.empty()) + { + curl_easy_setopt(curl, CURLOPT_PRIVATE, reinterpret_cast(static_cast(request_index))); + } + else + { + const auto& output = outputs[request_index]; + std::error_code ec; + auto& request_write_pointer = write_pointers.emplace_back(output, Append::NO, ec); + if (ec) + { + context.report_error(format_filesystem_call_error(ec, "fopen", {output})); + Checks::unreachable(VCPKG_LINE_INFO); + } + + curl_easy_setopt(curl, CURLOPT_PRIVATE, static_cast(&request_write_pointer)); + // note explicit cast to void* necessary to go through ... + curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&request_write_pointer)); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); + multi_handle.add_easy_handle(easy_handle); + } + } - // form a maximum length command line of operations: - auto batch_cmd = prefix_cmd; - size_t last_try_op = ret.size(); - while (last_try_op != operation_args.size() && batch_cmd.try_append(operation_args[last_try_op])) + int still_running = 0; + do + { + CURLMcode mc = curl_multi_perform(multi_handle.get(), &still_running); + if (mc != CURLM_OK) { - ++last_try_op; + Debug::println("curl_multi_perform failed:"); + Debug::println(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(mc)) + .append_raw(fmt::format(" ({}).", curl_multi_strerror(mc)))); + Checks::unreachable(VCPKG_LINE_INFO); } - // actually run curl - bool new_curl_seen = false; - std::vector debug_lines; - auto maybe_this_batch_exit_code = cmd_execute_and_stream_lines(context, batch_cmd, [&](StringView line) { - debug_lines.emplace_back(line.data(), line.size()); - new_curl_seen |= parse_curl_status_line(context, ret, guid_marker, line); - }); + // we use curl_multi_wait rather than curl_multi_poll for wider compatibility + mc = curl_multi_wait(multi_handle.get(), nullptr, 0, 1000, nullptr); + if (mc != CURLM_OK) + { + Debug::println("curl_multi_wait failed:"); + Debug::println(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(mc)) + .append_raw(fmt::format(" ({}).", curl_multi_strerror(mc)))); + Checks::unreachable(VCPKG_LINE_INFO); + } + } while (still_running); - if (auto this_batch_exit_code = maybe_this_batch_exit_code.get()) + // drain all messages + int messages_in_queue = 0; + while (auto* msg = curl_multi_info_read(multi_handle.get(), &messages_in_queue)) + { + if (msg->msg == CURLMSG_DONE) { - if (!new_curl_seen) + CURL* handle = msg->easy_handle; + if (msg->data.result == CURLE_OK) { - // old version of curl, we only have the result code for the last operation - context.report_error(msgCurlFailedGeneric, msg::exit_code = *this_batch_exit_code); - } + size_t idx; + void* curlinfo_private; + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &curlinfo_private); + if (outputs.empty()) + { + idx = reinterpret_cast(curlinfo_private); + } + else + { + if (!curlinfo_private) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + auto request_write_handle = static_cast(curlinfo_private); + idx = request_write_handle - write_pointers.data(); + } - if (ret.size() != last_try_op) + long response_code; + curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); + return_codes[idx] = static_cast(response_code); + } + else { - // curl didn't process everything we asked of it; this usually means curl crashed - auto command_line = std::move(batch_cmd).extract(); - replace_secrets(command_line, secrets); - context.report_error_with_log(Strings::join("\n", debug_lines), - msgCurlFailedToReturnExpectedNumberOfExitCodes, - msg::exit_code = *this_batch_exit_code, - msg::command_line = command_line); - return ret; + context.report_error( + msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(msg->data.result)) + .append_raw(fmt::format(" ({}).", curl_easy_strerror(msg->data.result)))); } } - else - { - // couldn't even launch curl, record this as the last fatal error and give up - return ret; - } } + return return_codes; + } - return ret; + static std::vector libcurl_bulk_check(DiagnosticContext& context, + View urls, + View headers) + { + return libcurl_bulk_operation(context, + urls, + {}, // no output + headers); } - std::vector url_heads(DiagnosticContext& context, - View urls, - View headers, - View secrets) + std::vector url_heads(DiagnosticContext& context, View urls, View headers) { - return curl_bulk_operation( - context, - Util::fmap(urls, [](const std::string& url) { return Command{}.string_arg(url_encode_spaces(url)); }), - "--head", - headers, - secrets); + return libcurl_bulk_check(context, urls, headers); } std::vector download_files_no_cache(DiagnosticContext& context, View> url_pairs, - View headers, - View secrets) + View headers) { - return curl_bulk_operation(context, - Util::fmap(url_pairs, - [](const std::pair& url_pair) { - return Command{} - .string_arg(url_encode_spaces(url_pair.first)) - .string_arg("-o") - .string_arg(url_pair.second); - }), - "--create-dirs", - headers, - secrets); + return libcurl_bulk_operation(context, + Util::fmap(url_pairs, [](auto&& kv) -> std::string { return kv.first; }), + Util::fmap(url_pairs, [](auto&& kv) -> Path { return kv.second; }), + headers); } bool submit_github_dependency_graph_snapshot(DiagnosticContext& context, @@ -781,8 +297,6 @@ namespace vcpkg const std::string& github_repository, const Json::Object& snapshot) { - static constexpr StringLiteral guid_marker = "fcfad8a3-bb68-4a54-ad00-dab1ff671ed2"; - std::string uri; if (auto github_server_url = maybe_github_server_url.get()) { @@ -797,93 +311,92 @@ namespace vcpkg fmt::format_to( std::back_inserter(uri), "/repos/{}/dependency-graph/snapshots", url_encode_spaces(github_repository)); - auto cmd = Command{"curl"}; - cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}"); - cmd.string_arg("-X").string_arg("POST"); - { - std::string headers[] = { - "Accept: application/vnd.github+json", - "Authorization: Bearer " + github_token, - "X-GitHub-Api-Version: 2022-11-28", - }; - add_curl_headers(cmd, headers); - } + CurlEasyHandle handle; + CURL* curl = handle.get(); - cmd.string_arg(uri); - cmd.string_arg("-d").string_arg("@-"); + std::string post_data = Json::stringify(snapshot); - RedirectedProcessLaunchSettings settings; - settings.stdin_content = Json::stringify(snapshot); - int code = 0; - auto result = cmd_execute_and_stream_lines(context, cmd, settings, [&code](StringView line) { - if (line.starts_with(guid_marker)) - { - code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); - } - }); + std::string headers[]{ + "Accept: application/vnd.github+json", + ("Authorization: Bearer " + github_token), + "X-GitHub-Api-Version: 2022-11-28", + "Content-Type: application/json", + }; + + CurlHeaders request_headers(headers); + set_common_curl_easy_options(handle, uri, request_headers); + curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.length()); + + CURLcode result = curl_easy_perform(curl); + long response_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); - auto r = result.get(); - if (r && *r == 0 && code >= 200 && code < 300) + if (result != CURLE_OK) { - return true; + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(result)) + .append_raw(fmt::format(" ({}).", curl_easy_strerror(result)))); + return false; } - return false; + + return response_code >= 200 && response_code < 300; + } + + static size_t read_file_callback(char* buffer, size_t size, size_t nitems, void* param) + { + auto* file = static_cast(param); + return file->read(buffer, size, nitems); } bool store_to_asset_cache(DiagnosticContext& context, StringView raw_url, const SanitizedUrl& sanitized_url, - StringLiteral method, View headers, const Path& file) { - static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; - - if (raw_url.starts_with("ftp://")) + std::error_code ec; + ReadFilePointer fileptr(file, ec); + if (ec) { - // HTTP headers are ignored for FTP clients - auto ftp_cmd = Command{"curl"}; - ftp_cmd.string_arg(url_encode_spaces(raw_url)); - ftp_cmd.string_arg("-T").string_arg(file); - auto maybe_res = cmd_execute_and_capture_output(context, ftp_cmd); - if (auto res = maybe_res.get()) - { - if (res->exit_code == 0) - { - return true; - } - - context.report_error_with_log( - res->output, msgCurlFailedToPut, msg::exit_code = res->exit_code, msg::url = sanitized_url); - return false; - } - + context.report_error(format_filesystem_call_error(ec, "fopen", {file})); + return false; + } + auto file_size = fileptr.size(ec); + if (ec) + { + context.report_error(format_filesystem_call_error(ec, "fstat", {file})); return false; } - auto http_cmd = Command{"curl"}.string_arg("-X").string_arg(method); - add_curl_headers(http_cmd, headers); - http_cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}"); - http_cmd.string_arg(raw_url); - http_cmd.string_arg("-T").string_arg(file); - int code = 0; - auto res = cmd_execute_and_stream_lines(context, http_cmd, [&code](StringView line) { - if (line.starts_with(guid_marker)) - { - code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); - } - }); + CurlEasyHandle handle; + CURL* curl = handle.get(); - auto pres = res.get(); - if (!pres) + auto request_headers = raw_url.starts_with("ftp://") ? CurlHeaders() : CurlHeaders(headers); + auto upload_url = url_encode_spaces(raw_url); + curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + curl_easy_setopt(curl, CURLOPT_URL, upload_url.c_str()); + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(curl, CURLOPT_READDATA, static_cast(&fileptr)); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, &read_file_callback); + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast(file_size)); + + auto result = curl_easy_perform(curl); + if (result != CURLE_OK) { + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(result)) + .append_raw(fmt::format(" ({}).", curl_easy_strerror(result)))); return false; } - if (*pres != 0 || (code >= 100 && code < 200) || code >= 300) + long response_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + + if ((response_code >= 100 && response_code < 200) || response_code >= 300) { - context.report_error(msg::format( - msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = sanitized_url, msg::value = code)); + context.report_error(msg::format(msgCurlFailedToPut, msg::url = sanitized_url, msg::value = response_code)); return false; } @@ -937,169 +450,14 @@ namespace vcpkg return fmt::format(FMT_COMPILE("{}?{}"), base_url, fmt::join(query_params, "&")); } - Optional invoke_http_request(DiagnosticContext& context, - StringLiteral method, - View headers, - StringView raw_url, - View secrets, - StringView data) - { - auto cmd = Command{"curl"}.string_arg("-s").string_arg("-L"); - add_curl_headers(cmd, headers); - - cmd.string_arg("-X").string_arg(method); - - if (!data.empty()) - { - cmd.string_arg("--data-raw").string_arg(data); - } - - cmd.string_arg(url_encode_spaces(raw_url)); - - auto maybe_output = cmd_execute_and_capture_output(context, cmd); - if (auto output = check_zero_exit_code( - context, cmd, maybe_output, RedirectedProcessLaunchSettings{}.echo_in_debug, secrets)) - { - return *output; - } - - return nullopt; - } - -#if defined(_WIN32) - static WinHttpTrialResult download_winhttp_trial(DiagnosticContext& context, - MessageSink& machine_readable_progress, - const Filesystem& fs, - const WinHttpSession& s, - const Path& download_path_part_path, - SplitUrlView split_uri_view, - StringView hostname, - INTERNET_PORT port, - const SanitizedUrl& sanitized_url) - { - WinHttpConnection conn; - if (!conn.connect(context, s, hostname, port, sanitized_url)) - { - return WinHttpTrialResult::retry; - } - - WinHttpRequest req; - if (!req.open( - context, conn, split_uri_view.path_query_fragment, sanitized_url, split_uri_view.scheme == "https")) - { - return WinHttpTrialResult::retry; - } - - auto maybe_status = req.query_status(context, sanitized_url); - const auto status = maybe_status.get(); - if (!status) - { - return WinHttpTrialResult::retry; - } - - if (*status < 200 || *status >= 300) - { - context.report_error(msgDownloadFailedStatusCode, msg::url = sanitized_url, msg::value = *status); - return WinHttpTrialResult::failed; - } - - return req.write_response_body(context, - machine_readable_progress, - sanitized_url, - fs.open_for_write(download_path_part_path, VCPKG_LINE_INFO)); - } - - /// - /// Download a file using WinHTTP -- only supports HTTP and HTTPS - /// - static bool download_winhttp(DiagnosticContext& context, - MessageSink& machine_readable_progress, - const Filesystem& fs, - const Path& download_path_part_path, - SplitUrlView split_url_view, - const SanitizedUrl& sanitized_url) - { - // `download_winhttp` does not support user or port syntax in authorities - auto hostname = split_url_view.authority.value_or_exit(VCPKG_LINE_INFO).substr(2); - INTERNET_PORT port; - if (split_url_view.scheme == "https") - { - port = INTERNET_DEFAULT_HTTPS_PORT; - } - else if (split_url_view.scheme == "http") - { - port = INTERNET_DEFAULT_HTTP_PORT; - } - else - { - Checks::unreachable(VCPKG_LINE_INFO); - } - - // Make sure the directories are present, otherwise fopen_s fails - const auto dir = download_path_part_path.parent_path(); - if (!dir.empty()) - { - fs.create_directories(dir, VCPKG_LINE_INFO); - } - - WinHttpSession s; - if (!s.open(context, sanitized_url)) - { - return false; - } - - AttemptDiagnosticContext adc{context}; - switch (download_winhttp_trial(adc, - machine_readable_progress, - fs, - s, - download_path_part_path, - split_url_view, - hostname, - port, - sanitized_url)) - { - case WinHttpTrialResult::succeeded: adc.commit(); return true; - case WinHttpTrialResult::failed: adc.commit(); return false; - case WinHttpTrialResult::retry: break; - } - - for (size_t trials = 1; trials < 4; ++trials) - { - // 1s, 2s, 4s - const auto trialMs = 500 << trials; - adc.handle(); - context.statusln( - DiagnosticLine(DiagKind::Warning, - msg::format(msgDownloadFailedRetrying, msg::value = trialMs, msg::url = sanitized_url)) - .to_message_line()); - std::this_thread::sleep_for(std::chrono::milliseconds(trialMs)); - switch (download_winhttp_trial(adc, - machine_readable_progress, - fs, - s, - download_path_part_path, - split_url_view, - hostname, - port, - sanitized_url)) - { - case WinHttpTrialResult::succeeded: adc.commit(); return true; - case WinHttpTrialResult::failed: adc.commit(); return false; - case WinHttpTrialResult::retry: break; - } - } - - adc.commit(); - return false; - } -#endif - enum class DownloadPrognosis { Success, OtherError, - NetworkErrorProxyMightHelp + NetworkErrorProxyMightHelp, + // Transient error means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500, 502, 503 or + // 504 response code. https://everything.curl.dev/usingcurl/downloads/retry.html#retry + TransientNetworkError }; static bool check_combine_download_prognosis(DownloadPrognosis& target, DownloadPrognosis individual_call) @@ -1133,6 +491,71 @@ namespace vcpkg } } + static DownloadPrognosis perform_download(DiagnosticContext& context, + MessageSink& machine_readable_progress, + StringView raw_url, + const Path& download_path, + View headers) + { + std::error_code ec; + WriteFilePointer fileptr(download_path, Append::NO, ec); + if (ec) + { + context.report_error(format_filesystem_call_error(ec, "fopen", {download_path})); + return DownloadPrognosis::OtherError; + } + + CurlHeaders request_headers(headers); + + CurlEasyHandle handle; + CURL* curl = handle.get(); + set_common_curl_easy_options(handle, raw_url, request_headers); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&fileptr)); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); // change from default to enable progress + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, &progress_callback); + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, static_cast(&machine_readable_progress)); + auto curl_code = curl_easy_perform(curl); + + if (curl_code == CURLE_OPERATION_TIMEDOUT) + { + context.report_error(msgCurlDownloadTimeout); + return DownloadPrognosis::TransientNetworkError; + } + + if (curl_code != CURLE_OK) + { + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(curl_code)) + .append_raw(fmt::format(" ({}).", curl_easy_strerror(curl_code)))); + return DownloadPrognosis::NetworkErrorProxyMightHelp; + } + + long response_code = -1; + auto get_info_code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (get_info_code != CURLE_OK) + { + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(get_info_code)) + .append_raw(fmt::format(" ({}).", curl_easy_strerror(get_info_code)))); + return DownloadPrognosis::NetworkErrorProxyMightHelp; + } + + if ((response_code >= 200 && response_code < 300) || (raw_url.starts_with("file://") && response_code == 0)) + { + return DownloadPrognosis::Success; + } + + if (response_code == 429 || response_code == 408 || response_code == 500 || response_code == 502 || + response_code == 503 || response_code == 504) + { + context.report_error( + msg::format(msgCurlFailedHttpResponse, msg::exit_code = static_cast(response_code))); + return DownloadPrognosis::TransientNetworkError; + } + + context.report_error(msg::format(msgCurlFailedHttpResponse, msg::exit_code = static_cast(response_code))); + return DownloadPrognosis::NetworkErrorProxyMightHelp; + } + static DownloadPrognosis try_download_file(DiagnosticContext& context, MessageSink& machine_readable_progress, const Filesystem& fs, @@ -1152,59 +575,6 @@ namespace vcpkg #endif download_path_part_path += ".part"; -#if defined(_WIN32) - auto maybe_https_proxy_env = get_environment_variable(EnvironmentVariableHttpsProxy); - bool needs_proxy_auth = false; - if (auto proxy_url = maybe_https_proxy_env.get()) - { - needs_proxy_auth = proxy_url->find('@') != std::string::npos; - } - if (headers.size() == 0 && !needs_proxy_auth) - { - auto maybe_split_uri_view = parse_split_url_view(raw_url); - auto split_uri_view = maybe_split_uri_view.get(); - if (!split_uri_view) - { - context.report_error(msgInvalidUri, msg::value = sanitized_url); - return DownloadPrognosis::OtherError; - } - - if (split_uri_view->scheme == "https" || split_uri_view->scheme == "http") - { - auto maybe_authority = split_uri_view->authority.get(); - if (!maybe_authority) - { - context.report_error(msg::format(msgInvalidUri, msg::value = sanitized_url)); - return DownloadPrognosis::OtherError; - } - - auto authority = StringView{*maybe_authority}.substr(2); - // This check causes complex URLs (non-default port, embedded basic auth) to be passed down to - // curl.exe - if (Strings::find_first_of(authority, ":@") == authority.end()) - { - if (!download_winhttp(context, - machine_readable_progress, - fs, - download_path_part_path, - *split_uri_view, - sanitized_url)) - { - return DownloadPrognosis::NetworkErrorProxyMightHelp; - } - - if (!check_downloaded_file_hash( - context, fs, sanitized_url, download_path_part_path, maybe_sha512, out_sha512)) - { - return DownloadPrognosis::OtherError; - } - - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); - return DownloadPrognosis::Success; - } - } - } -#endif // Create directory in advance, otherwise curl will create it in 750 mode on unix style file systems. const auto dir = download_path_part_path.parent_path(); if (!dir.empty()) @@ -1212,85 +582,37 @@ namespace vcpkg fs.create_directories(dir, VCPKG_LINE_INFO); } - auto cmd = Command{"curl"} - .string_arg("--fail") - .string_arg("--retry") - .string_arg("3") - .string_arg("-L") - .string_arg(url_encode_spaces(raw_url)) - .string_arg("--create-dirs") - .string_arg("--output") - .string_arg(download_path_part_path); - add_curl_headers(cmd, headers); - bool seen_any_curl_errors = false; - // if seen_any_curl_errors, contains the curl error lines starting with "curl:" - // otherwise, contains all curl's output unless it is the machine readable output - std::vector likely_curl_errors; - auto maybe_exit_code = cmd_execute_and_stream_lines(context, cmd, [&](StringView line) { - const auto maybe_parsed = try_parse_curl_progress_data(line); - if (const auto parsed = maybe_parsed.get()) - { - machine_readable_progress.println(Color::none, - LocalizedString::from_raw(fmt::format("{}%", parsed->total_percent))); - return; - } + // Retry on transient errors: + // Transient error means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500, 502, 503 or + // 504 response code. https://everything.curl.dev/usingcurl/downloads/retry.html#retry + using namespace std::chrono_literals; + static constexpr std::array attempt_delays = {0s, 1s, 2s}; + DownloadPrognosis prognosis = DownloadPrognosis::NetworkErrorProxyMightHelp; + for (size_t attempt_count = 0; attempt_count < attempt_delays.size(); attempt_count++) + { + std::this_thread::sleep_for(attempt_delays[attempt_count]); - static constexpr StringLiteral WarningColon = "warning: "; - if (Strings::case_insensitive_ascii_starts_with(line, WarningColon)) - { - context.statusln( - DiagnosticLine{DiagKind::Warning, LocalizedString::from_raw(line.substr(WarningColon.size()))} - .to_message_line()); - return; - } + prognosis = perform_download(context, machine_readable_progress, raw_url, download_path_part_path, headers); - // clang-format off - // example: - // 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (6) Could not resolve host: nonexistent.example.com - // clang-format on - static constexpr StringLiteral CurlColon = "curl:"; - auto curl_start = std::search(line.begin(), line.end(), CurlColon.begin(), CurlColon.end()); - if (curl_start == line.end()) + if (DownloadPrognosis::Success == prognosis) { - if (seen_any_curl_errors) - { - return; - } - - curl_start = line.begin(); + break; } - else + + if (DownloadPrognosis::TransientNetworkError != prognosis) { - if (!seen_any_curl_errors) - { - seen_any_curl_errors = true; - likely_curl_errors.clear(); - } + context.report_error(msg::format(msgDownloadNotTransientErrorWontRetry, msg::url = sanitized_url)); + return prognosis; } - likely_curl_errors.emplace_back(curl_start, line.end()); - }); - - const auto exit_code = maybe_exit_code.get(); - if (!exit_code) - { - return DownloadPrognosis::OtherError; + context.report_error(msg::format( + msgDownloadTransientErrorRetry, msg::count = attempt_count + 1, msg::value = attempt_delays.size())); } - if (*exit_code != 0) + if (DownloadPrognosis::Success != prognosis) { - std::set seen_errors; - for (StringView likely_curl_error : likely_curl_errors) - { - auto seen_position = seen_errors.lower_bound(likely_curl_error); - if (seen_position == seen_errors.end() || *seen_position != likely_curl_error) - { - seen_errors.emplace_hint(seen_position, likely_curl_error); - context.report(DiagnosticLine{DiagKind::Error, LocalizedString::from_raw(likely_curl_error)}); - } - } - - return DownloadPrognosis::NetworkErrorProxyMightHelp; + context.report_error(msg::format(msgDownloadTransientErrorRetriesExhausted, msg::url = sanitized_url)); + return prognosis; } if (!check_downloaded_file_hash(context, fs, sanitized_url, download_path_part_path, maybe_sha512, out_sha512)) @@ -1308,89 +630,6 @@ namespace vcpkg return s_headers; } - bool parse_curl_status_line(DiagnosticContext& context, - std::vector& http_codes, - StringLiteral prefix, - StringView this_line) - { - if (!this_line.starts_with(prefix)) - { - return false; - } - - auto first = this_line.begin(); - const auto last = this_line.end(); - first += prefix.size(); - const auto first_http_code = first; - - int http_code; - for (;; ++first) - { - if (first == last) - { - // this output is broken, even if we don't know %{exit_code} or ${errormsg}, the spaces in front - // of them should still be printed. - return false; - } - - if (!ParserBase::is_ascii_digit(*first)) - { - http_code = Strings::strto(StringView{first_http_code, first}).value_or_exit(VCPKG_LINE_INFO); - break; - } - } - - if (*first != ' ' || ++first == last) - { - // didn't see the space after the http_code - return false; - } - - if (*first == ' ') - { - // old curl that doesn't understand %{exit_code}, this is the space after it - http_codes.emplace_back(http_code); - return false; - } - - if (!ParserBase::is_ascii_digit(*first)) - { - // not exit_code - return false; - } - - const auto first_exit_code = first; - for (;;) - { - if (++first == last) - { - // didn't see the space after %{exit_code} - return false; - } - - if (*first == ' ') - { - // the space after exit_code, everything after this space is the error message if any - http_codes.emplace_back(http_code); - auto exit_code = Strings::strto(StringView{first_exit_code, first}).value_or_exit(VCPKG_LINE_INFO); - // note that this gets the space out of the output :) - if (exit_code != 0) - { - context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = exit_code) - .append_raw(StringView{first, last})); - } - - return true; - } - - if (!ParserBase::is_ascii_digit(*first)) - { - // non numeric exit_code? - return false; - } - } - } - static DownloadPrognosis download_file_azurl_asset_cache(DiagnosticContext& context, MessageSink& machine_readable_progress, const AssetCachingSettings& asset_cache_settings, @@ -1659,12 +898,8 @@ namespace vcpkg context.statusln( msg::format(msgDownloadSuccesfulUploading, msg::path = display_path, msg::url = sanitized_upload_url)); WarningDiagnosticContext wdc{context}; - if (!store_to_asset_cache(wdc, - raw_upload_url, - sanitized_upload_url, - "PUT", - asset_cache_settings.m_write_headers, - download_path)) + if (!store_to_asset_cache( + wdc, raw_upload_url, sanitized_upload_url, asset_cache_settings.m_write_headers, download_path)) { context.report(DiagnosticLine{DiagKind::Warning, msg::format(msgFailedToStoreBackToMirror, @@ -1869,6 +1104,11 @@ namespace vcpkg context, download_path, display_path, asset_cache_settings, maybe_sha512); return true; } + else + { + asset_cache_attempt_context.commit(); + authoritative_attempt_context.commit(); + } while (++first_sanitized_url, ++first_raw_url != last_raw_url) { @@ -1984,12 +1224,8 @@ namespace vcpkg auto raw_upload_url = Strings::replace_all(*url_template, "", sha512); SanitizedUrl sanitized_upload_url{raw_upload_url, asset_cache_settings.m_secrets}; - return store_to_asset_cache(context, - raw_upload_url, - sanitized_upload_url, - "PUT", - asset_cache_settings.m_write_headers, - file_to_put); + return store_to_asset_cache( + context, raw_upload_url, sanitized_upload_url, asset_cache_settings.m_write_headers, file_to_put); } return true; diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index 617250284b..e6eda6de4f 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -1562,6 +1562,35 @@ namespace vcpkg ec.clear(); } + uint64_t ReadFilePointer::size(LineInfo li) const + { + std::error_code ec; + auto result = this->size(ec); + if (ec) + { + exit_filesystem_call_error(li, ec, __func__, {m_path}); + } + + return result; + } + + uint64_t ReadFilePointer::size(std::error_code& ec) const + { + ec.clear(); +#if _WIN32 + return stdfs::file_size(to_stdfs_path(m_path), ec); +#else + struct stat st; + if (::fstat(::fileno(m_fs), &st) != 0) + { + ec.assign(errno, std::generic_category()); + return 0; + } + + return st.st_size; +#endif + } + WriteFilePointer::WriteFilePointer() noexcept = default; WriteFilePointer::WriteFilePointer(WriteFilePointer&&) noexcept = default; diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 39c4af661c..5540bf1722 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -491,7 +491,7 @@ namespace auto url = templ.instantiate_variables(request); WarningDiagnosticContext wdc{context}; auto maybe_success = - store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, "PUT", templ.headers, zip_path); + store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, templ.headers, zip_path); if (maybe_success) { count_stored++; @@ -536,7 +536,7 @@ namespace } WarningDiagnosticContext wdc{context}; - auto codes = download_files_no_cache(wdc, url_paths, m_url_template.headers, m_secrets); + auto codes = download_files_no_cache(wdc, url_paths, m_url_template.headers); for (size_t i = 0; i < codes.size(); ++i) { if (codes[i] == 200) @@ -558,7 +558,7 @@ namespace } WarningDiagnosticContext wdc{context}; - auto codes = url_heads(wdc, urls, {}, m_secrets); + auto codes = url_heads(wdc, urls, {}); for (size_t i = 0; i < codes.size(); ++i) { out_status[i] = codes[i] == 200 ? CacheAvailability::available : CacheAvailability::unavailable; @@ -602,7 +602,7 @@ namespace // cf. // https://learn.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json - constexpr size_t max_single_write = 5000000000; + constexpr size_t max_single_write = 5000000000u; bool use_azcopy = file_size > max_single_write; WarningDiagnosticContext wdc{context}; @@ -611,9 +611,8 @@ namespace { auto url = templ.instantiate_variables(request); auto maybe_success = - use_azcopy - ? azcopy_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, zip_path) - : store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, "PUT", templ.headers, zip_path); + use_azcopy ? azcopy_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, zip_path) + : store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, templ.headers, zip_path); if (maybe_success) { count_stored++; diff --git a/src/vcpkg/commands.cpp b/src/vcpkg/commands.cpp index 29430906b1..a90862b4fb 100644 --- a/src/vcpkg/commands.cpp +++ b/src/vcpkg/commands.cpp @@ -70,8 +70,8 @@ namespace vcpkg {CommandCheckToolsShaMetadata, command_check_tools_sha_and_exit}, {CommandInitRegistryMetadata, command_init_registry_and_exit}, {CommandVersionMetadata, command_version_and_exit}, -#if defined(_WIN32) {CommandZUploadMetricsMetadata, command_z_upload_metrics_and_exit}, +#if defined(_WIN32) {CommandZApplocalMetadata, command_z_applocal_and_exit}, #endif // defined(_WIN32) {CommandZGenerateDefaultMessageMapMetadata, command_z_generate_default_message_map_and_exit}, diff --git a/src/vcpkg/commands.z-check-tools-sha.cpp b/src/vcpkg/commands.z-check-tools-sha.cpp index 91050ff6ac..e2d335df15 100644 --- a/src/vcpkg/commands.z-check-tools-sha.cpp +++ b/src/vcpkg/commands.z-check-tools-sha.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -80,7 +81,7 @@ namespace vcpkg } msg::println(msgDownloadingTools, msg::count = urlAndPaths.size()); - auto result = download_files_no_cache(console_diagnostic_context, urlAndPaths, {}, {}); + auto result = download_files_no_cache(console_diagnostic_context, urlAndPaths, {}); // no headers std::unordered_map url_to_fixed_sha; auto http_codes_iter = result.begin(); @@ -111,7 +112,7 @@ namespace vcpkg ++http_codes_iter; } - if (!has_sha_error) + if (!has_http_error && !has_sha_error) { msg::println(msgAllShasValid); } diff --git a/src/vcpkg/commands.z-upload-metrics.cpp b/src/vcpkg/commands.z-upload-metrics.cpp index 21dcbd1ab5..7cb5523503 100644 --- a/src/vcpkg/commands.z-upload-metrics.cpp +++ b/src/vcpkg/commands.z-upload-metrics.cpp @@ -1,9 +1,9 @@ -#include - -#if defined(_WIN32) #include +#include #include +#include +#include #include #include @@ -26,8 +26,18 @@ namespace vcpkg const auto parsed = args.parse_arguments(CommandZUploadMetricsMetadata); const auto& payload_path = parsed.command_arguments[0]; auto payload = fs.read_contents(payload_path, VCPKG_LINE_INFO); - winhttp_upload_metrics(payload); + if (!curl_upload_metrics(payload)) + { + Debug::println("Failed to upload metrics"); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + std::error_code ec; + fs.remove(payload_path, ec); + if (ec) + { + Debug::println("Failed to remove file after upload: {}", ec.message()); + } Checks::exit_success(VCPKG_LINE_INFO); } } -#endif // defined(_WIN32) diff --git a/src/vcpkg/metrics.cpp b/src/vcpkg/metrics.cpp index a1d088e5a6..6e6caa5aad 100644 --- a/src/vcpkg/metrics.cpp +++ b/src/vcpkg/metrics.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -17,6 +19,7 @@ #include #include +#include #include #include @@ -477,99 +480,6 @@ namespace vcpkg std::atomic g_should_print_metrics = false; std::atomic g_metrics_enabled = false; -#if defined(_WIN32) - void winhttp_upload_metrics(StringView payload) - { - HINTERNET connect = nullptr, request = nullptr; - BOOL results = FALSE; - - const HINTERNET session = WinHttpOpen( - L"vcpkg/1.0", WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0); - - unsigned long secure_protocols = WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2; - if (session && WinHttpSetOption(session, WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(DWORD))) - { - connect = WinHttpConnect(session, L"dc.services.visualstudio.com", INTERNET_DEFAULT_HTTPS_PORT, 0); - } - - if (connect) - { - request = WinHttpOpenRequest(connect, - L"POST", - L"/v2/track", - nullptr, - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, - WINHTTP_FLAG_SECURE); - } - - if (request) - { - auto mutable_payload = payload.to_string(); - if (MAXDWORD <= mutable_payload.size()) abort(); - std::wstring hdrs = L"Content-Type: application/json\r\n"; - results = WinHttpSendRequest(request, - hdrs.c_str(), - static_cast(hdrs.size()), - static_cast(mutable_payload.data()), - static_cast(mutable_payload.size()), - static_cast(mutable_payload.size()), - 0); - } - - if (results) - { - results = WinHttpReceiveResponse(request, nullptr); - } - - DWORD http_code = 0, junk = sizeof(DWORD); - - if (results) - { - results = WinHttpQueryHeaders(request, - WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, - nullptr, - &http_code, - &junk, - WINHTTP_NO_HEADER_INDEX); - } - - std::vector response_buffer; - if (results) - { - DWORD available_data = 0, read_data = 0, total_data = 0; - while ((results = WinHttpQueryDataAvailable(request, &available_data)) == TRUE && available_data > 0) - { - response_buffer.resize(response_buffer.size() + available_data); - - results = WinHttpReadData(request, &response_buffer[total_data], available_data, &read_data); - - if (!results) - { - break; - } - - total_data += read_data; - - response_buffer.resize(total_data); - } - } - - if (!results) - { -#ifndef NDEBUG - __debugbreak(); - auto err = GetLastError(); - fprintf(stderr, "[DEBUG] failed to connect to server: %08lu\n", err); -#endif // NDEBUG - } - - if (request) WinHttpCloseHandle(request); - if (connect) WinHttpCloseHandle(connect); - if (session) WinHttpCloseHandle(session); - } -#endif // ^^^ _WIN32 - void flush_global_metrics(const Filesystem& fs) { if (!g_metrics_enabled.load()) @@ -608,32 +518,98 @@ namespace vcpkg fs.write_contents(vcpkg_metrics_txt_path, payload, ec); if (ec) return; -#if defined(_WIN32) - const Path temp_folder_path_exe = temp_folder_path / "vcpkg-" VCPKG_BASE_VERSION_AS_STRING ".exe"; + const Path temp_folder_path_exe = temp_folder_path / "vcpkg-" VCPKG_BASE_VERSION_AS_STRING +#if defined(WIN32) + ".exe" +#endif + ; fs.copy_file(get_exe_path_of_current_process(), temp_folder_path_exe, CopyOptions::skip_existing, ec); if (ec) return; + Command builder; builder.string_arg(temp_folder_path_exe); builder.string_arg("z-upload-metrics"); builder.string_arg(vcpkg_metrics_txt_path); cmd_execute_background(builder); -#else - cmd_execute_background(Command("curl") - .string_arg("https://dc.services.visualstudio.com/v2/track") - .string_arg("--max-time") - .string_arg("60") - .string_arg("-H") - .string_arg("Content-Type: application/json") - .string_arg("-X") - .string_arg("POST") - .string_arg("--tlsv1.2") - .string_arg("--data") - .string_arg(Strings::concat("@", vcpkg_metrics_txt_path)) - .raw_arg(">/dev/null") - .raw_arg("2>&1") - .raw_arg(";") - .string_arg("rm") - .string_arg(vcpkg_metrics_txt_path)); -#endif + } + + static size_t string_append_cb(void* buff, size_t size, size_t nmemb, void* param) + { + auto* str = reinterpret_cast(param); + if (!str || !buff) return 0; + if (size != 1) return 0; + str->append(reinterpret_cast(buff), nmemb); + return size * nmemb; + } + + bool parse_metrics_response(StringView response_body) + { + auto maybe_json = Json::parse_object(response_body, "metrics_response"); + auto json = maybe_json.get(); + if (!json) return false; + + auto maybe_received = json->get(AppInsightsResponseItemsReceived); + auto maybe_accepted = json->get(AppInsightsResponseItemsAccepted); + auto maybe_errors = json->get(AppInsightsResponseErrors); + + if (maybe_received && maybe_accepted && maybe_errors && maybe_received->is_integer() && + maybe_accepted->is_integer() && maybe_errors->is_array()) + { + auto item_received = maybe_received->integer(VCPKG_LINE_INFO); + auto item_accepted = maybe_accepted->integer(VCPKG_LINE_INFO); + auto errors = maybe_errors->array(VCPKG_LINE_INFO); + return (errors.size() == 0) && (item_received == item_accepted); + } + Debug::println("Metrics response has unexpected format"); + return false; + } + + bool curl_upload_metrics(const std::string& payload) + { + if (payload.length() > static_cast(std::numeric_limits::max())) + { + Debug::println("Metrics payload too large to upload"); + return false; + } + + CurlEasyHandle handle; + CURL* curl = handle.get(); + + std::string headers[] = { + "Content-Type: application/json", + }; + CurlHeaders request_headers(headers); + + curl_easy_setopt(curl, CURLOPT_URL, "https://dc.services.visualstudio.com/v2/track"); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.length())); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L); + curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // follow redirects + curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + + std::string buff; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&buff)); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &string_append_cb); + + long response_code = 0; + CURLcode res = curl_easy_perform(curl); + bool is_success = false; + if (res == CURLE_OK) + { + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + Debug::println(fmt::format("Metrics upload response code: {}", response_code)); + Debug::println("Metrics upload response body: ", buff); + if (response_code == 200) + { + is_success = parse_metrics_response(buff); + } + } + else + { + Debug::println("Metrics upload failed: ", curl_easy_strerror(res)); + } + return is_success; } } From 74fc3cc36c4ebb1838a7a5237e95e21edf3839b7 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 22 Jan 2026 16:55:21 -0800 Subject: [PATCH 02/40] Revert build lab changes to that used for official vcpkg releases without curl embedded. --- azure-pipelines/signing.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 0b7bf12519..5ee1f2b2d7 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -188,7 +188,7 @@ extends: inputs: failOnStderr: true script: | - cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=10.13 -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_CURL_URL=$(curl-tarball-url)" "-DVCPKG_BASE_VERSION=$VCPKG_BASE_VERSION" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$VCPKG_STANDALONE_BUNDLE_SHA" "-DVCPKG_ARTIFACTS_SHA=$VCPKG_ARTIFACTS_SHA" -B "$(Build.BinariesDirectory)/build" 2>&1 + cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=10.13 -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$VCPKG_BASE_VERSION" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$VCPKG_STANDALONE_BUNDLE_SHA" "-DVCPKG_ARTIFACTS_SHA=$VCPKG_ARTIFACTS_SHA" -B "$(Build.BinariesDirectory)/build" 2>&1 make -j 8 -C "$(Build.BinariesDirectory)/build" zip -j "$(Build.ArtifactStagingDirectory)/vcpkg-macos.zip" "$(Build.BinariesDirectory)/build/vcpkg" - job: glibc_build @@ -226,7 +226,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-amd64:2025-11-17 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-linux/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_CURL_URL=$(curl-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-amd64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-linux/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-glibc" - job: muslc_build displayName: 'muslc (Alpine) Build' @@ -263,7 +263,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16.1 sh -c "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_CURL_URL=$(curl-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-muslc" - job: glibc_arm64_build displayName: 'glibc Arm64 Build' @@ -301,7 +301,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-arm64:2025-11-17 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-arm64/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_CURL_URL=$(curl-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-arm64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-arm64/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-glibc-arm64" - job: windows_and_sign displayName: 'Build Windows binaries and Sign' @@ -371,7 +371,7 @@ extends: script: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 cmake.exe --version - cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_CURL_URL=$(curl-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\amd64" 2>&1 + cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\amd64" 2>&1 ninja.exe -C "$(Build.BinariesDirectory)\amd64" - task: CmdLine@2 displayName: "Build vcpkg arm64 with CMake" @@ -380,7 +380,7 @@ extends: script: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=arm64 -host_arch=amd64 cmake.exe --version - cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_PDB_SUFFIX="-arm64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_CURL_URL=$(curl-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\arm64" 2>&1 + cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_PDB_SUFFIX="-arm64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\arm64" 2>&1 ninja.exe -C "$(Build.BinariesDirectory)\arm64" - task: NuGetToolInstaller@1 inputs: From 114898f915ccb6928c6eacc03b6e20912aa622fd Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 22 Jan 2026 17:08:14 -0800 Subject: [PATCH 03/40] Delete try_parse_curl_progress_data and try_parse_curl_max5_size. --- include/vcpkg/base/downloads.h | 20 ----- src/vcpkg-test/downloads.cpp | 106 --------------------------- src/vcpkg/base/downloads.cpp | 130 --------------------------------- 3 files changed, 256 deletions(-) diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 50300cfd04..6d09161bf8 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -99,26 +99,6 @@ namespace vcpkg const SanitizedUrl& sanitized_url, const Path& file); - Optional try_parse_curl_max5_size(StringView sv); - - struct CurlProgressData - { - unsigned int total_percent; - unsigned long long total_size; - unsigned int received_percent; - unsigned long long received_size; - unsigned int transfer_percent; - unsigned long long transfer_size; - unsigned long long average_download_speed; // bytes per second - unsigned long long average_upload_speed; // bytes per second - // ElapsedTime total_time; - // ElapsedTime time_spent; - // ElapsedTime time_left; - unsigned long long current_speed; - }; - - Optional try_parse_curl_progress_data(StringView curl_progress_line); - // Replaces spaces with %20 for purposes of including in a URL. // This is typically used to filter a command line passed to `x-download` or similar which // might contain spaces that we, in turn, pass to curl. diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp index 1f601505b2..8b206f13ac 100644 --- a/src/vcpkg-test/downloads.cpp +++ b/src/vcpkg-test/downloads.cpp @@ -141,112 +141,6 @@ TEST_CASE ("download_files", "[downloads]") all_errors[1] == "error: curl operation failed with error code 7 (Couldn't connect to server).")); } -TEST_CASE ("try_parse_curl_max5_size", "[downloads]") -{ - REQUIRE(!try_parse_curl_max5_size("").has_value()); - REQUIRE(!try_parse_curl_max5_size("hi").has_value()); - REQUIRE(try_parse_curl_max5_size("0").value_or_exit(VCPKG_LINE_INFO) == 0ull); - REQUIRE(try_parse_curl_max5_size("1").value_or_exit(VCPKG_LINE_INFO) == 1ull); - REQUIRE(try_parse_curl_max5_size("10").value_or_exit(VCPKG_LINE_INFO) == 10ull); - REQUIRE(!try_parse_curl_max5_size("10 ").has_value()); // no unknown suffixes - REQUIRE(try_parse_curl_max5_size("100").value_or_exit(VCPKG_LINE_INFO) == 100ull); - REQUIRE(try_parse_curl_max5_size("100").value_or_exit(VCPKG_LINE_INFO) == 100ull); - REQUIRE(try_parse_curl_max5_size("1000").value_or_exit(VCPKG_LINE_INFO) == 1000ull); - REQUIRE(!try_parse_curl_max5_size("1000.").has_value()); // dot needs 1 or 2 digits - REQUIRE(!try_parse_curl_max5_size("1000.k").has_value()); - // fails in parsing the number: - REQUIRE(!try_parse_curl_max5_size("18446744073709551616").has_value()); - - // suffixes are 1024'd - REQUIRE(try_parse_curl_max5_size("1k").value_or_exit(VCPKG_LINE_INFO) == (1ull << 10)); - REQUIRE(try_parse_curl_max5_size("1M").value_or_exit(VCPKG_LINE_INFO) == (1ull << 20)); - REQUIRE(try_parse_curl_max5_size("1G").value_or_exit(VCPKG_LINE_INFO) == (1ull << 30)); - REQUIRE(try_parse_curl_max5_size("1T").value_or_exit(VCPKG_LINE_INFO) == (1ull << 40)); - REQUIRE(try_parse_curl_max5_size("1P").value_or_exit(VCPKG_LINE_INFO) == (1ull << 50)); - REQUIRE(!try_parse_curl_max5_size("1a").has_value()); - - // 1.3*1024 == 1'331.2 - REQUIRE(try_parse_curl_max5_size("1.3k").value_or_exit(VCPKG_LINE_INFO) == 1331ull); - // 1.33*1024 == 1'361.92 - REQUIRE(try_parse_curl_max5_size("1.33k").value_or_exit(VCPKG_LINE_INFO) == 1361ull); - - // 1.3*1024*1024 == 1'363'148.8 - REQUIRE(try_parse_curl_max5_size("1.3M").value_or_exit(VCPKG_LINE_INFO) == 1363148ull); - // 1.33*1024*1024 == 1'394'606.08 - REQUIRE(try_parse_curl_max5_size("1.33M").value_or_exit(VCPKG_LINE_INFO) == 1394606ull); - - // 1.3*1024*1024*1024 == 1'395'864'371.2 - REQUIRE(try_parse_curl_max5_size("1.3G").value_or_exit(VCPKG_LINE_INFO) == 1395864371ull); - // 1.33*1024*1024*1024 == 1'428'076'625.92 - REQUIRE(try_parse_curl_max5_size("1.33G").value_or_exit(VCPKG_LINE_INFO) == 1428076625ull); - - // 1.3*1024*1024*1024*1024 == 1'429'365'116'108.8 - REQUIRE(try_parse_curl_max5_size("1.3T").value_or_exit(VCPKG_LINE_INFO) == 1429365116108ull); - // 1.33*1024*1024*1024*1024 == 1'462'350'464'942.08 - REQUIRE(try_parse_curl_max5_size("1.33T").value_or_exit(VCPKG_LINE_INFO) == 1462350464942ull); - - // 1.3*1024*1024*1024*1024*1024 == 1'463'669'878'895'411.2 - REQUIRE(try_parse_curl_max5_size("1.3P").value_or_exit(VCPKG_LINE_INFO) == 1463669878895411ull); - // 1.33*1024*1024*1024*1024*1024 == 1'497'446'876'100'689.92 - REQUIRE(try_parse_curl_max5_size("1.33P").value_or_exit(VCPKG_LINE_INFO) == 1497446876100689ull); -} - -TEST_CASE ("try_parse_curl_progress_data", "[downloads]") -{ - // % Total % Received % Xferd Average Speed Time Time Time Current - // Dload Upload Total Spent Left Speed - // - // 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 - // 100 242 100 242 0 0 298 0 --:--:-- --:--:-- --:--:-- 298 - // 100 242 100 242 0 0 297 0 --:--:-- --:--:-- --:--:-- 297 - // - // 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 - // 0 190M 0 511k 0 0 199k 0 0:16:19 0:00:02 0:16:17 548k - // 0 190M 0 1423k 0 0 410k 0 0:07:55 0:00:03 0:07:52 776k - // 1 190M 1 2159k 0 0 468k 0 0:06:56 0:00:04 0:06:52 726k - // 1 190M 1 2767k 0 0 499k 0 0:06:30 0:00:05 0:06:25 709k - // 1 190M 1 3327k 0 0 507k 0 0:06:24 0:00:06 0:06:18 676k - // 2 190M 2 3935k 0 0 519k 0 0:06:15 0:00:07 0:06:08 683k - - REQUIRE( - !try_parse_curl_progress_data(" % Total % Received % Xferd Average Speed Time Time Time Current") - .has_value()); - - REQUIRE( - !try_parse_curl_progress_data(" Dload Upload Total Spent Left Speed") - .has_value()); - - { - const auto out = try_parse_curl_progress_data( - " 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0") - .value_or_exit(VCPKG_LINE_INFO); - REQUIRE(out.total_percent == 0); - REQUIRE(out.total_size == 0); - REQUIRE(out.received_percent == 0); - REQUIRE(out.received_size == 0); - REQUIRE(out.transfer_percent == 0); - REQUIRE(out.transfer_size == 0); - REQUIRE(out.average_upload_speed == 0); - REQUIRE(out.average_download_speed == 0); - REQUIRE(out.current_speed == 0); - } - - { - const auto out = try_parse_curl_progress_data( - " 2 190M 2 3935k 0 0 519k 0 0:06:15 0:00:07 0:06:08 683k") - .value_or_exit(VCPKG_LINE_INFO); - REQUIRE(out.total_percent == 2); - REQUIRE(out.total_size == 190 * 1024 * 1024); - REQUIRE(out.received_percent == 2); - REQUIRE(out.received_size == 3935 * 1024); - REQUIRE(out.transfer_percent == 0); - REQUIRE(out.transfer_size == 0); - REQUIRE(out.average_upload_speed == 0); - REQUIRE(out.average_download_speed == 519 * 1024); - REQUIRE(out.current_speed == 683 * 1024); - } -} - TEST_CASE ("url_encode_spaces", "[downloads]") { REQUIRE(url_encode_spaces("https://example.com?query=value&query2=value2") == diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 0515082bb7..685375068d 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -1231,135 +1231,5 @@ namespace vcpkg return true; } - Optional try_parse_curl_max5_size(StringView sv) - { - // \d+(\.\d{1, 2})?[kMGTP]? - std::size_t idx = 0; - while (idx < sv.size() && ParserBase::is_ascii_digit(sv[idx])) - { - ++idx; - } - - if (idx == 0) - { - return nullopt; - } - - unsigned long long accumulator; - { - const auto maybe_first_digits = Strings::strto(sv.substr(0, idx)); - if (auto p = maybe_first_digits.get()) - { - accumulator = *p; - } - else - { - return nullopt; - } - } - - unsigned long long after_digits = 0; - if (idx < sv.size() && sv[idx] == '.') - { - ++idx; - if (idx >= sv.size() || !ParserBase::is_ascii_digit(sv[idx])) - { - return nullopt; - } - - after_digits = (sv[idx] - '0') * 10u; - ++idx; - if (idx < sv.size() && ParserBase::is_ascii_digit(sv[idx])) - { - after_digits += sv[idx] - '0'; - ++idx; - } - } - - if (idx == sv.size()) - { - return accumulator; - } - - if (idx + 1 != sv.size()) - { - return nullopt; - } - - switch (sv[idx]) - { - case 'k': return (accumulator << 10) + (after_digits << 10) / 100; - case 'M': return (accumulator << 20) + (after_digits << 20) / 100; - case 'G': return (accumulator << 30) + (after_digits << 30) / 100; - case 'T': return (accumulator << 40) + (after_digits << 40) / 100; - case 'P': return (accumulator << 50) + (after_digits << 50) / 100; - default: return nullopt; - } - } - - static bool parse_curl_uint_impl(unsigned int& target, const char*& first, const char* const last) - { - first = std::find_if_not(first, last, ParserBase::is_whitespace); - const auto start = first; - first = std::find_if(first, last, ParserBase::is_whitespace); - const auto maybe_parsed = Strings::strto(StringView{start, first}); - if (const auto parsed = maybe_parsed.get()) - { - target = *parsed; - return false; - } - - return true; - } - - static bool parse_curl_max5_impl(unsigned long long& target, const char*& first, const char* const last) - { - first = std::find_if_not(first, last, ParserBase::is_whitespace); - const auto start = first; - first = std::find_if(first, last, ParserBase::is_whitespace); - const auto maybe_parsed = try_parse_curl_max5_size(StringView{start, first}); - if (const auto parsed = maybe_parsed.get()) - { - target = *parsed; - return false; - } - - return true; - } - - static bool skip_curl_time_impl(const char*& first, const char* const last) - { - first = std::find_if_not(first, last, ParserBase::is_whitespace); - first = std::find_if(first, last, ParserBase::is_whitespace); - return false; - } - - Optional try_parse_curl_progress_data(StringView curl_progress_line) - { - // Curl's maintainer Daniel Stenberg clarified that this output is semi-contractual - // here: https://twitter.com/bagder/status/1600615752725307400 - // % Total % Received % Xferd Average Speed Time Time Time Current - // Dload Upload Total Spent Left Speed - // https://github.com/curl/curl/blob/5ccddf64398c1186deb5769dac086d738e150e09/lib/progress.c#L546 - CurlProgressData result; - auto first = curl_progress_line.begin(); - const auto last = curl_progress_line.end(); - if (parse_curl_uint_impl(result.total_percent, first, last) || - parse_curl_max5_impl(result.total_size, first, last) || - parse_curl_uint_impl(result.received_percent, first, last) || - parse_curl_max5_impl(result.received_size, first, last) || - parse_curl_uint_impl(result.transfer_percent, first, last) || - parse_curl_max5_impl(result.transfer_size, first, last) || - parse_curl_max5_impl(result.average_download_speed, first, last) || - parse_curl_max5_impl(result.average_upload_speed, first, last) || skip_curl_time_impl(first, last) || - skip_curl_time_impl(first, last) || skip_curl_time_impl(first, last) || - parse_curl_max5_impl(result.current_speed, first, last)) - { - return nullopt; - } - - return result; - } - std::string url_encode_spaces(StringView url) { return Strings::replace_all(url, StringLiteral{" "}, "%20"); } } From eb13c3e93724de8ade0137d7e2ae911280c89fe8 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 22 Jan 2026 17:10:22 -0800 Subject: [PATCH 04/40] Delete parse_split_url_view and format_url_query. --- include/vcpkg/base/downloads.h | 12 ---- include/vcpkg/base/fwd/downloads.h | 2 - src/vcpkg-test/downloads.cpp | 102 ----------------------------- src/vcpkg/base/downloads.cpp | 29 -------- 4 files changed, 145 deletions(-) diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 6d09161bf8..e996024570 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -25,16 +25,6 @@ namespace vcpkg std::string m_sanitized_url; }; - struct SplitUrlView - { - StringView scheme; - Optional authority; - StringView path_query_fragment; - }; - - // e.g. {"https","//example.org", "/index.html"} - Optional parse_split_url_view(StringView raw_url); - View azure_blob_headers(); std::vector download_files_no_cache(DiagnosticContext& context, @@ -47,8 +37,6 @@ namespace vcpkg const std::string& github_repository, const Json::Object& snapshot); - std::string format_url_query(StringView base_url, View query_params); - std::vector url_heads(DiagnosticContext& context, View urls, View headers); struct AssetCachingSettings diff --git a/include/vcpkg/base/fwd/downloads.h b/include/vcpkg/base/fwd/downloads.h index c51accb232..5313a16292 100644 --- a/include/vcpkg/base/fwd/downloads.h +++ b/include/vcpkg/base/fwd/downloads.h @@ -3,7 +3,5 @@ namespace vcpkg { struct SanitizedUrl; - struct SplitUrlView; struct AssetCachingSettings; - struct CurlProgressData; } diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp index 8b206f13ac..dba4b05005 100644 --- a/src/vcpkg-test/downloads.cpp +++ b/src/vcpkg-test/downloads.cpp @@ -18,108 +18,6 @@ using namespace vcpkg; } \ } while (0) -TEST_CASE ("parse_split_url_view", "[downloads]") -{ - { - auto x = parse_split_url_view("https://github.com/Microsoft/vcpkg"); - if (auto v = x.get()) - { - REQUIRE(v->scheme == "https"); - REQUIRE(v->authority.value_or("") == "//github.com"); - REQUIRE(v->path_query_fragment == "/Microsoft/vcpkg"); - } - else - { - FAIL(); - } - } - { - REQUIRE(!parse_split_url_view("").has_value()); - REQUIRE(!parse_split_url_view("hello").has_value()); - } - { - auto x = parse_split_url_view("file:"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(!y->authority.has_value()); - REQUIRE(y->path_query_fragment == ""); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("file:path"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(!y->authority.has_value()); - REQUIRE(y->path_query_fragment == "path"); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("file:/path"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(!y->authority.has_value()); - REQUIRE(y->path_query_fragment == "/path"); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("file://user:pw@host"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(y->authority.value_or("") == "//user:pw@host"); - REQUIRE(y->path_query_fragment == ""); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("ftp://host:port/"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "ftp"); - REQUIRE(y->authority.value_or("") == "//host:port"); - REQUIRE(y->path_query_fragment == "/"); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("file://D:\\work\\testing\\asset-cache/" - "562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42b69e4aeb3" - "1a4879e1bcf6f7c2d41f2ace12180ea83ba7af48879d40ab"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(y->authority.value_or("") == "//D:\\work\\testing\\asset-cache"); - REQUIRE(y->path_query_fragment == "/562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42" - "b69e4aeb31a4879e1bcf6f7c2d41f2ace12180ea83ba7af48879d40ab"); - } - else - { - FAIL(); - } - } -} - TEST_CASE ("download_files", "[downloads]") { auto const dst = Test::base_temporary_directory() / "download_files"; diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 685375068d..95706d0b8e 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -44,25 +44,6 @@ namespace vcpkg replace_secrets(m_sanitized_url, secrets); } - Optional parse_split_url_view(StringView raw_url) - { - auto sep = std::find(raw_url.begin(), raw_url.end(), ':'); - if (sep == raw_url.end()) - { - return nullopt; - } - - StringView scheme(raw_url.begin(), sep); - if (Strings::starts_with({sep + 1, raw_url.end()}, "//")) - { - auto path_start = std::find(sep + 3, raw_url.end(), '/'); - return SplitUrlView{scheme, StringView{sep + 1, path_start}, StringView{path_start, raw_url.end()}}; - } - - // no authority - return SplitUrlView{scheme, {}, StringView{sep + 1, raw_url.end()}}; - } - static bool check_downloaded_file_hash(DiagnosticContext& context, const ReadOnlyFilesystem& fs, const SanitizedUrl& sanitized_url, @@ -440,16 +421,6 @@ namespace vcpkg return true; } - std::string format_url_query(StringView base_url, View query_params) - { - if (query_params.empty()) - { - return base_url.to_string(); - } - - return fmt::format(FMT_COMPILE("{}?{}"), base_url, fmt::join(query_params, "&")); - } - enum class DownloadPrognosis { Success, From 78a1462ee7028feef1ef640c4bf17925bdebb150 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 22 Jan 2026 17:23:23 -0800 Subject: [PATCH 05/40] Also try libcurl-gnutls.so.4 and libcurl-nss.so.4 --- src/vcpkg.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/vcpkg.cpp b/src/vcpkg.cpp index 3b3b18dda8..3bb8f3a5dd 100644 --- a/src/vcpkg.cpp +++ b/src/vcpkg.cpp @@ -66,7 +66,18 @@ namespace #if defined(TEST_LIBCURL_AVAILABLE) // calling dlclose() on the handle after calling curl_version() causes asan to // report a false leak, so we intentionally don't unload the library - if (auto handle = dlopen("libcurl.so.4", RTLD_NOW | RTLD_LOCAL)) + auto handle = dlopen("libcurl.so.4", RTLD_NOW | RTLD_LOCAL); + if (!handle) + { + // Ubuntu 16.04 has this if the user only installs `curl` + handle = dlopen("libcurl-gnutls.so.4", RTLD_NOW | RTLD_LOCAL); + } + if (!handle) + { + // It's possible that someone explicitly installs this one + handle = dlopen("libcurl-nss.so.4", RTLD_NOW | RTLD_LOCAL); + } + if (handle) { auto curl_version_fn = reinterpret_cast(dlsym(handle, "curl_version")); if (!dlerror() && curl_version_fn) From 079b9f4f80d61db19f1af5f854c109e7121f4e29 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 22 Jan 2026 17:30:35 -0800 Subject: [PATCH 06/40] Use system libcurl on macOS --- cmake/FindLibCURL.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake index ddd346c349..1e07b43d88 100644 --- a/cmake/FindLibCURL.cmake +++ b/cmake/FindLibCURL.cmake @@ -1,4 +1,4 @@ -if (WIN32) +if (WIN32 OR APPLE) option(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL "Use an external version of the libcurl library" OFF) else() option(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL "Use an external version of the libcurl library" ON) From 424b231b3bd7c9fae5ad57a8a04a5c45f1fb24aa Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 23 Jan 2026 18:10:52 -0800 Subject: [PATCH 07/40] Add CMAKE_EXPORT_COMPILE_COMMANDS --- CMakePresets.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakePresets.json b/CMakePresets.json index d9f8b340b3..73983af5c0 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -18,7 +18,8 @@ "installDir": "out/install/${presetName}", "cacheVariables": { "VCPKG_BUILD_BENCHMARKING": true, - "VCPKG_BUILD_FUZZING": true + "VCPKG_BUILD_FUZZING": true, + "CMAKE_EXPORT_COMPILE_COMMANDS": true } }, { From 2260bbd4eae75dcb655580f5f773ca7917f5d9ea Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 23 Jan 2026 20:44:22 -0800 Subject: [PATCH 08/40] Try to get curl into the official signed builds' netiso --- CMakePresets.json | 4 ++ azure-pipelines/signing.yml | 4 ++ cmake/FindLibCURL.cmake | 88 ++++++++++--------------------------- vcpkg-configuration.json | 7 +++ vcpkg.json | 15 +++++++ 5 files changed, 52 insertions(+), 66 deletions(-) create mode 100644 vcpkg-configuration.json create mode 100644 vcpkg.json diff --git a/CMakePresets.json b/CMakePresets.json index 73983af5c0..c3cecb26eb 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -16,6 +16,7 @@ "generator": "Ninja", "binaryDir": "out/build/${presetName}", "installDir": "out/install/${presetName}", + "toolchainFile": "D:/vcpkg/scripts/buildsystems/vcpkg.cmake", "cacheVariables": { "VCPKG_BUILD_BENCHMARKING": true, "VCPKG_BUILD_FUZZING": true, @@ -64,6 +65,9 @@ "architecture": { "value": "x64", "strategy": "external" + }, + "cacheVariables": { + "VCPKG_TARGET_TRIPLET": "x64-windows-static" } }, { diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 5ee1f2b2d7..2fe0dc21d1 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -19,6 +19,10 @@ resources: type: git name: 1ESPipelineTemplates/MicroBuildTemplate ref: refs/tags/release + - repository: vcpkg + type: git + name: microsoft/vcpkg + ref: 544a4c5c297e60e4ac4a5a1810df66748d908869 extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake index 1e07b43d88..e1c387e1b3 100644 --- a/cmake/FindLibCURL.cmake +++ b/cmake/FindLibCURL.cmake @@ -1,93 +1,49 @@ if (WIN32 OR APPLE) - option(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL "Use an external version of the libcurl library" OFF) + set(VCPKG_LIBCURL "SYSTEM" CACHE STRING "Select libcurl provider (SYSTEM|DLSYM)") else() - option(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL "Use an external version of the libcurl library" ON) + set(VCPKG_LIBCURL "DLSYM" CACHE STRING "Select libcurl provider (SYSTEM|DLSYM)") endif() if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() -if (VCPKG_DEPENDENCY_EXTERNAL_LIBCURL) +if (VCPKG_LIBCURL STREQUAL "SYSTEM") find_package(CURL REQUIRED) return() +elseif (NOT VCPKG_LIBCURL STREQUAL "DLSYM") + message(FATAL_ERROR "Unsupported VCPKG_LIBCURL value '${VCPKG_LIBCURL}'. Expected SYSTEM or DLSYM.") endif() # This option exists to allow the URI to be replaced with a Microsoft-internal URI in official # builds which have restricted internet access; see azure-pipelines/signing.yml # Note that the SHA512 is the same, so vcpkg-tool contributors need not be concerned that we built # with different content. +# +# We're using curl-7_23_0 headers here because that's what comes with RHEL 7 and curl has not broken +# ABI in a very long time, so that should still be acceptable on modern *nix if(NOT VCPKG_LIBCURL_URL) - set(VCPKG_LIBCURL_URL "https://github.com/curl/curl/releases/download/curl-8_17_0/curl-8.17.0.tar.gz") + set(VCPKG_LIBCURL_URL "https://github.com/curl/curl/archive/refs/tags/curl-7_23_0.tar.gz") endif() include(FetchContent) FetchContent_Declare( - LibCURL + LibCURLHeaders URL "${VCPKG_LIBCURL_URL}" - URL_HASH "SHA512=88ab4b7aac12b26a6ad32fb0e1a9675288a45894438cb031102ef5d4ab6b33c2bc99cae0c70b71bdfa12eb49762827e2490555114c5eb4a6876b95e1f2a4eb74" + URL_HASH "SHA512=d927d76c43b9803d260b2a400e3b5ac6bbd9517d00a2083dc55da066a7f7393ded22ccda3778ff4faa812461b779f92e4d0775d8985ceaddc28e349598fe8362" ) - -if(NOT LibCURL_FIND_REQUIRED) - message(FATAL_ERROR "LibCURL must be REQUIRED") +FetchContent_GetProperties(LibCURLHeaders) +# This dance is done rather than `FetchContent_MakeAvailable` because we only want to download +# curl's headers for use with dlopen/dlsym rather than building curl. +if(NOT LibCURLHeaders_POPULATED) + FetchContent_Populate(LibCURLHeaders) endif() -# This is in function() so no need to backup the variables -function(get_libcurl) - set(BUILD_CURL_EXE OFF) - set(BUILD_EXAMPLES OFF) - set(BUILD_LIBCURL_DOCS OFF) - set(BUILD_MISC_DOCS OFF) - set(BUILD_SHARED_LIBS OFF) - set(BUILD_TESTING OFF) - set(CURL_ENABLE_EXPORT_TARGET OFF) - set(CURL_USE_LIBSSH2 OFF) - set(CURL_USE_LIBPSL OFF) - if (WIN32) - set(CURL_USE_SCHANNEL ON) - endif() - set(ENABLE_CURL_MANUAL OFF) - set(ENABLE_UNICODE ON) - set(PICKY_COMPILER OFF) - set(USE_NGHTTP2 OFF) - set(USE_LIBIDN2 OFF) - set(CMAKE_DISABLE_FIND_PACKAGE_Perl ON) - set(CMAKE_DISABLE_FIND_PACKAGE_ZLIB ON) - set(CMAKE_DISABLE_FIND_PACKAGE_LibPSL ON) - set(CMAKE_DISABLE_FIND_PACKAGE_LibSSH2 ON) - set(CMAKE_DISABLE_FIND_PACKAGE_Brotli ON) - set(CMAKE_DISABLE_FIND_PACKAGE_Zstd ON) - set(CMAKE_DISABLE_FIND_PACKAGE_NGHTTP2 ON) - set(CMAKE_DISABLE_FIND_PACKAGE_Libidn2 ON) - if(MSVC) - string(APPEND CMAKE_C_FLAGS " /wd6101") - string(APPEND CMAKE_C_FLAGS " /wd6011") - string(APPEND CMAKE_C_FLAGS " /wd6054") - string(APPEND CMAKE_C_FLAGS " /wd6287") - string(APPEND CMAKE_C_FLAGS " /wd6323") - string(APPEND CMAKE_C_FLAGS " /wd6385") - string(APPEND CMAKE_C_FLAGS " /wd6387") - string(APPEND CMAKE_C_FLAGS " /wd28182") - string(APPEND CMAKE_C_FLAGS " /wd28251") - string(APPEND CMAKE_C_FLAGS " /wd28301") - else() - string(APPEND CMAKE_C_FLAGS " -Wno-error") - endif() - FetchContent_MakeAvailable(LibCURL) -endfunction() - -get_libcurl() - if(NOT TARGET CURL::libcurl) - if(TARGET libcurl_static) - add_library(CURL::libcurl ALIAS libcurl_static) - target_compile_definitions(libcurl_static INTERFACE CURL_STATICLIB) - elseif(TARGET libcurl) - add_library(CURL::libcurl ALIAS libcurl) - if(NOT BUILD_SHARED_LIBS) - target_compile_definitions(libcurl INTERFACE CURL_STATICLIB) - endif() - else() - message(FATAL_ERROR "After FetchContent_MakeAvailable(LibCURL) no suitable curl target (libcurl or libcurl_static) was found.") - endif() + add_library(CURL::libcurl INTERFACE IMPORTED) endif() + +set_target_properties(CURL::libcurl PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LibCURLHeaders_SOURCE_DIR}/include" + INTERFACE_COMPILE_DEFINITIONS "VCPKG_LIBCURL_DLSYM=1" +) diff --git a/vcpkg-configuration.json b/vcpkg-configuration.json new file mode 100644 index 0000000000..64c2236d12 --- /dev/null +++ b/vcpkg-configuration.json @@ -0,0 +1,7 @@ +{ + "default-registry": { + "kind": "git", + "baseline": "75a2e14262fc23c1f03bac18e135c4866dd25b51", + "repository": "https://github.com/microsoft/vcpkg" + } +} diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000000..117a787b1c --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,15 @@ +{ + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "brotli", + "http2", + "ssl", + "zstd" + ], + "platform": "windows" + } + ] +} From 4777da102be2dfb58555f2745706fc0daf9f4d1c Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 27 Jan 2026 00:57:42 +0000 Subject: [PATCH 09/40] Workaround older versions of curl. --- CMakePresets.json | 1 - cmake/FindLibCURL.cmake | 41 ++++++++++++++++++------------------ src/vcpkg/base/downloads.cpp | 35 +++++++++++++++++++----------- src/vcpkg/metrics.cpp | 18 +++++++++++++++- 4 files changed, 61 insertions(+), 34 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index c3cecb26eb..158075e553 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -16,7 +16,6 @@ "generator": "Ninja", "binaryDir": "out/build/${presetName}", "installDir": "out/install/${presetName}", - "toolchainFile": "D:/vcpkg/scripts/buildsystems/vcpkg.cmake", "cacheVariables": { "VCPKG_BUILD_BENCHMARKING": true, "VCPKG_BUILD_FUZZING": true, diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake index e1c387e1b3..308ae95178 100644 --- a/cmake/FindLibCURL.cmake +++ b/cmake/FindLibCURL.cmake @@ -20,30 +20,31 @@ endif() # Note that the SHA512 is the same, so vcpkg-tool contributors need not be concerned that we built # with different content. # -# We're using curl-7_23_0 headers here because that's what comes with RHEL 7 and curl has not broken -# ABI in a very long time, so that should still be acceptable on modern *nix +# We're using curl-7_29_0 headers here because that's what comes with RHEL 7 (and inherited by +# similarly ancient Oracle Linux 7) if(NOT VCPKG_LIBCURL_URL) - set(VCPKG_LIBCURL_URL "https://github.com/curl/curl/archive/refs/tags/curl-7_23_0.tar.gz") + set(VCPKG_LIBCURL_URL "https://curl.se/download/archeology/curl-7.29.0.tar.gz") endif() +if(NOT TARGET CURL::libcurl) include(FetchContent) -FetchContent_Declare( - LibCURLHeaders - URL "${VCPKG_LIBCURL_URL}" - URL_HASH "SHA512=d927d76c43b9803d260b2a400e3b5ac6bbd9517d00a2083dc55da066a7f7393ded22ccda3778ff4faa812461b779f92e4d0775d8985ceaddc28e349598fe8362" -) -FetchContent_GetProperties(LibCURLHeaders) -# This dance is done rather than `FetchContent_MakeAvailable` because we only want to download -# curl's headers for use with dlopen/dlsym rather than building curl. -if(NOT LibCURLHeaders_POPULATED) - FetchContent_Populate(LibCURLHeaders) -endif() + FetchContent_Declare( + LibCURLHeaders + URL "${VCPKG_LIBCURL_URL}" + URL_HASH "SHA512=08bafd09fa6d14362a426932fed8528c13133895477d8134c829e085637956d66d6be5a791057c1c04da04af6baa6496a6d59e00abf9ca6be5d29e798718b9bc" + ) + + FetchContent_GetProperties(LibCURLHeaders) + # This dance is done rather than `FetchContent_MakeAvailable` because we only want to download + # curl's headers for use with dlopen/dlsym rather than building curl. + if(NOT LibCURLHeaders_POPULATED) + FetchContent_Populate(LibCURLHeaders) + endif() -if(NOT TARGET CURL::libcurl) add_library(CURL::libcurl INTERFACE IMPORTED) -endif() -set_target_properties(CURL::libcurl PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${LibCURLHeaders_SOURCE_DIR}/include" - INTERFACE_COMPILE_DEFINITIONS "VCPKG_LIBCURL_DLSYM=1" -) + set_target_properties(CURL::libcurl PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${libcurlheaders_SOURCE_DIR}/include" + INTERFACE_COMPILE_DEFINITIONS "VCPKG_LIBCURL_DLSYM=1" + ) +endif() diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 95706d0b8e..d60dcc5e69 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -32,7 +32,9 @@ namespace 2L); // Follow redirects, change request method based on HTTP response code. // https://curl.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html#CURLFOLLOWOBEYCODE curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); - curl_easy_setopt(curl, CURLOPT_HEADEROPT, CURLHEADER_SEPARATE); // don't send headers to proxy CONNECT + + // don't send headers to proxy CONNECT ; this intentionally fails on older versions of libcurl + curl_easy_setopt(curl, static_cast(229) /* CURLOPT_HEADEROPT */, (1L << 0) /* CURLHEADER_SEPARATE */); } } @@ -118,19 +120,21 @@ namespace vcpkg return static_cast(param)->write(contents, size, nmemb); } - static size_t progress_callback( - void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) - { - (void)ultotal; + static int progress_callback(void *clientp, + double dltotal, + double dlnow, + double ultotal, + double ulnow) { + (void)ultotal; (void)ulnow; auto machine_readable_progress = static_cast(clientp); if (dltotal && machine_readable_progress) { - double percentage = static_cast(dlnow) / static_cast(dltotal) * 100.0; + double percentage = dlnow / dltotal * 100.0; machine_readable_progress->println(LocalizedString::from_raw(fmt::format("{:.2f}%", percentage))); } return 0; - } + } static std::vector libcurl_bulk_operation(DiagnosticContext& context, View urls, @@ -184,7 +188,7 @@ namespace vcpkg } int still_running = 0; - do + for (;;) { CURLMcode mc = curl_multi_perform(multi_handle.get(), &still_running); if (mc != CURLM_OK) @@ -195,7 +199,12 @@ namespace vcpkg Checks::unreachable(VCPKG_LINE_INFO); } - // we use curl_multi_wait rather than curl_multi_poll for wider compatibility + if (still_running == 0) + { + break; + } + + // FIXME use curl_multi_poll if available mc = curl_multi_wait(multi_handle.get(), nullptr, 0, 1000, nullptr); if (mc != CURLM_OK) { @@ -204,7 +213,7 @@ namespace vcpkg .append_raw(fmt::format(" ({}).", curl_multi_strerror(mc)))); Checks::unreachable(VCPKG_LINE_INFO); } - } while (still_running); + } // drain all messages int messages_in_queue = 0; @@ -484,8 +493,10 @@ namespace vcpkg curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&fileptr)); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); // change from default to enable progress - curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, &progress_callback); - curl_easy_setopt(curl, CURLOPT_XFERINFODATA, static_cast(&machine_readable_progress)); + // curlopt_progressfucntion is deprecated, but we want the values as doubles anyway and + // the replacement isn't available on all versions of libcurl we support + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, static_cast(&progress_callback)); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, static_cast(&machine_readable_progress)); auto curl_code = curl_easy_perform(curl); if (curl_code == CURLE_OPERATION_TIMEDOUT) diff --git a/src/vcpkg/metrics.cpp b/src/vcpkg/metrics.cpp index 6e6caa5aad..7532b56a79 100644 --- a/src/vcpkg/metrics.cpp +++ b/src/vcpkg/metrics.cpp @@ -28,6 +28,17 @@ #pragma comment(lib, "winhttp") #endif +// Polyfill this value from newer versions of libcurl +#ifndef CURL_SSLVERSION_TLSv1_0 +#define CURL_SSLVERSION_TLSv1_0 4L +#endif +#ifndef CURL_SSLVERSION_TLSv1_1 +#define CURL_SSLVERSION_TLSv1_1 5L +#endif +#ifndef CURL_SSLVERSION_TLSv1_2 +#define CURL_SSLVERSION_TLSv1_2 6L +#endif + namespace { using namespace vcpkg; @@ -585,7 +596,12 @@ namespace vcpkg curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.length())); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L); - curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); + if (curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2) != CURLE_OK) { + if (curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_1) != CURLE_OK) { + curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_0); + } + } + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // follow redirects curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); From d4b4839895ee54115959e9f76b4983c00e62ef0a Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 27 Jan 2026 02:19:42 +0000 Subject: [PATCH 10/40] Implement VCPKG_LIBCURL_DLSYM --- include/vcpkg/base/curl.h | 57 ++++++++++++++++- src/vcpkg.cpp | 45 +++---------- src/vcpkg/base/curl.cpp | 119 ++++++++++++++++++++++++++--------- src/vcpkg/base/downloads.cpp | 91 +++++++++++++-------------- src/vcpkg/metrics.cpp | 30 ++++----- 5 files changed, 211 insertions(+), 131 deletions(-) diff --git a/include/vcpkg/base/curl.h b/include/vcpkg/base/curl.h index 61acc9d377..5ce5fc30f1 100644 --- a/include/vcpkg/base/curl.h +++ b/include/vcpkg/base/curl.h @@ -15,13 +15,64 @@ VCPKG_MSVC_WARNING(disable : 6101) #include VCPKG_MSVC_WARNING(pop) +#ifdef VCPKG_LIBCURL_DLSYM +void vcpkg_curl_global_init(long flags); + +// these are filled in by the call to vcpkg_curl_global_init +extern decltype(&curl_easy_cleanup) vcpkg_curl_easy_cleanup; +extern decltype(&curl_easy_getinfo) vcpkg_curl_easy_getinfo; +extern decltype(&curl_easy_init) vcpkg_curl_easy_init; +extern decltype(&curl_easy_perform) vcpkg_curl_easy_perform; +extern decltype(&curl_easy_setopt) vcpkg_curl_easy_setopt; +extern decltype(&curl_easy_strerror) vcpkg_curl_easy_strerror; + +extern decltype(&curl_multi_add_handle) vcpkg_curl_multi_add_handle; +extern decltype(&curl_multi_cleanup) vcpkg_curl_multi_cleanup; +extern decltype(&curl_multi_info_read) vcpkg_curl_multi_info_read; +extern decltype(&curl_multi_init) vcpkg_curl_multi_init; +extern decltype(&curl_multi_remove_handle) vcpkg_curl_multi_remove_handle; +extern decltype(&curl_multi_strerror) vcpkg_curl_multi_strerror; +extern decltype(&curl_multi_perform) vcpkg_curl_multi_perform; +extern decltype(&curl_multi_wait) vcpkg_curl_multi_poll; // or _wait, if _poll is not present + +extern decltype(&curl_slist_append) vcpkg_curl_slist_append; +extern decltype(&curl_slist_free_all) vcpkg_curl_slist_free_all; + +extern decltype(&curl_version) vcpkg_curl_version; +#else // ^^^ VCPKG_LIBCURL_DLSYM / !VCPKG_LIBCURL_DLSYM vvv +#define vcpkg_curl_global_init(flags) curl_global_init(flags) + +#define vcpkg_curl_easy_cleanup(handle) curl_easy_cleanup(handle) +#define vcpkg_curl_easy_getinfo(handle, info, data) curl_easy_getinfo(handle, info, data) +#define vcpkg_curl_easy_init() curl_easy_init() +#define vcpkg_curl_easy_perform(handle) curl_easy_perform(handle) +#define vcpkg_curl_easy_setopt(handle, option, parameter) curl_easy_setopt(handle, option, parameter) +#define vcpkg_curl_easy_strerror(code) curl_easy_strerror(code) + +#define vcpkg_curl_multi_add_handle(multi_handle, easy_handle) curl_multi_add_handle(multi_handle, easy_handle) +#define vcpkg_curl_multi_cleanup(multi_handle) curl_multi_cleanup(multi_handle) +#define vcpkg_curl_multi_info_read(multi_handle, messages_in_queue) \ + curl_multi_info_read(multi_handle, messages_in_queue) +#define vcpkg_curl_multi_init() curl_multi_init() +#define vcpkg_curl_multi_remove_handle(multi_handle, easy_handle) \ + curl_multi_remove_handle(multi_handle, easy_handle) +#define vcpkg_curl_multi_strerror(code) curl_multi_strerror(code) +#define vcpkg_curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, numfds) \ + curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, numfds) +#define vcpkg_curl_multi_perform(multi_handle, running_handles) \ + curl_multi_perform(multi_handle, running_handles) + +#define vcpkg_curl_slist_append(list, string) curl_slist_append(list, string) +#define vcpkg_curl_slist_free_all(list) curl_slist_free_all(list) + +#define vcpkg_curl_version() curl_version() +#endif // ^^^ !VCPKG_LIBCURL_DLSYM + namespace vcpkg { - CURLcode get_curl_global_init_status() noexcept; - struct CurlEasyHandle { - CurlEasyHandle(); + CurlEasyHandle() = default; CurlEasyHandle(CurlEasyHandle&& other) noexcept; CurlEasyHandle& operator=(CurlEasyHandle&& other) noexcept; ~CurlEasyHandle(); diff --git a/src/vcpkg.cpp b/src/vcpkg.cpp index 3bb8f3a5dd..acb9a21651 100644 --- a/src/vcpkg.cpp +++ b/src/vcpkg.cpp @@ -32,11 +32,6 @@ #pragma comment(lib, "shell32") #endif -#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) -#define TEST_LIBCURL_AVAILABLE -#include -#endif - using namespace vcpkg; namespace @@ -58,37 +53,6 @@ namespace Checks::exit_fail(VCPKG_LINE_INFO); } - Optional detect_libcurl() - { - // Determine if libcurl.so.4 is installed in the system by attempting to load it - // At the moment we don't do anything with it, but we're tracking availability - // of libcurl to replace the current download/upload implementation -#if defined(TEST_LIBCURL_AVAILABLE) - // calling dlclose() on the handle after calling curl_version() causes asan to - // report a false leak, so we intentionally don't unload the library - auto handle = dlopen("libcurl.so.4", RTLD_NOW | RTLD_LOCAL); - if (!handle) - { - // Ubuntu 16.04 has this if the user only installs `curl` - handle = dlopen("libcurl-gnutls.so.4", RTLD_NOW | RTLD_LOCAL); - } - if (!handle) - { - // It's possible that someone explicitly installs this one - handle = dlopen("libcurl-nss.so.4", RTLD_NOW | RTLD_LOCAL); - } - if (handle) - { - auto curl_version_fn = reinterpret_cast(dlsym(handle, "curl_version")); - if (!dlerror() && curl_version_fn) - { - return {curl_version_fn()}; - } - } -#endif - return nullopt; - } - bool detect_container(const Filesystem& fs) { (void)fs; @@ -147,12 +111,19 @@ namespace void inner(const Filesystem& fs, const VcpkgCmdArguments& args, const BundleSettings& bundle) { + vcpkg_curl_global_init(CURL_GLOBAL_DEFAULT); + // track version on each invocation get_global_metrics_collector().track_string(StringMetric::VcpkgVersion, vcpkg_executable_version); get_global_metrics_collector().track_bool(BoolMetric::DetectedContainer, detect_container(fs)); + + const auto* detected_curl_version = vcpkg_curl_version(); + if (!detected_curl_version) { + detected_curl_version = "unknown"; + } get_global_metrics_collector().track_string(StringMetric::DetectedLibCurlVersion, - detect_libcurl().value_or("unknown")); + detected_curl_version); if (args.get_command().empty()) { diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp index 98305363c7..fe1c3d7633 100644 --- a/src/vcpkg/base/curl.cpp +++ b/src/vcpkg/base/curl.cpp @@ -1,34 +1,93 @@ #include #include -namespace -{ - struct CurlGlobalInit - { - CurlGlobalInit() : init_status(curl_global_init(CURL_GLOBAL_DEFAULT)) { } - ~CurlGlobalInit() { curl_global_cleanup(); } +#ifdef VCPKG_LIBCURL_DLSYM +#include + +// these are filled in by the first call to vcpkg_curl_global_init +decltype(&curl_easy_cleanup) vcpkg_curl_easy_cleanup; +decltype(&curl_easy_getinfo) vcpkg_curl_easy_getinfo; +decltype(&curl_easy_init) vcpkg_curl_easy_init; +decltype(&curl_easy_perform) vcpkg_curl_easy_perform; +decltype(&curl_easy_setopt) vcpkg_curl_easy_setopt; +decltype(&curl_easy_strerror) vcpkg_curl_easy_strerror; - CurlGlobalInit(const CurlGlobalInit&) = delete; - CurlGlobalInit(CurlGlobalInit&&) = delete; - CurlGlobalInit& operator=(const CurlGlobalInit&) = delete; - CurlGlobalInit& operator=(CurlGlobalInit&&) = delete; +decltype(&curl_multi_add_handle) vcpkg_curl_multi_add_handle; +decltype(&curl_multi_cleanup) vcpkg_curl_multi_cleanup; +decltype(&curl_multi_info_read) vcpkg_curl_multi_info_read; +decltype(&curl_multi_init) vcpkg_curl_multi_init; +decltype(&curl_multi_remove_handle) vcpkg_curl_multi_remove_handle; +decltype(&curl_multi_strerror) vcpkg_curl_multi_strerror; +decltype(&curl_multi_perform) vcpkg_curl_multi_perform; +decltype(&curl_multi_wait) vcpkg_curl_multi_poll; // or _wait, if _poll is not present - CURLcode get_init_status() const { return init_status; } +decltype(&curl_slist_append) vcpkg_curl_slist_append; +decltype(&curl_slist_free_all) vcpkg_curl_slist_free_all; - private: - CURLcode init_status; - }; +decltype(&curl_version) vcpkg_curl_version; + +template +static void load_symbol(FnT (&target), void* handle, const char* symbol_name) { + target = reinterpret_cast(dlsym(handle, symbol_name)); + if (!target) { + vcpkg::Checks::unreachable(VCPKG_LINE_INFO); + } } -namespace vcpkg -{ - CURLcode get_curl_global_init_status() noexcept - { - static CurlGlobalInit g_curl_global_init; - return g_curl_global_init.get_init_status(); + void vcpkg_curl_global_init(long flags) { + // calling dlclose() on the handle after calling curl_version() causes asan to + // report a false leak, so we intentionally don't unload the library + auto handle = dlopen("libcurl.so.4", RTLD_NOW | RTLD_LOCAL); + if (!handle) + { + // Ubuntu 16.04 has this if the user only installs `curl` + handle = dlopen("libcurl-gnutls.so.4", RTLD_NOW | RTLD_LOCAL); + } + if (!handle) + { + // It's possible that someone explicitly installs this one + handle = dlopen("libcurl-nss.so.4", RTLD_NOW | RTLD_LOCAL); + } + + if (!handle) { + vcpkg::Checks::unreachable(VCPKG_LINE_INFO); // FIXME emit an error for users + } + + { + decltype(&curl_global_init) global_init; + load_symbol(global_init, handle, "curl_global_init"); + global_init(flags); + } + + load_symbol(vcpkg_curl_easy_cleanup, handle, "curl_easy_cleanup"); + load_symbol(vcpkg_curl_easy_getinfo, handle, "curl_easy_getinfo"); + load_symbol(vcpkg_curl_easy_init, handle, "curl_easy_init"); + load_symbol(vcpkg_curl_easy_perform, handle, "curl_easy_perform"); + load_symbol(vcpkg_curl_easy_setopt, handle, "curl_easy_setopt"); + load_symbol(vcpkg_curl_easy_strerror, handle, "curl_easy_strerror"); + + load_symbol(vcpkg_curl_multi_add_handle, handle, "curl_multi_add_handle"); + load_symbol(vcpkg_curl_multi_cleanup, handle, "curl_multi_cleanup"); + load_symbol(vcpkg_curl_multi_info_read, handle, "curl_multi_info_read"); + load_symbol(vcpkg_curl_multi_init, handle, "curl_multi_init"); + load_symbol(vcpkg_curl_multi_remove_handle, handle, "curl_multi_remove_handle"); + load_symbol(vcpkg_curl_multi_strerror, handle, "curl_multi_strerror"); + load_symbol(vcpkg_curl_multi_perform, handle, "curl_multi_perform"); + // try to load curl_multi_poll first, fall back to curl_multi_wait + vcpkg_curl_multi_poll = reinterpret_cast(dlsym(handle, "curl_multi_poll")); + if (!vcpkg_curl_multi_poll) { + load_symbol(vcpkg_curl_multi_poll, handle, "curl_multi_wait"); + } + + load_symbol(vcpkg_curl_slist_append, handle, "curl_slist_append"); + load_symbol(vcpkg_curl_slist_free_all, handle, "curl_slist_free_all"); + + load_symbol(vcpkg_curl_version, handle, "curl_version"); } +#endif // ^^^ VCPKG_LIBCURL_DLSYM - CurlEasyHandle::CurlEasyHandle() { get_curl_global_init_status(); } +namespace vcpkg +{ CurlEasyHandle::CurlEasyHandle(CurlEasyHandle&& other) noexcept : m_ptr(std::exchange(other.m_ptr, nullptr)) { } CurlEasyHandle& CurlEasyHandle::operator=(CurlEasyHandle&& other) noexcept { @@ -39,14 +98,14 @@ namespace vcpkg { if (m_ptr) { - curl_easy_cleanup(m_ptr); + vcpkg_curl_easy_cleanup(m_ptr); } } CURL* CurlEasyHandle::get() { if (!m_ptr) { - m_ptr = curl_easy_init(); + m_ptr = vcpkg_curl_easy_init(); if (!m_ptr) { Checks::unreachable(VCPKG_LINE_INFO); @@ -55,7 +114,7 @@ namespace vcpkg return m_ptr; } - CurlMultiHandle::CurlMultiHandle() { get_curl_global_init_status(); } + CurlMultiHandle::CurlMultiHandle() = default; CurlMultiHandle::CurlMultiHandle(CurlMultiHandle&& other) noexcept : m_ptr(std::exchange(other.m_ptr, nullptr)), m_easy_handles(std::move(other.m_easy_handles)) { @@ -70,18 +129,18 @@ namespace vcpkg { for (auto* easy_handle : m_easy_handles) { - curl_multi_remove_handle(m_ptr, easy_handle); + vcpkg_curl_multi_remove_handle(m_ptr, easy_handle); } if (m_ptr) { - curl_multi_cleanup(m_ptr); + vcpkg_curl_multi_cleanup(m_ptr); } } void CurlMultiHandle::add_easy_handle(CurlEasyHandle& easy_handle) { auto* handle = easy_handle.get(); - if (curl_multi_add_handle(this->get(), handle) == CURLM_OK) + if (vcpkg_curl_multi_add_handle(this->get(), handle) == CURLM_OK) { m_easy_handles.push_back(handle); } @@ -90,7 +149,7 @@ namespace vcpkg { if (!m_ptr) { - m_ptr = curl_multi_init(); + m_ptr = vcpkg_curl_multi_init(); if (!m_ptr) { Checks::unreachable(VCPKG_LINE_INFO); @@ -103,7 +162,7 @@ namespace vcpkg { for (const auto& header : headers) { - m_headers = curl_slist_append(m_headers, header.c_str()); + m_headers = vcpkg_curl_slist_append(m_headers, header.c_str()); } } CurlHeaders::CurlHeaders(CurlHeaders&& other) noexcept : m_headers(std::exchange(other.m_headers, nullptr)) { } @@ -116,7 +175,7 @@ namespace vcpkg { if (m_headers) { - curl_slist_free_all(m_headers); + vcpkg_curl_slist_free_all(m_headers); } } curl_slist* CurlHeaders::get() const { return m_headers; } diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index d60dcc5e69..b600f4894a 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -25,16 +25,16 @@ namespace void set_common_curl_easy_options(CurlEasyHandle& easy_handle, StringView url, const CurlHeaders& request_headers) { auto* curl = easy_handle.get(); - curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); - curl_easy_setopt(curl, CURLOPT_URL, url_encode_spaces(url).c_str()); - curl_easy_setopt(curl, + vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + vcpkg_curl_easy_setopt(curl, CURLOPT_URL, url_encode_spaces(url).c_str()); + vcpkg_curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 2L); // Follow redirects, change request method based on HTTP response code. // https://curl.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html#CURLFOLLOWOBEYCODE - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); // don't send headers to proxy CONNECT ; this intentionally fails on older versions of libcurl - curl_easy_setopt(curl, static_cast(229) /* CURLOPT_HEADEROPT */, (1L << 0) /* CURLHEADER_SEPARATE */); + vcpkg_curl_easy_setopt(curl, static_cast(229) /* CURLOPT_HEADEROPT */, (1L << 0) /* CURLHEADER_SEPARATE */); } } @@ -166,7 +166,7 @@ namespace vcpkg set_common_curl_easy_options(easy_handle, url, request_headers); if (outputs.empty()) { - curl_easy_setopt(curl, CURLOPT_PRIVATE, reinterpret_cast(static_cast(request_index))); + vcpkg_curl_easy_setopt(curl, CURLOPT_PRIVATE, reinterpret_cast(static_cast(request_index))); } else { @@ -179,10 +179,10 @@ namespace vcpkg Checks::unreachable(VCPKG_LINE_INFO); } - curl_easy_setopt(curl, CURLOPT_PRIVATE, static_cast(&request_write_pointer)); + vcpkg_curl_easy_setopt(curl, CURLOPT_PRIVATE, static_cast(&request_write_pointer)); // note explicit cast to void* necessary to go through ... - curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&request_write_pointer)); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&request_write_pointer)); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); multi_handle.add_easy_handle(easy_handle); } } @@ -190,12 +190,12 @@ namespace vcpkg int still_running = 0; for (;;) { - CURLMcode mc = curl_multi_perform(multi_handle.get(), &still_running); + CURLMcode mc = vcpkg_curl_multi_perform(multi_handle.get(), &still_running); if (mc != CURLM_OK) { Debug::println("curl_multi_perform failed:"); Debug::println(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(mc)) - .append_raw(fmt::format(" ({}).", curl_multi_strerror(mc)))); + .append_raw(fmt::format(" ({}).", vcpkg_curl_multi_strerror(mc)))); Checks::unreachable(VCPKG_LINE_INFO); } @@ -204,20 +204,19 @@ namespace vcpkg break; } - // FIXME use curl_multi_poll if available - mc = curl_multi_wait(multi_handle.get(), nullptr, 0, 1000, nullptr); + mc = vcpkg_curl_multi_poll(multi_handle.get(), nullptr, 0, 1000, nullptr); if (mc != CURLM_OK) { - Debug::println("curl_multi_wait failed:"); + Debug::println("curl_multi_wait/poll failed:"); Debug::println(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(mc)) - .append_raw(fmt::format(" ({}).", curl_multi_strerror(mc)))); + .append_raw(fmt::format(" ({}).", vcpkg_curl_multi_strerror(mc)))); Checks::unreachable(VCPKG_LINE_INFO); } } // drain all messages int messages_in_queue = 0; - while (auto* msg = curl_multi_info_read(multi_handle.get(), &messages_in_queue)) + while (auto* msg = vcpkg_curl_multi_info_read(multi_handle.get(), &messages_in_queue)) { if (msg->msg == CURLMSG_DONE) { @@ -226,7 +225,7 @@ namespace vcpkg { size_t idx; void* curlinfo_private; - curl_easy_getinfo(handle, CURLINFO_PRIVATE, &curlinfo_private); + vcpkg_curl_easy_getinfo(handle, CURLINFO_PRIVATE, &curlinfo_private); if (outputs.empty()) { idx = reinterpret_cast(curlinfo_private); @@ -242,14 +241,14 @@ namespace vcpkg } long response_code; - curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); + vcpkg_curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); return_codes[idx] = static_cast(response_code); } else { context.report_error( msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(msg->data.result)) - .append_raw(fmt::format(" ({}).", curl_easy_strerror(msg->data.result)))); + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(msg->data.result)))); } } } @@ -315,19 +314,19 @@ namespace vcpkg CurlHeaders request_headers(headers); set_common_curl_easy_options(handle, uri, request_headers); - curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.length()); + vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + vcpkg_curl_easy_setopt(curl, CURLOPT_POST, 1L); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.length()); - CURLcode result = curl_easy_perform(curl); + CURLcode result = vcpkg_curl_easy_perform(curl); long response_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + vcpkg_curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); if (result != CURLE_OK) { context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(result)) - .append_raw(fmt::format(" ({}).", curl_easy_strerror(result)))); + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(result)))); return false; } @@ -365,24 +364,24 @@ namespace vcpkg auto request_headers = raw_url.starts_with("ftp://") ? CurlHeaders() : CurlHeaders(headers); auto upload_url = url_encode_spaces(raw_url); - curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); - curl_easy_setopt(curl, CURLOPT_URL, upload_url.c_str()); - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); - curl_easy_setopt(curl, CURLOPT_READDATA, static_cast(&fileptr)); - curl_easy_setopt(curl, CURLOPT_READFUNCTION, &read_file_callback); - curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast(file_size)); - - auto result = curl_easy_perform(curl); + vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + vcpkg_curl_easy_setopt(curl, CURLOPT_URL, upload_url.c_str()); + vcpkg_curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + vcpkg_curl_easy_setopt(curl, CURLOPT_READDATA, static_cast(&fileptr)); + vcpkg_curl_easy_setopt(curl, CURLOPT_READFUNCTION, &read_file_callback); + vcpkg_curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast(file_size)); + + auto result = vcpkg_curl_easy_perform(curl); if (result != CURLE_OK) { context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(result)) - .append_raw(fmt::format(" ({}).", curl_easy_strerror(result)))); + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(result)))); return false; } long response_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + vcpkg_curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); if ((response_code >= 100 && response_code < 200) || response_code >= 300) { @@ -490,14 +489,14 @@ namespace vcpkg CurlEasyHandle handle; CURL* curl = handle.get(); set_common_curl_easy_options(handle, raw_url, request_headers); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&fileptr)); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); // change from default to enable progress + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&fileptr)); + vcpkg_curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); // change from default to enable progress // curlopt_progressfucntion is deprecated, but we want the values as doubles anyway and // the replacement isn't available on all versions of libcurl we support - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, static_cast(&progress_callback)); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, static_cast(&machine_readable_progress)); - auto curl_code = curl_easy_perform(curl); + vcpkg_curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, static_cast(&progress_callback)); + vcpkg_curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, static_cast(&machine_readable_progress)); + auto curl_code = vcpkg_curl_easy_perform(curl); if (curl_code == CURLE_OPERATION_TIMEDOUT) { @@ -508,16 +507,16 @@ namespace vcpkg if (curl_code != CURLE_OK) { context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(curl_code)) - .append_raw(fmt::format(" ({}).", curl_easy_strerror(curl_code)))); + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(curl_code)))); return DownloadPrognosis::NetworkErrorProxyMightHelp; } long response_code = -1; - auto get_info_code = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + auto get_info_code = vcpkg_curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); if (get_info_code != CURLE_OK) { context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(get_info_code)) - .append_raw(fmt::format(" ({}).", curl_easy_strerror(get_info_code)))); + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(get_info_code)))); return DownloadPrognosis::NetworkErrorProxyMightHelp; } diff --git a/src/vcpkg/metrics.cpp b/src/vcpkg/metrics.cpp index 7532b56a79..0bc772eb20 100644 --- a/src/vcpkg/metrics.cpp +++ b/src/vcpkg/metrics.cpp @@ -591,30 +591,30 @@ namespace vcpkg }; CurlHeaders request_headers(headers); - curl_easy_setopt(curl, CURLOPT_URL, "https://dc.services.visualstudio.com/v2/track"); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.length())); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L); - if (curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2) != CURLE_OK) { - if (curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_1) != CURLE_OK) { - curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_0); + vcpkg_curl_easy_setopt(curl, CURLOPT_URL, "https://dc.services.visualstudio.com/v2/track"); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.length())); + vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + vcpkg_curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L); + if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2) != CURLE_OK) { + if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_1) != CURLE_OK) { + vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_0); } } - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // follow redirects - curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + vcpkg_curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // follow redirects + vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); std::string buff; - curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&buff)); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &string_append_cb); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&buff)); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &string_append_cb); long response_code = 0; - CURLcode res = curl_easy_perform(curl); + CURLcode res = vcpkg_curl_easy_perform(curl); bool is_success = false; if (res == CURLE_OK) { - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + vcpkg_curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); Debug::println(fmt::format("Metrics upload response code: {}", response_code)); Debug::println("Metrics upload response body: ", buff); if (response_code == 200) @@ -624,7 +624,7 @@ namespace vcpkg } else { - Debug::println("Metrics upload failed: ", curl_easy_strerror(res)); + Debug::println("Metrics upload failed: ", vcpkg_curl_easy_strerror(res)); } return is_success; } From 2a4cdbeb7dfddc3dbff988863b0f95797bdb149f Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 27 Jan 2026 15:00:43 -0800 Subject: [PATCH 11/40] Try selfhosting vcpkg to get curl on Windows. --- CMakePresets.json | 5 +++-- azure-pipelines/signing-nuget.config | 7 +++++++ azure-pipelines/signing.yml | 22 +++++++++++++--------- 3 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 azure-pipelines/signing-nuget.config diff --git a/CMakePresets.json b/CMakePresets.json index 158075e553..defaed2872 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -6,8 +6,9 @@ "hidden": true, "cacheVariables": { "VCPKG_OFFICIAL_BUILD": true, - "VCPKG_BASE_VERSION": "2023-09-15", - "VCPKG_STANDALONE_BUNDLE_SHA": "8b28c1829802a133941805c68004427052588ba6eefbdf9fb6061151a92c131491df7e29470309e804450c075d2f0673515d5d8c19997148ebcb2874493d304d" + "VCPKG_BASE_VERSION": "2025-12-16", + "VCPKG_STANDALONE_BUNDLE_SHA": "23c77d1dd70bf861328a8e35203aed2db0deb9a83aa924cadaf96ffaae42e8629363184b99168e33158b819d695c748bd7cb9eb39528bd374f8b7e2ab6d4f6de", + "VCPKG_ARTIFACTS_SHA": "def65b1f4a710c0b521603a275ff6bae31ad8c5b938cd4445fb69c4d0da97c21d753c274c9b9b1cc2f8a86ba694759d4f3f4f325f88be02a5d8ad10c8f56e5df" } }, { diff --git a/azure-pipelines/signing-nuget.config b/azure-pipelines/signing-nuget.config new file mode 100644 index 0000000000..7ac764c952 --- /dev/null +++ b/azure-pipelines/signing-nuget.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 2fe0dc21d1..df7d345e19 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -19,10 +19,6 @@ resources: type: git name: 1ESPipelineTemplates/MicroBuildTemplate ref: refs/tags/release - - repository: vcpkg - type: git - name: microsoft/vcpkg - ref: 544a4c5c297e60e4ac4a5a1810df66748d908869 extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: @@ -368,6 +364,15 @@ extends: packagesToPush: '$(Build.ArtifactStagingDirectory)/vs-insertion/drop/VS.Redist.Vcpkg.amd64.1.0.0-$(VCPKG_FULL_VERSION).nupkg' publishVstsFeed: '97a41293-2972-4f48-8c0e-05493ae82010' steps: + - task: NuGetToolInstaller@1 + inputs: + versionSpec: 5.7 + - task: NuGetCommand@2 + displayName: 'Extract Microsoft.Build.Vcpkg' + inputs: + command: custom + feedsToUse: config + arguments: 'install Microsoft.Build.Vcpkg -Version 2026.1.26.329-f14401ca0f -OutputDirectory "$(Build.BinariesDirectory)" -ConfigFile "$(Build.SourcesDirectory)\azure-pipelines\signing-nuget.config" -ExcludeVersion -NonInteractive' - task: CmdLine@2 displayName: "Build vcpkg amd64 with CMake" inputs: @@ -375,7 +380,8 @@ extends: script: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 cmake.exe --version - cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\amd64" 2>&1 + set X_VCPKG_ASSET_SOURCES=x-script,$(Build.BinariesDirectory)\Microsoft.Build.Vcpkg\trt\TerrapinRetrievalTool.exe -b https://vcpkg.storage.devpackages.microsoft.io/artifacts/ -a true -u Environment -p {url} -s {sha512} -d {dst};x-block-origin + cmake.exe -G Ninja -DCMAKE_TOOLCHAIN_FILE="$(Build.BinariesDirectory)\Microsoft.Build.Vcpkg\tools\scripts\buildsystems\vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=x64-windows-static-release -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\amd64" 2>&1 ninja.exe -C "$(Build.BinariesDirectory)\amd64" - task: CmdLine@2 displayName: "Build vcpkg arm64 with CMake" @@ -384,11 +390,9 @@ extends: script: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=arm64 -host_arch=amd64 cmake.exe --version - cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_PDB_SUFFIX="-arm64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\arm64" 2>&1 + set X_VCPKG_ASSET_SOURCES=x-script,$(Build.BinariesDirectory)\Microsoft.Build.Vcpkg\trt\TerrapinRetrievalTool.exe -b https://vcpkg.storage.devpackages.microsoft.io/artifacts/ -a true -u Environment -p {url} -s {sha512} -d {dst};x-block-origin + cmake.exe -G Ninja -DCMAKE_TOOLCHAIN_FILE="$(Build.BinariesDirectory)\Microsoft.Build.Vcpkg\tools\scripts\buildsystems\vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=arm64-windows-static-release -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_PDB_SUFFIX="-arm64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\arm64" 2>&1 ninja.exe -C "$(Build.BinariesDirectory)\arm64" - - task: NuGetToolInstaller@1 - inputs: - versionSpec: 5.7 - task: NuGetCommand@2 displayName: 'NuGet Restore MicroBuild Signing Extension' inputs: From 01a8033149ab506fbefef08f3bd1f2b214eb42e7 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 27 Jan 2026 16:32:05 -0800 Subject: [PATCH 12/40] Also fix scripts-SHA updating. --- vcpkg-configuration.json | 2 +- vcpkg-init/update-scripts-sha.ps1 | 15 ++++++++++++--- vcpkg-init/vcpkg-scripts-sha.txt | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/vcpkg-configuration.json b/vcpkg-configuration.json index 64c2236d12..3bb948f6d4 100644 --- a/vcpkg-configuration.json +++ b/vcpkg-configuration.json @@ -1,7 +1,7 @@ { "default-registry": { "kind": "git", - "baseline": "75a2e14262fc23c1f03bac18e135c4866dd25b51", + "baseline": "a2a478a93d582a4b395a4dc4b7052bfcb42c1f8e", "repository": "https://github.com/microsoft/vcpkg" } } diff --git a/vcpkg-init/update-scripts-sha.ps1 b/vcpkg-init/update-scripts-sha.ps1 index c4e165f583..97539278ce 100644 --- a/vcpkg-init/update-scripts-sha.ps1 +++ b/vcpkg-init/update-scripts-sha.ps1 @@ -1,3 +1,12 @@ -git ls-remote https://github.com/microsoft/vcpkg master | ` - ForEach-Object { $x = [regex]::Match($_, '^[0-9a-f]+').Value; "$x`n" } | ` - Out-File -LiteralPath "$PSScriptRoot/vcpkg-scripts-sha.txt" -Encoding Ascii -NoNewline +$sha = git ls-remote https://github.com/microsoft/vcpkg master | + ForEach-Object { [regex]::Match($_, '^[0-9a-f]+').Value } | + Select-Object -First 1 + +if (-not $sha) { throw "Failed to determine vcpkg scripts SHA." } + +"$sha`n" | Out-File -LiteralPath "$PSScriptRoot/vcpkg-scripts-sha.txt" -Encoding Ascii -NoNewline + +$configPath = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..\vcpkg-configuration.json')).Path +$configText = Get-Content -LiteralPath $configPath -Raw -Encoding Ascii +$newConfigText = $configText -replace '("baseline"\s*:\s*")[0-9a-f]+(")', "`$1$sha`$2" +$newConfigText | Out-File -LiteralPath $configPath -Encoding Ascii -NoNewline diff --git a/vcpkg-init/vcpkg-scripts-sha.txt b/vcpkg-init/vcpkg-scripts-sha.txt index 488f36c455..206f54c187 100644 --- a/vcpkg-init/vcpkg-scripts-sha.txt +++ b/vcpkg-init/vcpkg-scripts-sha.txt @@ -1 +1 @@ -544a4c5c297e60e4ac4a5a1810df66748d908869 +a2a478a93d582a4b395a4dc4b7052bfcb42c1f8e From 20b08e1130dd2482fcb19f68cc10ccf4507dad97 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 27 Jan 2026 17:15:59 -0800 Subject: [PATCH 13/40] Add toolchain for Windows. --- CMakePresets.json | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakePresets.json b/CMakePresets.json index defaed2872..3b816c3754 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -40,6 +40,7 @@ { "name": "windows", "hidden": true, + "toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake", "cacheVariables": { "VCPKG_BUILD_TLS12_DOWNLOADER": true }, From c7083f5b73f14f2836a9add66ff2cf4e5c4b25c6 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 27 Jan 2026 17:37:58 -0800 Subject: [PATCH 14/40] Supply VCPKG_LIBCURL_URL on glibc and use system libcurl on alpine. --- azure-pipelines/signing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index df7d345e19..394c8ab4ca 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -226,7 +226,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-amd64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-linux/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-amd64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-linux/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_LIBCURL_URL=$(curl-tarball-url) -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-glibc" - job: muslc_build displayName: 'muslc (Alpine) Build' @@ -263,7 +263,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "apk add curl-dev && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_LIBCURL=SYSTEM -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-muslc" - job: glibc_arm64_build displayName: 'glibc Arm64 Build' From af7b6c048721ebfd30cee298f439b8b5bb92c7f3 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 11:53:16 -0800 Subject: [PATCH 15/40] Fix VCPKG_LIBCURL to actually use options. --- cmake/FindLibCURL.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake index 308ae95178..5dd196d0f7 100644 --- a/cmake/FindLibCURL.cmake +++ b/cmake/FindLibCURL.cmake @@ -1,9 +1,11 @@ if (WIN32 OR APPLE) - set(VCPKG_LIBCURL "SYSTEM" CACHE STRING "Select libcurl provider (SYSTEM|DLSYM)") + set(VCPKG_LIBCURL_DEFAULT "SYSTEM") else() - set(VCPKG_LIBCURL "DLSYM" CACHE STRING "Select libcurl provider (SYSTEM|DLSYM)") + set(VCPKG_LIBCURL_DEFAULT "DLSYM") endif() +option(VCPKG_LIBCURL "Select libcurl provider (SYSTEM|DLSYM)" "${VCPKG_LIBCURL_DEFAULT}") + if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() From ce29b948c52a24713b33277d45368d07121cc7c0 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 12:26:48 -0800 Subject: [PATCH 16/40] More curl options. --- azure-pipelines/signing.yml | 2 +- cmake/FindLibCURL.cmake | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 394c8ab4ca..08745a1e74 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -263,7 +263,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "apk add curl-dev && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_LIBCURL=SYSTEM -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "apk add curl-dev && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_LIBCURL_DLSYM=OFF -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-muslc" - job: glibc_arm64_build displayName: 'glibc Arm64 Build' diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake index 5dd196d0f7..cb3b03c452 100644 --- a/cmake/FindLibCURL.cmake +++ b/cmake/FindLibCURL.cmake @@ -1,20 +1,18 @@ if (WIN32 OR APPLE) - set(VCPKG_LIBCURL_DEFAULT "SYSTEM") + set(VCPKG_LIBCURL_DEFAULT "OFF") else() - set(VCPKG_LIBCURL_DEFAULT "DLSYM") + set(VCPKG_LIBCURL_DEFAULT "ON") endif() -option(VCPKG_LIBCURL "Select libcurl provider (SYSTEM|DLSYM)" "${VCPKG_LIBCURL_DEFAULT}") +option(VCPKG_LIBCURL_DLSYM "Select libcurl provider (SYSTEM|DLSYM)" "${VCPKG_LIBCURL_DEFAULT}") if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() -if (VCPKG_LIBCURL STREQUAL "SYSTEM") +if (NOT VCPKG_LIBCURL_DLSYM) find_package(CURL REQUIRED) return() -elseif (NOT VCPKG_LIBCURL STREQUAL "DLSYM") - message(FATAL_ERROR "Unsupported VCPKG_LIBCURL value '${VCPKG_LIBCURL}'. Expected SYSTEM or DLSYM.") endif() # This option exists to allow the URI to be replaced with a Microsoft-internal URI in official From c661d5543a54e44fe7dbf16321a4812418cb7552 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 13:10:19 -0800 Subject: [PATCH 17/40] Fix default typo. --- cmake/FindLibCURL.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake index cb3b03c452..c9bd803e83 100644 --- a/cmake/FindLibCURL.cmake +++ b/cmake/FindLibCURL.cmake @@ -1,10 +1,10 @@ if (WIN32 OR APPLE) - set(VCPKG_LIBCURL_DEFAULT "OFF") + set(VCPKG_LIBCURL_DLSYM_DEFAULT "OFF") else() - set(VCPKG_LIBCURL_DEFAULT "ON") + set(VCPKG_LIBCURL_DLSYM_DEFAULT "ON") endif() -option(VCPKG_LIBCURL_DLSYM "Select libcurl provider (SYSTEM|DLSYM)" "${VCPKG_LIBCURL_DEFAULT}") +option(VCPKG_LIBCURL_DLSYM "Select libcurl provider (SYSTEM|DLSYM)" "${VCPKG_LIBCURL_DLSYM_DEFAULT}") if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) From ecbe8dbd86a38e4eb3ae59b79030f75029b97c45 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 13:40:35 -0800 Subject: [PATCH 18/40] Go back to dlsym on Alpine, try updated headers to fix arm64-linux --- azure-pipelines/signing.yml | 4 ++-- cmake/FindLibCURL.cmake | 23 ++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 08745a1e74..51dd7477d8 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -263,7 +263,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "apk add curl-dev && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_LIBCURL_DLSYM=OFF -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_LIBCURL_URL=$(curl-tarball-url) -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-muslc" - job: glibc_arm64_build displayName: 'glibc Arm64 Build' @@ -301,7 +301,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-arm64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-arm64/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-arm64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-arm64/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_LIBCURL_URL=$(curl-tarball-new-url) -DVCPKG_LIBCURL_DLSYM_UPDATED_HEADERS=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-glibc-arm64" - job: windows_and_sign displayName: 'Build Windows binaries and Sign' diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake index c9bd803e83..d0866c565a 100644 --- a/cmake/FindLibCURL.cmake +++ b/cmake/FindLibCURL.cmake @@ -5,6 +5,7 @@ else() endif() option(VCPKG_LIBCURL_DLSYM "Select libcurl provider (SYSTEM|DLSYM)" "${VCPKG_LIBCURL_DLSYM_DEFAULT}") +option(VCPKG_LIBCURL_DLSYM_UPDATED_HEADERS "Use more recent libcurl headers when using DLSYM than 7.29.0" OFF) if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) @@ -15,15 +16,23 @@ if (NOT VCPKG_LIBCURL_DLSYM) return() endif() -# This option exists to allow the URI to be replaced with a Microsoft-internal URI in official +# The URI option exists to allow the URI to be replaced with a Microsoft-internal URI in official # builds which have restricted internet access; see azure-pipelines/signing.yml # Note that the SHA512 is the same, so vcpkg-tool contributors need not be concerned that we built # with different content. -# -# We're using curl-7_29_0 headers here because that's what comes with RHEL 7 (and inherited by -# similarly ancient Oracle Linux 7) -if(NOT VCPKG_LIBCURL_URL) - set(VCPKG_LIBCURL_URL "https://curl.se/download/archeology/curl-7.29.0.tar.gz") +if (VCPKG_LIBCURL_DLSYM_UPDATED_HEADERS) + # curl-7_55_1 for https://daniel.haxx.se/blog/2017/06/15/target-independent-libcurl-headers/ + # (mainly for arm64 support) + set(VCPKG_LIBCURL_HASH 4b0bf36a978b8b5ba66aecedbc2ae8ae9230da63ba5b80f9553d96671e013ccd679ee9cc10946c50b94d640858d74f3ec5d4e198c6b9f8842c941986d275cf7a) + if(NOT VCPKG_LIBCURL_URL) + set(VCPKG_LIBCURL_URL "https://curl.se/download/curl-7.55.1.tar.gz") + endif() +else() + # curl-7_29_0 headers here because that's what comes with RHEL 7 (and inherited by similarly ancient Oracle Linux 7) + set(VCPKG_LIBCURL_HASH 08bafd09fa6d14362a426932fed8528c13133895477d8134c829e085637956d66d6be5a791057c1c04da04af6baa6496a6d59e00abf9ca6be5d29e798718b9bc) + if(NOT VCPKG_LIBCURL_URL) + set(VCPKG_LIBCURL_URL "https://curl.se/download/archeology/curl-7.29.0.tar.gz") + endif() endif() if(NOT TARGET CURL::libcurl) @@ -31,7 +40,7 @@ include(FetchContent) FetchContent_Declare( LibCURLHeaders URL "${VCPKG_LIBCURL_URL}" - URL_HASH "SHA512=08bafd09fa6d14362a426932fed8528c13133895477d8134c829e085637956d66d6be5a791057c1c04da04af6baa6496a6d59e00abf9ca6be5d29e798718b9bc" + URL_HASH "SHA512=${VCPKG_LIBCURL_HASH}" ) FetchContent_GetProperties(LibCURLHeaders) From c2bd2db85123729779d1bed3ec317eb755ca998c Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 14:49:39 -0800 Subject: [PATCH 19/40] Disable auto-baseline. --- azure-pipelines/signing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 51dd7477d8..3f7a13e90a 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -22,6 +22,8 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: + featureFlags: + autoBaseline: false sdl: sourceAnalysisPool: name: AzurePipelines-EO From 85bde1db22a60dac705ca933af0d3af97bff6495 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 15:26:41 -0800 Subject: [PATCH 20/40] Suppress CodeQL and format. --- include/vcpkg/base/curl.h | 10 ++-- src/vcpkg.cpp | 6 +- src/vcpkg/base/curl.cpp | 99 ++++++++++++++++--------------- src/vcpkg/base/downloads.cpp | 23 ++++--- src/vcpkg/base/system.process.cpp | 8 ++- src/vcpkg/metrics.cpp | 6 +- 6 files changed, 80 insertions(+), 72 deletions(-) diff --git a/include/vcpkg/base/curl.h b/include/vcpkg/base/curl.h index 5ce5fc30f1..a57fc9626a 100644 --- a/include/vcpkg/base/curl.h +++ b/include/vcpkg/base/curl.h @@ -51,16 +51,14 @@ extern decltype(&curl_version) vcpkg_curl_version; #define vcpkg_curl_multi_add_handle(multi_handle, easy_handle) curl_multi_add_handle(multi_handle, easy_handle) #define vcpkg_curl_multi_cleanup(multi_handle) curl_multi_cleanup(multi_handle) -#define vcpkg_curl_multi_info_read(multi_handle, messages_in_queue) \ +#define vcpkg_curl_multi_info_read(multi_handle, messages_in_queue) \ curl_multi_info_read(multi_handle, messages_in_queue) #define vcpkg_curl_multi_init() curl_multi_init() -#define vcpkg_curl_multi_remove_handle(multi_handle, easy_handle) \ - curl_multi_remove_handle(multi_handle, easy_handle) +#define vcpkg_curl_multi_remove_handle(multi_handle, easy_handle) curl_multi_remove_handle(multi_handle, easy_handle) #define vcpkg_curl_multi_strerror(code) curl_multi_strerror(code) -#define vcpkg_curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, numfds) \ +#define vcpkg_curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, numfds) \ curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, numfds) -#define vcpkg_curl_multi_perform(multi_handle, running_handles) \ - curl_multi_perform(multi_handle, running_handles) +#define vcpkg_curl_multi_perform(multi_handle, running_handles) curl_multi_perform(multi_handle, running_handles) #define vcpkg_curl_slist_append(list, string) curl_slist_append(list, string) #define vcpkg_curl_slist_free_all(list) curl_slist_free_all(list) diff --git a/src/vcpkg.cpp b/src/vcpkg.cpp index acb9a21651..b696e08221 100644 --- a/src/vcpkg.cpp +++ b/src/vcpkg.cpp @@ -119,11 +119,11 @@ namespace get_global_metrics_collector().track_bool(BoolMetric::DetectedContainer, detect_container(fs)); const auto* detected_curl_version = vcpkg_curl_version(); - if (!detected_curl_version) { + if (!detected_curl_version) + { detected_curl_version = "unknown"; } - get_global_metrics_collector().track_string(StringMetric::DetectedLibCurlVersion, - detected_curl_version); + get_global_metrics_collector().track_string(StringMetric::DetectedLibCurlVersion, detected_curl_version); if (args.get_command().empty()) { diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp index fe1c3d7633..6b0077aaea 100644 --- a/src/vcpkg/base/curl.cpp +++ b/src/vcpkg/base/curl.cpp @@ -27,63 +27,68 @@ decltype(&curl_slist_free_all) vcpkg_curl_slist_free_all; decltype(&curl_version) vcpkg_curl_version; template -static void load_symbol(FnT (&target), void* handle, const char* symbol_name) { +static void load_symbol(FnT(&target), void* handle, const char* symbol_name) +{ target = reinterpret_cast(dlsym(handle, symbol_name)); - if (!target) { + if (!target) + { vcpkg::Checks::unreachable(VCPKG_LINE_INFO); } } - void vcpkg_curl_global_init(long flags) { - // calling dlclose() on the handle after calling curl_version() causes asan to - // report a false leak, so we intentionally don't unload the library - auto handle = dlopen("libcurl.so.4", RTLD_NOW | RTLD_LOCAL); - if (!handle) - { - // Ubuntu 16.04 has this if the user only installs `curl` - handle = dlopen("libcurl-gnutls.so.4", RTLD_NOW | RTLD_LOCAL); - } - if (!handle) - { - // It's possible that someone explicitly installs this one - handle = dlopen("libcurl-nss.so.4", RTLD_NOW | RTLD_LOCAL); - } +void vcpkg_curl_global_init(long flags) +{ + // calling dlclose() on the handle after calling curl_version() causes asan to + // report a false leak, so we intentionally don't unload the library + auto handle = dlopen("libcurl.so.4", RTLD_NOW | RTLD_LOCAL); + if (!handle) + { + // Ubuntu 16.04 has this if the user only installs `curl` + handle = dlopen("libcurl-gnutls.so.4", RTLD_NOW | RTLD_LOCAL); + } + if (!handle) + { + // It's possible that someone explicitly installs this one + handle = dlopen("libcurl-nss.so.4", RTLD_NOW | RTLD_LOCAL); + } - if (!handle) { - vcpkg::Checks::unreachable(VCPKG_LINE_INFO); // FIXME emit an error for users - } + if (!handle) + { + vcpkg::Checks::unreachable(VCPKG_LINE_INFO); // FIXME emit an error for users + } - { - decltype(&curl_global_init) global_init; - load_symbol(global_init, handle, "curl_global_init"); - global_init(flags); - } + { + decltype(&curl_global_init) global_init; + load_symbol(global_init, handle, "curl_global_init"); + global_init(flags); + } - load_symbol(vcpkg_curl_easy_cleanup, handle, "curl_easy_cleanup"); - load_symbol(vcpkg_curl_easy_getinfo, handle, "curl_easy_getinfo"); - load_symbol(vcpkg_curl_easy_init, handle, "curl_easy_init"); - load_symbol(vcpkg_curl_easy_perform, handle, "curl_easy_perform"); - load_symbol(vcpkg_curl_easy_setopt, handle, "curl_easy_setopt"); - load_symbol(vcpkg_curl_easy_strerror, handle, "curl_easy_strerror"); - - load_symbol(vcpkg_curl_multi_add_handle, handle, "curl_multi_add_handle"); - load_symbol(vcpkg_curl_multi_cleanup, handle, "curl_multi_cleanup"); - load_symbol(vcpkg_curl_multi_info_read, handle, "curl_multi_info_read"); - load_symbol(vcpkg_curl_multi_init, handle, "curl_multi_init"); - load_symbol(vcpkg_curl_multi_remove_handle, handle, "curl_multi_remove_handle"); - load_symbol(vcpkg_curl_multi_strerror, handle, "curl_multi_strerror"); - load_symbol(vcpkg_curl_multi_perform, handle, "curl_multi_perform"); - // try to load curl_multi_poll first, fall back to curl_multi_wait - vcpkg_curl_multi_poll = reinterpret_cast(dlsym(handle, "curl_multi_poll")); - if (!vcpkg_curl_multi_poll) { - load_symbol(vcpkg_curl_multi_poll, handle, "curl_multi_wait"); - } + load_symbol(vcpkg_curl_easy_cleanup, handle, "curl_easy_cleanup"); + load_symbol(vcpkg_curl_easy_getinfo, handle, "curl_easy_getinfo"); + load_symbol(vcpkg_curl_easy_init, handle, "curl_easy_init"); + load_symbol(vcpkg_curl_easy_perform, handle, "curl_easy_perform"); + load_symbol(vcpkg_curl_easy_setopt, handle, "curl_easy_setopt"); + load_symbol(vcpkg_curl_easy_strerror, handle, "curl_easy_strerror"); + + load_symbol(vcpkg_curl_multi_add_handle, handle, "curl_multi_add_handle"); + load_symbol(vcpkg_curl_multi_cleanup, handle, "curl_multi_cleanup"); + load_symbol(vcpkg_curl_multi_info_read, handle, "curl_multi_info_read"); + load_symbol(vcpkg_curl_multi_init, handle, "curl_multi_init"); + load_symbol(vcpkg_curl_multi_remove_handle, handle, "curl_multi_remove_handle"); + load_symbol(vcpkg_curl_multi_strerror, handle, "curl_multi_strerror"); + load_symbol(vcpkg_curl_multi_perform, handle, "curl_multi_perform"); + // try to load curl_multi_poll first, fall back to curl_multi_wait + vcpkg_curl_multi_poll = reinterpret_cast(dlsym(handle, "curl_multi_poll")); + if (!vcpkg_curl_multi_poll) + { + load_symbol(vcpkg_curl_multi_poll, handle, "curl_multi_wait"); + } - load_symbol(vcpkg_curl_slist_append, handle, "curl_slist_append"); - load_symbol(vcpkg_curl_slist_free_all, handle, "curl_slist_free_all"); + load_symbol(vcpkg_curl_slist_append, handle, "curl_slist_append"); + load_symbol(vcpkg_curl_slist_free_all, handle, "curl_slist_free_all"); - load_symbol(vcpkg_curl_version, handle, "curl_version"); - } + load_symbol(vcpkg_curl_version, handle, "curl_version"); +} #endif // ^^^ VCPKG_LIBCURL_DLSYM namespace vcpkg diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index b600f4894a..c67118c1a1 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -28,13 +28,14 @@ namespace vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); vcpkg_curl_easy_setopt(curl, CURLOPT_URL, url_encode_spaces(url).c_str()); vcpkg_curl_easy_setopt(curl, - CURLOPT_FOLLOWLOCATION, - 2L); // Follow redirects, change request method based on HTTP response code. - // https://curl.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html#CURLFOLLOWOBEYCODE + CURLOPT_FOLLOWLOCATION, + 2L); // Follow redirects, change request method based on HTTP response code. + // https://curl.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html#CURLFOLLOWOBEYCODE vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); // don't send headers to proxy CONNECT ; this intentionally fails on older versions of libcurl - vcpkg_curl_easy_setopt(curl, static_cast(229) /* CURLOPT_HEADEROPT */, (1L << 0) /* CURLHEADER_SEPARATE */); + vcpkg_curl_easy_setopt( + curl, static_cast(229) /* CURLOPT_HEADEROPT */, (1L << 0) /* CURLHEADER_SEPARATE */); } } @@ -120,12 +121,9 @@ namespace vcpkg return static_cast(param)->write(contents, size, nmemb); } - static int progress_callback(void *clientp, - double dltotal, - double dlnow, - double ultotal, - double ulnow) { - (void)ultotal; + static int progress_callback(void* clientp, double dltotal, double dlnow, double ultotal, double ulnow) + { + (void)ultotal; (void)ulnow; auto machine_readable_progress = static_cast(clientp); if (dltotal && machine_readable_progress) @@ -134,7 +132,7 @@ namespace vcpkg machine_readable_progress->println(LocalizedString::from_raw(fmt::format("{:.2f}%", percentage))); } return 0; - } + } static std::vector libcurl_bulk_operation(DiagnosticContext& context, View urls, @@ -166,7 +164,8 @@ namespace vcpkg set_common_curl_easy_options(easy_handle, url, request_headers); if (outputs.empty()) { - vcpkg_curl_easy_setopt(curl, CURLOPT_PRIVATE, reinterpret_cast(static_cast(request_index))); + vcpkg_curl_easy_setopt( + curl, CURLOPT_PRIVATE, reinterpret_cast(static_cast(request_index))); } else { diff --git a/src/vcpkg/base/system.process.cpp b/src/vcpkg/base/system.process.cpp index 0b15a537b5..062a191b78 100644 --- a/src/vcpkg/base/system.process.cpp +++ b/src/vcpkg/base/system.process.cpp @@ -1099,7 +1099,9 @@ namespace DWORD bytes_read = 0; static constexpr DWORD buffer_size = 1024 * 32; - char buf[buffer_size]; + // the alignment is forced here for callers that are expecting the buffer to contain UTF-16 + // on Windows + alignas(wchar_t) char buf[buffer_size]; while (stdout_pipe.read_pipe != INVALID_HANDLE_VALUE) { switch (WaitForSingleObjectEx(stdout_pipe.read_pipe, INFINITE, TRUE)) @@ -1650,7 +1652,9 @@ namespace // Note: This doesn't handle unpaired surrogates or partial encoding units correctly in // order to be able to reuse Strings::to_utf8 which we believe will be fine 99% of the time. std::string encoded; - Strings::to_utf8(encoded, reinterpret_cast(buf), bytes_read / 2); + // clang-format off + Strings::to_utf8(encoded, reinterpret_cast(buf), bytes_read / 2); // CodeQL [SM02986] This is not an incorrect cast to shut up the compiler, the suggested "call an encoding conversion function instead" is exactly what this is doing. + // clang-format on std::replace(encoded.begin(), encoded.end(), '\0', '?'); if (settings.echo_in_debug == EchoInDebug::Show && Debug::g_debugging) { diff --git a/src/vcpkg/metrics.cpp b/src/vcpkg/metrics.cpp index 0bc772eb20..a0483a98c5 100644 --- a/src/vcpkg/metrics.cpp +++ b/src/vcpkg/metrics.cpp @@ -596,8 +596,10 @@ namespace vcpkg vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.length())); vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); vcpkg_curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L); - if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2) != CURLE_OK) { - if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_1) != CURLE_OK) { + if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2) != CURLE_OK) + { + if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_1) != CURLE_OK) + { vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_0); } } From 70318290ac7be6d1afb56bd9ef2a22778b9c0ab7 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 16:04:07 -0800 Subject: [PATCH 21/40] Revert "Disable auto-baseline." This reverts commit c2bd2db85123729779d1bed3ec317eb755ca998c. --- azure-pipelines/signing.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 3f7a13e90a..51dd7477d8 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -22,8 +22,6 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - featureFlags: - autoBaseline: false sdl: sourceAnalysisPool: name: AzurePipelines-EO From 373d292482915ed575294c1f7a388b285095a14b Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 16:13:56 -0800 Subject: [PATCH 22/40] Delete 'Install system libcurl' now that we no longer need it on *nix --- .github/workflows/build.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1596506fd2..7524e3d264 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -36,11 +36,6 @@ jobs: uses: github/codeql-action/init@v4 with: languages: javascript-typescript, c-cpp - - name: Install system libcurl - if: matrix.preset == 'linux-arm64-ci' || matrix.preset == 'linux-ci' - run: | - sudo apt update - sudo apt install -y libcurl4-openssl-dev - name: Configure and Build if: matrix.preset != 'windows-ci' run: | From 488a3fc006e129c8fa324ecb52df273cdf0d34e8 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 16:25:40 -0800 Subject: [PATCH 23/40] Revert no longer relevant build setup changes. --- azure-pipelines/vcpkg-alpine/Dockerfile | 2 +- azure-pipelines/vcpkg-arm64/Dockerfile | 14 +++++++------- azure-pipelines/vcpkg-linux/Dockerfile | 14 +++++++------- cgmanifest.json | 13 ------------- 4 files changed, 15 insertions(+), 28 deletions(-) diff --git a/azure-pipelines/vcpkg-alpine/Dockerfile b/azure-pipelines/vcpkg-alpine/Dockerfile index 07bdffc627..e7e85974ee 100644 --- a/azure-pipelines/vcpkg-alpine/Dockerfile +++ b/azure-pipelines/vcpkg-alpine/Dockerfile @@ -1,4 +1,4 @@ # The authoritative version of this file is in https://devdiv.visualstudio.com/DevDiv/_git/vcpkg-pme-utils FROM alpine:3.16 -RUN apk add alpine-sdk cmake ninja git curl tar gzip zip curl-dev && apk upgrade +RUN apk add alpine-sdk cmake ninja git curl tar gzip zip && apk upgrade diff --git a/azure-pipelines/vcpkg-arm64/Dockerfile b/azure-pipelines/vcpkg-arm64/Dockerfile index 484ccbc61c..77622dd235 100644 --- a/azure-pipelines/vcpkg-arm64/Dockerfile +++ b/azure-pipelines/vcpkg-arm64/Dockerfile @@ -51,13 +51,13 @@ RUN chroot /crossrootfs/arm64 apt-get clean && \ # Repeated runs of apt-get install workaround 'hash sum mismatch' errors # (This is also why these are a separate cache layer) -RUN chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev && \ +RUN chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/arm64 apt-get install -y build-essential symlinks "g++-9" && \ chroot /crossrootfs/arm64 symlinks -cr /usr && \ chroot /crossrootfs/arm64 apt-get clean diff --git a/azure-pipelines/vcpkg-linux/Dockerfile b/azure-pipelines/vcpkg-linux/Dockerfile index 9ef4c79ecf..92b848f396 100644 --- a/azure-pipelines/vcpkg-linux/Dockerfile +++ b/azure-pipelines/vcpkg-linux/Dockerfile @@ -50,13 +50,13 @@ RUN chroot /crossrootfs/x64 apt-get clean && \ # Repeated runs of apt-get install workaround 'hash sum mismatch' errors # (This is also why these are a separate cache layer) -RUN chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev || \ - chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" libcurl4-openssl-dev && \ +RUN chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" || \ + chroot /crossrootfs/x64 apt-get install -y build-essential symlinks "g++-9" && \ chroot /crossrootfs/x64 symlinks -cr /usr && \ chroot /crossrootfs/x64 apt-get clean diff --git a/cgmanifest.json b/cgmanifest.json index 1244e27517..19dcc509fe 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -37,19 +37,6 @@ }, "DevelopmentDependency": false, "DependencyRoots": [] - }, - { - "Component": { - "Type": "other", - "other": { - "name": "curl", - "version": "8.17.0", - "downloadUrl": "https://github.com/curl/curl/archive/refs/tags/curl-8_17_0.tar.gz", - "hash": "88ab4b7aac12b26a6ad32fb0e1a9675288a45894438cb031102ef5d4ab6b33c2bc99cae0c70b71bdfa12eb49762827e2490555114c5eb4a6876b95e1f2a4eb74" - } - }, - "DevelopmentDependency": false, - "DependencyRoots": [] } ] } From 908dfee2e03b6a744f6c31e2ef686900d1d80e25 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 16:34:33 -0800 Subject: [PATCH 24/40] Add an error message for when we can't find libcurl.so.4 --- include/vcpkg/base/message-data.inc.h | 5 +++++ locales/messages.json | 1 + src/vcpkg/base/curl.cpp | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 565554ed96..1b81dcbe14 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -2860,6 +2860,11 @@ DECLARE_MESSAGE(TwoFeatureFlagsSpecified, (msg::value), "'{value}' is a feature flag.", "Both '{value}' and -'{value}' were specified as feature flags.") +DECLARE_MESSAGE(UnableToFindCurl, + (), + "", + "vcpkg was unable to find a libcurl.so.4, libcurl-gnutls.so.4, or libcurl-nss.so.4 to use on this " + "system. Please install libcurl from your system package manager and retry vcpkg.") DECLARE_MESSAGE(UnableToReadAppDatas, (), "", "both %LOCALAPPDATA% and %APPDATA% were unreadable") DECLARE_MESSAGE(UnableToReadEnvironmentVariable, (msg::env_var), "", "unable to read {env_var}") DECLARE_MESSAGE(UndeterminedToolChainForTriplet, diff --git a/locales/messages.json b/locales/messages.json index 54488e7124..27e3d15da8 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -1492,6 +1492,7 @@ "TripletLabel": "Triplet:", "TwoFeatureFlagsSpecified": "Both '{value}' and -'{value}' were specified as feature flags.", "_TwoFeatureFlagsSpecified.comment": "'{value}' is a feature flag.", + "UnableToFindCurl": "vcpkg was unable to find a libcurl.so.4, libcurl-gnutls.so.4, or libcurl-nss.so.4 to use on this system. Please install libcurl from your system package manager and retry vcpkg.", "UnableToReadAppDatas": "both %LOCALAPPDATA% and %APPDATA% were unreadable", "UnableToReadEnvironmentVariable": "unable to read {env_var}", "_UnableToReadEnvironmentVariable.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp index 6b0077aaea..2128afefea 100644 --- a/src/vcpkg/base/curl.cpp +++ b/src/vcpkg/base/curl.cpp @@ -54,7 +54,7 @@ void vcpkg_curl_global_init(long flags) if (!handle) { - vcpkg::Checks::unreachable(VCPKG_LINE_INFO); // FIXME emit an error for users + vcpkg::Checks::msg_exit_with_error(vcpkg::msgUnableToFindCurl); } { From 0cc0003f29f0a3d33b03346e47603e17314615b4 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 16:45:43 -0800 Subject: [PATCH 25/40] Fix some Linux build errors. --- CMakePresets.json | 3 ++- src/vcpkg/base/curl.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index 3b816c3754..8e6d8cb62e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -170,7 +170,8 @@ "name": "linux-arm64-ci", "inherits": [ "linux-ci" ], "cacheVariables": { - "CMAKE_SYSTEM_PROCESSOR": "aarch64" + "CMAKE_SYSTEM_PROCESSOR": "aarch64", + "VCPKG_LIBCURL_DLSYM_UPDATED_HEADERS": true } }, { diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp index 2128afefea..51151ac243 100644 --- a/src/vcpkg/base/curl.cpp +++ b/src/vcpkg/base/curl.cpp @@ -54,7 +54,7 @@ void vcpkg_curl_global_init(long flags) if (!handle) { - vcpkg::Checks::msg_exit_with_error(vcpkg::msgUnableToFindCurl); + vcpkg::Checks::msg_exit_with_error(vcpkg::msg::msgUnableToFindCurl); } { From 9f353d5dc7c321feeb832044b1e792cdd961dccd Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 16:48:13 -0800 Subject: [PATCH 26/40] Move vcpkg's vcpkg manifest to a subdirectory to avoid having every e2e test fail due to thinking they are in manifest mode. --- CMakeLists.txt | 2 ++ vcpkg-configuration.json => src/vcpkg-configuration.json | 0 vcpkg.json => src/vcpkg.json | 0 3 files changed, 2 insertions(+) rename vcpkg-configuration.json => src/vcpkg-configuration.json (100%) rename vcpkg.json => src/vcpkg.json (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 11c2501111..880a9115fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,8 @@ if(DEFINE_DISABLE_METRICS OR VCPKG_DISABLE_METRICS) "file vcpkg.disable_metrics next to the binary.") endif() +set(VCPKG_MANIFEST_DIR "${CMAKE_CURRENT_LIST_DIR}/src" CACHE PATH "Path to vcpkg manifest directory") + project(vcpkg DESCRIPTION "vcpkg helps you manage C and C++ libraries on Windows, Linux and MacOS." HOMEPAGE_URL "https://github.com/microsoft/vcpkg" diff --git a/vcpkg-configuration.json b/src/vcpkg-configuration.json similarity index 100% rename from vcpkg-configuration.json rename to src/vcpkg-configuration.json diff --git a/vcpkg.json b/src/vcpkg.json similarity index 100% rename from vcpkg.json rename to src/vcpkg.json From ca27d9342c2865d19310c14135b43b0e38b2458f Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 16:55:42 -0800 Subject: [PATCH 27/40] =?UTF-8?q?Actually=20fix=20Linux=20build=20errors?= =?UTF-8?q?=20=F0=9F=98=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/vcpkg/base/curl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp index 51151ac243..342c06113f 100644 --- a/src/vcpkg/base/curl.cpp +++ b/src/vcpkg/base/curl.cpp @@ -54,7 +54,7 @@ void vcpkg_curl_global_init(long flags) if (!handle) { - vcpkg::Checks::msg_exit_with_error(vcpkg::msg::msgUnableToFindCurl); + vcpkg::Checks::msg_exit_with_error(VCPKG_LINE_INFO, vcpkg::msgUnableToFindCurl); } { From dbb019c680b1290bba26c35b6c6e55f00dfddc06 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 17:04:03 -0800 Subject: [PATCH 28/40] Turn on curl[non-http] because we have told customers to use file:// --- src/vcpkg.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vcpkg.json b/src/vcpkg.json index 117a787b1c..4d64541c44 100644 --- a/src/vcpkg.json +++ b/src/vcpkg.json @@ -6,6 +6,7 @@ "features": [ "brotli", "http2", + "non-http", "ssl", "zstd" ], From 594e0f192f52d8f5ec48a073655012ed9bebedca Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 17:12:06 -0800 Subject: [PATCH 29/40] Call vcpkg_curl_global_init(CURL_GLOBAL_DEFAULT); in vcpkg-test. --- src/vcpkg-test/catch.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vcpkg-test/catch.cpp b/src/vcpkg-test/catch.cpp index f296f80bc5..e3a03b712a 100644 --- a/src/vcpkg-test/catch.cpp +++ b/src/vcpkg-test/catch.cpp @@ -1,6 +1,7 @@ #define CATCH_CONFIG_RUNNER #include +#include #include #include @@ -11,6 +12,7 @@ namespace vcpkg::Checks int main(int argc, char** argv) { + vcpkg_curl_global_init(CURL_GLOBAL_DEFAULT); if (vcpkg::get_environment_variable("VCPKG_DEBUG").value_or("") == "1") vcpkg::Debug::g_debugging = true; // We set VCPKG_ROOT to an invalid value to ensure unit tests do not attempt to instantiate VcpkgRoot vcpkg::set_environment_variable("VCPKG_ROOT", "VCPKG_TESTS_SHOULD_NOT_USE_VCPKG_ROOT"); From c7866331a829754724c03b28031530861b22471e Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Wed, 28 Jan 2026 17:51:15 -0800 Subject: [PATCH 30/40] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- cmake/FindLibCURL.cmake | 2 +- src/vcpkg/base/curl.cpp | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake index d0866c565a..af4576b5e7 100644 --- a/cmake/FindLibCURL.cmake +++ b/cmake/FindLibCURL.cmake @@ -4,7 +4,7 @@ else() set(VCPKG_LIBCURL_DLSYM_DEFAULT "ON") endif() -option(VCPKG_LIBCURL_DLSYM "Select libcurl provider (SYSTEM|DLSYM)" "${VCPKG_LIBCURL_DLSYM_DEFAULT}") +option(VCPKG_LIBCURL_DLSYM "Use dlsym to dynamically load libcurl at runtime instead of using the system libcurl" "${VCPKG_LIBCURL_DLSYM_DEFAULT}") option(VCPKG_LIBCURL_DLSYM_UPDATED_HEADERS "Use more recent libcurl headers when using DLSYM than 7.29.0" OFF) if(POLICY CMP0135) diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp index 342c06113f..e4be64d336 100644 --- a/src/vcpkg/base/curl.cpp +++ b/src/vcpkg/base/curl.cpp @@ -132,15 +132,17 @@ namespace vcpkg } CurlMultiHandle::~CurlMultiHandle() { - for (auto* easy_handle : m_easy_handles) + if (!m_ptr) { - vcpkg_curl_multi_remove_handle(m_ptr, easy_handle); + return; } - if (m_ptr) + for (auto* easy_handle : m_easy_handles) { - vcpkg_curl_multi_cleanup(m_ptr); + vcpkg_curl_multi_remove_handle(m_ptr, easy_handle); } + + vcpkg_curl_multi_cleanup(m_ptr); } void CurlMultiHandle::add_easy_handle(CurlEasyHandle& easy_handle) { From 6da8d8b1604c98c273570748c3383a7635eb58fd Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 17:44:49 -0800 Subject: [PATCH 31/40] Implement move assignment operators correctly. --- include/vcpkg/base/curl.h | 6 ++++++ src/vcpkg/base/curl.cpp | 19 +++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/include/vcpkg/base/curl.h b/include/vcpkg/base/curl.h index a57fc9626a..e81d7e4107 100644 --- a/include/vcpkg/base/curl.h +++ b/include/vcpkg/base/curl.h @@ -74,6 +74,8 @@ namespace vcpkg CurlEasyHandle(CurlEasyHandle&& other) noexcept; CurlEasyHandle& operator=(CurlEasyHandle&& other) noexcept; ~CurlEasyHandle(); + + void swap(CurlEasyHandle& other) noexcept; CURL* get(); @@ -88,6 +90,8 @@ namespace vcpkg CurlMultiHandle& operator=(CurlMultiHandle&& other) noexcept; ~CurlMultiHandle(); + void swap(CurlMultiHandle& other) noexcept; + // Adds an easy handle to the multi handle but doesn't take ownership of it. // Makes sure that the easy handle is removed from the multi handle on cleanup. void add_easy_handle(CurlEasyHandle& easy_handle); @@ -106,6 +110,8 @@ namespace vcpkg CurlHeaders(CurlHeaders&& other) noexcept; CurlHeaders& operator=(CurlHeaders&& other) noexcept; ~CurlHeaders(); + + void swap(CurlHeaders& other) noexcept; curl_slist* get() const; diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp index e4be64d336..dc7daded97 100644 --- a/src/vcpkg/base/curl.cpp +++ b/src/vcpkg/base/curl.cpp @@ -96,7 +96,7 @@ namespace vcpkg CurlEasyHandle::CurlEasyHandle(CurlEasyHandle&& other) noexcept : m_ptr(std::exchange(other.m_ptr, nullptr)) { } CurlEasyHandle& CurlEasyHandle::operator=(CurlEasyHandle&& other) noexcept { - m_ptr = std::exchange(other.m_ptr, nullptr); + CurlEasyHandle{std::move(other)}.swap(*this); return *this; } CurlEasyHandle::~CurlEasyHandle() @@ -106,6 +106,7 @@ namespace vcpkg vcpkg_curl_easy_cleanup(m_ptr); } } + void CurlEasyHandle::swap(CurlEasyHandle& other) noexcept { std::swap(m_ptr, other.m_ptr); } CURL* CurlEasyHandle::get() { if (!m_ptr) @@ -126,8 +127,7 @@ namespace vcpkg } CurlMultiHandle& CurlMultiHandle::operator=(CurlMultiHandle&& other) noexcept { - m_ptr = std::exchange(other.m_ptr, nullptr); - m_easy_handles = std::move(other.m_easy_handles); + CurlMultiHandle{std::move(other)}.swap(*this); return *this; } CurlMultiHandle::~CurlMultiHandle() @@ -144,6 +144,12 @@ namespace vcpkg vcpkg_curl_multi_cleanup(m_ptr); } + void CurlMultiHandle::swap(CurlMultiHandle& other) noexcept + { + using std::swap; + swap(m_ptr, other.m_ptr); + swap(m_easy_handles, other.m_easy_handles); + } void CurlMultiHandle::add_easy_handle(CurlEasyHandle& easy_handle) { auto* handle = easy_handle.get(); @@ -175,7 +181,7 @@ namespace vcpkg CurlHeaders::CurlHeaders(CurlHeaders&& other) noexcept : m_headers(std::exchange(other.m_headers, nullptr)) { } CurlHeaders& CurlHeaders::operator=(CurlHeaders&& other) noexcept { - m_headers = std::exchange(other.m_headers, nullptr); + CurlHeaders{std::move(other)}.swap(*this); return *this; } CurlHeaders::~CurlHeaders() @@ -185,5 +191,10 @@ namespace vcpkg vcpkg_curl_slist_free_all(m_headers); } } + void CurlHeaders::swap(CurlHeaders& other) noexcept + { + using std::swap; + swap(m_headers, other.m_headers); + } curl_slist* CurlHeaders::get() const { return m_headers; } } From f9718a419cf9b34b3cf9dddebfbde086961aa1fe Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 17:50:46 -0800 Subject: [PATCH 32/40] Fix HEAD requests. --- src/vcpkg/base/downloads.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index c67118c1a1..22eb5bebf7 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -166,6 +166,7 @@ namespace vcpkg { vcpkg_curl_easy_setopt( curl, CURLOPT_PRIVATE, reinterpret_cast(static_cast(request_index))); + vcpkg_curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); } else { @@ -182,8 +183,9 @@ namespace vcpkg // note explicit cast to void* necessary to go through ... vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&request_write_pointer)); vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); - multi_handle.add_easy_handle(easy_handle); } + + multi_handle.add_easy_handle(easy_handle); } int still_running = 0; From 75bf930f043ba25ce6069068ff0100fd06063ff2 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 17:53:10 -0800 Subject: [PATCH 33/40] Format --- include/vcpkg/base/curl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/vcpkg/base/curl.h b/include/vcpkg/base/curl.h index e81d7e4107..d033da7483 100644 --- a/include/vcpkg/base/curl.h +++ b/include/vcpkg/base/curl.h @@ -74,7 +74,7 @@ namespace vcpkg CurlEasyHandle(CurlEasyHandle&& other) noexcept; CurlEasyHandle& operator=(CurlEasyHandle&& other) noexcept; ~CurlEasyHandle(); - + void swap(CurlEasyHandle& other) noexcept; CURL* get(); @@ -110,7 +110,7 @@ namespace vcpkg CurlHeaders(CurlHeaders&& other) noexcept; CurlHeaders& operator=(CurlHeaders&& other) noexcept; ~CurlHeaders(); - + void swap(CurlHeaders& other) noexcept; curl_slist* get() const; From 9d3d9a6a5ab536b350c307f4d5e27ac389393673 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 18:17:22 -0800 Subject: [PATCH 34/40] Resolve copilot PR suggestions. --- src/vcpkg/base/curl.cpp | 13 ++++++++++--- src/vcpkg/base/downloads.cpp | 4 ++-- src/vcpkg/metrics.cpp | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp index dc7daded97..6b829bc119 100644 --- a/src/vcpkg/base/curl.cpp +++ b/src/vcpkg/base/curl.cpp @@ -153,9 +153,10 @@ namespace vcpkg void CurlMultiHandle::add_easy_handle(CurlEasyHandle& easy_handle) { auto* handle = easy_handle.get(); - if (vcpkg_curl_multi_add_handle(this->get(), handle) == CURLM_OK) + m_easy_handles.push_back(handle); + if (vcpkg_curl_multi_add_handle(this->get(), handle) != CURLM_OK) { - m_easy_handles.push_back(handle); + Checks::unreachable(VCPKG_LINE_INFO); } } CURLM* CurlMultiHandle::get() @@ -175,7 +176,13 @@ namespace vcpkg { for (const auto& header : headers) { - m_headers = vcpkg_curl_slist_append(m_headers, header.c_str()); + auto new_headers = vcpkg_curl_slist_append(m_headers, header.c_str()); + if (!new_headers) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + + m_headers = new_headers; } } CurlHeaders::CurlHeaders(CurlHeaders&& other) noexcept : m_headers(std::exchange(other.m_headers, nullptr)) { } diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 22eb5bebf7..395c8950e8 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -318,7 +318,7 @@ namespace vcpkg vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); vcpkg_curl_easy_setopt(curl, CURLOPT_POST, 1L); vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); - vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, post_data.length()); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, static_cast(post_data.size())); CURLcode result = vcpkg_curl_easy_perform(curl); long response_code = 0; @@ -493,7 +493,7 @@ namespace vcpkg vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&fileptr)); vcpkg_curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); // change from default to enable progress - // curlopt_progressfucntion is deprecated, but we want the values as doubles anyway and + // curlopt_progressfunction is deprecated, but we want the values as doubles anyway and // the replacement isn't available on all versions of libcurl we support vcpkg_curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, static_cast(&progress_callback)); vcpkg_curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, static_cast(&machine_readable_progress)); diff --git a/src/vcpkg/metrics.cpp b/src/vcpkg/metrics.cpp index a0483a98c5..04ce6154b3 100644 --- a/src/vcpkg/metrics.cpp +++ b/src/vcpkg/metrics.cpp @@ -593,7 +593,7 @@ namespace vcpkg vcpkg_curl_easy_setopt(curl, CURLOPT_URL, "https://dc.services.visualstudio.com/v2/track"); vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); - vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.length())); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, static_cast(payload.length())); vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); vcpkg_curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L); if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2) != CURLE_OK) From 647594d423f663b5a6e5e019e6b8efe36bdcef3f Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 18:31:42 -0800 Subject: [PATCH 35/40] Implement NetworkErrorProxyMightHelp in the two switches speaking DownloadPrognosis --- src/vcpkg/base/downloads.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 395c8950e8..ee72e238ba 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -433,11 +433,11 @@ namespace vcpkg enum class DownloadPrognosis { Success, - OtherError, - NetworkErrorProxyMightHelp, // Transient error means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500, 502, 503 or // 504 response code. https://everything.curl.dev/usingcurl/downloads/retry.html#retry - TransientNetworkError + TransientNetworkError, + OtherError, + NetworkErrorProxyMightHelp, }; static bool check_combine_download_prognosis(DownloadPrognosis& target, DownloadPrognosis individual_call) @@ -445,15 +445,23 @@ namespace vcpkg switch (individual_call) { case DownloadPrognosis::Success: return true; - case DownloadPrognosis::OtherError: + case DownloadPrognosis::TransientNetworkError: if (target == DownloadPrognosis::Success) + { + target = DownloadPrognosis::TransientNetworkError; + } + + return false; + case DownloadPrognosis::OtherError: + if (target == DownloadPrognosis::Success || target == DownloadPrognosis::TransientNetworkError) { target = DownloadPrognosis::OtherError; } return false; case DownloadPrognosis::NetworkErrorProxyMightHelp: - if (target == DownloadPrognosis::Success || target == DownloadPrognosis::OtherError) + if (target == DownloadPrognosis::Success || target == DownloadPrognosis::TransientNetworkError || + target == DownloadPrognosis::OtherError) { target = DownloadPrognosis::NetworkErrorProxyMightHelp; } @@ -852,6 +860,7 @@ namespace vcpkg out_sha512)) { case DownloadPrognosis::Success: return DownloadPrognosis::Success; + case DownloadPrognosis::TransientNetworkError: return DownloadPrognosis::TransientNetworkError; case DownloadPrognosis::OtherError: return download_file_script_asset_cache(context, asset_cache_settings, From d9d10e5904662b5d1de9c5652c0ad681d23df4d8 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Wed, 28 Jan 2026 18:51:34 -0800 Subject: [PATCH 36/40] Use "warning" for retries. Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/vcpkg/base/downloads.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index ee72e238ba..72fda2101c 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -595,8 +595,10 @@ namespace vcpkg return prognosis; } - context.report_error(msg::format( - msgDownloadTransientErrorRetry, msg::count = attempt_count + 1, msg::value = attempt_delays.size())); + context.statusln(DiagnosticKind::Warning, + msg::format(msgDownloadTransientErrorRetry, + msg::count = attempt_count + 1, + msg::value = attempt_delays.size())); } if (DownloadPrognosis::Success != prognosis) From d07cf6b55f124ede856c8f7bf1a2f22c804de6bc Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 18:56:15 -0800 Subject: [PATCH 37/40] Improve ReadFilePointer::size. --- src/vcpkg/base/files.cpp | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index e6eda6de4f..e1ad6d72cd 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -37,6 +37,7 @@ #include #if defined(_WIN32) +#include #include #include @@ -1578,7 +1579,34 @@ namespace vcpkg { ec.clear(); #if _WIN32 - return stdfs::file_size(to_stdfs_path(m_path), ec); + if (!m_fs) + { + ec.assign(EBADF, std::generic_category()); + return 0; + } + + const auto file_number = ::_fileno(m_fs); + if (file_number == -1) + { + ec.assign(errno, std::generic_category()); + return 0; + } + + const auto os_handle = ::_get_osfhandle(file_number); + if (os_handle == -1) + { + ec.assign(errno, std::generic_category()); + return 0; + } + + LARGE_INTEGER size; + if (!::GetFileSizeEx(reinterpret_cast(os_handle), &size)) + { + ec.assign(::GetLastError(), std::system_category()); + return 0; + } + + return static_cast(size.QuadPart); #else struct stat st; if (::fstat(::fileno(m_fs), &st) != 0) From ea811880420e730253453defcadcfcc498c01931 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 28 Jan 2026 18:58:19 -0800 Subject: [PATCH 38/40] Fix copilot's status suggestion to actually build. --- src/vcpkg/base/downloads.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 72fda2101c..8c3efb335a 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -595,10 +595,8 @@ namespace vcpkg return prognosis; } - context.statusln(DiagnosticKind::Warning, - msg::format(msgDownloadTransientErrorRetry, - msg::count = attempt_count + 1, - msg::value = attempt_delays.size())); + context.statusln(msg::format( + msgDownloadTransientErrorRetry, msg::count = attempt_count + 1, msg::value = attempt_delays.size())); } if (DownloadPrognosis::Success != prognosis) From a87b9fd2b0d9b7a79e8cb289eaa685bf4b04ac37 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 29 Jan 2026 11:07:37 -0800 Subject: [PATCH 39/40] Reorder retries to avoid sleep(0) --- src/vcpkg/base/downloads.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 8c3efb335a..76755e4db2 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -576,27 +576,27 @@ namespace vcpkg // Transient error means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500, 502, 503 or // 504 response code. https://everything.curl.dev/usingcurl/downloads/retry.html#retry using namespace std::chrono_literals; - static constexpr std::array attempt_delays = {0s, 1s, 2s}; + static constexpr std::array attempt_delays = {1s, 2s}; DownloadPrognosis prognosis = DownloadPrognosis::NetworkErrorProxyMightHelp; for (size_t attempt_count = 0; attempt_count < attempt_delays.size(); attempt_count++) { - std::this_thread::sleep_for(attempt_delays[attempt_count]); - prognosis = perform_download(context, machine_readable_progress, raw_url, download_path_part_path, headers); - if (DownloadPrognosis::Success == prognosis) { break; } - if (DownloadPrognosis::TransientNetworkError != prognosis) + if (DownloadPrognosis::TransientNetworkError == prognosis) { - context.report_error(msg::format(msgDownloadNotTransientErrorWontRetry, msg::url = sanitized_url)); - return prognosis; + context.statusln(msg::format(msgDownloadTransientErrorRetry, + msg::count = attempt_count + 1, + msg::value = attempt_delays.size() + 1)); + std::this_thread::sleep_for(attempt_delays[attempt_count]); + continue; } - context.statusln(msg::format( - msgDownloadTransientErrorRetry, msg::count = attempt_count + 1, msg::value = attempt_delays.size())); + context.report_error(msg::format(msgDownloadNotTransientErrorWontRetry, msg::url = sanitized_url)); + return prognosis; } if (DownloadPrognosis::Success != prognosis) From eb21f0d1c1b6bf96631e39fde19db0bb44062545 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 10 Feb 2026 15:46:53 -0800 Subject: [PATCH 40/40] Treat all FTP 4xx errors as retry-able --- include/vcpkg/base/message-data.inc.h | 4 ++-- locales/messages.json | 4 ++-- src/vcpkg/base/downloads.cpp | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 1b81dcbe14..16bf0cd6dd 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -990,10 +990,10 @@ DECLARE_MESSAGE(CurlFailedGeneric, "curl is the name of a program, see curl.se.", "curl operation failed with error code {exit_code}") DECLARE_MESSAGE(CurlDownloadTimeout, (), "", "Download timed out.") -DECLARE_MESSAGE(CurlFailedHttpResponse, +DECLARE_MESSAGE(CurlFailedResponse, (msg::exit_code), "curl is the name of a program, see curl.se.", - "curl operation failed with HTTP response code {exit_code}.") + "curl operation failed with response code {exit_code}.") DECLARE_MESSAGE(CurlFailedToPut, (msg::url, msg::value), "curl is the name of a program, see curl.se. {value} is an HTTP status code", diff --git a/locales/messages.json b/locales/messages.json index 27e3d15da8..36e28899ec 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -573,8 +573,8 @@ "CurlDownloadTimeout": "Download timed out.", "CurlFailedGeneric": "curl operation failed with error code {exit_code}", "_CurlFailedGeneric.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.", - "CurlFailedHttpResponse": "curl operation failed with HTTP response code {exit_code}.", - "_CurlFailedHttpResponse.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.", + "CurlFailedResponse": "curl operation failed with response code {exit_code}.", + "_CurlFailedResponse.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.", "CurlFailedToPut": "curl failed to PUT file to {url} with response code {value}.", "_CurlFailedToPut.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.", "CurrentCommitBaseline": "You can use the current commit as a baseline, which is:\n\t\"builtin-baseline\": \"{commit_sha}\"", diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 76755e4db2..22f1bb9b3f 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -534,15 +534,15 @@ namespace vcpkg return DownloadPrognosis::Success; } - if (response_code == 429 || response_code == 408 || response_code == 500 || response_code == 502 || - response_code == 503 || response_code == 504) + context.report_error(msg::format(msgCurlFailedResponse, msg::exit_code = static_cast(response_code))); + + if ((raw_url.starts_with("ftp://") && response_code >= 400 && response_code < 500) || + (response_code == 429 || response_code == 408 || response_code == 500 || response_code == 502 || + response_code == 503 || response_code == 504)) { - context.report_error( - msg::format(msgCurlFailedHttpResponse, msg::exit_code = static_cast(response_code))); return DownloadPrognosis::TransientNetworkError; } - context.report_error(msg::format(msgCurlFailedHttpResponse, msg::exit_code = static_cast(response_code))); return DownloadPrognosis::NetworkErrorProxyMightHelp; }