diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d52ddf01a..880a9115fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,8 @@ if(DEFINE_DISABLE_METRICS OR VCPKG_DISABLE_METRICS) "file vcpkg.disable_metrics next to the binary.") endif() +set(VCPKG_MANIFEST_DIR "${CMAKE_CURRENT_LIST_DIR}/src" CACHE PATH "Path to vcpkg manifest directory") + project(vcpkg DESCRIPTION "vcpkg helps you manage C and C++ libraries on Windows, Linux and MacOS." HOMEPAGE_URL "https://github.com/microsoft/vcpkg" @@ -195,6 +197,7 @@ set(TEST_SCRIPT_ASSET_CACHE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/test-script find_package(fmt REQUIRED) find_package(CMakeRC REQUIRED) +find_package(LibCURL REQUIRED) # === Target: locale-resources === @@ -229,6 +232,8 @@ target_compile_definitions(vcpkglib PUBLIC _FILE_OFFSET_BITS=64 ) +target_link_libraries(vcpkglib PUBLIC CURL::libcurl) + if(VCPKG_STANDALONE_BUNDLE_SHA) target_compile_definitions(vcpkglib PUBLIC "VCPKG_STANDALONE_BUNDLE_SHA=${VCPKG_STANDALONE_BUNDLE_SHA}" diff --git a/CMakePresets.json b/CMakePresets.json index 73983af5c0..8e6d8cb62e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -6,8 +6,9 @@ "hidden": true, "cacheVariables": { "VCPKG_OFFICIAL_BUILD": true, - "VCPKG_BASE_VERSION": "2023-09-15", - "VCPKG_STANDALONE_BUNDLE_SHA": "8b28c1829802a133941805c68004427052588ba6eefbdf9fb6061151a92c131491df7e29470309e804450c075d2f0673515d5d8c19997148ebcb2874493d304d" + "VCPKG_BASE_VERSION": "2025-12-16", + "VCPKG_STANDALONE_BUNDLE_SHA": "23c77d1dd70bf861328a8e35203aed2db0deb9a83aa924cadaf96ffaae42e8629363184b99168e33158b819d695c748bd7cb9eb39528bd374f8b7e2ab6d4f6de", + "VCPKG_ARTIFACTS_SHA": "def65b1f4a710c0b521603a275ff6bae31ad8c5b938cd4445fb69c4d0da97c21d753c274c9b9b1cc2f8a86ba694759d4f3f4f325f88be02a5d8ad10c8f56e5df" } }, { @@ -39,6 +40,7 @@ { "name": "windows", "hidden": true, + "toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake", "cacheVariables": { "VCPKG_BUILD_TLS12_DOWNLOADER": true }, @@ -64,6 +66,9 @@ "architecture": { "value": "x64", "strategy": "external" + }, + "cacheVariables": { + "VCPKG_TARGET_TRIPLET": "x64-windows-static" } }, { @@ -165,7 +170,8 @@ "name": "linux-arm64-ci", "inherits": [ "linux-ci" ], "cacheVariables": { - "CMAKE_SYSTEM_PROCESSOR": "aarch64" + "CMAKE_SYSTEM_PROCESSOR": "aarch64", + "VCPKG_LIBCURL_DLSYM_UPDATED_HEADERS": true } }, { diff --git a/NOTICE.txt b/NOTICE.txt index 7b5c798535..3d26b1b4b5 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -71,6 +71,35 @@ SOFTWARE. ========================================= END OF CMakeRC NOTICES, INFORMATION, AND LICENSE +curl + +%% curl NOTICES, INFORMATION, AND LICENSE BEGIN HERE +========================================= +COPYRIGHT AND PERMISSION NOTICE + +Copyright (C) Daniel Stenberg, , and many +contributors, see the THANKS file. + +All rights reserved. + +Permission to use, copy, modify, and distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright +notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization of the copyright holder. +========================================= +END OF curl NOTICES, INFORMATION, AND LICENSE + The following third party software is incorporated into vcpkg-artifacts: --------------------------------------------------------- diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index ef7e956088..91ba717751 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -48,7 +48,8 @@ Throw-IfNotFailed $expected = @( "A suitable version of cmake was not found \(required v[0-9.]+\)\.", "Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-z]+", -"error: curl: \(37\) Couldn't open file [^\n]+", +"error: curl operation failed with error code 37 \((Couldn't|Could not) read a file:\/\/ file\)\.", +"error: Not a transient network error, won't retry download from file://$assetCacheRegex/[0-9a-z]+" "error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", @@ -100,7 +101,8 @@ if (-not ($actual -match $expected)) { Refresh-TestRoot $expected = @( "^Downloading https://localhost:1234/foobar\.html -> example3\.html", -"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"error: curl operation failed with error code 7 \((Couldn't|Could not) connect to server\)\.", +"error: Not a transient network error, won't retry download from https://localhost:1234/foobar\.html", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https://address:port``\.", @@ -123,9 +125,11 @@ if (-not ($actual -match $expected)) { Refresh-TestRoot $expected = @( "^Downloading example3\.html, trying https://localhost:1234/foobar\.html", +"error: curl operation failed with error code 7 \((Couldn't|Could not) connect to server\)\.", +"error: Not a transient network error, won't retry download from https://localhost:1234/foobar\.html", "Trying https://localhost:1235/baz\.html", -"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", -"error: curl: \(7\) Failed to connect to localhost port 1235( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"error: curl operation failed with error code 7 \((Couldn't|Could not) connect to server\)\.", +"error: Not a transient network error, won't retry download from https://localhost:1235/baz\.html", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https://address:port``\.", @@ -177,34 +181,12 @@ if (-not ($actual -match $expected)) { } # ... also with multiple authoritative URLs -if ($IsWindows) { - # WinHTTP - Refresh-TestRoot - $expected = @( - "^Downloading example3\.html, trying https://nonexistent\.example\.com", - "warning: Download https://nonexistent\.example\.com failed -- retrying after 1000ms", - "warning: Download https://nonexistent\.example\.com failed -- retrying after 2000ms", - "warning: Download https://nonexistent\.example\.com failed -- retrying after 4000ms", - "Trying https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt", - "Successfully downloaded example3\.html", - "$" - ) -join "`n" - - $actual = Run-VcpkgAndCaptureOutput @commonArgs x-download "$TestDownloadsRoot/example3.html" --sha512 65077997890f66f6041bb3284bb7b88e27631411ccbc253201ca4e00c4bcc58c0d77edffda4975498797cc10772c7fd68fbeb13cc4ac493a3471a9d49e5b6f24 --url https://nonexistent.example.com --url https://raw.githubusercontent.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE.txt - Throw-IfFailed - if (-not ($actual -match $expected)) { - throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" - } -} - -# Force curl with --header Refresh-TestRoot $expected = @( "^Downloading example3\.html, trying https://nonexistent\.example\.com", -"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in 1 seconds?\. 3 retries left\.", -"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in \d+ seconds?\. 2 retries left\.", -"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in \d+ seconds?\. 1 (retries|retry) left\.", -"Trying https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt", +"error: curl operation failed with error code 6 \((Couldn't|Could not) resolve (hostname|host name)\)\.", +"error: Not a transient network error, won't retry download from https://nonexistent\.example\.com", +"Trying https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt" "Successfully downloaded example3\.html", "$" ) -join "`n" @@ -212,7 +194,7 @@ $expected = @( $actual = Run-VcpkgAndCaptureOutput @commonArgs x-download "$TestDownloadsRoot/example3.html" --sha512 65077997890f66f6041bb3284bb7b88e27631411ccbc253201ca4e00c4bcc58c0d77edffda4975498797cc10772c7fd68fbeb13cc4ac493a3471a9d49e5b6f24 --url https://nonexistent.example.com --url https://raw.githubusercontent.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE.txt --header "Cache-Control: no-cache" Throw-IfFailed if (-not ($actual -match $expected)) { - throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" + throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed), headers (cache-control)" } # azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a) @@ -236,8 +218,10 @@ Refresh-TestRoot $expected = @( "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", "Asset cache miss; trying authoritative source https://localhost:1234/foobar\.html", -"error: curl: \(37\) Couldn't open file [^\n]+", -"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"error: curl operation failed with error code 37 \((Couldn't|Could not) read a file:// file\)\.", +"error: Not a transient network error, won't retry download from file://$assetCacheRegex/[0-9a-z]+", +"error: curl operation failed with error code 7 \((Couldn't|Could not) connect to server\)\.", +"error: Not a transient network error, won't retry download from https://localhost:1234/foobar\.html", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https://address:port``\.", @@ -296,7 +280,11 @@ if (-not ($actual -match $expected)) { $expected = @( "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", "Asset cache miss; trying authoritative source https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt", -"error: curl: \(37\) Couldn't open file [^\n]+", +"error: curl operation failed with error code 37 \((Couldn't|Could not) read a file:// file\)\.", +"error: Not a transient network error, won't retry download from file://$assetCacheRegex/[0-9a-z]+", +"[^\n]+example3\.html\.\d+\.part: error: download from https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt had an unexpected hash", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", +"note: Actual : 65077997890f66f6041bb3284bb7b88e27631411ccbc253201ca4e00c4bcc58c0d77edffda4975498797cc10772c7fd68fbeb13cc4ac493a3471a9d49e5b6f24", "note: If you are using a proxy, please ensure your proxy settings are correct\.", "Possible causes are:", "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https://address:port``\.", @@ -306,9 +294,6 @@ $expected = @( "The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", "3\. Your proxy's remote server is out of service\.", "If you believe this is not a temporary download server failure and vcpkg needs to be changed to download this file from a different location, please submit an issue to https://github\.com/Microsoft/vcpkg/issues", -"[^\n]+example3\.html\.\d+\.part: error: download from https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt had an unexpected hash", -"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", -"note: Actual : 65077997890f66f6041bb3284bb7b88e27631411ccbc253201ca4e00c4bcc58c0d77edffda4975498797cc10772c7fd68fbeb13cc4ac493a3471a9d49e5b6f24", "$" ) -join "`n" $actual = Run-VcpkgAndCaptureOutput @commonArgs x-download "$TestDownloadsRoot/example3.html" --sha512 d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b --url https://raw.githubusercontent.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE.txt "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite" @@ -358,7 +343,8 @@ if (-not ($actual -match $expected)) { Refresh-TestRoot $expected = @( "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", -"error: curl: \(37\) Couldn't open file [^\n]+", +"error: curl operation failed with error code 37 \((Couldn't|Could not) read a file:// file\)\.", +"error: Not a transient network error, won't retry download from file://$assetCacheRegex/[0-9a-z]+", "error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://raw\.githubusercontent\.com/microsoft/vcpkg-tool/1767aaee7b229c609f7ad5cf2f57b6a6cc309fb8/LICENSE\.txt", "note: or https://alternate\.example\.com", "note: If you are using a proxy, please ensure your proxy settings are correct\.", diff --git a/azure-pipelines/signing-nuget.config b/azure-pipelines/signing-nuget.config new file mode 100644 index 0000000000..7ac764c952 --- /dev/null +++ b/azure-pipelines/signing-nuget.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/azure-pipelines/signing.yml b/azure-pipelines/signing.yml index 17c080709d..51dd7477d8 100644 --- a/azure-pipelines/signing.yml +++ b/azure-pipelines/signing.yml @@ -226,7 +226,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-amd64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-linux/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-amd64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-linux/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_LIBCURL_URL=$(curl-tarball-url) -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-glibc" - job: muslc_build displayName: 'muslc (Alpine) Build' @@ -263,7 +263,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_CXX_FLAGS=\"-static -s -static-libgcc -static-libstdc++\" -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-alpine:3.16 sh -c "cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DCMAKE_CXX_FLAGS=\"-s -static-libgcc -static-libstdc++\" -DVCPKG_LIBCURL_URL=$(curl-tarball-url) -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-muslc" - job: glibc_arm64_build displayName: 'glibc Arm64 Build' @@ -301,7 +301,7 @@ extends: inlineScript: | az acr login --name vcpkgpmeofficialbuilders --resource-group vcpkg-tool-official-builds --subscription c0f11a1f-38f5-4908-8698-1aa5df75baf3 mkdir -p "$(Agent.TempDirectory)/build" - docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-arm64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-arm64/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" + docker run --rm --mount "type=bind,source=$(Build.Repository.LocalPath),target=/source,readonly" --mount "type=bind,source=$(Agent.TempDirectory)/build,target=/build" vcpkgpmeofficialbuilders-c7ajd0chdtfugffn.azurecr.io/vcpkg/vcpkg-build-linux-arm64:2025-07-28 sh -c "cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE=/source/azure-pipelines/vcpkg-arm64/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_LIBCURL_URL=$(curl-tarball-new-url) -DVCPKG_LIBCURL_DLSYM_UPDATED_HEADERS=ON -DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url) -DVCPKG_FMT_URL=$(fmt-tarball-url) -DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA) -DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA) -DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION) -DVCPKG_VERSION=$(Build.SourceVersion) -S /source -B /build 2>&1 && ninja -C /build" mv "$(Agent.TempDirectory)/build/vcpkg" "$(Build.ArtifactStagingDirectory)/vcpkg-glibc-arm64" - job: windows_and_sign displayName: 'Build Windows binaries and Sign' @@ -364,6 +364,15 @@ extends: packagesToPush: '$(Build.ArtifactStagingDirectory)/vs-insertion/drop/VS.Redist.Vcpkg.amd64.1.0.0-$(VCPKG_FULL_VERSION).nupkg' publishVstsFeed: '97a41293-2972-4f48-8c0e-05493ae82010' steps: + - task: NuGetToolInstaller@1 + inputs: + versionSpec: 5.7 + - task: NuGetCommand@2 + displayName: 'Extract Microsoft.Build.Vcpkg' + inputs: + command: custom + feedsToUse: config + arguments: 'install Microsoft.Build.Vcpkg -Version 2026.1.26.329-f14401ca0f -OutputDirectory "$(Build.BinariesDirectory)" -ConfigFile "$(Build.SourcesDirectory)\azure-pipelines\signing-nuget.config" -ExcludeVersion -NonInteractive' - task: CmdLine@2 displayName: "Build vcpkg amd64 with CMake" inputs: @@ -371,7 +380,8 @@ extends: script: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 cmake.exe --version - cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\amd64" 2>&1 + set X_VCPKG_ASSET_SOURCES=x-script,$(Build.BinariesDirectory)\Microsoft.Build.Vcpkg\trt\TerrapinRetrievalTool.exe -b https://vcpkg.storage.devpackages.microsoft.io/artifacts/ -a true -u Environment -p {url} -s {sha512} -d {dst};x-block-origin + cmake.exe -G Ninja -DCMAKE_TOOLCHAIN_FILE="$(Build.BinariesDirectory)\Microsoft.Build.Vcpkg\tools\scripts\buildsystems\vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=x64-windows-static-release -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\amd64" 2>&1 ninja.exe -C "$(Build.BinariesDirectory)\amd64" - task: CmdLine@2 displayName: "Build vcpkg arm64 with CMake" @@ -380,11 +390,9 @@ extends: script: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=arm64 -host_arch=amd64 cmake.exe --version - cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_PDB_SUFFIX="-arm64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\arm64" 2>&1 + set X_VCPKG_ASSET_SOURCES=x-script,$(Build.BinariesDirectory)\Microsoft.Build.Vcpkg\trt\TerrapinRetrievalTool.exe -b https://vcpkg.storage.devpackages.microsoft.io/artifacts/ -a true -u Environment -p {url} -s {sha512} -d {dst};x-block-origin + cmake.exe -G Ninja -DCMAKE_TOOLCHAIN_FILE="$(Build.BinariesDirectory)\Microsoft.Build.Vcpkg\tools\scripts\buildsystems\vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=arm64-windows-static-release -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -DVCPKG_BUILD_TLS12_DOWNLOADER=ON -DVCPKG_EMBED_GIT_SHA=ON -DVCPKG_OFFICIAL_BUILD=ON -DVCPKG_PDB_SUFFIX="-arm64" "-DVCPKG_FMT_URL=$(fmt-tarball-url)" "-DVCPKG_CMAKERC_URL=$(cmakerc-tarball-url)" "-DVCPKG_BASE_VERSION=$(VCPKG_BASE_VERSION)" "-DVCPKG_VERSION=$(Build.SourceVersion)" "-DVCPKG_STANDALONE_BUNDLE_SHA=$(VCPKG_STANDALONE_BUNDLE_SHA)" "-DVCPKG_ARTIFACTS_SHA=$(VCPKG_ARTIFACTS_SHA)" -B "$(Build.BinariesDirectory)\arm64" 2>&1 ninja.exe -C "$(Build.BinariesDirectory)\arm64" - - task: NuGetToolInstaller@1 - inputs: - versionSpec: 5.7 - task: NuGetCommand@2 displayName: 'NuGet Restore MicroBuild Signing Extension' inputs: diff --git a/cmake/FindLibCURL.cmake b/cmake/FindLibCURL.cmake new file mode 100644 index 0000000000..af4576b5e7 --- /dev/null +++ b/cmake/FindLibCURL.cmake @@ -0,0 +1,59 @@ +if (WIN32 OR APPLE) + set(VCPKG_LIBCURL_DLSYM_DEFAULT "OFF") +else() + set(VCPKG_LIBCURL_DLSYM_DEFAULT "ON") +endif() + +option(VCPKG_LIBCURL_DLSYM "Use dlsym to dynamically load libcurl at runtime instead of using the system libcurl" "${VCPKG_LIBCURL_DLSYM_DEFAULT}") +option(VCPKG_LIBCURL_DLSYM_UPDATED_HEADERS "Use more recent libcurl headers when using DLSYM than 7.29.0" OFF) + +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() + +if (NOT VCPKG_LIBCURL_DLSYM) + find_package(CURL REQUIRED) + return() +endif() + +# The URI option exists to allow the URI to be replaced with a Microsoft-internal URI in official +# builds which have restricted internet access; see azure-pipelines/signing.yml +# Note that the SHA512 is the same, so vcpkg-tool contributors need not be concerned that we built +# with different content. +if (VCPKG_LIBCURL_DLSYM_UPDATED_HEADERS) + # curl-7_55_1 for https://daniel.haxx.se/blog/2017/06/15/target-independent-libcurl-headers/ + # (mainly for arm64 support) + set(VCPKG_LIBCURL_HASH 4b0bf36a978b8b5ba66aecedbc2ae8ae9230da63ba5b80f9553d96671e013ccd679ee9cc10946c50b94d640858d74f3ec5d4e198c6b9f8842c941986d275cf7a) + if(NOT VCPKG_LIBCURL_URL) + set(VCPKG_LIBCURL_URL "https://curl.se/download/curl-7.55.1.tar.gz") + endif() +else() + # curl-7_29_0 headers here because that's what comes with RHEL 7 (and inherited by similarly ancient Oracle Linux 7) + set(VCPKG_LIBCURL_HASH 08bafd09fa6d14362a426932fed8528c13133895477d8134c829e085637956d66d6be5a791057c1c04da04af6baa6496a6d59e00abf9ca6be5d29e798718b9bc) + if(NOT VCPKG_LIBCURL_URL) + set(VCPKG_LIBCURL_URL "https://curl.se/download/archeology/curl-7.29.0.tar.gz") + endif() +endif() + +if(NOT TARGET CURL::libcurl) +include(FetchContent) + FetchContent_Declare( + LibCURLHeaders + URL "${VCPKG_LIBCURL_URL}" + URL_HASH "SHA512=${VCPKG_LIBCURL_HASH}" + ) + + FetchContent_GetProperties(LibCURLHeaders) + # This dance is done rather than `FetchContent_MakeAvailable` because we only want to download + # curl's headers for use with dlopen/dlsym rather than building curl. + if(NOT LibCURLHeaders_POPULATED) + FetchContent_Populate(LibCURLHeaders) + endif() + + add_library(CURL::libcurl INTERFACE IMPORTED) + + set_target_properties(CURL::libcurl PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${libcurlheaders_SOURCE_DIR}/include" + INTERFACE_COMPILE_DEFINITIONS "VCPKG_LIBCURL_DLSYM=1" + ) +endif() diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h index f00ef4f10b..2703c055ef 100644 --- a/include/vcpkg/base/contractual-constants.h +++ b/include/vcpkg/base/contractual-constants.h @@ -589,4 +589,9 @@ namespace vcpkg inline constexpr StringLiteral StatusInstalled = "installed"; inline constexpr StringLiteral StatusNotInstalled = "not-installed"; inline constexpr StringLiteral StatusPurge = "purge"; + + // App Insights JSON response fields + inline constexpr StringLiteral AppInsightsResponseItemsReceived = "itemsReceived"; + inline constexpr StringLiteral AppInsightsResponseItemsAccepted = "itemsAccepted"; + inline constexpr StringLiteral AppInsightsResponseErrors = "errors"; } diff --git a/include/vcpkg/base/curl.h b/include/vcpkg/base/curl.h new file mode 100644 index 0000000000..d033da7483 --- /dev/null +++ b/include/vcpkg/base/curl.h @@ -0,0 +1,124 @@ +#pragma once + +#include + +#include + +#include + +VCPKG_MSVC_WARNING(push) +// note: disable warning triggered by curl headers +// ws2tcpip.h(968): warning C6101: Returning uninitialized memory '*Mtu': A successful path through the function does +// not set the named _Out_ parameter. +VCPKG_MSVC_WARNING(disable : 6101) +#include +#include +VCPKG_MSVC_WARNING(pop) + +#ifdef VCPKG_LIBCURL_DLSYM +void vcpkg_curl_global_init(long flags); + +// these are filled in by the call to vcpkg_curl_global_init +extern decltype(&curl_easy_cleanup) vcpkg_curl_easy_cleanup; +extern decltype(&curl_easy_getinfo) vcpkg_curl_easy_getinfo; +extern decltype(&curl_easy_init) vcpkg_curl_easy_init; +extern decltype(&curl_easy_perform) vcpkg_curl_easy_perform; +extern decltype(&curl_easy_setopt) vcpkg_curl_easy_setopt; +extern decltype(&curl_easy_strerror) vcpkg_curl_easy_strerror; + +extern decltype(&curl_multi_add_handle) vcpkg_curl_multi_add_handle; +extern decltype(&curl_multi_cleanup) vcpkg_curl_multi_cleanup; +extern decltype(&curl_multi_info_read) vcpkg_curl_multi_info_read; +extern decltype(&curl_multi_init) vcpkg_curl_multi_init; +extern decltype(&curl_multi_remove_handle) vcpkg_curl_multi_remove_handle; +extern decltype(&curl_multi_strerror) vcpkg_curl_multi_strerror; +extern decltype(&curl_multi_perform) vcpkg_curl_multi_perform; +extern decltype(&curl_multi_wait) vcpkg_curl_multi_poll; // or _wait, if _poll is not present + +extern decltype(&curl_slist_append) vcpkg_curl_slist_append; +extern decltype(&curl_slist_free_all) vcpkg_curl_slist_free_all; + +extern decltype(&curl_version) vcpkg_curl_version; +#else // ^^^ VCPKG_LIBCURL_DLSYM / !VCPKG_LIBCURL_DLSYM vvv +#define vcpkg_curl_global_init(flags) curl_global_init(flags) + +#define vcpkg_curl_easy_cleanup(handle) curl_easy_cleanup(handle) +#define vcpkg_curl_easy_getinfo(handle, info, data) curl_easy_getinfo(handle, info, data) +#define vcpkg_curl_easy_init() curl_easy_init() +#define vcpkg_curl_easy_perform(handle) curl_easy_perform(handle) +#define vcpkg_curl_easy_setopt(handle, option, parameter) curl_easy_setopt(handle, option, parameter) +#define vcpkg_curl_easy_strerror(code) curl_easy_strerror(code) + +#define vcpkg_curl_multi_add_handle(multi_handle, easy_handle) curl_multi_add_handle(multi_handle, easy_handle) +#define vcpkg_curl_multi_cleanup(multi_handle) curl_multi_cleanup(multi_handle) +#define vcpkg_curl_multi_info_read(multi_handle, messages_in_queue) \ + curl_multi_info_read(multi_handle, messages_in_queue) +#define vcpkg_curl_multi_init() curl_multi_init() +#define vcpkg_curl_multi_remove_handle(multi_handle, easy_handle) curl_multi_remove_handle(multi_handle, easy_handle) +#define vcpkg_curl_multi_strerror(code) curl_multi_strerror(code) +#define vcpkg_curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, numfds) \ + curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, numfds) +#define vcpkg_curl_multi_perform(multi_handle, running_handles) curl_multi_perform(multi_handle, running_handles) + +#define vcpkg_curl_slist_append(list, string) curl_slist_append(list, string) +#define vcpkg_curl_slist_free_all(list) curl_slist_free_all(list) + +#define vcpkg_curl_version() curl_version() +#endif // ^^^ !VCPKG_LIBCURL_DLSYM + +namespace vcpkg +{ + struct CurlEasyHandle + { + CurlEasyHandle() = default; + CurlEasyHandle(CurlEasyHandle&& other) noexcept; + CurlEasyHandle& operator=(CurlEasyHandle&& other) noexcept; + ~CurlEasyHandle(); + + void swap(CurlEasyHandle& other) noexcept; + + CURL* get(); + + private: + CURL* m_ptr = nullptr; + }; + + struct CurlMultiHandle + { + CurlMultiHandle(); + CurlMultiHandle(CurlMultiHandle&& other) noexcept; + CurlMultiHandle& operator=(CurlMultiHandle&& other) noexcept; + ~CurlMultiHandle(); + + void swap(CurlMultiHandle& other) noexcept; + + // Adds an easy handle to the multi handle but doesn't take ownership of it. + // Makes sure that the easy handle is removed from the multi handle on cleanup. + void add_easy_handle(CurlEasyHandle& easy_handle); + + CURLM* get(); + + private: + CURLM* m_ptr = nullptr; + std::vector m_easy_handles; + }; + + struct CurlHeaders + { + CurlHeaders() = default; + CurlHeaders(View headers); + CurlHeaders(CurlHeaders&& other) noexcept; + CurlHeaders& operator=(CurlHeaders&& other) noexcept; + ~CurlHeaders(); + + void swap(CurlHeaders& other) noexcept; + + curl_slist* get() const; + + private: + curl_slist* m_headers = nullptr; + }; + + constexpr char vcpkg_curl_user_agent[] = + "vcpkg/" VCPKG_BASE_VERSION_AS_STRING "-" VCPKG_VERSION_AS_STRING " (curl)"; +} diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 068ab463b3..e996024570 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -25,32 +25,11 @@ namespace vcpkg std::string m_sanitized_url; }; - struct SplitUrlView - { - StringView scheme; - Optional authority; - StringView path_query_fragment; - }; - - // e.g. {"https","//example.org", "/index.html"} - Optional parse_split_url_view(StringView raw_url); - View azure_blob_headers(); - // Parses a curl output line for curl invoked with - // -w "PREFIX%{http_code} %{exitcode} %{errormsg}" - // with specific handling for curl version < 7.75.0 which does not understand %{exitcode} %{errormsg} - // If the line is malformed for any reason, no entry to http_codes is added. - // Returns: true if the new version of curl's output with exitcode and errormsg was parsed; otherwise, false. - bool parse_curl_status_line(DiagnosticContext& context, - std::vector& http_codes, - StringLiteral prefix, - StringView this_line); - std::vector download_files_no_cache(DiagnosticContext& context, View> url_pairs, - View headers, - View secrets); + View headers); bool submit_github_dependency_graph_snapshot(DiagnosticContext& context, const Optional& maybe_github_server_url, @@ -58,19 +37,7 @@ namespace vcpkg const std::string& github_repository, const Json::Object& snapshot); - Optional invoke_http_request(DiagnosticContext& context, - StringLiteral method, - View headers, - StringView url, - View secrets, - StringView data = {}); - - std::string format_url_query(StringView base_url, View query_params); - - std::vector url_heads(DiagnosticContext& context, - View urls, - View headers, - View secrets); + std::vector url_heads(DiagnosticContext& context, View urls, View headers); struct AssetCachingSettings { @@ -107,7 +74,6 @@ namespace vcpkg bool store_to_asset_cache(DiagnosticContext& context, StringView raw_url, const SanitizedUrl& sanitized_url, - StringLiteral method, View headers, const Path& file); @@ -121,26 +87,6 @@ namespace vcpkg const SanitizedUrl& sanitized_url, const Path& file); - Optional try_parse_curl_max5_size(StringView sv); - - struct CurlProgressData - { - unsigned int total_percent; - unsigned long long total_size; - unsigned int received_percent; - unsigned long long received_size; - unsigned int transfer_percent; - unsigned long long transfer_size; - unsigned long long average_download_speed; // bytes per second - unsigned long long average_upload_speed; // bytes per second - // ElapsedTime total_time; - // ElapsedTime time_spent; - // ElapsedTime time_left; - unsigned long long current_speed; - }; - - Optional try_parse_curl_progress_data(StringView curl_progress_line); - // Replaces spaces with %20 for purposes of including in a URL. // This is typically used to filter a command line passed to `x-download` or similar which // might contain spaces that we, in turn, pass to curl. diff --git a/include/vcpkg/base/files.h b/include/vcpkg/base/files.h index d9b08a46fc..546e0c2975 100644 --- a/include/vcpkg/base/files.h +++ b/include/vcpkg/base/files.h @@ -103,6 +103,8 @@ namespace vcpkg // reads any remaining chunks of the file; used to implement read_to_end void read_to_end_suffix( std::string& output, std::error_code& ec, char* buffer, size_t buffer_size, size_t last_read); + uint64_t size(LineInfo li) const; + uint64_t size(std::error_code& ec) const; }; struct WriteFilePointer : FilePointer diff --git a/include/vcpkg/base/fwd/downloads.h b/include/vcpkg/base/fwd/downloads.h index c51accb232..5313a16292 100644 --- a/include/vcpkg/base/fwd/downloads.h +++ b/include/vcpkg/base/fwd/downloads.h @@ -3,7 +3,5 @@ namespace vcpkg { struct SanitizedUrl; - struct SplitUrlView; struct AssetCachingSettings; - struct CurlProgressData; } diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index fbdd6932e6..16bf0cd6dd 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -988,22 +988,16 @@ DECLARE_MESSAGE(CreationFailed, (msg::path), "", "Creating {path} failed.") DECLARE_MESSAGE(CurlFailedGeneric, (msg::exit_code), "curl is the name of a program, see curl.se.", - "curl operation failed with error code {exit_code}.") + "curl operation failed with error code {exit_code}") +DECLARE_MESSAGE(CurlDownloadTimeout, (), "", "Download timed out.") +DECLARE_MESSAGE(CurlFailedResponse, + (msg::exit_code), + "curl is the name of a program, see curl.se.", + "curl operation failed with response code {exit_code}.") DECLARE_MESSAGE(CurlFailedToPut, - (msg::exit_code, msg::url), - "curl is the name of a program, see curl.se", - "curl failed to put file to {url} with exit code {exit_code}.") -DECLARE_MESSAGE(CurlFailedToPutHttp, - (msg::exit_code, msg::url, msg::value), + (msg::url, msg::value), "curl is the name of a program, see curl.se. {value} is an HTTP status code", - "curl failed to put file to {url} with exit code {exit_code} and http code {value}.") -DECLARE_MESSAGE( - CurlFailedToReturnExpectedNumberOfExitCodes, - (msg::exit_code, msg::command_line), - "", - "curl failed to return the expected number of exit codes; this can happen if something terminates curl " - "before it has finished. curl exited with {exit_code} which is normally the result code for the last operation, " - "but may be the result of a crash. The command line was {command_line}, and all output is below:") + "curl failed to PUT file to {url} with response code {value}.") DECLARE_MESSAGE(CurrentCommitBaseline, (msg::commit_sha), "", @@ -1085,10 +1079,6 @@ DECLARE_MESSAGE( (msg::sha), "", "failing download because the expected SHA512 was all zeros, please change the expected SHA512 to: {sha}") -DECLARE_MESSAGE(DownloadFailedRetrying, - (msg::value, msg::url), - "{value} is a number of milliseconds", - "Download {url} failed -- retrying after {value}ms") DECLARE_MESSAGE(DownloadFailedStatusCode, (msg::url, msg::value), "{value} is an HTTP status code", @@ -1125,6 +1115,18 @@ DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundle, (msg::version), "", "Downloadi DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundleLatest, (), "", "Downloading latest standalone bundle.") DECLARE_MESSAGE(DownloadingTools, (msg::count), "", "Downloading {count} tools") DECLARE_MESSAGE(DownloadOrUrl, (msg::url), "", "or {url}") +DECLARE_MESSAGE(DownloadTransientErrorRetry, + (msg::count, msg::value), + "{value} is the maximum number of attempts to download a file", + "Attempt {count} of {value}, retrying download.") +DECLARE_MESSAGE(DownloadTransientErrorRetriesExhausted, + (msg::url), + "", + "Reached maximum number of attempts, won't retry download from {url}.") +DECLARE_MESSAGE(DownloadNotTransientErrorWontRetry, + (msg::url), + "", + "Not a transient network error, won't retry download from {url}") DECLARE_MESSAGE(DownloadTryingAuthoritativeSource, (msg::url), "", "Trying {url}") DECLARE_MESSAGE(DownloadRootsDir, (msg::env_var), "", "Downloads directory (default: {env_var})") DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}") @@ -1132,10 +1134,6 @@ DECLARE_MESSAGE(DownloadSuccesfulUploading, (msg::path, msg::url), "", "Successfully downloaded {path}, storing to {url}") -DECLARE_MESSAGE(DownloadWinHttpError, - (msg::system_api, msg::exit_code, msg::url), - "", - "{url}: {system_api} failed with exit code {exit_code}.") DECLARE_MESSAGE(DuplicateDependencyOverride, (msg::package_name), "", "{package_name} already has an override") DECLARE_MESSAGE(DuplicatedKeyInObj, (msg::value), @@ -2600,7 +2598,7 @@ DECLARE_MESSAGE( "the license is not installed to ${{CURRENT_PACKAGES_DIR}}/share/${{PORT}}/copyright . This can be fixed by adding " "a call to vcpkg_install_copyright. To suppress this message, add set(VCPKG_POLICY_SKIP_COPYRIGHT_CHECK enabled)") DECLARE_MESSAGE(PortBugMissingLicenseFixIt, - (msg ::value), + (msg::value), "{value} is a CMake function call for the user to paste into their file, for example: " "vcpkg_install_copyright(FILE_LIST ${{SOURCE_PATH}}/COPYING ${{SOURCE_PATH}}/LICENSE.txt)", "Consider adding: {value}") @@ -2862,6 +2860,11 @@ DECLARE_MESSAGE(TwoFeatureFlagsSpecified, (msg::value), "'{value}' is a feature flag.", "Both '{value}' and -'{value}' were specified as feature flags.") +DECLARE_MESSAGE(UnableToFindCurl, + (), + "", + "vcpkg was unable to find a libcurl.so.4, libcurl-gnutls.so.4, or libcurl-nss.so.4 to use on this " + "system. Please install libcurl from your system package manager and retry vcpkg.") DECLARE_MESSAGE(UnableToReadAppDatas, (), "", "both %LOCALAPPDATA% and %APPDATA% were unreadable") DECLARE_MESSAGE(UnableToReadEnvironmentVariable, (msg::env_var), "", "unable to read {env_var}") DECLARE_MESSAGE(UndeterminedToolChainForTriplet, @@ -3160,7 +3163,6 @@ DECLARE_MESSAGE(VcpkgUsage, "[]s, or --s should be preserved. @response_file should be localized to be consistent with the message " "named 'ResponseFileCode'.", "usage: vcpkg [--switches] [--options=values] [arguments] @response_file") -DECLARE_MESSAGE(InvalidUri, (msg::value), "{value} is the URI we attempted to parse.", "unable to parse uri: {value}") DECLARE_MESSAGE(VcpkgInVsPrompt, (msg::value, msg::triplet), "'{value}' is a VS prompt", diff --git a/include/vcpkg/metrics.h b/include/vcpkg/metrics.h index 07d750439d..e81ea0c95b 100644 --- a/include/vcpkg/metrics.h +++ b/include/vcpkg/metrics.h @@ -198,7 +198,6 @@ namespace vcpkg extern std::atomic g_should_send_metrics; void flush_global_metrics(const Filesystem&); -#if defined(_WIN32) - void winhttp_upload_metrics(StringView payload); -#endif // ^^^ _WIN32 + bool curl_upload_metrics(const std::string& payload); + bool parse_metrics_response(StringView response_body); } diff --git a/locales/messages.json b/locales/messages.json index c58fe8a1b7..36e28899ec 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -570,14 +570,13 @@ "CreatingZipArchive": "Creating zip archive...", "CreationFailed": "Creating {path} failed.", "_CreationFailed.comment": "An example of {path} is /foo/bar.", - "CurlFailedGeneric": "curl operation failed with error code {exit_code}.", + "CurlDownloadTimeout": "Download timed out.", + "CurlFailedGeneric": "curl operation failed with error code {exit_code}", "_CurlFailedGeneric.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.", - "CurlFailedToPut": "curl failed to put file to {url} with exit code {exit_code}.", - "_CurlFailedToPut.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", - "CurlFailedToPutHttp": "curl failed to put file to {url} with exit code {exit_code} and http code {value}.", - "_CurlFailedToPutHttp.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", - "CurlFailedToReturnExpectedNumberOfExitCodes": "curl failed to return the expected number of exit codes; this can happen if something terminates curl before it has finished. curl exited with {exit_code} which is normally the result code for the last operation, but may be the result of a crash. The command line was {command_line}, and all output is below:", - "_CurlFailedToReturnExpectedNumberOfExitCodes.comment": "An example of {exit_code} is 127. An example of {command_line} is vcpkg install zlib.", + "CurlFailedResponse": "curl operation failed with response code {exit_code}.", + "_CurlFailedResponse.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.", + "CurlFailedToPut": "curl failed to PUT file to {url} with response code {value}.", + "_CurlFailedToPut.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.", "CurrentCommitBaseline": "You can use the current commit as a baseline, which is:\n\t\"builtin-baseline\": \"{commit_sha}\"", "_CurrentCommitBaseline.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.", "CycleDetectedDuring": "cycle detected during {spec}:", @@ -625,10 +624,10 @@ "DownloadFailedHashMismatchZero": "failing download because the expected SHA512 was all zeros, please change the expected SHA512 to: {sha}", "_DownloadFailedHashMismatchZero.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", "DownloadFailedProxySettings": "If you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https://address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See: https://github.com/microsoft/vcpkg-tool/pull/77\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is out of service.\nIf you believe this is not a temporary download server failure and vcpkg needs to be changed to download this file from a different location, please submit an issue to https://github.com/Microsoft/vcpkg/issues", - "DownloadFailedRetrying": "Download {url} failed -- retrying after {value}ms", - "_DownloadFailedRetrying.comment": "{value} is a number of milliseconds An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadFailedStatusCode": "{url}: failed: status code {value}", "_DownloadFailedStatusCode.comment": "{value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadNotTransientErrorWontRetry": "Not a transient network error, won't retry download from {url}", + "_DownloadNotTransientErrorWontRetry.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadOrUrl": "or {url}", "_DownloadOrUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadRootsDir": "Downloads directory (default: {env_var})", @@ -637,10 +636,12 @@ "_DownloadSuccesful.comment": "An example of {path} is /foo/bar.", "DownloadSuccesfulUploading": "Successfully downloaded {path}, storing to {url}", "_DownloadSuccesfulUploading.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadTransientErrorRetriesExhausted": "Reached maximum number of attempts, won't retry download from {url}.", + "_DownloadTransientErrorRetriesExhausted.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadTransientErrorRetry": "Attempt {count} of {value}, retrying download.", + "_DownloadTransientErrorRetry.comment": "{value} is the maximum number of attempts to download a file An example of {count} is 42.", "DownloadTryingAuthoritativeSource": "Trying {url}", "_DownloadTryingAuthoritativeSource.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", - "DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}.", - "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadedSources": "Downloaded sources for {spec}", "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.", "DownloadingAssetShaToFile": "Downloading asset cache entry {sha} -> {path}", @@ -1104,8 +1105,6 @@ "InvalidToolVersion": "Invalid tool version; expected a string containing a substring of between 1 and 3 numbers separated by dots.", "InvalidTriplet": "Invalid triplet: {triplet}", "_InvalidTriplet.comment": "An example of {triplet} is x64-windows.", - "InvalidUri": "unable to parse uri: {value}", - "_InvalidUri.comment": "{value} is the URI we attempted to parse.", "InvalidValueHashAdditionalFiles": "Variable VCPKG_HASH_ADDITIONAL_FILES contains invalid file path: '{path}'. The value must be an absolute path to an existent file.", "_InvalidValueHashAdditionalFiles.comment": "An example of {path} is /foo/bar.", "InvalidValuePostPortfileIncludes": "Variable VCPKG_POST_PORTFILE_INCLUDES contains invalid file path: '{path}'. The value must be an absolute path to an existent cmake file.", @@ -1493,6 +1492,7 @@ "TripletLabel": "Triplet:", "TwoFeatureFlagsSpecified": "Both '{value}' and -'{value}' were specified as feature flags.", "_TwoFeatureFlagsSpecified.comment": "'{value}' is a feature flag.", + "UnableToFindCurl": "vcpkg was unable to find a libcurl.so.4, libcurl-gnutls.so.4, or libcurl-nss.so.4 to use on this system. Please install libcurl from your system package manager and retry vcpkg.", "UnableToReadAppDatas": "both %LOCALAPPDATA% and %APPDATA% were unreadable", "UnableToReadEnvironmentVariable": "unable to read {env_var}", "_UnableToReadEnvironmentVariable.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", diff --git a/src/vcpkg-configuration.json b/src/vcpkg-configuration.json new file mode 100644 index 0000000000..3bb948f6d4 --- /dev/null +++ b/src/vcpkg-configuration.json @@ -0,0 +1,7 @@ +{ + "default-registry": { + "kind": "git", + "baseline": "a2a478a93d582a4b395a4dc4b7052bfcb42c1f8e", + "repository": "https://github.com/microsoft/vcpkg" + } +} diff --git a/src/vcpkg-test/catch.cpp b/src/vcpkg-test/catch.cpp index f296f80bc5..e3a03b712a 100644 --- a/src/vcpkg-test/catch.cpp +++ b/src/vcpkg-test/catch.cpp @@ -1,6 +1,7 @@ #define CATCH_CONFIG_RUNNER #include +#include #include #include @@ -11,6 +12,7 @@ namespace vcpkg::Checks int main(int argc, char** argv) { + vcpkg_curl_global_init(CURL_GLOBAL_DEFAULT); if (vcpkg::get_environment_variable("VCPKG_DEBUG").value_or("") == "1") vcpkg::Debug::g_debugging = true; // We set VCPKG_ROOT to an invalid value to ensure unit tests do not attempt to instantiate VcpkgRoot vcpkg::set_environment_variable("VCPKG_ROOT", "VCPKG_TESTS_SHOULD_NOT_USE_VCPKG_ROOT"); diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp index 3df1949a8e..dba4b05005 100644 --- a/src/vcpkg-test/downloads.cpp +++ b/src/vcpkg-test/downloads.cpp @@ -18,300 +18,25 @@ using namespace vcpkg; } \ } while (0) -TEST_CASE ("parse_split_url_view", "[downloads]") -{ - { - auto x = parse_split_url_view("https://github.com/Microsoft/vcpkg"); - if (auto v = x.get()) - { - REQUIRE(v->scheme == "https"); - REQUIRE(v->authority.value_or("") == "//github.com"); - REQUIRE(v->path_query_fragment == "/Microsoft/vcpkg"); - } - else - { - FAIL(); - } - } - { - REQUIRE(!parse_split_url_view("").has_value()); - REQUIRE(!parse_split_url_view("hello").has_value()); - } - { - auto x = parse_split_url_view("file:"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(!y->authority.has_value()); - REQUIRE(y->path_query_fragment == ""); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("file:path"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(!y->authority.has_value()); - REQUIRE(y->path_query_fragment == "path"); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("file:/path"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(!y->authority.has_value()); - REQUIRE(y->path_query_fragment == "/path"); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("file://user:pw@host"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(y->authority.value_or("") == "//user:pw@host"); - REQUIRE(y->path_query_fragment == ""); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("ftp://host:port/"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "ftp"); - REQUIRE(y->authority.value_or("") == "//host:port"); - REQUIRE(y->path_query_fragment == "/"); - } - else - { - FAIL(); - } - } - { - auto x = parse_split_url_view("file://D:\\work\\testing\\asset-cache/" - "562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42b69e4aeb3" - "1a4879e1bcf6f7c2d41f2ace12180ea83ba7af48879d40ab"); - if (auto y = x.get()) - { - REQUIRE(y->scheme == "file"); - REQUIRE(y->authority.value_or("") == "//D:\\work\\testing\\asset-cache"); - REQUIRE(y->path_query_fragment == "/562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42" - "b69e4aeb31a4879e1bcf6f7c2d41f2ace12180ea83ba7af48879d40ab"); - } - else - { - FAIL(); - } - } -} - -TEST_CASE ("parse_curl_status_line", "[downloads]") -{ - std::vector http_codes; - StringLiteral malformed_examples[] = { - "asdfasdf", // wrong prefix - "curl: unknown --write-out variable: 'exitcode'", // wrong prefixes, and also what old curl does - "curl: unknown --write-out variable: 'errormsg'", - "prefix", // missing spaces - "prefix42", // missing spaces - "prefix42 2", // missing space - "prefix42 2a", // non numeric exitcode - }; - - FullyBufferedDiagnosticContext bdc; - for (auto&& malformed : malformed_examples) - { - REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", malformed)); - REQUIRE(http_codes.empty()); - REQUIRE(bdc.empty()); - } - - // old curl output - REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix200 ")); - REQUIRE(http_codes == std::vector{200}); - REQUIRE(bdc.empty()); - http_codes.clear(); - - REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix404 ")); - REQUIRE(http_codes == std::vector{404}); - REQUIRE(bdc.empty()); - http_codes.clear(); - - REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix0 ")); // a failure, but we don't know that yet - REQUIRE(http_codes == std::vector{0}); - REQUIRE(bdc.empty()); - http_codes.clear(); - - // current curl output - REQUIRE(parse_curl_status_line(bdc, http_codes, "prefix", "prefix200 0 ")); - REQUIRE(http_codes == std::vector{200}); - REQUIRE(bdc.empty()); - http_codes.clear(); - - REQUIRE(parse_curl_status_line( - bdc, - http_codes, - "prefix", - "prefix0 60 schannel: SNI or certificate check failed: SEC_E_WRONG_PRINCIPAL (0x80090322) " - "- The target principal name is incorrect.")); - REQUIRE(http_codes == std::vector{0}); - REQUIRE(bdc.to_string() == - "error: curl operation failed with error code 60. schannel: SNI or certificate check failed: " - "SEC_E_WRONG_PRINCIPAL (0x80090322) - The target principal name is incorrect."); -} - TEST_CASE ("download_files", "[downloads]") { auto const dst = Test::base_temporary_directory() / "download_files"; - auto const url = [&](std::string l) -> auto { return std::pair(l, dst); }; + real_filesystem.create_directories(dst, VCPKG_LINE_INFO); + + static const std::vector> test_downloads{ + {"unknown://localhost:9/secret", dst / "test1"}, + {"http://localhost:9/not-exists/secret", dst / "test2"}, + }; FullyBufferedDiagnosticContext bdc; std::vector headers; - std::vector secrets; - auto results = download_files_no_cache( - bdc, - std::vector{url("unknown://localhost:9/secret"), url("http://localhost:9/not-exists/secret")}, - headers, - secrets); - REQUIRE(results == std::vector{0, 0}); - auto all_errors = bdc.to_string(); - if (all_errors == "error: curl operation failed with error code 7.") - { - // old curl, this is OK! - } - else - { - // new curl - REQUIRE_THAT( - all_errors, - Catch::Matches("error: curl operation failed with error code 1\\. Protocol \"unknown\" not supported( or " - "disabled in libcurl)?\n" - "error: curl operation failed with error code 7\\. ((Failed to connect to localhost port 9 " - "after [0-9]+ ms: ((Could not|Couldn't) connect to server|Connection refused))" - "|(getsockname\\(\\) failed with errno 22: Invalid argument))", - Catch::CaseSensitive::Yes)); - } -} - -TEST_CASE ("try_parse_curl_max5_size", "[downloads]") -{ - REQUIRE(!try_parse_curl_max5_size("").has_value()); - REQUIRE(!try_parse_curl_max5_size("hi").has_value()); - REQUIRE(try_parse_curl_max5_size("0").value_or_exit(VCPKG_LINE_INFO) == 0ull); - REQUIRE(try_parse_curl_max5_size("1").value_or_exit(VCPKG_LINE_INFO) == 1ull); - REQUIRE(try_parse_curl_max5_size("10").value_or_exit(VCPKG_LINE_INFO) == 10ull); - REQUIRE(!try_parse_curl_max5_size("10 ").has_value()); // no unknown suffixes - REQUIRE(try_parse_curl_max5_size("100").value_or_exit(VCPKG_LINE_INFO) == 100ull); - REQUIRE(try_parse_curl_max5_size("100").value_or_exit(VCPKG_LINE_INFO) == 100ull); - REQUIRE(try_parse_curl_max5_size("1000").value_or_exit(VCPKG_LINE_INFO) == 1000ull); - REQUIRE(!try_parse_curl_max5_size("1000.").has_value()); // dot needs 1 or 2 digits - REQUIRE(!try_parse_curl_max5_size("1000.k").has_value()); - // fails in parsing the number: - REQUIRE(!try_parse_curl_max5_size("18446744073709551616").has_value()); - - // suffixes are 1024'd - REQUIRE(try_parse_curl_max5_size("1k").value_or_exit(VCPKG_LINE_INFO) == (1ull << 10)); - REQUIRE(try_parse_curl_max5_size("1M").value_or_exit(VCPKG_LINE_INFO) == (1ull << 20)); - REQUIRE(try_parse_curl_max5_size("1G").value_or_exit(VCPKG_LINE_INFO) == (1ull << 30)); - REQUIRE(try_parse_curl_max5_size("1T").value_or_exit(VCPKG_LINE_INFO) == (1ull << 40)); - REQUIRE(try_parse_curl_max5_size("1P").value_or_exit(VCPKG_LINE_INFO) == (1ull << 50)); - REQUIRE(!try_parse_curl_max5_size("1a").has_value()); - - // 1.3*1024 == 1'331.2 - REQUIRE(try_parse_curl_max5_size("1.3k").value_or_exit(VCPKG_LINE_INFO) == 1331ull); - // 1.33*1024 == 1'361.92 - REQUIRE(try_parse_curl_max5_size("1.33k").value_or_exit(VCPKG_LINE_INFO) == 1361ull); - - // 1.3*1024*1024 == 1'363'148.8 - REQUIRE(try_parse_curl_max5_size("1.3M").value_or_exit(VCPKG_LINE_INFO) == 1363148ull); - // 1.33*1024*1024 == 1'394'606.08 - REQUIRE(try_parse_curl_max5_size("1.33M").value_or_exit(VCPKG_LINE_INFO) == 1394606ull); - - // 1.3*1024*1024*1024 == 1'395'864'371.2 - REQUIRE(try_parse_curl_max5_size("1.3G").value_or_exit(VCPKG_LINE_INFO) == 1395864371ull); - // 1.33*1024*1024*1024 == 1'428'076'625.92 - REQUIRE(try_parse_curl_max5_size("1.33G").value_or_exit(VCPKG_LINE_INFO) == 1428076625ull); - - // 1.3*1024*1024*1024*1024 == 1'429'365'116'108.8 - REQUIRE(try_parse_curl_max5_size("1.3T").value_or_exit(VCPKG_LINE_INFO) == 1429365116108ull); - // 1.33*1024*1024*1024*1024 == 1'462'350'464'942.08 - REQUIRE(try_parse_curl_max5_size("1.33T").value_or_exit(VCPKG_LINE_INFO) == 1462350464942ull); - - // 1.3*1024*1024*1024*1024*1024 == 1'463'669'878'895'411.2 - REQUIRE(try_parse_curl_max5_size("1.3P").value_or_exit(VCPKG_LINE_INFO) == 1463669878895411ull); - // 1.33*1024*1024*1024*1024*1024 == 1'497'446'876'100'689.92 - REQUIRE(try_parse_curl_max5_size("1.33P").value_or_exit(VCPKG_LINE_INFO) == 1497446876100689ull); -} - -TEST_CASE ("try_parse_curl_progress_data", "[downloads]") -{ - // % Total % Received % Xferd Average Speed Time Time Time Current - // Dload Upload Total Spent Left Speed - // - // 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 - // 100 242 100 242 0 0 298 0 --:--:-- --:--:-- --:--:-- 298 - // 100 242 100 242 0 0 297 0 --:--:-- --:--:-- --:--:-- 297 - // - // 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 - // 0 190M 0 511k 0 0 199k 0 0:16:19 0:00:02 0:16:17 548k - // 0 190M 0 1423k 0 0 410k 0 0:07:55 0:00:03 0:07:52 776k - // 1 190M 1 2159k 0 0 468k 0 0:06:56 0:00:04 0:06:52 726k - // 1 190M 1 2767k 0 0 499k 0 0:06:30 0:00:05 0:06:25 709k - // 1 190M 1 3327k 0 0 507k 0 0:06:24 0:00:06 0:06:18 676k - // 2 190M 2 3935k 0 0 519k 0 0:06:15 0:00:07 0:06:08 683k - - REQUIRE( - !try_parse_curl_progress_data(" % Total % Received % Xferd Average Speed Time Time Time Current") - .has_value()); - - REQUIRE( - !try_parse_curl_progress_data(" Dload Upload Total Spent Left Speed") - .has_value()); - - { - const auto out = try_parse_curl_progress_data( - " 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0") - .value_or_exit(VCPKG_LINE_INFO); - REQUIRE(out.total_percent == 0); - REQUIRE(out.total_size == 0); - REQUIRE(out.received_percent == 0); - REQUIRE(out.received_size == 0); - REQUIRE(out.transfer_percent == 0); - REQUIRE(out.transfer_size == 0); - REQUIRE(out.average_upload_speed == 0); - REQUIRE(out.average_download_speed == 0); - REQUIRE(out.current_speed == 0); - } - - { - const auto out = try_parse_curl_progress_data( - " 2 190M 2 3935k 0 0 519k 0 0:06:15 0:00:07 0:06:08 683k") - .value_or_exit(VCPKG_LINE_INFO); - REQUIRE(out.total_percent == 2); - REQUIRE(out.total_size == 190 * 1024 * 1024); - REQUIRE(out.received_percent == 2); - REQUIRE(out.received_size == 3935 * 1024); - REQUIRE(out.transfer_percent == 0); - REQUIRE(out.transfer_size == 0); - REQUIRE(out.average_upload_speed == 0); - REQUIRE(out.average_download_speed == 519 * 1024); - REQUIRE(out.current_speed == 683 * 1024); - } + auto results = download_files_no_cache(bdc, test_downloads, headers); + REQUIRE(results == std::vector{-1, -1}); + auto all_errors = Strings::split(bdc.to_string(), '\n'); + REQUIRE(all_errors[0] == "error: curl operation failed with error code 1 (Unsupported protocol)."); + // Old versions of libcurl use "Couldn't" on error messages + REQUIRE((all_errors[1] == "error: curl operation failed with error code 7 (Could not connect to server)." || + all_errors[1] == "error: curl operation failed with error code 7 (Couldn't connect to server).")); } TEST_CASE ("url_encode_spaces", "[downloads]") @@ -374,7 +99,7 @@ TEST_CASE ("azblob", "[.][azblob]") FullyBufferedDiagnosticContext diagnostics{}; auto plain_put_success = store_to_asset_cache( - diagnostics, plain_put_url, SanitizedUrl{url, {}}, "PUT", azure_blob_headers(), data_filepath); + diagnostics, plain_put_url, SanitizedUrl{url, {}}, azure_blob_headers(), data_filepath); INFO(diagnostics.to_string()); CHECK(plain_put_success); } @@ -393,7 +118,7 @@ TEST_CASE ("azblob", "[.][azblob]") { FullyBufferedDiagnosticContext diagnostics{}; - auto results = download_files_no_cache(diagnostics, url_pairs, azure_blob_headers(), {}); + auto results = download_files_no_cache(diagnostics, url_pairs, azure_blob_headers()); INFO(diagnostics.to_string()); CHECK(results == std::vector{200, 200}); } diff --git a/src/vcpkg-test/metrics.cpp b/src/vcpkg-test/metrics.cpp index dbd24d08a2..e069ae4826 100644 --- a/src/vcpkg-test/metrics.cpp +++ b/src/vcpkg-test/metrics.cpp @@ -256,3 +256,39 @@ TEST_CASE ("payload smoke test", "[metrics]") )json"; REQUIRE(expected == actual); } + +TEST_CASE ("parse metrics response", "[metrics]") +{ + const std::string response = R"json( +{ + "itemsReceived": 1, + "itemsAccepted": 1, + "errors": [] +} +)json"; + auto parsed = parse_metrics_response(response); + CHECK(parsed); + + const std::string response_with_errors = R"json( +{ + "itemsReceived": 2, + "itemsAccepted": 1, + "errors": [ + { + "message": "Invalid payload" + } + ] +} +)json"; + auto parsed_with_errors = parse_metrics_response(response_with_errors); + CHECK(!parsed_with_errors); + + const std::string response_with_errors2 = R"json( +{ + "itemsReceived": 2, + "errors": [] +} +)json"; + auto parsed_with_errors2 = parse_metrics_response(response_with_errors2); + CHECK(!parsed_with_errors2); +} diff --git a/src/vcpkg.cpp b/src/vcpkg.cpp index 0dc64e15ea..b696e08221 100644 --- a/src/vcpkg.cpp +++ b/src/vcpkg.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -31,11 +32,6 @@ #pragma comment(lib, "shell32") #endif -#if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) -#define TEST_LIBCURL_AVAILABLE -#include -#endif - using namespace vcpkg; namespace @@ -57,26 +53,6 @@ namespace Checks::exit_fail(VCPKG_LINE_INFO); } - Optional detect_libcurl() - { - // Determine if libcurl.so.4 is installed in the system by attempting to load it - // At the moment we don't do anything with it, but we're tracking availability - // of libcurl to replace the current download/upload implementation -#if defined(TEST_LIBCURL_AVAILABLE) - // calling dlclose() on the handle after calling curl_version() causes asan to - // report a false leak, so we intentionally don't unload the library - if (auto handle = dlopen("libcurl.so.4", RTLD_NOW | RTLD_LOCAL)) - { - auto curl_version_fn = reinterpret_cast(dlsym(handle, "curl_version")); - if (!dlerror() && curl_version_fn) - { - return {curl_version_fn()}; - } - } -#endif - return nullopt; - } - bool detect_container(const Filesystem& fs) { (void)fs; @@ -135,12 +111,19 @@ namespace void inner(const Filesystem& fs, const VcpkgCmdArguments& args, const BundleSettings& bundle) { + vcpkg_curl_global_init(CURL_GLOBAL_DEFAULT); + // track version on each invocation get_global_metrics_collector().track_string(StringMetric::VcpkgVersion, vcpkg_executable_version); get_global_metrics_collector().track_bool(BoolMetric::DetectedContainer, detect_container(fs)); - get_global_metrics_collector().track_string(StringMetric::DetectedLibCurlVersion, - detect_libcurl().value_or("unknown")); + + const auto* detected_curl_version = vcpkg_curl_version(); + if (!detected_curl_version) + { + detected_curl_version = "unknown"; + } + get_global_metrics_collector().track_string(StringMetric::DetectedLibCurlVersion, detected_curl_version); if (args.get_command().empty()) { diff --git a/src/vcpkg.json b/src/vcpkg.json new file mode 100644 index 0000000000..4d64541c44 --- /dev/null +++ b/src/vcpkg.json @@ -0,0 +1,16 @@ +{ + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "brotli", + "http2", + "non-http", + "ssl", + "zstd" + ], + "platform": "windows" + } + ] +} diff --git a/src/vcpkg/base/curl.cpp b/src/vcpkg/base/curl.cpp new file mode 100644 index 0000000000..6b829bc119 --- /dev/null +++ b/src/vcpkg/base/curl.cpp @@ -0,0 +1,207 @@ +#include +#include + +#ifdef VCPKG_LIBCURL_DLSYM +#include + +// these are filled in by the first call to vcpkg_curl_global_init +decltype(&curl_easy_cleanup) vcpkg_curl_easy_cleanup; +decltype(&curl_easy_getinfo) vcpkg_curl_easy_getinfo; +decltype(&curl_easy_init) vcpkg_curl_easy_init; +decltype(&curl_easy_perform) vcpkg_curl_easy_perform; +decltype(&curl_easy_setopt) vcpkg_curl_easy_setopt; +decltype(&curl_easy_strerror) vcpkg_curl_easy_strerror; + +decltype(&curl_multi_add_handle) vcpkg_curl_multi_add_handle; +decltype(&curl_multi_cleanup) vcpkg_curl_multi_cleanup; +decltype(&curl_multi_info_read) vcpkg_curl_multi_info_read; +decltype(&curl_multi_init) vcpkg_curl_multi_init; +decltype(&curl_multi_remove_handle) vcpkg_curl_multi_remove_handle; +decltype(&curl_multi_strerror) vcpkg_curl_multi_strerror; +decltype(&curl_multi_perform) vcpkg_curl_multi_perform; +decltype(&curl_multi_wait) vcpkg_curl_multi_poll; // or _wait, if _poll is not present + +decltype(&curl_slist_append) vcpkg_curl_slist_append; +decltype(&curl_slist_free_all) vcpkg_curl_slist_free_all; + +decltype(&curl_version) vcpkg_curl_version; + +template +static void load_symbol(FnT(&target), void* handle, const char* symbol_name) +{ + target = reinterpret_cast(dlsym(handle, symbol_name)); + if (!target) + { + vcpkg::Checks::unreachable(VCPKG_LINE_INFO); + } +} + +void vcpkg_curl_global_init(long flags) +{ + // calling dlclose() on the handle after calling curl_version() causes asan to + // report a false leak, so we intentionally don't unload the library + auto handle = dlopen("libcurl.so.4", RTLD_NOW | RTLD_LOCAL); + if (!handle) + { + // Ubuntu 16.04 has this if the user only installs `curl` + handle = dlopen("libcurl-gnutls.so.4", RTLD_NOW | RTLD_LOCAL); + } + if (!handle) + { + // It's possible that someone explicitly installs this one + handle = dlopen("libcurl-nss.so.4", RTLD_NOW | RTLD_LOCAL); + } + + if (!handle) + { + vcpkg::Checks::msg_exit_with_error(VCPKG_LINE_INFO, vcpkg::msgUnableToFindCurl); + } + + { + decltype(&curl_global_init) global_init; + load_symbol(global_init, handle, "curl_global_init"); + global_init(flags); + } + + load_symbol(vcpkg_curl_easy_cleanup, handle, "curl_easy_cleanup"); + load_symbol(vcpkg_curl_easy_getinfo, handle, "curl_easy_getinfo"); + load_symbol(vcpkg_curl_easy_init, handle, "curl_easy_init"); + load_symbol(vcpkg_curl_easy_perform, handle, "curl_easy_perform"); + load_symbol(vcpkg_curl_easy_setopt, handle, "curl_easy_setopt"); + load_symbol(vcpkg_curl_easy_strerror, handle, "curl_easy_strerror"); + + load_symbol(vcpkg_curl_multi_add_handle, handle, "curl_multi_add_handle"); + load_symbol(vcpkg_curl_multi_cleanup, handle, "curl_multi_cleanup"); + load_symbol(vcpkg_curl_multi_info_read, handle, "curl_multi_info_read"); + load_symbol(vcpkg_curl_multi_init, handle, "curl_multi_init"); + load_symbol(vcpkg_curl_multi_remove_handle, handle, "curl_multi_remove_handle"); + load_symbol(vcpkg_curl_multi_strerror, handle, "curl_multi_strerror"); + load_symbol(vcpkg_curl_multi_perform, handle, "curl_multi_perform"); + // try to load curl_multi_poll first, fall back to curl_multi_wait + vcpkg_curl_multi_poll = reinterpret_cast(dlsym(handle, "curl_multi_poll")); + if (!vcpkg_curl_multi_poll) + { + load_symbol(vcpkg_curl_multi_poll, handle, "curl_multi_wait"); + } + + load_symbol(vcpkg_curl_slist_append, handle, "curl_slist_append"); + load_symbol(vcpkg_curl_slist_free_all, handle, "curl_slist_free_all"); + + load_symbol(vcpkg_curl_version, handle, "curl_version"); +} +#endif // ^^^ VCPKG_LIBCURL_DLSYM + +namespace vcpkg +{ + CurlEasyHandle::CurlEasyHandle(CurlEasyHandle&& other) noexcept : m_ptr(std::exchange(other.m_ptr, nullptr)) { } + CurlEasyHandle& CurlEasyHandle::operator=(CurlEasyHandle&& other) noexcept + { + CurlEasyHandle{std::move(other)}.swap(*this); + return *this; + } + CurlEasyHandle::~CurlEasyHandle() + { + if (m_ptr) + { + vcpkg_curl_easy_cleanup(m_ptr); + } + } + void CurlEasyHandle::swap(CurlEasyHandle& other) noexcept { std::swap(m_ptr, other.m_ptr); } + CURL* CurlEasyHandle::get() + { + if (!m_ptr) + { + m_ptr = vcpkg_curl_easy_init(); + if (!m_ptr) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + } + return m_ptr; + } + + CurlMultiHandle::CurlMultiHandle() = default; + CurlMultiHandle::CurlMultiHandle(CurlMultiHandle&& other) noexcept + : m_ptr(std::exchange(other.m_ptr, nullptr)), m_easy_handles(std::move(other.m_easy_handles)) + { + } + CurlMultiHandle& CurlMultiHandle::operator=(CurlMultiHandle&& other) noexcept + { + CurlMultiHandle{std::move(other)}.swap(*this); + return *this; + } + CurlMultiHandle::~CurlMultiHandle() + { + if (!m_ptr) + { + return; + } + + for (auto* easy_handle : m_easy_handles) + { + vcpkg_curl_multi_remove_handle(m_ptr, easy_handle); + } + + vcpkg_curl_multi_cleanup(m_ptr); + } + void CurlMultiHandle::swap(CurlMultiHandle& other) noexcept + { + using std::swap; + swap(m_ptr, other.m_ptr); + swap(m_easy_handles, other.m_easy_handles); + } + void CurlMultiHandle::add_easy_handle(CurlEasyHandle& easy_handle) + { + auto* handle = easy_handle.get(); + m_easy_handles.push_back(handle); + if (vcpkg_curl_multi_add_handle(this->get(), handle) != CURLM_OK) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + } + CURLM* CurlMultiHandle::get() + { + if (!m_ptr) + { + m_ptr = vcpkg_curl_multi_init(); + if (!m_ptr) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + } + return m_ptr; + } + + CurlHeaders::CurlHeaders(View headers) + { + for (const auto& header : headers) + { + auto new_headers = vcpkg_curl_slist_append(m_headers, header.c_str()); + if (!new_headers) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + + m_headers = new_headers; + } + } + CurlHeaders::CurlHeaders(CurlHeaders&& other) noexcept : m_headers(std::exchange(other.m_headers, nullptr)) { } + CurlHeaders& CurlHeaders::operator=(CurlHeaders&& other) noexcept + { + CurlHeaders{std::move(other)}.swap(*this); + return *this; + } + CurlHeaders::~CurlHeaders() + { + if (m_headers) + { + vcpkg_curl_slist_free_all(m_headers); + } + } + void CurlHeaders::swap(CurlHeaders& other) noexcept + { + using std::swap; + swap(m_headers, other.m_headers); + } + curl_slist* CurlHeaders::get() const { return m_headers; } +} diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 4019e87cae..22f1bb9b3f 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -15,24 +16,26 @@ #include #include -#include - #include using namespace vcpkg; namespace { - constexpr StringLiteral vcpkg_curl_user_agent_header = - "User-Agent: vcpkg/" VCPKG_BASE_VERSION_AS_STRING "-" VCPKG_VERSION_AS_STRING " (curl)"; - - void add_curl_headers(Command& cmd, View headers) + void set_common_curl_easy_options(CurlEasyHandle& easy_handle, StringView url, const CurlHeaders& request_headers) { - cmd.string_arg("-H").string_arg(vcpkg_curl_user_agent_header); - for (auto&& header : headers) - { - cmd.string_arg("-H").string_arg(header); - } + auto* curl = easy_handle.get(); + vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + vcpkg_curl_easy_setopt(curl, CURLOPT_URL, url_encode_spaces(url).c_str()); + vcpkg_curl_easy_setopt(curl, + CURLOPT_FOLLOWLOCATION, + 2L); // Follow redirects, change request method based on HTTP response code. + // https://curl.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html#CURLFOLLOWOBEYCODE + vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + + // don't send headers to proxy CONNECT ; this intentionally fails on older versions of libcurl + vcpkg_curl_easy_setopt( + curl, static_cast(229) /* CURLOPT_HEADEROPT */, (1L << 0) /* CURLHEADER_SEPARATE */); } } @@ -44,566 +47,6 @@ namespace vcpkg replace_secrets(m_sanitized_url, secrets); } -#if defined(_WIN32) - struct FormatMessageHLocalAlloc - { - LPWSTR buffer = nullptr; - - ~FormatMessageHLocalAlloc() - { - if (buffer) - { - LocalFree(buffer); - } - } - }; - - static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, - const SanitizedUrl& sanitized_url, - DWORD last_error) - { - const HMODULE winhttp_module = GetModuleHandleW(L"winhttp.dll"); - FormatMessageHLocalAlloc alloc; - DWORD tchars_excluding_terminating_null = 0; - if (winhttp_module) - { - tchars_excluding_terminating_null = - FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_HMODULE, - winhttp_module, - last_error, - 0, - reinterpret_cast(&alloc.buffer), - 0, - nullptr); - } - - auto result = msg::format( - msgDownloadWinHttpError, msg::system_api = api_name, msg::exit_code = last_error, msg::url = sanitized_url); - if (tchars_excluding_terminating_null && alloc.buffer) - { - while (tchars_excluding_terminating_null != 0 && - (alloc.buffer[tchars_excluding_terminating_null - 1] == L'\r' || - alloc.buffer[tchars_excluding_terminating_null - 1] == L'\n')) - { - --tchars_excluding_terminating_null; - } - - tchars_excluding_terminating_null = static_cast( - std::remove(alloc.buffer, alloc.buffer + tchars_excluding_terminating_null, L'\r') - alloc.buffer); - result.append_raw(' ').append_raw(Strings::to_utf8(alloc.buffer, tchars_excluding_terminating_null)); - } - - return result; - } - - static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, const SanitizedUrl& sanitized_url) - { - return format_winhttp_last_error_message(api_name, sanitized_url, GetLastError()); - } - - static void maybe_emit_winhttp_progress(MessageSink& machine_readable_progress, - const Optional& maybe_content_length, - std::chrono::steady_clock::time_point& last_write, - unsigned long long total_downloaded_size) - { - if (const auto content_length = maybe_content_length.get()) - { - const auto now = std::chrono::steady_clock::now(); - if ((now - last_write) >= std::chrono::milliseconds(100)) - { - const double percent = - (static_cast(total_downloaded_size) / static_cast(*content_length)) * 100; - machine_readable_progress.println(LocalizedString::from_raw(fmt::format("{:.2f}%", percent))); - last_write = now; - } - } - } - - struct WinHttpHandle - { - WinHttpHandle() = default; - WinHttpHandle(const WinHttpHandle&) = delete; - WinHttpHandle& operator=(const WinHttpHandle&) = delete; - - void require_null_handle() const - { - if (h) - { - Checks::unreachable(VCPKG_LINE_INFO, "WinHTTP handle type confusion"); - } - } - - void require_created_handle() const - { - if (!h) - { - Checks::unreachable(VCPKG_LINE_INFO, "WinHTTP handle not created"); - } - } - - bool Connect(DiagnosticContext& context, - const WinHttpHandle& session, - StringView hostname, - INTERNET_PORT port, - const SanitizedUrl& sanitized_url) - { - require_null_handle(); - session.require_created_handle(); - h = WinHttpConnect(session.h, Strings::to_utf16(hostname).c_str(), port, 0); - if (h) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpConnect", sanitized_url)); - return false; - } - - bool Open(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - _In_opt_z_ LPCWSTR pszAgentW, - _In_ DWORD dwAccessType, - _In_opt_z_ LPCWSTR pszProxyW, - _In_opt_z_ LPCWSTR pszProxyBypassW, - _In_ DWORD dwFlags) - { - require_null_handle(); - h = WinHttpOpen(pszAgentW, dwAccessType, pszProxyW, pszProxyBypassW, dwFlags); - if (h) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpOpen", sanitized_url)); - return false; - } - - bool OpenRequest(DiagnosticContext& context, - const WinHttpHandle& hConnect, - const SanitizedUrl& sanitized_url, - IN LPCWSTR pwszVerb, - StringView path_query_fragment, - IN LPCWSTR pwszVersion, - IN LPCWSTR pwszReferrer OPTIONAL, - IN LPCWSTR FAR* ppwszAcceptTypes OPTIONAL, - IN DWORD dwFlags) - { - require_null_handle(); - h = WinHttpOpenRequest(hConnect.h, - pwszVerb, - Strings::to_utf16(path_query_fragment).c_str(), - pwszVersion, - pwszReferrer, - ppwszAcceptTypes, - dwFlags); - if (h) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpOpenRequest", sanitized_url)); - return false; - } - - bool SendRequest(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - _In_reads_opt_(dwHeadersLength) LPCWSTR lpszHeaders, - IN DWORD dwHeadersLength, - _In_reads_bytes_opt_(dwOptionalLength) LPVOID lpOptional, - IN DWORD dwOptionalLength, - IN DWORD dwTotalLength, - IN DWORD_PTR dwContext) const - { - require_created_handle(); - if (WinHttpSendRequest( - h, lpszHeaders, dwHeadersLength, lpOptional, dwOptionalLength, dwTotalLength, dwContext)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpSendRequest", sanitized_url)); - return false; - } - - bool ReceiveResponse(DiagnosticContext& context, const SanitizedUrl& url) - { - require_created_handle(); - if (WinHttpReceiveResponse(h, NULL)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpReceiveResponse", url)); - return false; - } - - bool SetTimeouts(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - int nResolveTimeout, - int nConnectTimeout, - int nSendTimeout, - int nReceiveTimeout) const - { - require_created_handle(); - if (WinHttpSetTimeouts(h, nResolveTimeout, nConnectTimeout, nSendTimeout, nReceiveTimeout)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpSetTimeouts", sanitized_url)); - return false; - } - - bool SetOption(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - DWORD dwOption, - LPVOID lpBuffer, - DWORD dwBufferLength) const - { - require_created_handle(); - if (WinHttpSetOption(h, dwOption, lpBuffer, dwBufferLength)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpSetOption", sanitized_url)); - return false; - } - - DWORD QueryHeaders(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - DWORD dwInfoLevel, - LPWSTR pwszName, - LPVOID lpBuffer, - LPDWORD lpdwBufferLength, - LPDWORD lpdwIndex) const - { - require_created_handle(); - if (WinHttpQueryHeaders(h, dwInfoLevel, pwszName, lpBuffer, lpdwBufferLength, lpdwIndex)) - { - return 0; - } - - DWORD last_error = GetLastError(); - context.report_error(format_winhttp_last_error_message("WinHttpQueryHeaders", sanitized_url, last_error)); - return last_error; - } - - bool ReadData(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - LPVOID buffer, - DWORD dwNumberOfBytesToRead, - DWORD* numberOfBytesRead) - { - require_created_handle(); - if (WinHttpReadData(h, buffer, dwNumberOfBytesToRead, numberOfBytesRead)) - { - return true; - } - - context.report_error(format_winhttp_last_error_message("WinHttpReadData", sanitized_url)); - return false; - } - - ~WinHttpHandle() - { - if (h) - { - // intentionally ignore failures - (void)WinHttpCloseHandle(h); - } - } - - private: - HINTERNET h{}; - }; - - enum class WinHttpTrialResult - { - failed, - succeeded, - retry - }; - - struct WinHttpSession - { - bool open(DiagnosticContext& context, const SanitizedUrl& sanitized_url) - { - if (!m_hSession.Open(context, - sanitized_url, - L"vcpkg/1.0", - WINHTTP_ACCESS_TYPE_NO_PROXY, - WINHTTP_NO_PROXY_NAME, - WINHTTP_NO_PROXY_BYPASS, - 0)) - { - return false; - } - - // Increase default timeouts to help connections behind proxies - // WinHttpSetTimeouts(HINTERNET hInternet, int nResolveTimeout, int nConnectTimeout, int nSendTimeout, int - // nReceiveTimeout); - if (!m_hSession.SetTimeouts(context, sanitized_url, 0, 120000, 120000, 120000)) - { - return false; - } - - // If the environment variable HTTPS_PROXY is set - // use that variable as proxy. This situation might exist when user is in a company network - // with restricted network/proxy settings - auto maybe_https_proxy_env = get_environment_variable(EnvironmentVariableHttpsProxy); - if (auto p_https_proxy = maybe_https_proxy_env.get()) - { - StringView p_https_proxy_view = *p_https_proxy; - if (p_https_proxy_view.size() != 0 && p_https_proxy_view.back() == '/') - { - // remove trailing slash - p_https_proxy_view = p_https_proxy_view.substr(0, p_https_proxy_view.size() - 1); - } - - std::wstring env_proxy_settings = Strings::to_utf16(p_https_proxy_view); - WINHTTP_PROXY_INFO proxy; - proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; - proxy.lpszProxy = env_proxy_settings.data(); - - // Try to get bypass list from environment variable - auto maybe_no_proxy_env = get_environment_variable(EnvironmentVariableNoProxy); - std::wstring env_noproxy_settings; - if (auto p_no_proxy = maybe_no_proxy_env.get()) - { - env_noproxy_settings = Strings::to_utf16(*p_no_proxy); - proxy.lpszProxyBypass = env_noproxy_settings.data(); - } - else - { - proxy.lpszProxyBypass = nullptr; - } - - if (!m_hSession.SetOption(context, sanitized_url, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy))) - { - return false; - } - } - // IE Proxy fallback, this works on Windows 10 - else - { - // We do not use WPAD anymore - // Directly read IE Proxy setting - auto ieProxy = get_windows_ie_proxy_server(); - if (ieProxy.has_value()) - { - WINHTTP_PROXY_INFO proxy; - proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; - proxy.lpszProxy = ieProxy.get()->server.data(); - proxy.lpszProxyBypass = ieProxy.get()->bypass.data(); - if (!m_hSession.SetOption(context, sanitized_url, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy))) - { - return false; - } - } - } - - // Use Windows 10 defaults on Windows 7 - DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | - WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2); - if (!m_hSession.SetOption(context, - sanitized_url, - WINHTTP_OPTION_SECURE_PROTOCOLS, - &secure_protocols, - sizeof(secure_protocols))) - { - return false; - } - - // Many open source mirrors such as https://download.gnome.org/ will redirect to http mirrors. - // `curl.exe -L` does follow https -> http redirection. - // Additionally, vcpkg hash checks the resulting archive. - DWORD redirect_policy(WINHTTP_OPTION_REDIRECT_POLICY_ALWAYS); - if (!m_hSession.SetOption( - context, sanitized_url, WINHTTP_OPTION_REDIRECT_POLICY, &redirect_policy, sizeof(redirect_policy))) - { - return false; - } - - return true; - } - - WinHttpHandle m_hSession; - }; - - struct WinHttpConnection - { - bool connect(DiagnosticContext& context, - const WinHttpSession& hSession, - StringView hostname, - INTERNET_PORT port, - const SanitizedUrl& sanitized_url) - { - // Specify an HTTP server. - return m_hConnect.Connect(context, hSession.m_hSession, hostname, port, sanitized_url); - } - - WinHttpHandle m_hConnect; - }; - - struct WinHttpRequest - { - bool open(DiagnosticContext& context, - const WinHttpConnection& hConnect, - StringView path_query_fragment, - const SanitizedUrl& sanitized_url, - bool https, - const wchar_t* method = L"GET") - { - if (!m_hRequest.OpenRequest(context, - hConnect.m_hConnect, - sanitized_url, - method, - path_query_fragment, - nullptr, - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, - https ? WINHTTP_FLAG_SECURE : 0)) - { - return false; - } - - // Send a request. - if (!m_hRequest.SendRequest( - context, sanitized_url, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0)) - { - return false; - } - - // End the request. - if (!m_hRequest.ReceiveResponse(context, sanitized_url)) - { - return false; - } - - return true; - } - - Optional query_status(DiagnosticContext& context, const SanitizedUrl& sanitized_url) const - { - DWORD status_code; - DWORD size = sizeof(status_code); - DWORD last_error = m_hRequest.QueryHeaders(context, - sanitized_url, - WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, - WINHTTP_HEADER_NAME_BY_INDEX, - &status_code, - &size, - WINHTTP_NO_HEADER_INDEX); - if (last_error) - { - return nullopt; - } - - return status_code; - } - - bool query_content_length(DiagnosticContext& context, - const SanitizedUrl& sanitized_url, - Optional& result) const - { - static constexpr DWORD buff_characters = 21; // 18446744073709551615 - wchar_t buff[buff_characters]; - DWORD size = sizeof(buff); - AttemptDiagnosticContext adc{context}; - DWORD last_error = m_hRequest.QueryHeaders(adc, - sanitized_url, - WINHTTP_QUERY_CONTENT_LENGTH, - WINHTTP_HEADER_NAME_BY_INDEX, - buff, - &size, - WINHTTP_NO_HEADER_INDEX); - if (!last_error) - { - adc.commit(); - result = Strings::strto(Strings::to_utf8(buff, size >> 1)); - return true; - } - - if (last_error == ERROR_WINHTTP_HEADER_NOT_FOUND) - { - adc.handle(); - return true; - } - - adc.commit(); - return false; - } - - WinHttpTrialResult write_response_body(DiagnosticContext& context, - MessageSink& machine_readable_progress, - const SanitizedUrl& sanitized_url, - const WriteFilePointer& file) - { - static constexpr DWORD buff_size = 65535; - std::unique_ptr buff{new char[buff_size]}; - Optional maybe_content_length; - auto last_write = std::chrono::steady_clock::now(); - if (!query_content_length(context, sanitized_url, maybe_content_length)) - { - return WinHttpTrialResult::retry; - } - - unsigned long long total_downloaded_size = 0; - for (;;) - { - DWORD this_read; - if (!m_hRequest.ReadData(context, sanitized_url, buff.get(), buff_size, &this_read)) - { - return WinHttpTrialResult::retry; - } - - if (this_read == 0) - { - return WinHttpTrialResult::succeeded; - } - - do - { - const auto this_write = static_cast(file.write(buff.get(), 1, this_read)); - if (this_write == 0) - { - context.report_error(format_filesystem_call_error( - std::error_code{errno, std::generic_category()}, "fwrite", {file.path()})); - return WinHttpTrialResult::failed; - } - - maybe_emit_winhttp_progress( - machine_readable_progress, maybe_content_length, last_write, total_downloaded_size); - this_read -= this_write; - total_downloaded_size += this_write; - } while (this_read > 0); - } - } - - WinHttpHandle m_hRequest; - }; -#endif - - Optional parse_split_url_view(StringView raw_url) - { - auto sep = std::find(raw_url.begin(), raw_url.end(), ':'); - if (sep == raw_url.end()) - { - return nullopt; - } - - StringView scheme(raw_url.begin(), sep); - if (Strings::starts_with({sep + 1, raw_url.end()}, "//")) - { - auto path_start = std::find(sep + 3, raw_url.end(), '/'); - return SplitUrlView{scheme, StringView{sep + 1, path_start}, StringView{path_start, raw_url.end()}}; - } - - // no authority - return SplitUrlView{scheme, {}, StringView{sep + 1, raw_url.end()}}; - } - static bool check_downloaded_file_hash(DiagnosticContext& context, const ReadOnlyFilesystem& fs, const SanitizedUrl& sanitized_url, @@ -672,107 +115,170 @@ namespace vcpkg return true; } - static std::vector curl_bulk_operation(DiagnosticContext& context, - View operation_args, - StringLiteral prefixArgs, - View headers, - View secrets) + static size_t write_file_callback(void* contents, size_t size, size_t nmemb, void* param) + { + if (!param) return 0; + return static_cast(param)->write(contents, size, nmemb); + } + + static int progress_callback(void* clientp, double dltotal, double dlnow, double ultotal, double ulnow) { -#define GUID_MARKER "5ec47b8e-6776-4d70-b9b3-ac2a57bc0a1c" - static constexpr StringLiteral guid_marker = GUID_MARKER; - Command prefix_cmd{"curl"}; - if (!prefixArgs.empty()) + (void)ultotal; + (void)ulnow; + auto machine_readable_progress = static_cast(clientp); + if (dltotal && machine_readable_progress) { - prefix_cmd.raw_arg(prefixArgs); + double percentage = dlnow / dltotal * 100.0; + machine_readable_progress->println(LocalizedString::from_raw(fmt::format("{:.2f}%", percentage))); + } + return 0; + } + + static std::vector libcurl_bulk_operation(DiagnosticContext& context, + View urls, + View outputs, + View headers) + { + if (!outputs.empty() && outputs.size() != urls.size()) + { + Checks::unreachable(VCPKG_LINE_INFO); } - prefix_cmd.string_arg("--retry").string_arg("3").string_arg("-L").string_arg("-sS").string_arg("-w").string_arg( - GUID_MARKER "%{http_code} %{exitcode} %{errormsg}\\n"); -#undef GUID_MARKER + std::vector return_codes(urls.size(), -1); - std::vector ret; - ret.reserve(operation_args.size()); - add_curl_headers(prefix_cmd, headers); - while (ret.size() != operation_args.size()) + CurlHeaders request_headers(headers); + + std::vector write_pointers; + write_pointers.reserve(urls.size()); + + std::vector easy_handles; + easy_handles.resize(urls.size()); + + CurlMultiHandle multi_handle; + for (size_t request_index = 0; request_index < urls.size(); ++request_index) + { + const auto& url = urls[request_index]; + auto& easy_handle = easy_handles[request_index]; + auto* curl = easy_handle.get(); + + set_common_curl_easy_options(easy_handle, url, request_headers); + if (outputs.empty()) + { + vcpkg_curl_easy_setopt( + curl, CURLOPT_PRIVATE, reinterpret_cast(static_cast(request_index))); + vcpkg_curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + } + else + { + const auto& output = outputs[request_index]; + std::error_code ec; + auto& request_write_pointer = write_pointers.emplace_back(output, Append::NO, ec); + if (ec) + { + context.report_error(format_filesystem_call_error(ec, "fopen", {output})); + Checks::unreachable(VCPKG_LINE_INFO); + } + + vcpkg_curl_easy_setopt(curl, CURLOPT_PRIVATE, static_cast(&request_write_pointer)); + // note explicit cast to void* necessary to go through ... + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&request_write_pointer)); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); + } + + multi_handle.add_easy_handle(easy_handle); + } + + int still_running = 0; + for (;;) { - // there's an edge case that we aren't handling here where not even one operation fits with the configured - // headers but this seems unlikely + CURLMcode mc = vcpkg_curl_multi_perform(multi_handle.get(), &still_running); + if (mc != CURLM_OK) + { + Debug::println("curl_multi_perform failed:"); + Debug::println(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(mc)) + .append_raw(fmt::format(" ({}).", vcpkg_curl_multi_strerror(mc)))); + Checks::unreachable(VCPKG_LINE_INFO); + } - // form a maximum length command line of operations: - auto batch_cmd = prefix_cmd; - size_t last_try_op = ret.size(); - while (last_try_op != operation_args.size() && batch_cmd.try_append(operation_args[last_try_op])) + if (still_running == 0) { - ++last_try_op; + break; } - // actually run curl - bool new_curl_seen = false; - std::vector debug_lines; - auto maybe_this_batch_exit_code = cmd_execute_and_stream_lines(context, batch_cmd, [&](StringView line) { - debug_lines.emplace_back(line.data(), line.size()); - new_curl_seen |= parse_curl_status_line(context, ret, guid_marker, line); - }); + mc = vcpkg_curl_multi_poll(multi_handle.get(), nullptr, 0, 1000, nullptr); + if (mc != CURLM_OK) + { + Debug::println("curl_multi_wait/poll failed:"); + Debug::println(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(mc)) + .append_raw(fmt::format(" ({}).", vcpkg_curl_multi_strerror(mc)))); + Checks::unreachable(VCPKG_LINE_INFO); + } + } - if (auto this_batch_exit_code = maybe_this_batch_exit_code.get()) + // drain all messages + int messages_in_queue = 0; + while (auto* msg = vcpkg_curl_multi_info_read(multi_handle.get(), &messages_in_queue)) + { + if (msg->msg == CURLMSG_DONE) { - if (!new_curl_seen) + CURL* handle = msg->easy_handle; + if (msg->data.result == CURLE_OK) { - // old version of curl, we only have the result code for the last operation - context.report_error(msgCurlFailedGeneric, msg::exit_code = *this_batch_exit_code); - } + size_t idx; + void* curlinfo_private; + vcpkg_curl_easy_getinfo(handle, CURLINFO_PRIVATE, &curlinfo_private); + if (outputs.empty()) + { + idx = reinterpret_cast(curlinfo_private); + } + else + { + if (!curlinfo_private) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + auto request_write_handle = static_cast(curlinfo_private); + idx = request_write_handle - write_pointers.data(); + } - if (ret.size() != last_try_op) + long response_code; + vcpkg_curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); + return_codes[idx] = static_cast(response_code); + } + else { - // curl didn't process everything we asked of it; this usually means curl crashed - auto command_line = std::move(batch_cmd).extract(); - replace_secrets(command_line, secrets); - context.report_error_with_log(Strings::join("\n", debug_lines), - msgCurlFailedToReturnExpectedNumberOfExitCodes, - msg::exit_code = *this_batch_exit_code, - msg::command_line = command_line); - return ret; + context.report_error( + msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(msg->data.result)) + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(msg->data.result)))); } } - else - { - // couldn't even launch curl, record this as the last fatal error and give up - return ret; - } } + return return_codes; + } - return ret; + static std::vector libcurl_bulk_check(DiagnosticContext& context, + View urls, + View headers) + { + return libcurl_bulk_operation(context, + urls, + {}, // no output + headers); } - std::vector url_heads(DiagnosticContext& context, - View urls, - View headers, - View secrets) + std::vector url_heads(DiagnosticContext& context, View urls, View headers) { - return curl_bulk_operation( - context, - Util::fmap(urls, [](const std::string& url) { return Command{}.string_arg(url_encode_spaces(url)); }), - "--head", - headers, - secrets); + return libcurl_bulk_check(context, urls, headers); } std::vector download_files_no_cache(DiagnosticContext& context, View> url_pairs, - View headers, - View secrets) + View headers) { - return curl_bulk_operation(context, - Util::fmap(url_pairs, - [](const std::pair& url_pair) { - return Command{} - .string_arg(url_encode_spaces(url_pair.first)) - .string_arg("-o") - .string_arg(url_pair.second); - }), - "--create-dirs", - headers, - secrets); + return libcurl_bulk_operation(context, + Util::fmap(url_pairs, [](auto&& kv) -> std::string { return kv.first; }), + Util::fmap(url_pairs, [](auto&& kv) -> Path { return kv.second; }), + headers); } bool submit_github_dependency_graph_snapshot(DiagnosticContext& context, @@ -781,8 +287,6 @@ namespace vcpkg const std::string& github_repository, const Json::Object& snapshot) { - static constexpr StringLiteral guid_marker = "fcfad8a3-bb68-4a54-ad00-dab1ff671ed2"; - std::string uri; if (auto github_server_url = maybe_github_server_url.get()) { @@ -797,309 +301,143 @@ namespace vcpkg fmt::format_to( std::back_inserter(uri), "/repos/{}/dependency-graph/snapshots", url_encode_spaces(github_repository)); - auto cmd = Command{"curl"}; - cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}"); - cmd.string_arg("-X").string_arg("POST"); - { - std::string headers[] = { - "Accept: application/vnd.github+json", - "Authorization: Bearer " + github_token, - "X-GitHub-Api-Version: 2022-11-28", - }; - add_curl_headers(cmd, headers); - } - - cmd.string_arg(uri); - cmd.string_arg("-d").string_arg("@-"); - - RedirectedProcessLaunchSettings settings; - settings.stdin_content = Json::stringify(snapshot); - int code = 0; - auto result = cmd_execute_and_stream_lines(context, cmd, settings, [&code](StringView line) { - if (line.starts_with(guid_marker)) - { - code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); - } - }); - - auto r = result.get(); - if (r && *r == 0 && code >= 200 && code < 300) - { - return true; - } - return false; - } - - bool store_to_asset_cache(DiagnosticContext& context, - StringView raw_url, - const SanitizedUrl& sanitized_url, - StringLiteral method, - View headers, - const Path& file) - { - static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; - - if (raw_url.starts_with("ftp://")) - { - // HTTP headers are ignored for FTP clients - auto ftp_cmd = Command{"curl"}; - ftp_cmd.string_arg(url_encode_spaces(raw_url)); - ftp_cmd.string_arg("-T").string_arg(file); - auto maybe_res = cmd_execute_and_capture_output(context, ftp_cmd); - if (auto res = maybe_res.get()) - { - if (res->exit_code == 0) - { - return true; - } - - context.report_error_with_log( - res->output, msgCurlFailedToPut, msg::exit_code = res->exit_code, msg::url = sanitized_url); - return false; - } - - return false; - } - - auto http_cmd = Command{"curl"}.string_arg("-X").string_arg(method); - add_curl_headers(http_cmd, headers); - http_cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}"); - http_cmd.string_arg(raw_url); - http_cmd.string_arg("-T").string_arg(file); - int code = 0; - auto res = cmd_execute_and_stream_lines(context, http_cmd, [&code](StringView line) { - if (line.starts_with(guid_marker)) - { - code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); - } - }); - - auto pres = res.get(); - if (!pres) - { - return false; - } - - if (*pres != 0 || (code >= 100 && code < 200) || code >= 300) - { - context.report_error(msg::format( - msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = sanitized_url, msg::value = code)); - return false; - } - - return true; - } - - bool azcopy_to_asset_cache(DiagnosticContext& context, - StringView raw_url, - const SanitizedUrl& sanitized_url, - const Path& file) - { - auto azcopy_cmd = Command{"azcopy"}; - azcopy_cmd.string_arg("copy"); - azcopy_cmd.string_arg("--from-to").string_arg("LocalBlob"); - azcopy_cmd.string_arg("--log-level").string_arg("NONE"); - azcopy_cmd.string_arg(file); - azcopy_cmd.string_arg(raw_url.to_string()); - - int code = 0; - auto res = cmd_execute_and_stream_lines(context, azcopy_cmd, [&code](StringView line) { - static constexpr StringLiteral response_marker = "RESPONSE "; - if (line.starts_with(response_marker)) - { - code = std::strtol(line.data() + response_marker.size(), nullptr, 10); - } - }); + CurlEasyHandle handle; + CURL* curl = handle.get(); - auto pres = res.get(); - if (!pres) - { - return false; - } + std::string post_data = Json::stringify(snapshot); - if (*pres != 0) - { - context.report_error(msg::format( - msgAzcopyFailedToPutBlob, msg::exit_code = *pres, msg::url = sanitized_url, msg::value = code)); - return false; - } + std::string headers[]{ + "Accept: application/vnd.github+json", + ("Authorization: Bearer " + github_token), + "X-GitHub-Api-Version: 2022-11-28", + "Content-Type: application/json", + }; - return true; - } - - std::string format_url_query(StringView base_url, View query_params) - { - if (query_params.empty()) - { - return base_url.to_string(); - } - - return fmt::format(FMT_COMPILE("{}?{}"), base_url, fmt::join(query_params, "&")); - } - - Optional invoke_http_request(DiagnosticContext& context, - StringLiteral method, - View headers, - StringView raw_url, - View secrets, - StringView data) - { - auto cmd = Command{"curl"}.string_arg("-s").string_arg("-L"); - add_curl_headers(cmd, headers); - - cmd.string_arg("-X").string_arg(method); - - if (!data.empty()) - { - cmd.string_arg("--data-raw").string_arg(data); - } - - cmd.string_arg(url_encode_spaces(raw_url)); - - auto maybe_output = cmd_execute_and_capture_output(context, cmd); - if (auto output = check_zero_exit_code( - context, cmd, maybe_output, RedirectedProcessLaunchSettings{}.echo_in_debug, secrets)) - { - return *output; - } - - return nullopt; - } - -#if defined(_WIN32) - static WinHttpTrialResult download_winhttp_trial(DiagnosticContext& context, - MessageSink& machine_readable_progress, - const Filesystem& fs, - const WinHttpSession& s, - const Path& download_path_part_path, - SplitUrlView split_uri_view, - StringView hostname, - INTERNET_PORT port, - const SanitizedUrl& sanitized_url) - { - WinHttpConnection conn; - if (!conn.connect(context, s, hostname, port, sanitized_url)) - { - return WinHttpTrialResult::retry; - } + CurlHeaders request_headers(headers); + set_common_curl_easy_options(handle, uri, request_headers); + vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + vcpkg_curl_easy_setopt(curl, CURLOPT_POST, 1L); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, static_cast(post_data.size())); - WinHttpRequest req; - if (!req.open( - context, conn, split_uri_view.path_query_fragment, sanitized_url, split_uri_view.scheme == "https")) - { - return WinHttpTrialResult::retry; - } + CURLcode result = vcpkg_curl_easy_perform(curl); + long response_code = 0; + vcpkg_curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); - auto maybe_status = req.query_status(context, sanitized_url); - const auto status = maybe_status.get(); - if (!status) + if (result != CURLE_OK) { - return WinHttpTrialResult::retry; + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(result)) + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(result)))); + return false; } - if (*status < 200 || *status >= 300) - { - context.report_error(msgDownloadFailedStatusCode, msg::url = sanitized_url, msg::value = *status); - return WinHttpTrialResult::failed; - } + return response_code >= 200 && response_code < 300; + } - return req.write_response_body(context, - machine_readable_progress, - sanitized_url, - fs.open_for_write(download_path_part_path, VCPKG_LINE_INFO)); + static size_t read_file_callback(char* buffer, size_t size, size_t nitems, void* param) + { + auto* file = static_cast(param); + return file->read(buffer, size, nitems); } - /// - /// Download a file using WinHTTP -- only supports HTTP and HTTPS - /// - static bool download_winhttp(DiagnosticContext& context, - MessageSink& machine_readable_progress, - const Filesystem& fs, - const Path& download_path_part_path, - SplitUrlView split_url_view, - const SanitizedUrl& sanitized_url) + bool store_to_asset_cache(DiagnosticContext& context, + StringView raw_url, + const SanitizedUrl& sanitized_url, + View headers, + const Path& file) { - // `download_winhttp` does not support user or port syntax in authorities - auto hostname = split_url_view.authority.value_or_exit(VCPKG_LINE_INFO).substr(2); - INTERNET_PORT port; - if (split_url_view.scheme == "https") + std::error_code ec; + ReadFilePointer fileptr(file, ec); + if (ec) { - port = INTERNET_DEFAULT_HTTPS_PORT; - } - else if (split_url_view.scheme == "http") - { - port = INTERNET_DEFAULT_HTTP_PORT; + context.report_error(format_filesystem_call_error(ec, "fopen", {file})); + return false; } - else + auto file_size = fileptr.size(ec); + if (ec) { - Checks::unreachable(VCPKG_LINE_INFO); + context.report_error(format_filesystem_call_error(ec, "fstat", {file})); + return false; } - // Make sure the directories are present, otherwise fopen_s fails - const auto dir = download_path_part_path.parent_path(); - if (!dir.empty()) + CurlEasyHandle handle; + CURL* curl = handle.get(); + + auto request_headers = raw_url.starts_with("ftp://") ? CurlHeaders() : CurlHeaders(headers); + auto upload_url = url_encode_spaces(raw_url); + vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + vcpkg_curl_easy_setopt(curl, CURLOPT_URL, upload_url.c_str()); + vcpkg_curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + vcpkg_curl_easy_setopt(curl, CURLOPT_READDATA, static_cast(&fileptr)); + vcpkg_curl_easy_setopt(curl, CURLOPT_READFUNCTION, &read_file_callback); + vcpkg_curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, static_cast(file_size)); + + auto result = vcpkg_curl_easy_perform(curl); + if (result != CURLE_OK) { - fs.create_directories(dir, VCPKG_LINE_INFO); + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(result)) + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(result)))); + return false; } - WinHttpSession s; - if (!s.open(context, sanitized_url)) + long response_code = 0; + vcpkg_curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + + if ((response_code >= 100 && response_code < 200) || response_code >= 300) { + context.report_error(msg::format(msgCurlFailedToPut, msg::url = sanitized_url, msg::value = response_code)); return false; } - AttemptDiagnosticContext adc{context}; - switch (download_winhttp_trial(adc, - machine_readable_progress, - fs, - s, - download_path_part_path, - split_url_view, - hostname, - port, - sanitized_url)) + return true; + } + + bool azcopy_to_asset_cache(DiagnosticContext& context, + StringView raw_url, + const SanitizedUrl& sanitized_url, + const Path& file) + { + auto azcopy_cmd = Command{"azcopy"}; + azcopy_cmd.string_arg("copy"); + azcopy_cmd.string_arg("--from-to").string_arg("LocalBlob"); + azcopy_cmd.string_arg("--log-level").string_arg("NONE"); + azcopy_cmd.string_arg(file); + azcopy_cmd.string_arg(raw_url.to_string()); + + int code = 0; + auto res = cmd_execute_and_stream_lines(context, azcopy_cmd, [&code](StringView line) { + static constexpr StringLiteral response_marker = "RESPONSE "; + if (line.starts_with(response_marker)) + { + code = std::strtol(line.data() + response_marker.size(), nullptr, 10); + } + }); + + auto pres = res.get(); + if (!pres) { - case WinHttpTrialResult::succeeded: adc.commit(); return true; - case WinHttpTrialResult::failed: adc.commit(); return false; - case WinHttpTrialResult::retry: break; + return false; } - for (size_t trials = 1; trials < 4; ++trials) + if (*pres != 0) { - // 1s, 2s, 4s - const auto trialMs = 500 << trials; - adc.handle(); - context.statusln( - DiagnosticLine(DiagKind::Warning, - msg::format(msgDownloadFailedRetrying, msg::value = trialMs, msg::url = sanitized_url)) - .to_message_line()); - std::this_thread::sleep_for(std::chrono::milliseconds(trialMs)); - switch (download_winhttp_trial(adc, - machine_readable_progress, - fs, - s, - download_path_part_path, - split_url_view, - hostname, - port, - sanitized_url)) - { - case WinHttpTrialResult::succeeded: adc.commit(); return true; - case WinHttpTrialResult::failed: adc.commit(); return false; - case WinHttpTrialResult::retry: break; - } + context.report_error(msg::format( + msgAzcopyFailedToPutBlob, msg::exit_code = *pres, msg::url = sanitized_url, msg::value = code)); + return false; } - adc.commit(); - return false; + return true; } -#endif enum class DownloadPrognosis { Success, + // Transient error means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500, 502, 503 or + // 504 response code. https://everything.curl.dev/usingcurl/downloads/retry.html#retry + TransientNetworkError, OtherError, - NetworkErrorProxyMightHelp + NetworkErrorProxyMightHelp, }; static bool check_combine_download_prognosis(DownloadPrognosis& target, DownloadPrognosis individual_call) @@ -1107,15 +445,23 @@ namespace vcpkg switch (individual_call) { case DownloadPrognosis::Success: return true; - case DownloadPrognosis::OtherError: + case DownloadPrognosis::TransientNetworkError: if (target == DownloadPrognosis::Success) + { + target = DownloadPrognosis::TransientNetworkError; + } + + return false; + case DownloadPrognosis::OtherError: + if (target == DownloadPrognosis::Success || target == DownloadPrognosis::TransientNetworkError) { target = DownloadPrognosis::OtherError; } return false; case DownloadPrognosis::NetworkErrorProxyMightHelp: - if (target == DownloadPrognosis::Success || target == DownloadPrognosis::OtherError) + if (target == DownloadPrognosis::Success || target == DownloadPrognosis::TransientNetworkError || + target == DownloadPrognosis::OtherError) { target = DownloadPrognosis::NetworkErrorProxyMightHelp; } @@ -1133,6 +479,73 @@ namespace vcpkg } } + static DownloadPrognosis perform_download(DiagnosticContext& context, + MessageSink& machine_readable_progress, + StringView raw_url, + const Path& download_path, + View headers) + { + std::error_code ec; + WriteFilePointer fileptr(download_path, Append::NO, ec); + if (ec) + { + context.report_error(format_filesystem_call_error(ec, "fopen", {download_path})); + return DownloadPrognosis::OtherError; + } + + CurlHeaders request_headers(headers); + + CurlEasyHandle handle; + CURL* curl = handle.get(); + set_common_curl_easy_options(handle, raw_url, request_headers); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_file_callback); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&fileptr)); + vcpkg_curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); // change from default to enable progress + // curlopt_progressfunction is deprecated, but we want the values as doubles anyway and + // the replacement isn't available on all versions of libcurl we support + vcpkg_curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, static_cast(&progress_callback)); + vcpkg_curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, static_cast(&machine_readable_progress)); + auto curl_code = vcpkg_curl_easy_perform(curl); + + if (curl_code == CURLE_OPERATION_TIMEDOUT) + { + context.report_error(msgCurlDownloadTimeout); + return DownloadPrognosis::TransientNetworkError; + } + + if (curl_code != CURLE_OK) + { + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(curl_code)) + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(curl_code)))); + return DownloadPrognosis::NetworkErrorProxyMightHelp; + } + + long response_code = -1; + auto get_info_code = vcpkg_curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + if (get_info_code != CURLE_OK) + { + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = static_cast(get_info_code)) + .append_raw(fmt::format(" ({}).", vcpkg_curl_easy_strerror(get_info_code)))); + return DownloadPrognosis::NetworkErrorProxyMightHelp; + } + + if ((response_code >= 200 && response_code < 300) || (raw_url.starts_with("file://") && response_code == 0)) + { + return DownloadPrognosis::Success; + } + + context.report_error(msg::format(msgCurlFailedResponse, msg::exit_code = static_cast(response_code))); + + if ((raw_url.starts_with("ftp://") && response_code >= 400 && response_code < 500) || + (response_code == 429 || response_code == 408 || response_code == 500 || response_code == 502 || + response_code == 503 || response_code == 504)) + { + return DownloadPrognosis::TransientNetworkError; + } + + return DownloadPrognosis::NetworkErrorProxyMightHelp; + } + static DownloadPrognosis try_download_file(DiagnosticContext& context, MessageSink& machine_readable_progress, const Filesystem& fs, @@ -1152,59 +565,6 @@ namespace vcpkg #endif download_path_part_path += ".part"; -#if defined(_WIN32) - auto maybe_https_proxy_env = get_environment_variable(EnvironmentVariableHttpsProxy); - bool needs_proxy_auth = false; - if (auto proxy_url = maybe_https_proxy_env.get()) - { - needs_proxy_auth = proxy_url->find('@') != std::string::npos; - } - if (headers.size() == 0 && !needs_proxy_auth) - { - auto maybe_split_uri_view = parse_split_url_view(raw_url); - auto split_uri_view = maybe_split_uri_view.get(); - if (!split_uri_view) - { - context.report_error(msgInvalidUri, msg::value = sanitized_url); - return DownloadPrognosis::OtherError; - } - - if (split_uri_view->scheme == "https" || split_uri_view->scheme == "http") - { - auto maybe_authority = split_uri_view->authority.get(); - if (!maybe_authority) - { - context.report_error(msg::format(msgInvalidUri, msg::value = sanitized_url)); - return DownloadPrognosis::OtherError; - } - - auto authority = StringView{*maybe_authority}.substr(2); - // This check causes complex URLs (non-default port, embedded basic auth) to be passed down to - // curl.exe - if (Strings::find_first_of(authority, ":@") == authority.end()) - { - if (!download_winhttp(context, - machine_readable_progress, - fs, - download_path_part_path, - *split_uri_view, - sanitized_url)) - { - return DownloadPrognosis::NetworkErrorProxyMightHelp; - } - - if (!check_downloaded_file_hash( - context, fs, sanitized_url, download_path_part_path, maybe_sha512, out_sha512)) - { - return DownloadPrognosis::OtherError; - } - - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); - return DownloadPrognosis::Success; - } - } - } -#endif // Create directory in advance, otherwise curl will create it in 750 mode on unix style file systems. const auto dir = download_path_part_path.parent_path(); if (!dir.empty()) @@ -1212,85 +572,37 @@ namespace vcpkg fs.create_directories(dir, VCPKG_LINE_INFO); } - auto cmd = Command{"curl"} - .string_arg("--fail") - .string_arg("--retry") - .string_arg("3") - .string_arg("-L") - .string_arg(url_encode_spaces(raw_url)) - .string_arg("--create-dirs") - .string_arg("--output") - .string_arg(download_path_part_path); - add_curl_headers(cmd, headers); - bool seen_any_curl_errors = false; - // if seen_any_curl_errors, contains the curl error lines starting with "curl:" - // otherwise, contains all curl's output unless it is the machine readable output - std::vector likely_curl_errors; - auto maybe_exit_code = cmd_execute_and_stream_lines(context, cmd, [&](StringView line) { - const auto maybe_parsed = try_parse_curl_progress_data(line); - if (const auto parsed = maybe_parsed.get()) - { - machine_readable_progress.println(Color::none, - LocalizedString::from_raw(fmt::format("{}%", parsed->total_percent))); - return; - } - - static constexpr StringLiteral WarningColon = "warning: "; - if (Strings::case_insensitive_ascii_starts_with(line, WarningColon)) + // Retry on transient errors: + // Transient error means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500, 502, 503 or + // 504 response code. https://everything.curl.dev/usingcurl/downloads/retry.html#retry + using namespace std::chrono_literals; + static constexpr std::array attempt_delays = {1s, 2s}; + DownloadPrognosis prognosis = DownloadPrognosis::NetworkErrorProxyMightHelp; + for (size_t attempt_count = 0; attempt_count < attempt_delays.size(); attempt_count++) + { + prognosis = perform_download(context, machine_readable_progress, raw_url, download_path_part_path, headers); + if (DownloadPrognosis::Success == prognosis) { - context.statusln( - DiagnosticLine{DiagKind::Warning, LocalizedString::from_raw(line.substr(WarningColon.size()))} - .to_message_line()); - return; + break; } - // clang-format off - // example: - // 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (6) Could not resolve host: nonexistent.example.com - // clang-format on - static constexpr StringLiteral CurlColon = "curl:"; - auto curl_start = std::search(line.begin(), line.end(), CurlColon.begin(), CurlColon.end()); - if (curl_start == line.end()) - { - if (seen_any_curl_errors) - { - return; - } - - curl_start = line.begin(); - } - else + if (DownloadPrognosis::TransientNetworkError == prognosis) { - if (!seen_any_curl_errors) - { - seen_any_curl_errors = true; - likely_curl_errors.clear(); - } + context.statusln(msg::format(msgDownloadTransientErrorRetry, + msg::count = attempt_count + 1, + msg::value = attempt_delays.size() + 1)); + std::this_thread::sleep_for(attempt_delays[attempt_count]); + continue; } - likely_curl_errors.emplace_back(curl_start, line.end()); - }); - - const auto exit_code = maybe_exit_code.get(); - if (!exit_code) - { - return DownloadPrognosis::OtherError; + context.report_error(msg::format(msgDownloadNotTransientErrorWontRetry, msg::url = sanitized_url)); + return prognosis; } - if (*exit_code != 0) + if (DownloadPrognosis::Success != prognosis) { - std::set seen_errors; - for (StringView likely_curl_error : likely_curl_errors) - { - auto seen_position = seen_errors.lower_bound(likely_curl_error); - if (seen_position == seen_errors.end() || *seen_position != likely_curl_error) - { - seen_errors.emplace_hint(seen_position, likely_curl_error); - context.report(DiagnosticLine{DiagKind::Error, LocalizedString::from_raw(likely_curl_error)}); - } - } - - return DownloadPrognosis::NetworkErrorProxyMightHelp; + context.report_error(msg::format(msgDownloadTransientErrorRetriesExhausted, msg::url = sanitized_url)); + return prognosis; } if (!check_downloaded_file_hash(context, fs, sanitized_url, download_path_part_path, maybe_sha512, out_sha512)) @@ -1308,89 +620,6 @@ namespace vcpkg return s_headers; } - bool parse_curl_status_line(DiagnosticContext& context, - std::vector& http_codes, - StringLiteral prefix, - StringView this_line) - { - if (!this_line.starts_with(prefix)) - { - return false; - } - - auto first = this_line.begin(); - const auto last = this_line.end(); - first += prefix.size(); - const auto first_http_code = first; - - int http_code; - for (;; ++first) - { - if (first == last) - { - // this output is broken, even if we don't know %{exit_code} or ${errormsg}, the spaces in front - // of them should still be printed. - return false; - } - - if (!ParserBase::is_ascii_digit(*first)) - { - http_code = Strings::strto(StringView{first_http_code, first}).value_or_exit(VCPKG_LINE_INFO); - break; - } - } - - if (*first != ' ' || ++first == last) - { - // didn't see the space after the http_code - return false; - } - - if (*first == ' ') - { - // old curl that doesn't understand %{exit_code}, this is the space after it - http_codes.emplace_back(http_code); - return false; - } - - if (!ParserBase::is_ascii_digit(*first)) - { - // not exit_code - return false; - } - - const auto first_exit_code = first; - for (;;) - { - if (++first == last) - { - // didn't see the space after %{exit_code} - return false; - } - - if (*first == ' ') - { - // the space after exit_code, everything after this space is the error message if any - http_codes.emplace_back(http_code); - auto exit_code = Strings::strto(StringView{first_exit_code, first}).value_or_exit(VCPKG_LINE_INFO); - // note that this gets the space out of the output :) - if (exit_code != 0) - { - context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = exit_code) - .append_raw(StringView{first, last})); - } - - return true; - } - - if (!ParserBase::is_ascii_digit(*first)) - { - // non numeric exit_code? - return false; - } - } - } - static DownloadPrognosis download_file_azurl_asset_cache(DiagnosticContext& context, MessageSink& machine_readable_progress, const AssetCachingSettings& asset_cache_settings, @@ -1631,6 +860,7 @@ namespace vcpkg out_sha512)) { case DownloadPrognosis::Success: return DownloadPrognosis::Success; + case DownloadPrognosis::TransientNetworkError: return DownloadPrognosis::TransientNetworkError; case DownloadPrognosis::OtherError: return download_file_script_asset_cache(context, asset_cache_settings, @@ -1659,12 +889,8 @@ namespace vcpkg context.statusln( msg::format(msgDownloadSuccesfulUploading, msg::path = display_path, msg::url = sanitized_upload_url)); WarningDiagnosticContext wdc{context}; - if (!store_to_asset_cache(wdc, - raw_upload_url, - sanitized_upload_url, - "PUT", - asset_cache_settings.m_write_headers, - download_path)) + if (!store_to_asset_cache( + wdc, raw_upload_url, sanitized_upload_url, asset_cache_settings.m_write_headers, download_path)) { context.report(DiagnosticLine{DiagKind::Warning, msg::format(msgFailedToStoreBackToMirror, @@ -1869,6 +1095,11 @@ namespace vcpkg context, download_path, display_path, asset_cache_settings, maybe_sha512); return true; } + else + { + asset_cache_attempt_context.commit(); + authoritative_attempt_context.commit(); + } while (++first_sanitized_url, ++first_raw_url != last_raw_url) { @@ -1984,146 +1215,12 @@ namespace vcpkg auto raw_upload_url = Strings::replace_all(*url_template, "", sha512); SanitizedUrl sanitized_upload_url{raw_upload_url, asset_cache_settings.m_secrets}; - return store_to_asset_cache(context, - raw_upload_url, - sanitized_upload_url, - "PUT", - asset_cache_settings.m_write_headers, - file_to_put); - } - - return true; - } - - Optional try_parse_curl_max5_size(StringView sv) - { - // \d+(\.\d{1, 2})?[kMGTP]? - std::size_t idx = 0; - while (idx < sv.size() && ParserBase::is_ascii_digit(sv[idx])) - { - ++idx; - } - - if (idx == 0) - { - return nullopt; - } - - unsigned long long accumulator; - { - const auto maybe_first_digits = Strings::strto(sv.substr(0, idx)); - if (auto p = maybe_first_digits.get()) - { - accumulator = *p; - } - else - { - return nullopt; - } - } - - unsigned long long after_digits = 0; - if (idx < sv.size() && sv[idx] == '.') - { - ++idx; - if (idx >= sv.size() || !ParserBase::is_ascii_digit(sv[idx])) - { - return nullopt; - } - - after_digits = (sv[idx] - '0') * 10u; - ++idx; - if (idx < sv.size() && ParserBase::is_ascii_digit(sv[idx])) - { - after_digits += sv[idx] - '0'; - ++idx; - } - } - - if (idx == sv.size()) - { - return accumulator; - } - - if (idx + 1 != sv.size()) - { - return nullopt; - } - - switch (sv[idx]) - { - case 'k': return (accumulator << 10) + (after_digits << 10) / 100; - case 'M': return (accumulator << 20) + (after_digits << 20) / 100; - case 'G': return (accumulator << 30) + (after_digits << 30) / 100; - case 'T': return (accumulator << 40) + (after_digits << 40) / 100; - case 'P': return (accumulator << 50) + (after_digits << 50) / 100; - default: return nullopt; - } - } - - static bool parse_curl_uint_impl(unsigned int& target, const char*& first, const char* const last) - { - first = std::find_if_not(first, last, ParserBase::is_whitespace); - const auto start = first; - first = std::find_if(first, last, ParserBase::is_whitespace); - const auto maybe_parsed = Strings::strto(StringView{start, first}); - if (const auto parsed = maybe_parsed.get()) - { - target = *parsed; - return false; - } - - return true; - } - - static bool parse_curl_max5_impl(unsigned long long& target, const char*& first, const char* const last) - { - first = std::find_if_not(first, last, ParserBase::is_whitespace); - const auto start = first; - first = std::find_if(first, last, ParserBase::is_whitespace); - const auto maybe_parsed = try_parse_curl_max5_size(StringView{start, first}); - if (const auto parsed = maybe_parsed.get()) - { - target = *parsed; - return false; + return store_to_asset_cache( + context, raw_upload_url, sanitized_upload_url, asset_cache_settings.m_write_headers, file_to_put); } return true; } - static bool skip_curl_time_impl(const char*& first, const char* const last) - { - first = std::find_if_not(first, last, ParserBase::is_whitespace); - first = std::find_if(first, last, ParserBase::is_whitespace); - return false; - } - - Optional try_parse_curl_progress_data(StringView curl_progress_line) - { - // Curl's maintainer Daniel Stenberg clarified that this output is semi-contractual - // here: https://twitter.com/bagder/status/1600615752725307400 - // % Total % Received % Xferd Average Speed Time Time Time Current - // Dload Upload Total Spent Left Speed - // https://github.com/curl/curl/blob/5ccddf64398c1186deb5769dac086d738e150e09/lib/progress.c#L546 - CurlProgressData result; - auto first = curl_progress_line.begin(); - const auto last = curl_progress_line.end(); - if (parse_curl_uint_impl(result.total_percent, first, last) || - parse_curl_max5_impl(result.total_size, first, last) || - parse_curl_uint_impl(result.received_percent, first, last) || - parse_curl_max5_impl(result.received_size, first, last) || - parse_curl_uint_impl(result.transfer_percent, first, last) || - parse_curl_max5_impl(result.transfer_size, first, last) || - parse_curl_max5_impl(result.average_download_speed, first, last) || - parse_curl_max5_impl(result.average_upload_speed, first, last) || skip_curl_time_impl(first, last) || - skip_curl_time_impl(first, last) || skip_curl_time_impl(first, last) || - parse_curl_max5_impl(result.current_speed, first, last)) - { - return nullopt; - } - - return result; - } - std::string url_encode_spaces(StringView url) { return Strings::replace_all(url, StringLiteral{" "}, "%20"); } } diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index 617250284b..e1ad6d72cd 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -37,6 +37,7 @@ #include #if defined(_WIN32) +#include #include #include @@ -1562,6 +1563,62 @@ namespace vcpkg ec.clear(); } + uint64_t ReadFilePointer::size(LineInfo li) const + { + std::error_code ec; + auto result = this->size(ec); + if (ec) + { + exit_filesystem_call_error(li, ec, __func__, {m_path}); + } + + return result; + } + + uint64_t ReadFilePointer::size(std::error_code& ec) const + { + ec.clear(); +#if _WIN32 + if (!m_fs) + { + ec.assign(EBADF, std::generic_category()); + return 0; + } + + const auto file_number = ::_fileno(m_fs); + if (file_number == -1) + { + ec.assign(errno, std::generic_category()); + return 0; + } + + const auto os_handle = ::_get_osfhandle(file_number); + if (os_handle == -1) + { + ec.assign(errno, std::generic_category()); + return 0; + } + + LARGE_INTEGER size; + if (!::GetFileSizeEx(reinterpret_cast(os_handle), &size)) + { + ec.assign(::GetLastError(), std::system_category()); + return 0; + } + + return static_cast(size.QuadPart); +#else + struct stat st; + if (::fstat(::fileno(m_fs), &st) != 0) + { + ec.assign(errno, std::generic_category()); + return 0; + } + + return st.st_size; +#endif + } + WriteFilePointer::WriteFilePointer() noexcept = default; WriteFilePointer::WriteFilePointer(WriteFilePointer&&) noexcept = default; diff --git a/src/vcpkg/base/system.process.cpp b/src/vcpkg/base/system.process.cpp index 0b15a537b5..062a191b78 100644 --- a/src/vcpkg/base/system.process.cpp +++ b/src/vcpkg/base/system.process.cpp @@ -1099,7 +1099,9 @@ namespace DWORD bytes_read = 0; static constexpr DWORD buffer_size = 1024 * 32; - char buf[buffer_size]; + // the alignment is forced here for callers that are expecting the buffer to contain UTF-16 + // on Windows + alignas(wchar_t) char buf[buffer_size]; while (stdout_pipe.read_pipe != INVALID_HANDLE_VALUE) { switch (WaitForSingleObjectEx(stdout_pipe.read_pipe, INFINITE, TRUE)) @@ -1650,7 +1652,9 @@ namespace // Note: This doesn't handle unpaired surrogates or partial encoding units correctly in // order to be able to reuse Strings::to_utf8 which we believe will be fine 99% of the time. std::string encoded; - Strings::to_utf8(encoded, reinterpret_cast(buf), bytes_read / 2); + // clang-format off + Strings::to_utf8(encoded, reinterpret_cast(buf), bytes_read / 2); // CodeQL [SM02986] This is not an incorrect cast to shut up the compiler, the suggested "call an encoding conversion function instead" is exactly what this is doing. + // clang-format on std::replace(encoded.begin(), encoded.end(), '\0', '?'); if (settings.echo_in_debug == EchoInDebug::Show && Debug::g_debugging) { diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 39c4af661c..5540bf1722 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -491,7 +491,7 @@ namespace auto url = templ.instantiate_variables(request); WarningDiagnosticContext wdc{context}; auto maybe_success = - store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, "PUT", templ.headers, zip_path); + store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, templ.headers, zip_path); if (maybe_success) { count_stored++; @@ -536,7 +536,7 @@ namespace } WarningDiagnosticContext wdc{context}; - auto codes = download_files_no_cache(wdc, url_paths, m_url_template.headers, m_secrets); + auto codes = download_files_no_cache(wdc, url_paths, m_url_template.headers); for (size_t i = 0; i < codes.size(); ++i) { if (codes[i] == 200) @@ -558,7 +558,7 @@ namespace } WarningDiagnosticContext wdc{context}; - auto codes = url_heads(wdc, urls, {}, m_secrets); + auto codes = url_heads(wdc, urls, {}); for (size_t i = 0; i < codes.size(); ++i) { out_status[i] = codes[i] == 200 ? CacheAvailability::available : CacheAvailability::unavailable; @@ -602,7 +602,7 @@ namespace // cf. // https://learn.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs?toc=%2Fazure%2Fstorage%2Fblobs%2Ftoc.json - constexpr size_t max_single_write = 5000000000; + constexpr size_t max_single_write = 5000000000u; bool use_azcopy = file_size > max_single_write; WarningDiagnosticContext wdc{context}; @@ -611,9 +611,8 @@ namespace { auto url = templ.instantiate_variables(request); auto maybe_success = - use_azcopy - ? azcopy_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, zip_path) - : store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, "PUT", templ.headers, zip_path); + use_azcopy ? azcopy_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, zip_path) + : store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, templ.headers, zip_path); if (maybe_success) { count_stored++; diff --git a/src/vcpkg/commands.cpp b/src/vcpkg/commands.cpp index 29430906b1..a90862b4fb 100644 --- a/src/vcpkg/commands.cpp +++ b/src/vcpkg/commands.cpp @@ -70,8 +70,8 @@ namespace vcpkg {CommandCheckToolsShaMetadata, command_check_tools_sha_and_exit}, {CommandInitRegistryMetadata, command_init_registry_and_exit}, {CommandVersionMetadata, command_version_and_exit}, -#if defined(_WIN32) {CommandZUploadMetricsMetadata, command_z_upload_metrics_and_exit}, +#if defined(_WIN32) {CommandZApplocalMetadata, command_z_applocal_and_exit}, #endif // defined(_WIN32) {CommandZGenerateDefaultMessageMapMetadata, command_z_generate_default_message_map_and_exit}, diff --git a/src/vcpkg/commands.z-check-tools-sha.cpp b/src/vcpkg/commands.z-check-tools-sha.cpp index 91050ff6ac..e2d335df15 100644 --- a/src/vcpkg/commands.z-check-tools-sha.cpp +++ b/src/vcpkg/commands.z-check-tools-sha.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -80,7 +81,7 @@ namespace vcpkg } msg::println(msgDownloadingTools, msg::count = urlAndPaths.size()); - auto result = download_files_no_cache(console_diagnostic_context, urlAndPaths, {}, {}); + auto result = download_files_no_cache(console_diagnostic_context, urlAndPaths, {}); // no headers std::unordered_map url_to_fixed_sha; auto http_codes_iter = result.begin(); @@ -111,7 +112,7 @@ namespace vcpkg ++http_codes_iter; } - if (!has_sha_error) + if (!has_http_error && !has_sha_error) { msg::println(msgAllShasValid); } diff --git a/src/vcpkg/commands.z-upload-metrics.cpp b/src/vcpkg/commands.z-upload-metrics.cpp index 21dcbd1ab5..7cb5523503 100644 --- a/src/vcpkg/commands.z-upload-metrics.cpp +++ b/src/vcpkg/commands.z-upload-metrics.cpp @@ -1,9 +1,9 @@ -#include - -#if defined(_WIN32) #include +#include #include +#include +#include #include #include @@ -26,8 +26,18 @@ namespace vcpkg const auto parsed = args.parse_arguments(CommandZUploadMetricsMetadata); const auto& payload_path = parsed.command_arguments[0]; auto payload = fs.read_contents(payload_path, VCPKG_LINE_INFO); - winhttp_upload_metrics(payload); + if (!curl_upload_metrics(payload)) + { + Debug::println("Failed to upload metrics"); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + std::error_code ec; + fs.remove(payload_path, ec); + if (ec) + { + Debug::println("Failed to remove file after upload: {}", ec.message()); + } Checks::exit_success(VCPKG_LINE_INFO); } } -#endif // defined(_WIN32) diff --git a/src/vcpkg/metrics.cpp b/src/vcpkg/metrics.cpp index a1d088e5a6..04ce6154b3 100644 --- a/src/vcpkg/metrics.cpp +++ b/src/vcpkg/metrics.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -17,6 +19,7 @@ #include #include +#include #include #include @@ -25,6 +28,17 @@ #pragma comment(lib, "winhttp") #endif +// Polyfill this value from newer versions of libcurl +#ifndef CURL_SSLVERSION_TLSv1_0 +#define CURL_SSLVERSION_TLSv1_0 4L +#endif +#ifndef CURL_SSLVERSION_TLSv1_1 +#define CURL_SSLVERSION_TLSv1_1 5L +#endif +#ifndef CURL_SSLVERSION_TLSv1_2 +#define CURL_SSLVERSION_TLSv1_2 6L +#endif + namespace { using namespace vcpkg; @@ -477,99 +491,6 @@ namespace vcpkg std::atomic g_should_print_metrics = false; std::atomic g_metrics_enabled = false; -#if defined(_WIN32) - void winhttp_upload_metrics(StringView payload) - { - HINTERNET connect = nullptr, request = nullptr; - BOOL results = FALSE; - - const HINTERNET session = WinHttpOpen( - L"vcpkg/1.0", WINHTTP_ACCESS_TYPE_DEFAULT_PROXY, WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0); - - unsigned long secure_protocols = WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2; - if (session && WinHttpSetOption(session, WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(DWORD))) - { - connect = WinHttpConnect(session, L"dc.services.visualstudio.com", INTERNET_DEFAULT_HTTPS_PORT, 0); - } - - if (connect) - { - request = WinHttpOpenRequest(connect, - L"POST", - L"/v2/track", - nullptr, - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, - WINHTTP_FLAG_SECURE); - } - - if (request) - { - auto mutable_payload = payload.to_string(); - if (MAXDWORD <= mutable_payload.size()) abort(); - std::wstring hdrs = L"Content-Type: application/json\r\n"; - results = WinHttpSendRequest(request, - hdrs.c_str(), - static_cast(hdrs.size()), - static_cast(mutable_payload.data()), - static_cast(mutable_payload.size()), - static_cast(mutable_payload.size()), - 0); - } - - if (results) - { - results = WinHttpReceiveResponse(request, nullptr); - } - - DWORD http_code = 0, junk = sizeof(DWORD); - - if (results) - { - results = WinHttpQueryHeaders(request, - WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, - nullptr, - &http_code, - &junk, - WINHTTP_NO_HEADER_INDEX); - } - - std::vector response_buffer; - if (results) - { - DWORD available_data = 0, read_data = 0, total_data = 0; - while ((results = WinHttpQueryDataAvailable(request, &available_data)) == TRUE && available_data > 0) - { - response_buffer.resize(response_buffer.size() + available_data); - - results = WinHttpReadData(request, &response_buffer[total_data], available_data, &read_data); - - if (!results) - { - break; - } - - total_data += read_data; - - response_buffer.resize(total_data); - } - } - - if (!results) - { -#ifndef NDEBUG - __debugbreak(); - auto err = GetLastError(); - fprintf(stderr, "[DEBUG] failed to connect to server: %08lu\n", err); -#endif // NDEBUG - } - - if (request) WinHttpCloseHandle(request); - if (connect) WinHttpCloseHandle(connect); - if (session) WinHttpCloseHandle(session); - } -#endif // ^^^ _WIN32 - void flush_global_metrics(const Filesystem& fs) { if (!g_metrics_enabled.load()) @@ -608,32 +529,105 @@ namespace vcpkg fs.write_contents(vcpkg_metrics_txt_path, payload, ec); if (ec) return; -#if defined(_WIN32) - const Path temp_folder_path_exe = temp_folder_path / "vcpkg-" VCPKG_BASE_VERSION_AS_STRING ".exe"; + const Path temp_folder_path_exe = temp_folder_path / "vcpkg-" VCPKG_BASE_VERSION_AS_STRING +#if defined(WIN32) + ".exe" +#endif + ; fs.copy_file(get_exe_path_of_current_process(), temp_folder_path_exe, CopyOptions::skip_existing, ec); if (ec) return; + Command builder; builder.string_arg(temp_folder_path_exe); builder.string_arg("z-upload-metrics"); builder.string_arg(vcpkg_metrics_txt_path); cmd_execute_background(builder); -#else - cmd_execute_background(Command("curl") - .string_arg("https://dc.services.visualstudio.com/v2/track") - .string_arg("--max-time") - .string_arg("60") - .string_arg("-H") - .string_arg("Content-Type: application/json") - .string_arg("-X") - .string_arg("POST") - .string_arg("--tlsv1.2") - .string_arg("--data") - .string_arg(Strings::concat("@", vcpkg_metrics_txt_path)) - .raw_arg(">/dev/null") - .raw_arg("2>&1") - .raw_arg(";") - .string_arg("rm") - .string_arg(vcpkg_metrics_txt_path)); -#endif + } + + static size_t string_append_cb(void* buff, size_t size, size_t nmemb, void* param) + { + auto* str = reinterpret_cast(param); + if (!str || !buff) return 0; + if (size != 1) return 0; + str->append(reinterpret_cast(buff), nmemb); + return size * nmemb; + } + + bool parse_metrics_response(StringView response_body) + { + auto maybe_json = Json::parse_object(response_body, "metrics_response"); + auto json = maybe_json.get(); + if (!json) return false; + + auto maybe_received = json->get(AppInsightsResponseItemsReceived); + auto maybe_accepted = json->get(AppInsightsResponseItemsAccepted); + auto maybe_errors = json->get(AppInsightsResponseErrors); + + if (maybe_received && maybe_accepted && maybe_errors && maybe_received->is_integer() && + maybe_accepted->is_integer() && maybe_errors->is_array()) + { + auto item_received = maybe_received->integer(VCPKG_LINE_INFO); + auto item_accepted = maybe_accepted->integer(VCPKG_LINE_INFO); + auto errors = maybe_errors->array(VCPKG_LINE_INFO); + return (errors.size() == 0) && (item_received == item_accepted); + } + Debug::println("Metrics response has unexpected format"); + return false; + } + + bool curl_upload_metrics(const std::string& payload) + { + if (payload.length() > static_cast(std::numeric_limits::max())) + { + Debug::println("Metrics payload too large to upload"); + return false; + } + + CurlEasyHandle handle; + CURL* curl = handle.get(); + + std::string headers[] = { + "Content-Type: application/json", + }; + CurlHeaders request_headers(headers); + + vcpkg_curl_easy_setopt(curl, CURLOPT_URL, "https://dc.services.visualstudio.com/v2/track"); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); + vcpkg_curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, static_cast(payload.length())); + vcpkg_curl_easy_setopt(curl, CURLOPT_HTTPHEADER, request_headers.get()); + vcpkg_curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L); + if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2) != CURLE_OK) + { + if (vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_1) != CURLE_OK) + { + vcpkg_curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_0); + } + } + + vcpkg_curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // follow redirects + vcpkg_curl_easy_setopt(curl, CURLOPT_USERAGENT, vcpkg_curl_user_agent); + + std::string buff; + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&buff)); + vcpkg_curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &string_append_cb); + + long response_code = 0; + CURLcode res = vcpkg_curl_easy_perform(curl); + bool is_success = false; + if (res == CURLE_OK) + { + vcpkg_curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + Debug::println(fmt::format("Metrics upload response code: {}", response_code)); + Debug::println("Metrics upload response body: ", buff); + if (response_code == 200) + { + is_success = parse_metrics_response(buff); + } + } + else + { + Debug::println("Metrics upload failed: ", vcpkg_curl_easy_strerror(res)); + } + return is_success; } } diff --git a/vcpkg-init/update-scripts-sha.ps1 b/vcpkg-init/update-scripts-sha.ps1 index c4e165f583..97539278ce 100644 --- a/vcpkg-init/update-scripts-sha.ps1 +++ b/vcpkg-init/update-scripts-sha.ps1 @@ -1,3 +1,12 @@ -git ls-remote https://github.com/microsoft/vcpkg master | ` - ForEach-Object { $x = [regex]::Match($_, '^[0-9a-f]+').Value; "$x`n" } | ` - Out-File -LiteralPath "$PSScriptRoot/vcpkg-scripts-sha.txt" -Encoding Ascii -NoNewline +$sha = git ls-remote https://github.com/microsoft/vcpkg master | + ForEach-Object { [regex]::Match($_, '^[0-9a-f]+').Value } | + Select-Object -First 1 + +if (-not $sha) { throw "Failed to determine vcpkg scripts SHA." } + +"$sha`n" | Out-File -LiteralPath "$PSScriptRoot/vcpkg-scripts-sha.txt" -Encoding Ascii -NoNewline + +$configPath = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..\vcpkg-configuration.json')).Path +$configText = Get-Content -LiteralPath $configPath -Raw -Encoding Ascii +$newConfigText = $configText -replace '("baseline"\s*:\s*")[0-9a-f]+(")', "`$1$sha`$2" +$newConfigText | Out-File -LiteralPath $configPath -Encoding Ascii -NoNewline diff --git a/vcpkg-init/vcpkg-scripts-sha.txt b/vcpkg-init/vcpkg-scripts-sha.txt index 488f36c455..206f54c187 100644 --- a/vcpkg-init/vcpkg-scripts-sha.txt +++ b/vcpkg-init/vcpkg-scripts-sha.txt @@ -1 +1 @@ -544a4c5c297e60e4ac4a5a1810df66748d908869 +a2a478a93d582a4b395a4dc4b7052bfcb42c1f8e