[vcpkg-tool] Replace curl command calls with libcurl

.github/workflows/build.yaml

@@ -36,6 +36,11 @@ jobs:
         uses: github/codeql-action/init@v3
         with:
           languages: javascript-typescript, c-cpp
+      - name: Install system libcurl
+        if: matrix.preset == 'linux-arm64-ci' || matrix.preset == 'linux-ci'
+        run: |
+          sudo apt update
+          sudo apt install -y libcurl4-openssl-dev
       - name: Configure and Build
         if: matrix.preset != 'windows-ci'
         run: |

CMakeLists.txt

@@ -195,6 +195,7 @@ set(TEST_SCRIPT_ASSET_CACHE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/test-script
 
 find_package(fmt REQUIRED)
 find_package(CMakeRC REQUIRED)
+find_package(LibCURL REQUIRED)
 
 # === Target: locale-resources ===
 
@@ -229,6 +230,8 @@ target_compile_definitions(vcpkglib PUBLIC
     _FILE_OFFSET_BITS=64
 )
 
+target_link_libraries(vcpkglib PUBLIC CURL::libcurl)
+
 if(VCPKG_STANDALONE_BUNDLE_SHA)
     target_compile_definitions(vcpkglib PUBLIC
         "VCPKG_STANDALONE_BUNDLE_SHA=${VCPKG_STANDALONE_BUNDLE_SHA}"

azure-pipelines/end-to-end-tests-dir/asset-caching.ps1

@@ -53,7 +53,7 @@ Throw-IfNotFailed
 $expected = @(
 "A suitable version of cmake was not found \(required v[0-9.]+\)\.",
 "Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-z]+",
-"error: curl: \(37\) Couldn't open file [^\n]+",
+"error: curl operation failed with error code 37 \(Couldn't read a file:// file\)\.",
 "error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+",
 "note: If you are using a proxy, please ensure your proxy settings are correct\.",
 "Possible causes are:",
@@ -105,7 +105,7 @@ if (-not ($actual -match $expected)) {
 Refresh-TestRoot
 $expected = @(
 "^Downloading https://localhost:1234/foobar\.html -> example3\.html",
-"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)",
+"error: curl operation failed with error code 7 \(Couldn't connect to server\)\.",
 "note: If you are using a proxy, please ensure your proxy settings are correct\.",
 "Possible causes are:",
 "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.",
@@ -129,8 +129,8 @@ Refresh-TestRoot
 $expected = @(
 "^Downloading example3\.html, trying https://localhost:1234/foobar\.html",
 "Trying https://localhost:1235/baz\.html",
-"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)",
-"error: curl: \(7\) Failed to connect to localhost port 1235( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)",
+"error: curl operation failed with error code 7 \(Couldn't connect to server\)\.",
+"error: curl operation failed with error code 7 \(Couldn't connect to server\)\.",
 "note: If you are using a proxy, please ensure your proxy settings are correct\.",
 "Possible causes are:",
 "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.",
@@ -206,9 +206,6 @@ if ($IsWindows) {
 Refresh-TestRoot
 $expected = @(
 "^Downloading example3\.html, trying https://nonexistent\.example\.com",
-"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in 1 seconds?\. 3 retries left\.",
-"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in 2 seconds\. 2 retries left\.",
-"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in 4 seconds\. 1 (retries|retry) left\.",
 "Trying https://example\.com",
 "Successfully downloaded example3\.html",
 "$"
@@ -217,7 +214,7 @@ $expected = @(
 $actual = Run-VcpkgAndCaptureOutput @commonArgs x-download "$TestDownloadsRoot/example3.html" --sha512 d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a --url https://nonexistent.example.com --url https://example.com --header "Cache-Control: no-cache"
 Throw-IfFailed
 if (-not ($actual -match $expected)) {
-    throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)"
+    throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed), headers (cache-control)"
 }
 
 # azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)
@@ -241,8 +238,8 @@ Refresh-TestRoot
 $expected = @(
 "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+",
 "Asset cache miss; trying authoritative source https://localhost:1234/foobar\.html",
-"error: curl: \(37\) Couldn't open file [^\n]+",
-"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)",
+"error: curl operation failed with error code 37 \(Couldn't read a file:// file\)\.",
+"error: curl operation failed with error code 7 \(Couldn't connect to server\)\.",
 "note: If you are using a proxy, please ensure your proxy settings are correct\.",
 "Possible causes are:",
 "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.",
@@ -301,7 +298,7 @@ if (-not ($actual -match $expected)) {
 $expected = @(
 "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+",
 "Asset cache miss; trying authoritative source https://example\.com",
-"error: curl: \(37\) Couldn't open file [^\n]+",
+"error: curl operation failed with error code 37 \(Couldn't read a file:// file\)\.",
 "note: If you are using a proxy, please ensure your proxy settings are correct\.",
 "Possible causes are:",
 "1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.",
@@ -363,7 +360,7 @@ if (-not ($actual -match $expected)) {
 Refresh-TestRoot
 $expected = @(
 "^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+",
-"error: curl: \(37\) Couldn't open file [^\n]+",
+"error: curl operation failed with error code 37 \(Couldn't read a file:// file\)\.",
 "error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com",
 "note: or https://alternate\.example\.com",
 "note: If you are using a proxy, please ensure your proxy settings are correct\.",

cmake/FindLibCURL.cmake

@@ -0,0 +1,82 @@
+if (WIN32)
+    option(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL "Use an external version of the libcurl library" OFF)
+else()
+    option(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL "Use an external version of the libcurl library" ON)
+endif()
+
+# This option exists to allow the URI to be replaced with a Microsoft-internal URI in official
+# builds which have restricted internet access; see azure-pipelines/signing.yml
+# Note that the SHA512 is the same, so vcpkg-tool contributors need not be concerned that we built
+# with different content.
+if(NOT VCPKG_LIBCURL_URL)
+    set(VCPKG_LIBCURL_URL "https://github.com/curl/curl/archive/refs/tags/curl-8_8_0.tar.gz")
+endif()
+
+if(POLICY CMP0135)
+    cmake_policy(SET CMP0135 NEW)
+endif()
+
+include(FetchContent)
+FetchContent_Declare(
+    LibCURL
+    URL "${VCPKG_LIBCURL_URL}"
+    URL_HASH "SHA512=e66cbf9bd3ae7b9b031475210b80b883b6a133042fbbc7cf2413f399d1b38aa54ab7322626abd3c6f1af56e0d540221f618aa903bd6b463ac8324f2c4e92dfa8"
+)
+
+if(NOT LibCURL_FIND_REQUIRED)
+    message(FATAL_ERROR "LibCURL must be REQUIRED")
+endif()
+
+if(VCPKG_DEPENDENCY_EXTERNAL_LIBCURL)
+    find_package(CURL REQUIRED)
+else()
+    function(get_libcurl)
+        set(BUILD_SHARED_LIBS OFF)
+        set(BUILD_CURL_EXE OFF)
+        set(CURL_ENABLE_EXPORT_TARGET OFF)
+        set(ENABLE_UNICODE ON)
+        set(BUILD_LIBCURL_DOCS  OFF)
+        set(BUILD_MISC_DOCS OFF)
+        set(ENABLE_CURL_MANUAL OFF)
+        set(PICKY_COMPILER OFF)
+        set(CMAKE_DISABLE_FIND_PACKAGE_Perl ON)
+        set(CMAKE_DISABLE_FIND_PACKAGE_ZLIB ON)
+        set(CMAKE_DISABLE_FIND_PACKAGE_LibPSL ON)
+        set(CMAKE_DISABLE_FIND_PACKAGE_LibSSH2 ON)
+        if(MSVC) # This is in function() so no need to backup the variables
+            string(APPEND CMAKE_C_FLAGS " /wd6101")
+            string(APPEND CMAKE_C_FLAGS " /wd6011")
+            string(APPEND CMAKE_C_FLAGS " /wd6054")
+            string(APPEND CMAKE_C_FLAGS " /wd6240")
+            string(APPEND CMAKE_C_FLAGS " /wd6239")
+            string(APPEND CMAKE_C_FLAGS " /wd6323")
+            string(APPEND CMAKE_C_FLAGS " /wd6387")
+            string(APPEND CMAKE_C_FLAGS " /wd28182")
+            string(APPEND CMAKE_C_FLAGS " /wd28183")
+            string(APPEND CMAKE_C_FLAGS " /wd28251")
+            string(APPEND CMAKE_C_FLAGS " /wd28301")
+        else()
+            string(APPEND CMAKE_C_FLAGS " -Wno-error")
+        endif()
+        if (WIN32)
+            set(CURL_USE_SCHANNEL ON)
+        endif()
+        FetchContent_MakeAvailable(LibCURL)
+    endfunction()
+
+    get_libcurl()
+
+    if(NOT TARGET CURL::libcurl)
+        if(TARGET libcurl_static)
+            add_library(CURL::libcurl ALIAS libcurl_static)
+            target_compile_definitions(libcurl_static INTERFACE CURL_STATICLIB)
+        elseif(TARGET libcurl)
+            add_library(CURL::libcurl ALIAS libcurl)
+            if(NOT BUILD_SHARED_LIBS)
+                target_compile_definitions(libcurl INTERFACE CURL_STATICLIB)
+            endif()
+        else()
+            message(FATAL_ERROR "After FetchContent_MakeAvailable(LibCURL) no suitable curl target (libcurl or libcurl_static) was found.")
+        endif()
+    endif()
+endif()

include/vcpkg/base/contractual-constants.h

@@ -206,6 +206,7 @@ namespace vcpkg
     inline constexpr StringLiteral SwitchDebug = "debug";
     inline constexpr StringLiteral SwitchDebugBin = "debug-bin";
     inline constexpr StringLiteral SwitchDebugEnv = "debug-env";
+    inline constexpr StringLiteral SwitchDeleteFileAfterUpload = "delete-file-after-upload";
     inline constexpr StringLiteral SwitchDgml = "dgml";
     inline constexpr StringLiteral SwitchDisableMetrics = "disable-metrics";
     inline constexpr StringLiteral SwitchDot = "dot";

include/vcpkg/base/curl.h

@@ -0,0 +1,17 @@
+
+#ifdef _MSC_VER
+#pragma warning(push)           // Save current warning state
+#pragma warning(disable : 6101) // Disable specific warning (e.g., warning 4996)
+#endif
+
+#include <curl/curl.h>
+#include <curl/multi.h>
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+namespace vcpkg
+{
+    CURLcode curl_global_init_status() noexcept;
+}

include/vcpkg/base/downloads.h

@@ -37,16 +37,6 @@ namespace vcpkg
 
     View<std::string> azure_blob_headers();
 
-    // Parses a curl output line for curl invoked with
-    // -w "PREFIX%{http_code} %{exitcode} %{errormsg}"
-    // with specific handling for curl version < 7.75.0 which does not understand %{exitcode} %{errormsg}
-    // If the line is malformed for any reason, no entry to http_codes is added.
-    // Returns: true if the new version of curl's output with exitcode and errormsg was parsed; otherwise, false.
-    bool parse_curl_status_line(DiagnosticContext& context,
-                                std::vector<int>& http_codes,
-                                StringLiteral prefix,
-                                StringView this_line);
-
     std::vector<int> download_files_no_cache(DiagnosticContext& context,
                                              View<std::pair<std::string, Path>> url_pairs,
                                              View<std::string> headers,
@@ -58,13 +48,6 @@ namespace vcpkg
                                                  const std::string& github_repository,
                                                  const Json::Object& snapshot);
 
-    Optional<std::string> invoke_http_request(DiagnosticContext& context,
-                                              StringLiteral method,
-                                              View<std::string> headers,
-                                              StringView url,
-                                              View<std::string> secrets,
-                                              StringView data = {});
-
     std::string format_url_query(StringView base_url, View<std::string> query_params);
 
     std::vector<int> url_heads(DiagnosticContext& context,

include/vcpkg/base/files.h

@@ -103,6 +103,8 @@ namespace vcpkg
         // reads any remaining chunks of the file; used to implement read_to_end
         void read_to_end_suffix(
             std::string& output, std::error_code& ec, char* buffer, size_t buffer_size, size_t last_read);
+        uint64_t size(LineInfo li) const;
+        uint64_t size(std::error_code& ec) const;
     };
 
     struct WriteFilePointer : FilePointer

include/vcpkg/base/message-data.inc.h

@@ -870,6 +870,7 @@ DECLARE_MESSAGE(CmdUpdateRegistryAllOrTargets,
                 (),
                 "",
                 "Update registry requires either a list of artifact registry names or URiIs to update, or --all.")
+DECLARE_MESSAGE(CmdUploadMetricsDeleteFileAfterUpload, (), "", "Delete metrics payload file after upload")
 DECLARE_MESSAGE(CmdUpgradeOptNoDryRun, (), "", "Actually upgrade")
 DECLARE_MESSAGE(CmdUpgradeOptNoKeepGoing, (), "", "Stop installing packages on failure")
 DECLARE_MESSAGE(CmdUseExample1,
@@ -966,24 +967,25 @@ DECLARE_MESSAGE(CreatingNugetPackage, (), "", "Creating NuGet package...")
 DECLARE_MESSAGE(CreatingZipArchive, (), "", "Creating zip archive...")
 DECLARE_MESSAGE(CreationFailed, (msg::path), "", "Creating {path} failed.")
 DECLARE_MESSAGE(CurlFailedGeneric,
+                (msg::exit_code, msg::error_msg),
+                "curl is the name of a program, see curl.se.",
+                "curl operation failed with error code {exit_code} ({error_msg}).")
+DECLARE_MESSAGE(CurlFailedGenericWithRetry,
+                (msg::exit_code, msg::error_msg, msg::count, msg::value),
+                "curl is the name of a program, see curl.se. {value} is the maximum amount of retries.",
+                "curl operation failed with error code {exit_code} ({error_msg}) retry {count} of {value}.")
+DECLARE_MESSAGE(CurlFailedHttpResponse,
                 (msg::exit_code),
                 "curl is the name of a program, see curl.se.",
-                "curl operation failed with error code {exit_code}.")
-DECLARE_MESSAGE(CurlFailedToPut,
-                (msg::exit_code, msg::url),
-                "curl is the name of a program, see curl.se",
-                "curl failed to put file to {url} with exit code {exit_code}.")
+                "curl operation failed with HTTP response code {exit_code}.")
+DECLARE_MESSAGE(CurlFailedHttpResponseWithRetry,
+                (msg::exit_code, msg::count, msg::value),
+                "curl is the name of a program, see curl.se. {value} is the maximum amount of retries.",
+                "curl operation failed with HTTP response code {exit_code} retry {count} of {value}.")
 DECLARE_MESSAGE(CurlFailedToPutHttp,
-                (msg::exit_code, msg::url, msg::value),
+                (msg::exit_code, msg::error_msg, msg::url, msg::value),
                 "curl is the name of a program, see curl.se. {value} is an HTTP status code",
-                "curl failed to put file to {url} with exit code {exit_code} and http code {value}.")
-DECLARE_MESSAGE(
-    CurlFailedToReturnExpectedNumberOfExitCodes,
-    (msg::exit_code, msg::command_line),
-    "",
-    "curl failed to return the expected number of exit codes; this can happen if something terminates curl "
-    "before it has finished. curl exited with {exit_code} which is normally the result code for the last operation, "
-    "but may be the result of a crash. The command line was {command_line}, and all output is below:")
+                "curl failed to put file to {url} with exit code {exit_code} ({error_msg}) and http code {value}.")
 DECLARE_MESSAGE(CurrentCommitBaseline,
                 (msg::commit_sha),
                 "",

include/vcpkg/metrics.h

@@ -198,7 +198,5 @@ namespace vcpkg
     extern std::atomic<bool> g_should_send_metrics;
 
     void flush_global_metrics(const Filesystem&);
-#if defined(_WIN32)
-    void winhttp_upload_metrics(StringView payload);
-#endif // ^^^ _WIN32
+    bool curl_upload_metrics(StringView payload);
 }

locales/messages.json

@@ -506,6 +506,7 @@
   "CmdUpdateRegistrySynopsis": "Re-downloads an artifact registry",
   "CmdUpgradeOptNoDryRun": "Actually upgrade",
   "CmdUpgradeOptNoKeepGoing": "Stop installing packages on failure",
+  "CmdUploadMetricsDeleteFileAfterUpload": "Delete metrics payload file after upload",
   "CmdUseExample1": "vcpkg use <artifact name>",
   "_CmdUseExample1.comment": "This is a command line, only the <artifact name> part should be localized.",
   "CmdUseSynopsis": "Activate a single artifact in this shell",
@@ -563,14 +564,16 @@
   "CreatingZipArchive": "Creating zip archive...",
   "CreationFailed": "Creating {path} failed.",
   "_CreationFailed.comment": "An example of {path} is /foo/bar.",
-  "CurlFailedGeneric": "curl operation failed with error code {exit_code}.",
-  "_CurlFailedGeneric.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.",
-  "CurlFailedToPut": "curl failed to put file to {url} with exit code {exit_code}.",
-  "_CurlFailedToPut.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
-  "CurlFailedToPutHttp": "curl failed to put file to {url} with exit code {exit_code} and http code {value}.",
-  "_CurlFailedToPutHttp.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
-  "CurlFailedToReturnExpectedNumberOfExitCodes": "curl failed to return the expected number of exit codes; this can happen if something terminates curl before it has finished. curl exited with {exit_code} which is normally the result code for the last operation, but may be the result of a crash. The command line was {command_line}, and all output is below:",
-  "_CurlFailedToReturnExpectedNumberOfExitCodes.comment": "An example of {exit_code} is 127. An example of {command_line} is vcpkg install zlib.",
+  "CurlFailedGeneric": "curl operation failed with error code {exit_code} ({error_msg}).",
+  "_CurlFailedGeneric.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127. An example of {error_msg} is File Not Found.",
+  "CurlFailedGenericWithRetry": "curl operation failed with error code {exit_code} ({error_msg}) retry {count} of {value}.",
+  "_CurlFailedGenericWithRetry.comment": "curl is the name of a program, see curl.se. {value} is the maximum amount of retries. An example of {exit_code} is 127. An example of {error_msg} is File Not Found. An example of {count} is 42.",
+  "CurlFailedHttpResponse": "curl operation failed with HTTP response code {exit_code}.",
+  "_CurlFailedHttpResponse.comment": "curl is the name of a program, see curl.se. An example of {exit_code} is 127.",
+  "CurlFailedHttpResponseWithRetry": "curl operation failed with HTTP response code {exit_code} retry {count} of {value}.",
+  "_CurlFailedHttpResponseWithRetry.comment": "curl is the name of a program, see curl.se. {value} is the maximum amount of retries. An example of {exit_code} is 127. An example of {count} is 42.",
+  "CurlFailedToPutHttp": "curl failed to put file to {url} with exit code {exit_code} ({error_msg}) and http code {value}.",
+  "_CurlFailedToPutHttp.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {exit_code} is 127. An example of {error_msg} is File Not Found. An example of {url} is https://github.com/microsoft/vcpkg.",
   "CurrentCommitBaseline": "You can use the current commit as a baseline, which is:\n\t\"builtin-baseline\": \"{commit_sha}\"",
   "_CurrentCommitBaseline.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.",
   "CycleDetectedDuring": "cycle detected during {spec}:",