Avoid curl command line length limits

include/vcpkg/base/downloads.h

@@ -32,9 +32,9 @@ namespace vcpkg
 
     View<std::string> azure_blob_headers();
 
-    std::vector<int> download_files(const Filesystem& fs,
-                                    View<std::pair<std::string, Path>> url_pairs,
-                                    View<std::string> headers);
+    std::vector<int> download_files(View<std::pair<std::string, Path>> url_pairs,
+                                    View<std::string> headers,
+                                    View<std::string> secrets);
 
     bool send_snapshot_to_api(const std::string& github_token,
                               const std::string& github_repository,

include/vcpkg/base/message-data.inc.h

@@ -837,7 +837,13 @@ DECLARE_MESSAGE(CommandFailed,
                 "",
                 "command:\n"
                 "{command_line}\n"
-                "failed with the following results:")
+                "failed with the following output:")
+DECLARE_MESSAGE(CommandFailedCode,
+                (msg::command_line, msg::exit_code),
+                "",
+                "command:\n"
+                "{command_line}\n"
+                "failed with exit code {exit_code} and the following output:")
 DECLARE_MESSAGE(CommunityTriplets, (), "", "Community Triplets:")
 DECLARE_MESSAGE(CompilerPath, (msg::path), "", "Compiler found: {path}")
 DECLARE_MESSAGE(CompressFolderFailed, (msg::path), "", "Failed to compress folder \"{path}\":")
@@ -902,10 +908,6 @@ DECLARE_MESSAGE(Creating7ZipArchive, (), "", "Creating 7zip archive...")
 DECLARE_MESSAGE(CreatingNugetPackage, (), "", "Creating NuGet package...")
 DECLARE_MESSAGE(CreatingZipArchive, (), "", "Creating zip archive...")
 DECLARE_MESSAGE(CreationFailed, (msg::path), "", "Creating {path} failed.")
-DECLARE_MESSAGE(CurlFailedToExecute,
-                (msg::exit_code),
-                "curl is the name of a program, see curl.se",
-                "curl failed to execute with exit code {exit_code}.")
 DECLARE_MESSAGE(CurlFailedToPut,
                 (msg::exit_code, msg::url),
                 "curl is the name of a program, see curl.se",
@@ -914,22 +916,15 @@ DECLARE_MESSAGE(CurlFailedToPutHttp,
                 (msg::exit_code, msg::url, msg::value),
                 "curl is the name of a program, see curl.se. {value} is an HTTP status code",
                 "curl failed to put file to {url} with exit code {exit_code} and http code {value}.")
-DECLARE_MESSAGE(CurlReportedUnexpectedResults,
-                (msg::command_line, msg::actual),
-                "{command_line} is the command line to call curl.exe, {actual} is the console output "
-                "of curl.exe locale-invariant download results.",
-                "curl has reported unexpected results to vcpkg and vcpkg cannot continue.\n"
-                "Please review the following text for sensitive information and open an issue on the "
-                "Microsoft/vcpkg GitHub to help fix this problem!\n"
-                "cmd: {command_line}\n"
-                "=== curl output ===\n"
-                "{actual}\n"
-                "=== end curl output ===")
-DECLARE_MESSAGE(CurlReturnedUnexpectedResponseCodes,
-                (msg::actual, msg::expected),
-                "{actual} and {expected} are integers, curl is the name of a program, see curl.se",
-                "curl returned a different number of response codes than were expected for the request ({actual} "
-                "vs expected {expected}).")
+DECLARE_MESSAGE(CurlResponseTruncatedRetrying,
+                (msg::value),
+                "{value} is the number of milliseconds for which we are waiting this time",
+                "curl returned a partial response; waiting {value} milliseconds and trying again")
+DECLARE_MESSAGE(CurlTimeout,
+                (msg::command_line),
+                "",
+                "curl was unable to perform all requested HTTP operations, even after timeout and retries. The last "
+                "command line was: {command_line}")
 DECLARE_MESSAGE(CurrentCommitBaseline,
                 (msg::commit_sha),
                 "",
@@ -2608,7 +2603,6 @@ DECLARE_MESSAGE(UnexpectedEOFMidArray, (), "", "Unexpected EOF in middle of arra
 DECLARE_MESSAGE(UnexpectedEOFMidKeyword, (), "", "Unexpected EOF in middle of keyword")
 DECLARE_MESSAGE(UnexpectedEOFMidString, (), "", "Unexpected EOF in middle of string")
 DECLARE_MESSAGE(UnexpectedEOFMidUnicodeEscape, (), "", "Unexpected end of file in middle of unicode escape")
-DECLARE_MESSAGE(UnexpectedErrorDuringBulkDownload, (), "", "an unexpected error occurred during bulk download.")
 DECLARE_MESSAGE(UnexpectedEscapeSequence, (), "", "Unexpected escape sequence continuation")
 DECLARE_MESSAGE(UnexpectedField, (msg::json_field), "", "unexpected field '{json_field}'")
 DECLARE_MESSAGE(UnexpectedFieldSuggest,

include/vcpkg/base/system.process.h

@@ -35,26 +35,8 @@ namespace vcpkg
         explicit Command(StringView s) { string_arg(s); }
 
         Command& string_arg(StringView s) &;
-        Command& raw_arg(StringView s) &
-        {
-            if (!buf.empty())
-            {
-                buf.push_back(' ');
-            }
-
-            buf.append(s.data(), s.size());
-            return *this;
-        }
-
-        Command& forwarded_args(View<std::string> args) &
-        {
-            for (auto&& arg : args)
-            {
-                string_arg(arg);
-            }
-
-            return *this;
-        }
+        Command& raw_arg(StringView s) &;
+        Command& forwarded_args(View<std::string> args) &;
 
         Command&& string_arg(StringView s) && { return std::move(string_arg(s)); };
         Command&& raw_arg(StringView s) && { return std::move(raw_arg(s)); }
@@ -67,6 +49,13 @@ namespace vcpkg
         void clear() { buf.clear(); }
         bool empty() const { return buf.empty(); }
 
+        // maximum UNICODE_STRING, with enough space for one MAX_PATH prepended
+        static constexpr size_t maximum_allowed = 32768 - 260 - 1;
+
+        // if `other` can be appended to this command without exceeding `maximum_allowed`, appends `other` and returns
+        // true; otherwise, returns false
+        bool try_append(const Command& other);
+
     private:
         std::string buf;
     };

locales/messages.json

@@ -493,8 +493,10 @@
   "CmdZExtractOptStrip": "The number of leading directories to strip from all paths",
   "CommandEnvExample2": "vcpkg env \"ninja -C <path>\" --triplet x64-windows",
   "_CommandEnvExample2.comment": "This is a command line, only the <path> part should be localized",
-  "CommandFailed": "command:\n{command_line}\nfailed with the following results:",
+  "CommandFailed": "command:\n{command_line}\nfailed with the following output:",
   "_CommandFailed.comment": "An example of {command_line} is vcpkg install zlib.",
+  "CommandFailedCode": "command:\n{command_line}\nfailed with exit code {exit_code} and the following output:",
+  "_CommandFailedCode.comment": "An example of {command_line} is vcpkg install zlib. An example of {exit_code} is 127.",
   "CommunityTriplets": "Community Triplets:",
   "CompilerPath": "Compiler found: {path}",
   "_CompilerPath.comment": "An example of {path} is /foo/bar.",
@@ -538,16 +540,14 @@
   "CreatingZipArchive": "Creating zip archive...",
   "CreationFailed": "Creating {path} failed.",
   "_CreationFailed.comment": "An example of {path} is /foo/bar.",
-  "CurlFailedToExecute": "curl failed to execute with exit code {exit_code}.",
-  "_CurlFailedToExecute.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127.",
   "CurlFailedToPut": "curl failed to put file to {url} with exit code {exit_code}.",
   "_CurlFailedToPut.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
   "CurlFailedToPutHttp": "curl failed to put file to {url} with exit code {exit_code} and http code {value}.",
   "_CurlFailedToPutHttp.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
-  "CurlReportedUnexpectedResults": "curl has reported unexpected results to vcpkg and vcpkg cannot continue.\nPlease review the following text for sensitive information and open an issue on the Microsoft/vcpkg GitHub to help fix this problem!\ncmd: {command_line}\n=== curl output ===\n{actual}\n=== end curl output ===",
-  "_CurlReportedUnexpectedResults.comment": "{command_line} is the command line to call curl.exe, {actual} is the console output of curl.exe locale-invariant download results. An example of {command_line} is vcpkg install zlib.",
-  "CurlReturnedUnexpectedResponseCodes": "curl returned a different number of response codes than were expected for the request ({actual} vs expected {expected}).",
-  "_CurlReturnedUnexpectedResponseCodes.comment": "{actual} and {expected} are integers, curl is the name of a program, see curl.se",
+  "CurlResponseTruncatedRetrying": "curl returned a partial response; waiting {value} milliseconds and trying again",
+  "_CurlResponseTruncatedRetrying.comment": "{value} is the number of milliseconds for which we are waiting this time",
+  "CurlTimeout": "curl was unable to perform all requested HTTP operations, even after timeout and retries. The last command line was: {command_line}",
+  "_CurlTimeout.comment": "An example of {command_line} is vcpkg install zlib.",
   "CurrentCommitBaseline": "You can use the current commit as a baseline, which is:\n\t\"builtin-baseline\": \"{commit_sha}\"",
   "_CurrentCommitBaseline.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.",
   "CycleDetectedDuring": "cycle detected during {spec}:",
@@ -1466,7 +1466,6 @@
   "UnexpectedEOFMidKeyword": "Unexpected EOF in middle of keyword",
   "UnexpectedEOFMidString": "Unexpected EOF in middle of string",
   "UnexpectedEOFMidUnicodeEscape": "Unexpected end of file in middle of unicode escape",
-  "UnexpectedErrorDuringBulkDownload": "an unexpected error occurred during bulk download.",
   "UnexpectedEscapeSequence": "Unexpected escape sequence continuation",
   "UnexpectedField": "unexpected field '{json_field}'",
   "_UnexpectedField.comment": "An example of {json_field} is identifer.",

src/vcpkg-test/system.process.cpp

@@ -60,3 +60,66 @@ TEST_CASE ("no closes-stdout crash", "[system.process]")
     REQUIRE(run.exit_code == 0);
     REQUIRE(run.output == "hello world");
 }
+
+TEST_CASE ("command try_append", "[system.process]")
+{
+    {
+        Command a;
+        REQUIRE(a.try_append(Command{"b"}));
+        REQUIRE(a.command_line() == "b");
+    }
+
+    {
+        Command a{"a"};
+        REQUIRE(a.try_append(Command{}));
+        REQUIRE(a.command_line() == "a");
+    }
+
+    {
+        Command a{"a"};
+        REQUIRE(a.try_append(Command{"b"}));
+        REQUIRE(a.command_line() == "a b");
+    }
+
+    // size limits
+
+    std::string one_string(1, 'a');
+    std::string big_string(Command::maximum_allowed, 'a');
+    std::string bigger_string(Command::maximum_allowed + 1, 'a');
+    Command empty_cmd;
+    Command one_cmd{one_string};
+    Command big_cmd{big_string};
+    Command bigger_cmd{bigger_string};
+
+    REQUIRE(!bigger_cmd.try_append(empty_cmd));
+    REQUIRE(bigger_cmd.command_line() == bigger_string);
+
+    REQUIRE(big_cmd.try_append(empty_cmd));
+    REQUIRE(big_cmd.command_line() == big_string);
+
+    {
+        auto cmd = empty_cmd;
+        REQUIRE(!cmd.try_append(bigger_cmd));
+        REQUIRE(cmd.empty());
+        REQUIRE(cmd.try_append(big_cmd));
+        REQUIRE(cmd.command_line() == big_string);
+    }
+
+    {
+        auto cmd = one_cmd;
+        REQUIRE(!cmd.try_append(big_cmd));
+        REQUIRE(cmd.command_line() == one_string);
+        // does not fit due to the needed space
+        std::string almost_string(Command::maximum_allowed - 1, 'a');
+        Command almost_cmd{almost_string};
+        REQUIRE(!cmd.try_append(almost_cmd));
+        REQUIRE(cmd.command_line() == one_string);
+        // fits exactly
+        std::string ok_string(Command::maximum_allowed - 2, 'a');
+        Command ok_cmd{ok_string};
+        REQUIRE(cmd.try_append(ok_cmd));
+        auto expected = big_string;
+        expected[1] = ' ';
+        REQUIRE(cmd.command_line() == expected);
+    }
+}