diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index 09b43d4761..a361e24f76 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -32,9 +32,9 @@ namespace vcpkg View azure_blob_headers(); - std::vector download_files(const Filesystem& fs, - View> url_pairs, - View headers); + std::vector download_files(View> url_pairs, + View headers, + View secrets); bool send_snapshot_to_api(const std::string& github_token, const std::string& github_repository, diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 4015aedc93..cb2a0b60d0 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -837,7 +837,13 @@ DECLARE_MESSAGE(CommandFailed, "", "command:\n" "{command_line}\n" - "failed with the following results:") + "failed with the following output:") +DECLARE_MESSAGE(CommandFailedCode, + (msg::command_line, msg::exit_code), + "", + "command:\n" + "{command_line}\n" + "failed with exit code {exit_code} and the following output:") DECLARE_MESSAGE(CommunityTriplets, (), "", "Community Triplets:") DECLARE_MESSAGE(CompilerPath, (msg::path), "", "Compiler found: {path}") DECLARE_MESSAGE(CompressFolderFailed, (msg::path), "", "Failed to compress folder \"{path}\":") @@ -902,10 +908,6 @@ DECLARE_MESSAGE(Creating7ZipArchive, (), "", "Creating 7zip archive...") DECLARE_MESSAGE(CreatingNugetPackage, (), "", "Creating NuGet package...") DECLARE_MESSAGE(CreatingZipArchive, (), "", "Creating zip archive...") DECLARE_MESSAGE(CreationFailed, (msg::path), "", "Creating {path} failed.") -DECLARE_MESSAGE(CurlFailedToExecute, - (msg::exit_code), - "curl is the name of a program, see curl.se", - "curl failed to execute with exit code {exit_code}.") DECLARE_MESSAGE(CurlFailedToPut, (msg::exit_code, msg::url), "curl is the name of a program, see curl.se", @@ -914,22 +916,15 @@ DECLARE_MESSAGE(CurlFailedToPutHttp, (msg::exit_code, msg::url, msg::value), "curl is the name of a program, see curl.se. {value} is an HTTP status code", "curl failed to put file to {url} with exit code {exit_code} and http code {value}.") -DECLARE_MESSAGE(CurlReportedUnexpectedResults, - (msg::command_line, msg::actual), - "{command_line} is the command line to call curl.exe, {actual} is the console output " - "of curl.exe locale-invariant download results.", - "curl has reported unexpected results to vcpkg and vcpkg cannot continue.\n" - "Please review the following text for sensitive information and open an issue on the " - "Microsoft/vcpkg GitHub to help fix this problem!\n" - "cmd: {command_line}\n" - "=== curl output ===\n" - "{actual}\n" - "=== end curl output ===") -DECLARE_MESSAGE(CurlReturnedUnexpectedResponseCodes, - (msg::actual, msg::expected), - "{actual} and {expected} are integers, curl is the name of a program, see curl.se", - "curl returned a different number of response codes than were expected for the request ({actual} " - "vs expected {expected}).") +DECLARE_MESSAGE(CurlResponseTruncatedRetrying, + (msg::value), + "{value} is the number of milliseconds for which we are waiting this time", + "curl returned a partial response; waiting {value} milliseconds and trying again") +DECLARE_MESSAGE(CurlTimeout, + (msg::command_line), + "", + "curl was unable to perform all requested HTTP operations, even after timeout and retries. The last " + "command line was: {command_line}") DECLARE_MESSAGE(CurrentCommitBaseline, (msg::commit_sha), "", @@ -2608,7 +2603,6 @@ DECLARE_MESSAGE(UnexpectedEOFMidArray, (), "", "Unexpected EOF in middle of arra DECLARE_MESSAGE(UnexpectedEOFMidKeyword, (), "", "Unexpected EOF in middle of keyword") DECLARE_MESSAGE(UnexpectedEOFMidString, (), "", "Unexpected EOF in middle of string") DECLARE_MESSAGE(UnexpectedEOFMidUnicodeEscape, (), "", "Unexpected end of file in middle of unicode escape") -DECLARE_MESSAGE(UnexpectedErrorDuringBulkDownload, (), "", "an unexpected error occurred during bulk download.") DECLARE_MESSAGE(UnexpectedEscapeSequence, (), "", "Unexpected escape sequence continuation") DECLARE_MESSAGE(UnexpectedField, (msg::json_field), "", "unexpected field '{json_field}'") DECLARE_MESSAGE(UnexpectedFieldSuggest, diff --git a/include/vcpkg/base/system.process.h b/include/vcpkg/base/system.process.h index 5a7378a46e..568738de33 100644 --- a/include/vcpkg/base/system.process.h +++ b/include/vcpkg/base/system.process.h @@ -35,26 +35,8 @@ namespace vcpkg explicit Command(StringView s) { string_arg(s); } Command& string_arg(StringView s) &; - Command& raw_arg(StringView s) & - { - if (!buf.empty()) - { - buf.push_back(' '); - } - - buf.append(s.data(), s.size()); - return *this; - } - - Command& forwarded_args(View args) & - { - for (auto&& arg : args) - { - string_arg(arg); - } - - return *this; - } + Command& raw_arg(StringView s) &; + Command& forwarded_args(View args) &; Command&& string_arg(StringView s) && { return std::move(string_arg(s)); }; Command&& raw_arg(StringView s) && { return std::move(raw_arg(s)); } @@ -67,6 +49,13 @@ namespace vcpkg void clear() { buf.clear(); } bool empty() const { return buf.empty(); } + // maximum UNICODE_STRING, with enough space for one MAX_PATH prepended + static constexpr size_t maximum_allowed = 32768 - 260 - 1; + + // if `other` can be appended to this command without exceeding `maximum_allowed`, appends `other` and returns + // true; otherwise, returns false + bool try_append(const Command& other); + private: std::string buf; }; diff --git a/locales/messages.json b/locales/messages.json index 2a246d6760..10115581f7 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -493,8 +493,10 @@ "CmdZExtractOptStrip": "The number of leading directories to strip from all paths", "CommandEnvExample2": "vcpkg env \"ninja -C \" --triplet x64-windows", "_CommandEnvExample2.comment": "This is a command line, only the part should be localized", - "CommandFailed": "command:\n{command_line}\nfailed with the following results:", + "CommandFailed": "command:\n{command_line}\nfailed with the following output:", "_CommandFailed.comment": "An example of {command_line} is vcpkg install zlib.", + "CommandFailedCode": "command:\n{command_line}\nfailed with exit code {exit_code} and the following output:", + "_CommandFailedCode.comment": "An example of {command_line} is vcpkg install zlib. An example of {exit_code} is 127.", "CommunityTriplets": "Community Triplets:", "CompilerPath": "Compiler found: {path}", "_CompilerPath.comment": "An example of {path} is /foo/bar.", @@ -538,16 +540,14 @@ "CreatingZipArchive": "Creating zip archive...", "CreationFailed": "Creating {path} failed.", "_CreationFailed.comment": "An example of {path} is /foo/bar.", - "CurlFailedToExecute": "curl failed to execute with exit code {exit_code}.", - "_CurlFailedToExecute.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127.", "CurlFailedToPut": "curl failed to put file to {url} with exit code {exit_code}.", "_CurlFailedToPut.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", "CurlFailedToPutHttp": "curl failed to put file to {url} with exit code {exit_code} and http code {value}.", "_CurlFailedToPutHttp.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", - "CurlReportedUnexpectedResults": "curl has reported unexpected results to vcpkg and vcpkg cannot continue.\nPlease review the following text for sensitive information and open an issue on the Microsoft/vcpkg GitHub to help fix this problem!\ncmd: {command_line}\n=== curl output ===\n{actual}\n=== end curl output ===", - "_CurlReportedUnexpectedResults.comment": "{command_line} is the command line to call curl.exe, {actual} is the console output of curl.exe locale-invariant download results. An example of {command_line} is vcpkg install zlib.", - "CurlReturnedUnexpectedResponseCodes": "curl returned a different number of response codes than were expected for the request ({actual} vs expected {expected}).", - "_CurlReturnedUnexpectedResponseCodes.comment": "{actual} and {expected} are integers, curl is the name of a program, see curl.se", + "CurlResponseTruncatedRetrying": "curl returned a partial response; waiting {value} milliseconds and trying again", + "_CurlResponseTruncatedRetrying.comment": "{value} is the number of milliseconds for which we are waiting this time", + "CurlTimeout": "curl was unable to perform all requested HTTP operations, even after timeout and retries. The last command line was: {command_line}", + "_CurlTimeout.comment": "An example of {command_line} is vcpkg install zlib.", "CurrentCommitBaseline": "You can use the current commit as a baseline, which is:\n\t\"builtin-baseline\": \"{commit_sha}\"", "_CurrentCommitBaseline.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.", "CycleDetectedDuring": "cycle detected during {spec}:", @@ -1466,7 +1466,6 @@ "UnexpectedEOFMidKeyword": "Unexpected EOF in middle of keyword", "UnexpectedEOFMidString": "Unexpected EOF in middle of string", "UnexpectedEOFMidUnicodeEscape": "Unexpected end of file in middle of unicode escape", - "UnexpectedErrorDuringBulkDownload": "an unexpected error occurred during bulk download.", "UnexpectedEscapeSequence": "Unexpected escape sequence continuation", "UnexpectedField": "unexpected field '{json_field}'", "_UnexpectedField.comment": "An example of {json_field} is identifer.", diff --git a/src/vcpkg-test/system.process.cpp b/src/vcpkg-test/system.process.cpp index 565f289d60..f2b031df36 100644 --- a/src/vcpkg-test/system.process.cpp +++ b/src/vcpkg-test/system.process.cpp @@ -60,3 +60,66 @@ TEST_CASE ("no closes-stdout crash", "[system.process]") REQUIRE(run.exit_code == 0); REQUIRE(run.output == "hello world"); } + +TEST_CASE ("command try_append", "[system.process]") +{ + { + Command a; + REQUIRE(a.try_append(Command{"b"})); + REQUIRE(a.command_line() == "b"); + } + + { + Command a{"a"}; + REQUIRE(a.try_append(Command{})); + REQUIRE(a.command_line() == "a"); + } + + { + Command a{"a"}; + REQUIRE(a.try_append(Command{"b"})); + REQUIRE(a.command_line() == "a b"); + } + + // size limits + + std::string one_string(1, 'a'); + std::string big_string(Command::maximum_allowed, 'a'); + std::string bigger_string(Command::maximum_allowed + 1, 'a'); + Command empty_cmd; + Command one_cmd{one_string}; + Command big_cmd{big_string}; + Command bigger_cmd{bigger_string}; + + REQUIRE(!bigger_cmd.try_append(empty_cmd)); + REQUIRE(bigger_cmd.command_line() == bigger_string); + + REQUIRE(big_cmd.try_append(empty_cmd)); + REQUIRE(big_cmd.command_line() == big_string); + + { + auto cmd = empty_cmd; + REQUIRE(!cmd.try_append(bigger_cmd)); + REQUIRE(cmd.empty()); + REQUIRE(cmd.try_append(big_cmd)); + REQUIRE(cmd.command_line() == big_string); + } + + { + auto cmd = one_cmd; + REQUIRE(!cmd.try_append(big_cmd)); + REQUIRE(cmd.command_line() == one_string); + // does not fit due to the needed space + std::string almost_string(Command::maximum_allowed - 1, 'a'); + Command almost_cmd{almost_string}; + REQUIRE(!cmd.try_append(almost_cmd)); + REQUIRE(cmd.command_line() == one_string); + // fits exactly + std::string ok_string(Command::maximum_allowed - 2, 'a'); + Command ok_cmd{ok_string}; + REQUIRE(cmd.try_append(ok_cmd)); + auto expected = big_string; + expected[1] = ' '; + REQUIRE(cmd.command_line() == expected); + } +} diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 3d70de6953..572318a832 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -371,150 +372,115 @@ namespace vcpkg return Unit{}; } - static void url_heads_inner(View urls, - View headers, - std::vector* out, - View secrets) + static std::vector curl_bulk_operation(View operation_args, + StringLiteral prefixArgs, + View headers, + View secrets) { - static constexpr StringLiteral guid_marker = "8a1db05f-a65d-419b-aa72-037fb4d0672e"; - - const size_t start_size = out->size(); - - auto cmd = Command{"curl"} - .string_arg("--head") - .string_arg("--location") - .string_arg("-w") - .string_arg(guid_marker.to_string() + " %{http_code}\\n"); - for (auto&& header : headers) +#define GUID_MARKER "5ec47b8e-6776-4d70-b9b3-ac2a57bc0a1c" + static constexpr StringLiteral guid_marker = GUID_MARKER; + Command prefix_cmd{"curl"}; + if (!prefixArgs.empty()) { - cmd.string_arg("-H").string_arg(header); + prefix_cmd.raw_arg(prefixArgs); } - for (auto&& url : urls) - { - cmd.string_arg(url_encode_spaces(url)); - } - - std::vector lines; - - auto res = cmd_execute_and_stream_lines(cmd, [out, &lines](StringView line) { - lines.push_back(line.to_string()); - if (Strings::starts_with(line, guid_marker)) - { - out->push_back(std::strtol(line.data() + guid_marker.size(), nullptr, 10)); - } - }).value_or_exit(VCPKG_LINE_INFO); - if (res != 0) - { - Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgCurlFailedToExecute, msg::exit_code = res); - } - - if (out->size() != start_size + urls.size()) - { - auto command_line = replace_secrets(std::move(cmd).extract(), secrets); - auto actual = replace_secrets(Strings::join("\n", lines), secrets); - Checks::msg_exit_with_error(VCPKG_LINE_INFO, - msgCurlReportedUnexpectedResults, - msg::command_line = command_line, - msg::actual = actual); - } - } - std::vector url_heads(View urls, View headers, View secrets) - { - static constexpr size_t batch_size = 100; + prefix_cmd.string_arg("-L").string_arg("-w").string_arg(GUID_MARKER "%{http_code}\\n"); +#undef GUID_MARKER std::vector ret; + ret.reserve(operation_args.size()); - size_t i = 0; - for (; i + batch_size <= urls.size(); i += batch_size) - { - url_heads_inner({urls.data() + i, batch_size}, headers, &ret, secrets); - } - - if (i != urls.size()) + for (auto&& header : headers) { - url_heads_inner({urls.begin() + i, urls.end()}, headers, &ret, secrets); + prefix_cmd.string_arg("-H").string_arg(header); } - Checks::check_exit(VCPKG_LINE_INFO, ret.size() == urls.size()); - return ret; - } + static constexpr auto initial_timeout_delay_ms = 100; + auto timeout_delay_ms = initial_timeout_delay_ms; + static constexpr auto maximum_timeout_delay_ms = 100000; - static void download_files_inner(const Filesystem&, - View> url_pairs, - View headers, - std::vector* out) - { - for (auto i : {100, 1000, 10000, 0}) + while (ret.size() != operation_args.size()) { - size_t start_size = out->size(); - static constexpr StringLiteral guid_marker = "5ec47b8e-6776-4d70-b9b3-ac2a57bc0a1c"; - - auto cmd = Command{"curl"} - .string_arg("--create-dirs") - .string_arg("--location") - .string_arg("-w") - .string_arg(guid_marker.to_string() + " %{http_code}\\n"); - for (StringView header : headers) + // there's an edge case that we aren't handling here where not even one operation fits with the configured + // headers but this seems unlikely + + // form a maximum length command line of operations: + auto batch_cmd = prefix_cmd; + size_t last_try_op = ret.size(); + while (last_try_op != operation_args.size() && batch_cmd.try_append(operation_args[last_try_op])) { - cmd.string_arg("-H").string_arg(header); + ++last_try_op; } - for (auto&& url : url_pairs) + + // actually run curl + auto this_batch_result = cmd_execute_and_capture_output(batch_cmd).value_or_exit(VCPKG_LINE_INFO); + if (this_batch_result.exit_code != 0) { - cmd.string_arg(url_encode_spaces(url.first)).string_arg("-o").string_arg(url.second); + Checks::msg_exit_with_error(VCPKG_LINE_INFO, + msgCommandFailedCode, + msg::command_line = + replace_secrets(std::move(batch_cmd).extract(), secrets), + msg::exit_code = this_batch_result.exit_code); } - auto res = - cmd_execute_and_stream_lines(cmd, [out](StringView line) { - if (Strings::starts_with(line, guid_marker)) - { - out->push_back(static_cast(std::strtol(line.data() + guid_marker.size(), nullptr, 10))); - } - }).value_or_exit(VCPKG_LINE_INFO); - if (res != 0) + // extract HTTP response codes + for (auto&& line : Strings::split(this_batch_result.output, '\n')) { - Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgCurlFailedToExecute, msg::exit_code = res); + if (Strings::starts_with(line, guid_marker)) + { + ret.push_back(static_cast(std::strtol(line.data() + guid_marker.size(), nullptr, 10))); + } } - if (start_size + url_pairs.size() > out->size()) + // check if we got a partial response, and, if so, issue a timed delay + if (ret.size() == last_try_op) { - // curl stopped before finishing all downloads; retry after some time - msg::println_warning(msgUnexpectedErrorDuringBulkDownload); - std::this_thread::sleep_for(std::chrono::milliseconds(i)); - url_pairs = - View>{url_pairs.begin() + out->size() - start_size, url_pairs.end()}; + timeout_delay_ms = initial_timeout_delay_ms; } else { - break; + // curl stopped before finishing all operations; retry after some time + if (timeout_delay_ms >= maximum_timeout_delay_ms) + { + Checks::msg_exit_with_error(VCPKG_LINE_INFO, + msgCurlTimeout, + msg::command_line = + replace_secrets(std::move(batch_cmd).extract(), secrets)); + } + + msg::println_warning(msgCurlResponseTruncatedRetrying, msg::value = timeout_delay_ms); + std::this_thread::sleep_for(std::chrono::milliseconds(timeout_delay_ms)); + timeout_delay_ms *= 10; } } - } - std::vector download_files(const Filesystem& fs, - View> url_pairs, - View headers) - { - static constexpr size_t batch_size = 50; - - std::vector ret; - size_t i = 0; - for (; i + batch_size <= url_pairs.size(); i += batch_size) - { - download_files_inner(fs, {url_pairs.data() + i, batch_size}, headers, &ret); - } + return ret; + } - if (i != url_pairs.size()) - { - download_files_inner(fs, {url_pairs.begin() + i, url_pairs.end()}, headers, &ret); - } + std::vector url_heads(View urls, View headers, View secrets) + { + return curl_bulk_operation( + Util::fmap(urls, [](const std::string& url) { return Command{}.string_arg(url_encode_spaces(url)); }), + "--head", + headers, + secrets); + } - Checks::msg_check_exit(VCPKG_LINE_INFO, - ret.size() == url_pairs.size(), - msgCurlReturnedUnexpectedResponseCodes, - msg::actual = ret.size(), - msg::expected = url_pairs.size()); - return ret; + std::vector download_files(View> url_pairs, + View headers, + View secrets) + { + return curl_bulk_operation(Util::fmap(url_pairs, + [](const std::pair& url_pair) { + return Command{} + .string_arg(url_encode_spaces(url_pair.first)) + .string_arg("-o") + .string_arg(url_pair.second); + }), + "--create-dirs", + headers, + secrets); } bool send_snapshot_to_api(const std::string& github_token, diff --git a/src/vcpkg/base/system.process.cpp b/src/vcpkg/base/system.process.cpp index 64e5fd3a6e..38f003fefc 100644 --- a/src/vcpkg/base/system.process.cpp +++ b/src/vcpkg/base/system.process.cpp @@ -474,6 +474,67 @@ namespace vcpkg return *this; } + Command& Command::raw_arg(StringView s) & + { + if (!buf.empty()) + { + buf.push_back(' '); + } + + buf.append(s.data(), s.size()); + return *this; + } + + Command& Command::forwarded_args(View args) & + { + for (auto&& arg : args) + { + string_arg(arg); + } + + return *this; + } + + bool Command::try_append(const Command& other) + { + if (buf.size() > maximum_allowed) + { + return false; + } + + if (other.buf.empty()) + { + return true; + } + + if (buf.empty()) + { + if (other.buf.size() > maximum_allowed) + { + return false; + } + + buf = other.buf; + return true; + } + + size_t leftover = maximum_allowed - buf.size(); + if (leftover == 0) + { + return false; + } + + --leftover; // for the space + if (other.buf.size() > leftover) + { + return false; + } + + buf.push_back(' '); + buf.append(other.buf); + return true; + } + #if defined(_WIN32) Environment get_modified_clean_environment(const std::unordered_map& extra_env, StringView prepend_to_path) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 0ea770b391..3559fd4275 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -443,7 +443,7 @@ namespace make_temp_archive_path(m_buildtrees, action.spec)); } - auto codes = download_files(m_fs, url_paths, m_url_template.headers); + auto codes = download_files(url_paths, m_url_template.headers, m_secrets); for (size_t i = 0; i < codes.size(); ++i) { @@ -791,8 +791,10 @@ namespace : ZipReadBinaryProvider(std::move(zip), fs) , m_buildtrees(buildtrees) , m_url(url + "_apis/artifactcache/cache") + , m_secrets() , m_token_header("Authorization: Bearer " + token) { + m_secrets.emplace_back(token); } std::string lookup_cache_entry(StringView name, const std::string& abi) const @@ -835,7 +837,7 @@ namespace url_indices.push_back(idx); } - const auto codes = download_files(m_fs, url_paths, {}); + const auto codes = download_files(url_paths, {}, m_secrets); for (size_t i = 0; i < codes.size(); ++i) { @@ -856,6 +858,7 @@ namespace Path m_buildtrees; std::string m_url; + std::vector m_secrets; std::string m_token_header; static constexpr StringLiteral m_accept_header = "Accept: application/json;api-version=6.0-preview.1"; static constexpr StringLiteral m_content_type_header = "Content-Type: application/json";