Skip to content

Commit

Permalink
Avoid curl command line length limits (#1390)
Browse files Browse the repository at this point in the history
BillyONeal authored Apr 22, 2024
1 parent 87adc3f commit 62ef030
Showing 8 changed files with 245 additions and 170 deletions.
6 changes: 3 additions & 3 deletions include/vcpkg/base/downloads.h
Original file line number Diff line number Diff line change
@@ -32,9 +32,9 @@ namespace vcpkg

View<std::string> azure_blob_headers();

std::vector<int> download_files(const Filesystem& fs,
View<std::pair<std::string, Path>> url_pairs,
View<std::string> headers);
std::vector<int> download_files(View<std::pair<std::string, Path>> url_pairs,
View<std::string> headers,
View<std::string> secrets);

bool send_snapshot_to_api(const std::string& github_token,
const std::string& github_repository,
38 changes: 16 additions & 22 deletions include/vcpkg/base/message-data.inc.h
Original file line number Diff line number Diff line change
@@ -837,7 +837,13 @@ DECLARE_MESSAGE(CommandFailed,
"",
"command:\n"
"{command_line}\n"
"failed with the following results:")
"failed with the following output:")
DECLARE_MESSAGE(CommandFailedCode,
(msg::command_line, msg::exit_code),
"",
"command:\n"
"{command_line}\n"
"failed with exit code {exit_code} and the following output:")
DECLARE_MESSAGE(CommunityTriplets, (), "", "Community Triplets:")
DECLARE_MESSAGE(CompilerPath, (msg::path), "", "Compiler found: {path}")
DECLARE_MESSAGE(CompressFolderFailed, (msg::path), "", "Failed to compress folder \"{path}\":")
@@ -902,10 +908,6 @@ DECLARE_MESSAGE(Creating7ZipArchive, (), "", "Creating 7zip archive...")
DECLARE_MESSAGE(CreatingNugetPackage, (), "", "Creating NuGet package...")
DECLARE_MESSAGE(CreatingZipArchive, (), "", "Creating zip archive...")
DECLARE_MESSAGE(CreationFailed, (msg::path), "", "Creating {path} failed.")
DECLARE_MESSAGE(CurlFailedToExecute,
(msg::exit_code),
"curl is the name of a program, see curl.se",
"curl failed to execute with exit code {exit_code}.")
DECLARE_MESSAGE(CurlFailedToPut,
(msg::exit_code, msg::url),
"curl is the name of a program, see curl.se",
@@ -914,22 +916,15 @@ DECLARE_MESSAGE(CurlFailedToPutHttp,
(msg::exit_code, msg::url, msg::value),
"curl is the name of a program, see curl.se. {value} is an HTTP status code",
"curl failed to put file to {url} with exit code {exit_code} and http code {value}.")
DECLARE_MESSAGE(CurlReportedUnexpectedResults,
(msg::command_line, msg::actual),
"{command_line} is the command line to call curl.exe, {actual} is the console output "
"of curl.exe locale-invariant download results.",
"curl has reported unexpected results to vcpkg and vcpkg cannot continue.\n"
"Please review the following text for sensitive information and open an issue on the "
"Microsoft/vcpkg GitHub to help fix this problem!\n"
"cmd: {command_line}\n"
"=== curl output ===\n"
"{actual}\n"
"=== end curl output ===")
DECLARE_MESSAGE(CurlReturnedUnexpectedResponseCodes,
(msg::actual, msg::expected),
"{actual} and {expected} are integers, curl is the name of a program, see curl.se",
"curl returned a different number of response codes than were expected for the request ({actual} "
"vs expected {expected}).")
DECLARE_MESSAGE(CurlResponseTruncatedRetrying,
(msg::value),
"{value} is the number of milliseconds for which we are waiting this time",
"curl returned a partial response; waiting {value} milliseconds and trying again")
DECLARE_MESSAGE(CurlTimeout,
(msg::command_line),
"",
"curl was unable to perform all requested HTTP operations, even after timeout and retries. The last "
"command line was: {command_line}")
DECLARE_MESSAGE(CurrentCommitBaseline,
(msg::commit_sha),
"",
@@ -2608,7 +2603,6 @@ DECLARE_MESSAGE(UnexpectedEOFMidArray, (), "", "Unexpected EOF in middle of arra
DECLARE_MESSAGE(UnexpectedEOFMidKeyword, (), "", "Unexpected EOF in middle of keyword")
DECLARE_MESSAGE(UnexpectedEOFMidString, (), "", "Unexpected EOF in middle of string")
DECLARE_MESSAGE(UnexpectedEOFMidUnicodeEscape, (), "", "Unexpected end of file in middle of unicode escape")
DECLARE_MESSAGE(UnexpectedErrorDuringBulkDownload, (), "", "an unexpected error occurred during bulk download.")
DECLARE_MESSAGE(UnexpectedEscapeSequence, (), "", "Unexpected escape sequence continuation")
DECLARE_MESSAGE(UnexpectedField, (msg::json_field), "", "unexpected field '{json_field}'")
DECLARE_MESSAGE(UnexpectedFieldSuggest,
29 changes: 9 additions & 20 deletions include/vcpkg/base/system.process.h
Original file line number Diff line number Diff line change
@@ -35,26 +35,8 @@ namespace vcpkg
explicit Command(StringView s) { string_arg(s); }

Command& string_arg(StringView s) &;
Command& raw_arg(StringView s) &
{
if (!buf.empty())
{
buf.push_back(' ');
}

buf.append(s.data(), s.size());
return *this;
}

Command& forwarded_args(View<std::string> args) &
{
for (auto&& arg : args)
{
string_arg(arg);
}

return *this;
}
Command& raw_arg(StringView s) &;
Command& forwarded_args(View<std::string> args) &;

Command&& string_arg(StringView s) && { return std::move(string_arg(s)); };
Command&& raw_arg(StringView s) && { return std::move(raw_arg(s)); }
@@ -67,6 +49,13 @@ namespace vcpkg
void clear() { buf.clear(); }
bool empty() const { return buf.empty(); }

// maximum UNICODE_STRING, with enough space for one MAX_PATH prepended
static constexpr size_t maximum_allowed = 32768 - 260 - 1;

// if `other` can be appended to this command without exceeding `maximum_allowed`, appends `other` and returns
// true; otherwise, returns false
bool try_append(const Command& other);

private:
std::string buf;
};
15 changes: 7 additions & 8 deletions locales/messages.json
Original file line number Diff line number Diff line change
@@ -493,8 +493,10 @@
"CmdZExtractOptStrip": "The number of leading directories to strip from all paths",
"CommandEnvExample2": "vcpkg env \"ninja -C <path>\" --triplet x64-windows",
"_CommandEnvExample2.comment": "This is a command line, only the <path> part should be localized",
"CommandFailed": "command:\n{command_line}\nfailed with the following results:",
"CommandFailed": "command:\n{command_line}\nfailed with the following output:",
"_CommandFailed.comment": "An example of {command_line} is vcpkg install zlib.",
"CommandFailedCode": "command:\n{command_line}\nfailed with exit code {exit_code} and the following output:",
"_CommandFailedCode.comment": "An example of {command_line} is vcpkg install zlib. An example of {exit_code} is 127.",
"CommunityTriplets": "Community Triplets:",
"CompilerPath": "Compiler found: {path}",
"_CompilerPath.comment": "An example of {path} is /foo/bar.",
@@ -538,16 +540,14 @@
"CreatingZipArchive": "Creating zip archive...",
"CreationFailed": "Creating {path} failed.",
"_CreationFailed.comment": "An example of {path} is /foo/bar.",
"CurlFailedToExecute": "curl failed to execute with exit code {exit_code}.",
"_CurlFailedToExecute.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127.",
"CurlFailedToPut": "curl failed to put file to {url} with exit code {exit_code}.",
"_CurlFailedToPut.comment": "curl is the name of a program, see curl.se An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
"CurlFailedToPutHttp": "curl failed to put file to {url} with exit code {exit_code} and http code {value}.",
"_CurlFailedToPutHttp.comment": "curl is the name of a program, see curl.se. {value} is an HTTP status code An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
"CurlReportedUnexpectedResults": "curl has reported unexpected results to vcpkg and vcpkg cannot continue.\nPlease review the following text for sensitive information and open an issue on the Microsoft/vcpkg GitHub to help fix this problem!\ncmd: {command_line}\n=== curl output ===\n{actual}\n=== end curl output ===",
"_CurlReportedUnexpectedResults.comment": "{command_line} is the command line to call curl.exe, {actual} is the console output of curl.exe locale-invariant download results. An example of {command_line} is vcpkg install zlib.",
"CurlReturnedUnexpectedResponseCodes": "curl returned a different number of response codes than were expected for the request ({actual} vs expected {expected}).",
"_CurlReturnedUnexpectedResponseCodes.comment": "{actual} and {expected} are integers, curl is the name of a program, see curl.se",
"CurlResponseTruncatedRetrying": "curl returned a partial response; waiting {value} milliseconds and trying again",
"_CurlResponseTruncatedRetrying.comment": "{value} is the number of milliseconds for which we are waiting this time",
"CurlTimeout": "curl was unable to perform all requested HTTP operations, even after timeout and retries. The last command line was: {command_line}",
"_CurlTimeout.comment": "An example of {command_line} is vcpkg install zlib.",
"CurrentCommitBaseline": "You can use the current commit as a baseline, which is:\n\t\"builtin-baseline\": \"{commit_sha}\"",
"_CurrentCommitBaseline.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.",
"CycleDetectedDuring": "cycle detected during {spec}:",
@@ -1466,7 +1466,6 @@
"UnexpectedEOFMidKeyword": "Unexpected EOF in middle of keyword",
"UnexpectedEOFMidString": "Unexpected EOF in middle of string",
"UnexpectedEOFMidUnicodeEscape": "Unexpected end of file in middle of unicode escape",
"UnexpectedErrorDuringBulkDownload": "an unexpected error occurred during bulk download.",
"UnexpectedEscapeSequence": "Unexpected escape sequence continuation",
"UnexpectedField": "unexpected field '{json_field}'",
"_UnexpectedField.comment": "An example of {json_field} is identifer.",
63 changes: 63 additions & 0 deletions src/vcpkg-test/system.process.cpp
Original file line number Diff line number Diff line change
@@ -60,3 +60,66 @@ TEST_CASE ("no closes-stdout crash", "[system.process]")
REQUIRE(run.exit_code == 0);
REQUIRE(run.output == "hello world");
}

TEST_CASE ("command try_append", "[system.process]")
{
{
Command a;
REQUIRE(a.try_append(Command{"b"}));
REQUIRE(a.command_line() == "b");
}

{
Command a{"a"};
REQUIRE(a.try_append(Command{}));
REQUIRE(a.command_line() == "a");
}

{
Command a{"a"};
REQUIRE(a.try_append(Command{"b"}));
REQUIRE(a.command_line() == "a b");
}

// size limits

std::string one_string(1, 'a');
std::string big_string(Command::maximum_allowed, 'a');
std::string bigger_string(Command::maximum_allowed + 1, 'a');
Command empty_cmd;
Command one_cmd{one_string};
Command big_cmd{big_string};
Command bigger_cmd{bigger_string};

REQUIRE(!bigger_cmd.try_append(empty_cmd));
REQUIRE(bigger_cmd.command_line() == bigger_string);

REQUIRE(big_cmd.try_append(empty_cmd));
REQUIRE(big_cmd.command_line() == big_string);

{
auto cmd = empty_cmd;
REQUIRE(!cmd.try_append(bigger_cmd));
REQUIRE(cmd.empty());
REQUIRE(cmd.try_append(big_cmd));
REQUIRE(cmd.command_line() == big_string);
}

{
auto cmd = one_cmd;
REQUIRE(!cmd.try_append(big_cmd));
REQUIRE(cmd.command_line() == one_string);
// does not fit due to the needed space
std::string almost_string(Command::maximum_allowed - 1, 'a');
Command almost_cmd{almost_string};
REQUIRE(!cmd.try_append(almost_cmd));
REQUIRE(cmd.command_line() == one_string);
// fits exactly
std::string ok_string(Command::maximum_allowed - 2, 'a');
Command ok_cmd{ok_string};
REQUIRE(cmd.try_append(ok_cmd));
auto expected = big_string;
expected[1] = ' ';
REQUIRE(cmd.command_line() == expected);
}
}
196 changes: 81 additions & 115 deletions src/vcpkg/base/downloads.cpp
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@
#include <vcpkg/base/system.h>
#include <vcpkg/base/system.process.h>
#include <vcpkg/base/system.proxy.h>
#include <vcpkg/base/util.h>

#include <vcpkg/commands.version.h>

@@ -371,150 +372,115 @@ namespace vcpkg
return Unit{};
}

static void url_heads_inner(View<std::string> urls,
View<std::string> headers,
std::vector<int>* out,
View<std::string> secrets)
static std::vector<int> curl_bulk_operation(View<Command> operation_args,
StringLiteral prefixArgs,
View<std::string> headers,
View<std::string> secrets)
{
static constexpr StringLiteral guid_marker = "8a1db05f-a65d-419b-aa72-037fb4d0672e";

const size_t start_size = out->size();

auto cmd = Command{"curl"}
.string_arg("--head")
.string_arg("--location")
.string_arg("-w")
.string_arg(guid_marker.to_string() + " %{http_code}\\n");
for (auto&& header : headers)
#define GUID_MARKER "5ec47b8e-6776-4d70-b9b3-ac2a57bc0a1c"
static constexpr StringLiteral guid_marker = GUID_MARKER;
Command prefix_cmd{"curl"};
if (!prefixArgs.empty())
{
cmd.string_arg("-H").string_arg(header);
prefix_cmd.raw_arg(prefixArgs);
}
for (auto&& url : urls)
{
cmd.string_arg(url_encode_spaces(url));
}

std::vector<std::string> lines;

auto res = cmd_execute_and_stream_lines(cmd, [out, &lines](StringView line) {
lines.push_back(line.to_string());
if (Strings::starts_with(line, guid_marker))
{
out->push_back(std::strtol(line.data() + guid_marker.size(), nullptr, 10));
}
}).value_or_exit(VCPKG_LINE_INFO);

if (res != 0)
{
Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgCurlFailedToExecute, msg::exit_code = res);
}

if (out->size() != start_size + urls.size())
{
auto command_line = replace_secrets(std::move(cmd).extract(), secrets);
auto actual = replace_secrets(Strings::join("\n", lines), secrets);
Checks::msg_exit_with_error(VCPKG_LINE_INFO,
msgCurlReportedUnexpectedResults,
msg::command_line = command_line,
msg::actual = actual);
}
}
std::vector<int> url_heads(View<std::string> urls, View<std::string> headers, View<std::string> secrets)
{
static constexpr size_t batch_size = 100;
prefix_cmd.string_arg("-L").string_arg("-w").string_arg(GUID_MARKER "%{http_code}\\n");
#undef GUID_MARKER

std::vector<int> ret;
ret.reserve(operation_args.size());

size_t i = 0;
for (; i + batch_size <= urls.size(); i += batch_size)
{
url_heads_inner({urls.data() + i, batch_size}, headers, &ret, secrets);
}

if (i != urls.size())
for (auto&& header : headers)
{
url_heads_inner({urls.begin() + i, urls.end()}, headers, &ret, secrets);
prefix_cmd.string_arg("-H").string_arg(header);
}

Checks::check_exit(VCPKG_LINE_INFO, ret.size() == urls.size());
return ret;
}
static constexpr auto initial_timeout_delay_ms = 100;
auto timeout_delay_ms = initial_timeout_delay_ms;
static constexpr auto maximum_timeout_delay_ms = 100000;

static void download_files_inner(const Filesystem&,
View<std::pair<std::string, Path>> url_pairs,
View<std::string> headers,
std::vector<int>* out)
{
for (auto i : {100, 1000, 10000, 0})
while (ret.size() != operation_args.size())
{
size_t start_size = out->size();
static constexpr StringLiteral guid_marker = "5ec47b8e-6776-4d70-b9b3-ac2a57bc0a1c";

auto cmd = Command{"curl"}
.string_arg("--create-dirs")
.string_arg("--location")
.string_arg("-w")
.string_arg(guid_marker.to_string() + " %{http_code}\\n");
for (StringView header : headers)
// there's an edge case that we aren't handling here where not even one operation fits with the configured
// headers but this seems unlikely

// form a maximum length command line of operations:
auto batch_cmd = prefix_cmd;
size_t last_try_op = ret.size();
while (last_try_op != operation_args.size() && batch_cmd.try_append(operation_args[last_try_op]))
{
cmd.string_arg("-H").string_arg(header);
++last_try_op;
}
for (auto&& url : url_pairs)

// actually run curl
auto this_batch_result = cmd_execute_and_capture_output(batch_cmd).value_or_exit(VCPKG_LINE_INFO);
if (this_batch_result.exit_code != 0)
{
cmd.string_arg(url_encode_spaces(url.first)).string_arg("-o").string_arg(url.second);
Checks::msg_exit_with_error(VCPKG_LINE_INFO,
msgCommandFailedCode,
msg::command_line =
replace_secrets(std::move(batch_cmd).extract(), secrets),
msg::exit_code = this_batch_result.exit_code);
}
auto res =
cmd_execute_and_stream_lines(cmd, [out](StringView line) {
if (Strings::starts_with(line, guid_marker))
{
out->push_back(static_cast<int>(std::strtol(line.data() + guid_marker.size(), nullptr, 10)));
}
}).value_or_exit(VCPKG_LINE_INFO);

if (res != 0)
// extract HTTP response codes
for (auto&& line : Strings::split(this_batch_result.output, '\n'))
{
Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgCurlFailedToExecute, msg::exit_code = res);
if (Strings::starts_with(line, guid_marker))
{
ret.push_back(static_cast<int>(std::strtol(line.data() + guid_marker.size(), nullptr, 10)));
}
}

if (start_size + url_pairs.size() > out->size())
// check if we got a partial response, and, if so, issue a timed delay
if (ret.size() == last_try_op)
{
// curl stopped before finishing all downloads; retry after some time
msg::println_warning(msgUnexpectedErrorDuringBulkDownload);
std::this_thread::sleep_for(std::chrono::milliseconds(i));
url_pairs =
View<std::pair<std::string, Path>>{url_pairs.begin() + out->size() - start_size, url_pairs.end()};
timeout_delay_ms = initial_timeout_delay_ms;
}
else
{
break;
// curl stopped before finishing all operations; retry after some time
if (timeout_delay_ms >= maximum_timeout_delay_ms)
{
Checks::msg_exit_with_error(VCPKG_LINE_INFO,
msgCurlTimeout,
msg::command_line =
replace_secrets(std::move(batch_cmd).extract(), secrets));
}

msg::println_warning(msgCurlResponseTruncatedRetrying, msg::value = timeout_delay_ms);
std::this_thread::sleep_for(std::chrono::milliseconds(timeout_delay_ms));
timeout_delay_ms *= 10;
}
}
}
std::vector<int> download_files(const Filesystem& fs,
View<std::pair<std::string, Path>> url_pairs,
View<std::string> headers)
{
static constexpr size_t batch_size = 50;

std::vector<int> ret;

size_t i = 0;
for (; i + batch_size <= url_pairs.size(); i += batch_size)
{
download_files_inner(fs, {url_pairs.data() + i, batch_size}, headers, &ret);
}
return ret;
}

if (i != url_pairs.size())
{
download_files_inner(fs, {url_pairs.begin() + i, url_pairs.end()}, headers, &ret);
}
std::vector<int> url_heads(View<std::string> urls, View<std::string> headers, View<std::string> secrets)
{
return curl_bulk_operation(
Util::fmap(urls, [](const std::string& url) { return Command{}.string_arg(url_encode_spaces(url)); }),
"--head",
headers,
secrets);
}

Checks::msg_check_exit(VCPKG_LINE_INFO,
ret.size() == url_pairs.size(),
msgCurlReturnedUnexpectedResponseCodes,
msg::actual = ret.size(),
msg::expected = url_pairs.size());
return ret;
std::vector<int> download_files(View<std::pair<std::string, Path>> url_pairs,
View<std::string> headers,
View<std::string> secrets)
{
return curl_bulk_operation(Util::fmap(url_pairs,
[](const std::pair<std::string, Path>& url_pair) {
return Command{}
.string_arg(url_encode_spaces(url_pair.first))
.string_arg("-o")
.string_arg(url_pair.second);
}),
"--create-dirs",
headers,
secrets);
}

bool send_snapshot_to_api(const std::string& github_token,
61 changes: 61 additions & 0 deletions src/vcpkg/base/system.process.cpp
Original file line number Diff line number Diff line change
@@ -474,6 +474,67 @@ namespace vcpkg
return *this;
}

Command& Command::raw_arg(StringView s) &
{
if (!buf.empty())
{
buf.push_back(' ');
}

buf.append(s.data(), s.size());
return *this;
}

Command& Command::forwarded_args(View<std::string> args) &
{
for (auto&& arg : args)
{
string_arg(arg);
}

return *this;
}

bool Command::try_append(const Command& other)
{
if (buf.size() > maximum_allowed)
{
return false;
}

if (other.buf.empty())
{
return true;
}

if (buf.empty())
{
if (other.buf.size() > maximum_allowed)
{
return false;
}

buf = other.buf;
return true;
}

size_t leftover = maximum_allowed - buf.size();
if (leftover == 0)
{
return false;
}

--leftover; // for the space
if (other.buf.size() > leftover)
{
return false;
}

buf.push_back(' ');
buf.append(other.buf);
return true;
}

#if defined(_WIN32)
Environment get_modified_clean_environment(const std::unordered_map<std::string, std::string>& extra_env,
StringView prepend_to_path)
7 changes: 5 additions & 2 deletions src/vcpkg/binarycaching.cpp
Original file line number Diff line number Diff line change
@@ -443,7 +443,7 @@ namespace
make_temp_archive_path(m_buildtrees, action.spec));
}

auto codes = download_files(m_fs, url_paths, m_url_template.headers);
auto codes = download_files(url_paths, m_url_template.headers, m_secrets);

for (size_t i = 0; i < codes.size(); ++i)
{
@@ -791,8 +791,10 @@ namespace
: ZipReadBinaryProvider(std::move(zip), fs)
, m_buildtrees(buildtrees)
, m_url(url + "_apis/artifactcache/cache")
, m_secrets()
, m_token_header("Authorization: Bearer " + token)
{
m_secrets.emplace_back(token);
}

std::string lookup_cache_entry(StringView name, const std::string& abi) const
@@ -835,7 +837,7 @@ namespace
url_indices.push_back(idx);
}

const auto codes = download_files(m_fs, url_paths, {});
const auto codes = download_files(url_paths, {}, m_secrets);

for (size_t i = 0; i < codes.size(); ++i)
{
@@ -856,6 +858,7 @@ namespace

Path m_buildtrees;
std::string m_url;
std::vector<std::string> m_secrets;
std::string m_token_header;
static constexpr StringLiteral m_accept_header = "Accept: application/json;api-version=6.0-preview.1";
static constexpr StringLiteral m_content_type_header = "Content-Type: application/json";

0 comments on commit 62ef030

Please sign in to comment.