From d9215bfda9913e0ff73732f925dd1654344506d7 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 6 Sep 2024 15:11:37 -0700 Subject: [PATCH 01/14] Overhaul StatusParagraph parsing to make status parsing testable. Also introduce adapt_to_string to help with the several to_string overload pairs. --- include/vcpkg/base/contractual-constants.h | 9 ++ include/vcpkg/base/fmt.h | 8 ++ include/vcpkg/base/message-data.inc.h | 14 +- include/vcpkg/base/parse.h | 1 + include/vcpkg/binaryparagraph.h | 4 +- include/vcpkg/fwd/statusparagraph.h | 2 +- include/vcpkg/paragraphs.h | 3 + include/vcpkg/statusparagraph.h | 46 ++++-- include/vcpkg/versions.h | 6 +- locales/messages.json | 6 + src/vcpkg-test/statusparagraphs.cpp | 49 +++++-- src/vcpkg-test/update.cpp | 3 +- src/vcpkg/base/parse.cpp | 21 +++ src/vcpkg/binaryparagraph.cpp | 26 ++-- src/vcpkg/commands.install.cpp | 12 +- src/vcpkg/commands.remove.cpp | 6 +- src/vcpkg/commands.upgrade.cpp | 2 + src/vcpkg/export.prefab.cpp | 7 +- src/vcpkg/metrics.cpp | 7 +- src/vcpkg/packagespec.cpp | 9 +- src/vcpkg/paragraphs.cpp | 10 ++ src/vcpkg/statusparagraph.cpp | 156 +++++++++++++-------- src/vcpkg/versions.cpp | 19 +-- 23 files changed, 281 insertions(+), 145 deletions(-) diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h index e637a800db..0b86b23290 100644 --- a/include/vcpkg/base/contractual-constants.h +++ b/include/vcpkg/base/contractual-constants.h @@ -558,4 +558,13 @@ namespace vcpkg inline constexpr StringLiteral AbiTagPostBuildChecks = "post_build_checks"; inline constexpr StringLiteral AbiTagPowershell = "powershell"; inline constexpr StringLiteral AbiTagPublicAbiOverride = "public_abi_override"; + + inline constexpr StringLiteral StatusDeinstall = "deinstall"; + inline constexpr StringLiteral StatusError = "error"; + inline constexpr StringLiteral StatusHalfInstalled = "half-installed"; + inline constexpr StringLiteral StatusHold = "hold"; + inline constexpr StringLiteral StatusInstall = "install"; + inline constexpr StringLiteral StatusInstalled = "installed"; + inline constexpr StringLiteral StatusNotInstalled = "not-installed"; + inline constexpr StringLiteral StatusPurge = "purge"; } diff --git a/include/vcpkg/base/fmt.h b/include/vcpkg/base/fmt.h index 416e6dcd38..2129c8c996 100644 --- a/include/vcpkg/base/fmt.h +++ b/include/vcpkg/base/fmt.h @@ -18,3 +18,11 @@ VCPKG_MSVC_WARNING(disable : 6239 4702) #include #include VCPKG_MSVC_WARNING(pop) + +template +std::string adapt_to_string(const T& val) +{ + std::string result; + val.to_string(result); + return result; +} diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index c4db50bea3..3545690ee1 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -1168,10 +1168,14 @@ DECLARE_MESSAGE(ExpectedCharacterHere, DECLARE_MESSAGE(ExpectedDefaultFeaturesList, (), "", "expected ',' or end of text in default features list") DECLARE_MESSAGE(ExpectedDependenciesList, (), "", "expected ',' or end of text in dependencies list") DECLARE_MESSAGE(ExpectedDigitsAfterDecimal, (), "", "Expected digits after the decimal point") +DECLARE_MESSAGE(ExpectedExplicitTriplet, (), "", "expected an explicit triplet") DECLARE_MESSAGE(ExpectedFailOrSkip, (), "", "expected 'fail', 'skip', or 'pass' here") DECLARE_MESSAGE(ExpectedFeatureListTerminal, (), "", "expected ',' or ']' in feature list") DECLARE_MESSAGE(ExpectedFeatureName, (), "", "expected feature name (must be lowercase, digits, '-')") -DECLARE_MESSAGE(ExpectedExplicitTriplet, (), "", "expected an explicit triplet") +DECLARE_MESSAGE(ExpectedInstallStateField, + (), + "The values in ''s are locale-invariant", + "expected one of 'not-installed', 'half-installed', or 'installed'") DECLARE_MESSAGE(ExpectedOneSetOfTags, (msg::count, msg::old_value, msg::new_value, msg::value), "{old_value} is a left tag and {new_value} is the right tag. {value} is the input.", @@ -1181,7 +1185,15 @@ DECLARE_MESSAGE(ExpectedPathToExist, (msg::path), "", "Expected {path} to exist DECLARE_MESSAGE(ExpectedPortName, (), "", "expected a port name here (must be lowercase, digits, '-')") DECLARE_MESSAGE(ExpectedReadWriteReadWrite, (), "", "unexpected argument: expected 'read', readwrite', or 'write'") DECLARE_MESSAGE(ExpectedStatusField, (), "", "Expected 'status' field in status paragraph") +DECLARE_MESSAGE(ExpectedTextHere, + (msg::expected), + "{expected} is a locale-invariant string a parser was searching for", + "expected '{expected}' here") DECLARE_MESSAGE(ExpectedTripletName, (), "", "expected a triplet name here (must be lowercase, digits, '-')") +DECLARE_MESSAGE(ExpectedWantField, + (), + "The values in ''s are locale-invariant", + "expected one of 'install', 'hold', 'deinstall', or 'purge' here") DECLARE_MESSAGE(ExportArchitectureReq, (), "", diff --git a/include/vcpkg/base/parse.h b/include/vcpkg/base/parse.h index 5ddd329adc..bf01590697 100644 --- a/include/vcpkg/base/parse.h +++ b/include/vcpkg/base/parse.h @@ -90,6 +90,7 @@ namespace vcpkg } bool require_character(char ch); + bool require_text(StringLiteral keyword); bool try_match_keyword(StringView keyword_content); diff --git a/include/vcpkg/binaryparagraph.h b/include/vcpkg/binaryparagraph.h index b9ad5b13c8..f1f2650ebe 100644 --- a/include/vcpkg/binaryparagraph.h +++ b/include/vcpkg/binaryparagraph.h @@ -8,9 +8,7 @@ namespace vcpkg { - /// - /// Built package metadata - /// + // metadata for a package in the 'packages' tree struct BinaryParagraph { BinaryParagraph() = default; diff --git a/include/vcpkg/fwd/statusparagraph.h b/include/vcpkg/fwd/statusparagraph.h index 5dd5567e89..8dd3893206 100644 --- a/include/vcpkg/fwd/statusparagraph.h +++ b/include/vcpkg/fwd/statusparagraph.h @@ -13,13 +13,13 @@ namespace vcpkg enum class Want { ERROR_STATE, - UNKNOWN, INSTALL, HOLD, DEINSTALL, PURGE }; + struct StatusLine; struct StatusParagraph; struct InstalledPackageView; } diff --git a/include/vcpkg/paragraphs.h b/include/vcpkg/paragraphs.h index efd7b64e3f..b5a6a06112 100644 --- a/include/vcpkg/paragraphs.h +++ b/include/vcpkg/paragraphs.h @@ -9,6 +9,7 @@ #include +#include #include #include @@ -24,6 +25,8 @@ namespace vcpkg::Paragraphs ExpectedL> parse_paragraphs(StringView str, StringView origin); + void append_paragraph_field(StringView name, StringView field, std::string& out_str); + bool is_port_directory(const ReadOnlyFilesystem& fs, const Path& maybe_directory); struct PortLoadResult diff --git a/include/vcpkg/statusparagraph.h b/include/vcpkg/statusparagraph.h index 4e4fa98f91..1abfb431dd 100644 --- a/include/vcpkg/statusparagraph.h +++ b/include/vcpkg/statusparagraph.h @@ -5,6 +5,8 @@ #include #include +#include + #include #include @@ -13,31 +15,45 @@ namespace vcpkg { - /// - /// Installed package metadata - /// + struct StatusLine + { + Want want = Want::ERROR_STATE; + InstallState state = InstallState::ERROR_STATE; + + bool is_installed() const noexcept { return want == Want::INSTALL && state == InstallState::INSTALLED; } + void to_string(std::string& out) const; + std::string to_string() const; + + friend bool operator==(const StatusLine& lhs, const StatusLine& rhs) + { + return lhs.want == rhs.want && lhs.state == rhs.state; + } + + friend bool operator!=(const StatusLine& lhs, const StatusLine& rhs) { return !(lhs == rhs); } + }; + + ExpectedL parse_status_line(StringView text, Optional origin); + + // metadata for a package's representation in the 'installed' tree struct StatusParagraph { - StatusParagraph() noexcept; + StatusParagraph() = default; StatusParagraph(StringView origin, Paragraph&& fields); - bool is_installed() const { return want == Want::INSTALL && state == InstallState::INSTALLED; } + bool is_installed() const noexcept { return status.is_installed(); } BinaryParagraph package; - Want want; - InstallState state; + StatusLine status; }; void serialize(const StatusParagraph& pgh, std::string& out_str); - std::string to_string(InstallState f); - - std::string to_string(Want f); + StringLiteral to_string_literal(InstallState f); + StringLiteral to_string_literal(Want f); struct InstalledPackageView { - InstalledPackageView() noexcept : core(nullptr) { } - + InstalledPackageView() = default; InstalledPackageView(const StatusParagraph* c, std::vector&& fs) : core(c), features(std::move(fs)) { @@ -51,7 +67,7 @@ namespace vcpkg std::vector all_status_paragraphs() const; - const StatusParagraph* core; + const StatusParagraph* core = nullptr; std::vector features; }; @@ -59,3 +75,7 @@ namespace vcpkg const InstalledPaths& installed, const ReadOnlyFilesystem& fs); } + +VCPKG_FORMAT_WITH_TO_STRING_LITERAL_NONMEMBER(vcpkg::InstallState); +VCPKG_FORMAT_WITH_TO_STRING_LITERAL_NONMEMBER(vcpkg::Want); +VCPKG_FORMAT_WITH_TO_STRING(vcpkg::StatusLine); diff --git a/include/vcpkg/versions.h b/include/vcpkg/versions.h index cb3b1a512f..8a54cbf0bc 100644 --- a/include/vcpkg/versions.h +++ b/include/vcpkg/versions.h @@ -48,6 +48,7 @@ namespace vcpkg VersionDiff(const Version& left, const Version& right); std::string to_string() const; + void to_string(std::string& out) const; }; struct VersionMapLess @@ -85,6 +86,7 @@ namespace vcpkg VersionSpec(const std::string& port_name, const std::string& version_string, int port_version); std::string to_string() const; + void to_string(std::string& out) const; friend bool operator==(const VersionSpec& lhs, const VersionSpec& rhs); friend bool operator!=(const VersionSpec& lhs, const VersionSpec& rhs); @@ -97,7 +99,7 @@ namespace vcpkg struct DotVersion { - DotVersion() { } // intentionally disable making this type an aggregate + DotVersion() noexcept { } // intentionally disable making this type an aggregate std::string original_string; std::string version_string; @@ -122,7 +124,7 @@ namespace vcpkg struct DateVersion { - DateVersion() { } // intentionally disable making this type an aggregate + DateVersion() noexcept { } // intentionally disable making this type an aggregate std::string original_string; std::string version_string; diff --git a/locales/messages.json b/locales/messages.json index be4a9a9054..1b4622a7d9 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -691,6 +691,8 @@ "ExpectedFailOrSkip": "expected 'fail', 'skip', or 'pass' here", "ExpectedFeatureListTerminal": "expected ',' or ']' in feature list", "ExpectedFeatureName": "expected feature name (must be lowercase, digits, '-')", + "ExpectedInstallStateField": "expected one of 'not-installed', 'half-installed', or 'installed'", + "_ExpectedInstallStateField.comment": "The values in ''s are locale-invariant", "ExpectedOneSetOfTags": "Found {count} sets of {old_value}.*{new_value} but expected exactly 1, in block:\n{value}", "_ExpectedOneSetOfTags.comment": "{old_value} is a left tag and {new_value} is the right tag. {value} is the input. An example of {count} is 42.", "ExpectedOneVersioningField": "expected only one versioning field", @@ -699,7 +701,11 @@ "ExpectedPortName": "expected a port name here (must be lowercase, digits, '-')", "ExpectedReadWriteReadWrite": "unexpected argument: expected 'read', readwrite', or 'write'", "ExpectedStatusField": "Expected 'status' field in status paragraph", + "ExpectedTextHere": "expected '{expected}' here", + "_ExpectedTextHere.comment": "{expected} is a locale-invariant string a parser was searching for", "ExpectedTripletName": "expected a triplet name here (must be lowercase, digits, '-')", + "ExpectedWantField": "expected one of 'install', 'hold', 'deinstall', or 'purge' here", + "_ExpectedWantField.comment": "The values in ''s are locale-invariant", "ExportArchitectureReq": "Export prefab requires targeting at least one of the following architectures arm64-v8a, armeabi-v7a, x86_64, x86 to be present.", "ExportPrefabRequiresAndroidTriplet": "export prefab requires an Android triplet.", "Exported7zipArchive": "7zip archive exported at: {path}", diff --git a/src/vcpkg-test/statusparagraphs.cpp b/src/vcpkg-test/statusparagraphs.cpp index 00c8a904ab..7183724baf 100644 --- a/src/vcpkg-test/statusparagraphs.cpp +++ b/src/vcpkg-test/statusparagraphs.cpp @@ -9,6 +9,35 @@ using namespace vcpkg; using namespace vcpkg::Paragraphs; using namespace vcpkg::Test; +static constexpr StringLiteral test_origin = "test"; + +TEST_CASE ("parse status lines", "[statusparagraphs]") +{ + REQUIRE(parse_status_line("install ok installed", test_origin).value_or_exit(VCPKG_LINE_INFO) == + StatusLine{Want::INSTALL, InstallState::INSTALLED}); + REQUIRE(parse_status_line("hold ok installed", test_origin).value_or_exit(VCPKG_LINE_INFO) == + StatusLine{Want::HOLD, InstallState::INSTALLED}); + REQUIRE(parse_status_line("deinstall ok installed", test_origin).value_or_exit(VCPKG_LINE_INFO) == + StatusLine{Want::DEINSTALL, InstallState::INSTALLED}); + REQUIRE(parse_status_line("purge ok installed", test_origin).value_or_exit(VCPKG_LINE_INFO) == + StatusLine{Want::PURGE, InstallState::INSTALLED}); + + REQUIRE(parse_status_line("install ok not-installed", test_origin).value_or_exit(VCPKG_LINE_INFO) == + StatusLine{Want::INSTALL, InstallState::NOT_INSTALLED}); + REQUIRE(parse_status_line("install ok half-installed", test_origin).value_or_exit(VCPKG_LINE_INFO) == + StatusLine{Want::INSTALL, InstallState::HALF_INSTALLED}); + + REQUIRE(parse_status_line("meow ok installed", test_origin).error() == + LocalizedString::from_raw("test:1:1: error: expected one of 'install', 'hold', 'deinstall', or 'purge' " + "here\n on expression: meow ok installed\n ^")); + REQUIRE(parse_status_line("install ko half-installed", test_origin).error() == + LocalizedString::from_raw("test:1:8: error: expected ' ok ' here\n on expression: install ko " + "half-installed\n ^")); + REQUIRE(parse_status_line("install ok meow", test_origin).error() == + LocalizedString::from_raw("test:1:12: error: expected one of 'not-installed', 'half-installed', or " + "'installed'\n on expression: install ok meow\n ^")); +} + TEST_CASE ("find installed", "[statusparagraphs]") { auto pghs = parse_paragraphs(R"( @@ -23,9 +52,8 @@ Status: install ok installed REQUIRE(pghs); - StatusParagraphs status_db(Util::fmap(*pghs.get(), [](Paragraph& rpgh) { - return std::make_unique(StringLiteral{"test"}, std::move(rpgh)); - })); + StatusParagraphs status_db(Util::fmap( + *pghs.get(), [](Paragraph& rpgh) { return std::make_unique(test_origin, std::move(rpgh)); })); auto it = status_db.find_installed({"ffmpeg", Test::X64_WINDOWS}); REQUIRE(it != status_db.end()); @@ -45,9 +73,8 @@ Status: purge ok not-installed REQUIRE(pghs); - StatusParagraphs status_db(Util::fmap(*pghs.get(), [](Paragraph& rpgh) { - return std::make_unique(StringLiteral{"test"}, std::move(rpgh)); - })); + StatusParagraphs status_db(Util::fmap( + *pghs.get(), [](Paragraph& rpgh) { return std::make_unique(test_origin, std::move(rpgh)); })); auto it = status_db.find_installed({"ffmpeg", Test::X64_WINDOWS}); REQUIRE(it == status_db.end()); @@ -75,9 +102,8 @@ Status: purge ok not-installed REQUIRE(pghs); - StatusParagraphs status_db(Util::fmap(*pghs.get(), [](Paragraph& rpgh) { - return std::make_unique(StringLiteral{"test"}, std::move(rpgh)); - })); + StatusParagraphs status_db(Util::fmap( + *pghs.get(), [](Paragraph& rpgh) { return std::make_unique(test_origin, std::move(rpgh)); })); auto it = status_db.find_installed({"ffmpeg", Test::X64_WINDOWS}); REQUIRE(it != status_db.end()); @@ -108,9 +134,8 @@ Status: install ok installed "test-origin"); REQUIRE(pghs); - StatusParagraphs status_db(Util::fmap(*pghs.get(), [](Paragraph& rpgh) { - return std::make_unique(StringLiteral{"test"}, std::move(rpgh)); - })); + StatusParagraphs status_db(Util::fmap( + *pghs.get(), [](Paragraph& rpgh) { return std::make_unique(test_origin, std::move(rpgh)); })); // Feature "openssl" is installed and should therefore be found auto it = status_db.find_installed({{"ffmpeg", Test::X64_WINDOWS}, "openssl"}); diff --git a/src/vcpkg-test/update.cpp b/src/vcpkg-test/update.cpp index 59eb818a1c..d1bbcc9f01 100644 --- a/src/vcpkg-test/update.cpp +++ b/src/vcpkg-test/update.cpp @@ -64,8 +64,7 @@ TEST_CASE ("find outdated packages features 2", "[update]") status_paragraphs.push_back(make_status_feature_pgh("a", "b")); status_paragraphs.back()->package.version = Version{"0", 0}; - status_paragraphs.back()->state = InstallState::NOT_INSTALLED; - status_paragraphs.back()->want = Want::PURGE; + status_paragraphs.back()->status = {Want::PURGE, InstallState::NOT_INSTALLED}; StatusParagraphs status_db(std::move(status_paragraphs)); diff --git a/src/vcpkg/base/parse.cpp b/src/vcpkg/base/parse.cpp index 7f9b5e2293..b1ded452c1 100644 --- a/src/vcpkg/base/parse.cpp +++ b/src/vcpkg/base/parse.cpp @@ -146,6 +146,27 @@ namespace vcpkg return true; } + bool ParserBase::require_text(StringLiteral text) + { + auto encoded = m_it; + // check that the encoded stream matches the keyword: + for (const char ch : text) + { + if (encoded.is_eof() || *encoded != static_cast(ch)) + { + add_error(msg::format(msgExpectedTextHere, msg::expected = text)); + return false; + } + + ++encoded; + } + + // success + m_it = encoded; + m_column += static_cast(text.size()); + return true; + } + bool ParserBase::try_match_keyword(StringView keyword_content) { auto encoded = m_it; diff --git a/src/vcpkg/binaryparagraph.cpp b/src/vcpkg/binaryparagraph.cpp index e997c43e58..47b637bd0d 100644 --- a/src/vcpkg/binaryparagraph.cpp +++ b/src/vcpkg/binaryparagraph.cpp @@ -7,6 +7,8 @@ #include #include +using namespace vcpkg::Paragraphs; + namespace vcpkg { BinaryParagraph::BinaryParagraph(StringView origin, Paragraph&& fields) @@ -177,15 +179,6 @@ namespace vcpkg bool operator!=(const BinaryParagraph& lhs, const BinaryParagraph& rhs) { return !(lhs == rhs); } - static void serialize_string(StringView name, const std::string& field, std::string& out_str) - { - if (field.empty()) - { - return; - } - - out_str.append(name.data(), name.size()).append(": ").append(field).push_back('\n'); - } static void serialize_array(StringView name, const std::vector& array, std::string& out_str, @@ -223,9 +216,9 @@ namespace vcpkg { const size_t initial_end = out_str.size(); - serialize_string(ParagraphIdPackage, pgh.spec.name(), out_str); + append_paragraph_field(ParagraphIdPackage, pgh.spec.name(), out_str); - serialize_string(ParagraphIdVersion, pgh.version.text, out_str); + append_paragraph_field(ParagraphIdVersion, pgh.version.text, out_str); if (pgh.version.port_version != 0) { fmt::format_to(std::back_inserter(out_str), "{}: {}\n", ParagraphIdPortVersion, pgh.version.port_version); @@ -233,20 +226,21 @@ namespace vcpkg if (pgh.is_feature()) { - serialize_string(ParagraphIdFeature, pgh.feature, out_str); + append_paragraph_field(ParagraphIdFeature, pgh.feature, out_str); } if (!pgh.dependencies.empty()) { - serialize_string(ParagraphIdDepends, serialize_deps_list(pgh.dependencies, pgh.spec.triplet()), out_str); + append_paragraph_field( + ParagraphIdDepends, serialize_deps_list(pgh.dependencies, pgh.spec.triplet()), out_str); } - serialize_string(ParagraphIdArchitecture, pgh.spec.triplet().to_string(), out_str); - serialize_string(ParagraphIdMultiArch, "same", out_str); + append_paragraph_field(ParagraphIdArchitecture, pgh.spec.triplet().to_string(), out_str); + append_paragraph_field(ParagraphIdMultiArch, "same", out_str); serialize_paragraph(ParagraphIdMaintainer, pgh.maintainers, out_str); - serialize_string(ParagraphIdAbi, pgh.abi, out_str); + append_paragraph_field(ParagraphIdAbi, pgh.abi, out_str); serialize_paragraph(ParagraphIdDescription, pgh.description, out_str); diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 2087e62835..82c91158d8 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -280,8 +280,7 @@ namespace vcpkg StatusParagraph source_paragraph; source_paragraph.package = bcf.core_paragraph; - source_paragraph.want = Want::INSTALL; - source_paragraph.state = InstallState::HALF_INSTALLED; + source_paragraph.status = StatusLine{Want::INSTALL, InstallState::HALF_INSTALLED}; write_update(fs, installed, source_paragraph); status_db->insert(std::make_unique(source_paragraph)); @@ -291,8 +290,7 @@ namespace vcpkg { StatusParagraph& feature_paragraph = features_spghs.emplace_back(); feature_paragraph.package = feature; - feature_paragraph.want = Want::INSTALL; - feature_paragraph.state = InstallState::HALF_INSTALLED; + feature_paragraph.status = StatusLine{Want::INSTALL, InstallState::HALF_INSTALLED}; write_update(fs, installed, feature_paragraph); status_db->insert(std::make_unique(feature_paragraph)); @@ -303,13 +301,13 @@ namespace vcpkg install_package_and_write_listfile(fs, package_dir, install_dir); - source_paragraph.state = InstallState::INSTALLED; + source_paragraph.status.state = InstallState::INSTALLED; write_update(fs, installed, source_paragraph); status_db->insert(std::make_unique(source_paragraph)); for (auto&& feature_paragraph : features_spghs) { - feature_paragraph.state = InstallState::INSTALLED; + feature_paragraph.status.state = InstallState::INSTALLED; write_update(fs, installed, feature_paragraph); status_db->insert(std::make_unique(feature_paragraph)); } @@ -1373,6 +1371,8 @@ namespace vcpkg binary_cache, null_build_logs_recorder()); + // Skip printing the summary without --keep-going because the status without it is 'obvious': everything was a + // success. if (keep_going == KeepGoing::Yes) { msg::print(summary.format()); diff --git a/src/vcpkg/commands.remove.cpp b/src/vcpkg/commands.remove.cpp index 9b53b9cd1b..9d3dbe6e5f 100644 --- a/src/vcpkg/commands.remove.cpp +++ b/src/vcpkg/commands.remove.cpp @@ -30,8 +30,7 @@ namespace vcpkg for (auto&& spgh : spghs) { - spgh.want = Want::PURGE; - spgh.state = InstallState::HALF_INSTALLED; + spgh.status = {Want::PURGE, InstallState::HALF_INSTALLED}; write_update(fs, installed, spgh); } @@ -93,9 +92,8 @@ namespace vcpkg for (auto&& spgh : spghs) { - spgh.state = InstallState::NOT_INSTALLED; + spgh.status.state = InstallState::NOT_INSTALLED; write_update(fs, installed, spgh); - status_db.insert(std::make_unique(std::move(spgh))); } } diff --git a/src/vcpkg/commands.upgrade.cpp b/src/vcpkg/commands.upgrade.cpp index a27ea0a6e6..a113231a56 100644 --- a/src/vcpkg/commands.upgrade.cpp +++ b/src/vcpkg/commands.upgrade.cpp @@ -211,6 +211,8 @@ namespace vcpkg const InstallSummary summary = install_execute_plan( args, paths, host_triplet, build_options, action_plan, status_db, binary_cache, null_build_logs_recorder()); + // Skip printing the summary without --keep-going because the status without it is 'obvious': everything was a + // success. if (keep_going == KeepGoing::Yes) { msg::print(summary.format()); diff --git a/src/vcpkg/export.prefab.cpp b/src/vcpkg/export.prefab.cpp index f6f02a461d..22f9fcad01 100644 --- a/src/vcpkg/export.prefab.cpp +++ b/src/vcpkg/export.prefab.cpp @@ -36,12 +36,7 @@ namespace vcpkg::Prefab return paths; } - std::string NdkVersion::to_string() const - { - std::string ret; - this->to_string(ret); - return ret; - } + std::string NdkVersion::to_string() const { return adapt_to_string(*this); } void NdkVersion::to_string(std::string& out) const { out.append("NdkVersion{major=") diff --git a/src/vcpkg/metrics.cpp b/src/vcpkg/metrics.cpp index c9ba1e212d..77b9c26db7 100644 --- a/src/vcpkg/metrics.cpp +++ b/src/vcpkg/metrics.cpp @@ -281,12 +281,7 @@ namespace vcpkg last_completed_survey); } - std::string MetricsUserConfig::to_string() const - { - std::string ret; - to_string(ret); - return ret; - } + std::string MetricsUserConfig::to_string() const { return adapt_to_string(*this); } void MetricsUserConfig::try_write(const Filesystem& fs) const { diff --git a/src/vcpkg/packagespec.cpp b/src/vcpkg/packagespec.cpp index eca04f0aa3..53030973c0 100644 --- a/src/vcpkg/packagespec.cpp +++ b/src/vcpkg/packagespec.cpp @@ -70,12 +70,7 @@ namespace namespace vcpkg { - std::string FeatureSpec::to_string() const - { - std::string ret; - this->to_string(ret); - return ret; - } + std::string FeatureSpec::to_string() const { return adapt_to_string(*this); } void FeatureSpec::to_string(std::string& out) const { if (feature().empty()) return spec().to_string(out); @@ -130,7 +125,7 @@ namespace vcpkg std::string PackageSpec::dir() const { return fmt::format("{}_{}", this->m_name, this->m_triplet); } - std::string PackageSpec::to_string() const { return fmt::format("{}:{}", this->name(), this->triplet()); } + std::string PackageSpec::to_string() const { return adapt_to_string(*this); } void PackageSpec::to_string(std::string& s) const { fmt::format_to(std::back_inserter(s), "{}:{}", this->name(), this->triplet()); diff --git a/src/vcpkg/paragraphs.cpp b/src/vcpkg/paragraphs.cpp index 93183c7906..bb30fcda80 100644 --- a/src/vcpkg/paragraphs.cpp +++ b/src/vcpkg/paragraphs.cpp @@ -352,6 +352,16 @@ namespace vcpkg::Paragraphs return PghParser(str, origin).get_paragraphs(); } + void append_paragraph_field(StringView name, StringView field, std::string& out_str) + { + if (field.empty()) + { + return; + } + + out_str.append(name.data(), name.size()).append(": ").append(field.data(), field.size()).push_back('\n'); + } + bool is_port_directory(const ReadOnlyFilesystem& fs, const Path& maybe_directory) { return fs.exists(maybe_directory / "CONTROL", IgnoreErrors{}) || diff --git a/src/vcpkg/statusparagraph.cpp b/src/vcpkg/statusparagraph.cpp index f0a0748ecb..b7a8894998 100644 --- a/src/vcpkg/statusparagraph.cpp +++ b/src/vcpkg/statusparagraph.cpp @@ -1,94 +1,128 @@ #include #include +#include #include +using namespace vcpkg::Paragraphs; + namespace vcpkg { - StatusParagraph::StatusParagraph() noexcept : want(Want::ERROR_STATE), state(InstallState::ERROR_STATE) { } + void StatusLine::to_string(std::string& out) const + { + fmt::format_to(std::back_inserter(out), "{} ok {}", want, state); + } - void serialize(const StatusParagraph& pgh, std::string& out_str) + std::string StatusLine::to_string() const { return adapt_to_string(*this); } + + ExpectedL parse_status_line(StringView text, Optional origin) { - serialize(pgh.package, out_str); - out_str.append("Status: ") - .append(to_string(pgh.want)) - .append(" ok ") - .append(to_string(pgh.state)) - .push_back('\n'); + ParserBase parser{text, origin}; + StatusLine result; + const auto want_start = parser.cur_loc(); + auto want_text = parser.match_until(ParserBase::is_whitespace); + if (want_text == StatusInstall) + { + result.want = Want::INSTALL; + } + else if (want_text == StatusHold) + { + result.want = Want::HOLD; + } + else if (want_text == StatusDeinstall) + { + result.want = Want::DEINSTALL; + } + else if (want_text == StatusPurge) + { + result.want = Want::PURGE; + } + else + { + parser.add_error(msg::format(msgExpectedWantField), want_start); + return parser.extract_messages().error.value_or_exit(VCPKG_LINE_INFO); + } + + if (parser.require_text(" ok ")) + { + auto state_start = parser.cur_loc(); + auto state_text = parser.match_until(ParserBase::is_whitespace); + if (state_text == StatusNotInstalled) + { + result.state = InstallState::NOT_INSTALLED; + } + else if (state_text == StatusInstalled) + { + result.state = InstallState::INSTALLED; + } + else if (state_text == StatusHalfInstalled) + { + result.state = InstallState::HALF_INSTALLED; + } + else + { + parser.add_error(msg::format(msgExpectedInstallStateField), state_start); + return parser.extract_messages().error.value_or_exit(VCPKG_LINE_INFO); + } + + if (parser.messages().good()) + { + return result; + } + } + + return parser.extract_messages().error.value_or_exit(VCPKG_LINE_INFO); + } + + void serialize(const StatusParagraph& pgh, std::string& out) + { + serialize(pgh.package, out); + append_paragraph_field(ParagraphIdStatus, pgh.status.to_string(), out); } StatusParagraph::StatusParagraph(StringView origin, Paragraph&& fields) - : want(Want::ERROR_STATE), state(InstallState::ERROR_STATE) { auto status_it = fields.find(ParagraphIdStatus); Checks::msg_check_maybe_upgrade(VCPKG_LINE_INFO, status_it != fields.end(), msgExpectedStatusField); - std::string status_field = std::move(status_it->second.first); + auto status_field = std::move(status_it->second); fields.erase(status_it); - this->package = BinaryParagraph(origin, std::move(fields)); - - auto b = status_field.begin(); - const auto mark = b; - const auto e = status_field.end(); - - // Todo: improve error handling - while (b != e && *b != ' ') - ++b; - - want = [](const std::string& text) { - if (text == "unknown") return Want::UNKNOWN; - if (text == "install") return Want::INSTALL; - if (text == "hold") return Want::HOLD; - if (text == "deinstall") return Want::DEINSTALL; - if (text == "purge") return Want::PURGE; - return Want::ERROR_STATE; - }(std::string(mark, b)); - - if (std::distance(b, e) < 4) return; - b += 4; - - state = [](const std::string& text) { - if (text == "not-installed") return InstallState::NOT_INSTALLED; - if (text == "installed") return InstallState::INSTALLED; - if (text == "half-installed") return InstallState::HALF_INSTALLED; - return InstallState::ERROR_STATE; - }(std::string(b, e)); + this->status = parse_status_line(status_field.first, origin).value_or_exit(VCPKG_LINE_INFO); } - std::string to_string(InstallState f) + StringLiteral to_string_literal(InstallState f) { switch (f) { - case InstallState::HALF_INSTALLED: return "half-installed"; - case InstallState::INSTALLED: return "installed"; - case InstallState::NOT_INSTALLED: return "not-installed"; - default: return "error"; + case InstallState::HALF_INSTALLED: return StatusHalfInstalled; + case InstallState::INSTALLED: return StatusInstalled; + case InstallState::NOT_INSTALLED: return StatusNotInstalled; + default: Checks::unreachable(VCPKG_LINE_INFO); } } - std::string to_string(Want f) + StringLiteral to_string_literal(Want f) { switch (f) { - case Want::DEINSTALL: return "deinstall"; - case Want::HOLD: return "hold"; - case Want::INSTALL: return "install"; - case Want::PURGE: return "purge"; - case Want::UNKNOWN: return "unknown"; - default: return "error"; + case Want::DEINSTALL: return StatusDeinstall; + case Want::HOLD: return StatusHold; + case Want::INSTALL: return StatusInstall; + case Want::PURGE: return StatusPurge; + default: Checks::unreachable(VCPKG_LINE_INFO); } } std::map> InstalledPackageView::feature_dependencies() const { - auto extract_deps = [&](const PackageSpec& spec) { return FeatureSpec{spec, FeatureNameCore.to_string()}; }; + auto extract_deps = [](const PackageSpec& spec) { return FeatureSpec{spec, FeatureNameCore}; }; std::map> deps; - deps.emplace(FeatureNameCore, Util::fmap(core->package.dependencies, extract_deps)); - - for (const StatusParagraph* const& feature : features) + for (const StatusParagraph* feature : features) + { deps.emplace(feature->package.feature, Util::fmap(feature->package.dependencies, extract_deps)); + } return deps; } @@ -97,10 +131,11 @@ namespace vcpkg { InternalFeatureSet ret; ret.emplace_back(FeatureNameCore); - for (const auto& f : features) + for (const StatusParagraph* f : features) { ret.emplace_back(f->package.feature); } + return ret; } @@ -111,18 +146,23 @@ namespace vcpkg // accumulate all features in installed dependencies // Todo: make this unneeded by collapsing all package dependencies into the core package std::vector deps; - for (auto&& feature : features) + for (const StatusParagraph* feature : features) + { for (auto&& dep : feature->package.dependencies) + { deps.push_back(dep); + } + } // Add the core paragraph dependencies to the list for (auto&& dep : core->package.dependencies) + { deps.push_back(dep); + } - auto this_spec = this->spec(); + const auto& this_spec = this->spec(); Util::erase_remove_if(deps, [this_spec](const PackageSpec& pspec) { return pspec == this_spec; }); Util::sort_unique_erase(deps); - return deps; } diff --git a/src/vcpkg/versions.cpp b/src/vcpkg/versions.cpp index e37351f086..00c1dd26e4 100644 --- a/src/vcpkg/versions.cpp +++ b/src/vcpkg/versions.cpp @@ -18,12 +18,7 @@ namespace vcpkg { } - std::string Version::to_string() const - { - std::string result; - to_string(result); - return result; - } + std::string Version::to_string() const { return adapt_to_string(*this); } void Version::to_string(std::string& out) const { @@ -81,9 +76,17 @@ namespace vcpkg VersionDiff::VersionDiff() noexcept : left(), right() { } VersionDiff::VersionDiff(const Version& left, const Version& right) : left(left), right(right) { } - std::string VersionDiff::to_string() const { return fmt::format("{} -> {}", left, right); } + std::string VersionDiff::to_string() const { return adapt_to_string(*this); } + void VersionDiff::to_string(std::string& out) const + { + fmt::format_to(std::back_inserter(out), "{} -> {}", left, right); + } - std::string VersionSpec::to_string() const { return fmt::format("{}@{}", port_name, version); } + std::string VersionSpec::to_string() const { return adapt_to_string(*this); } + void VersionSpec::to_string(std::string& out) const + { + fmt::format_to(std::back_inserter(out), "{}@{}", port_name, version); + } namespace { From 21f1d2e9f57e22abbe8213cfc8b0552e354e7fa9 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 27 Sep 2024 19:11:08 -0700 Subject: [PATCH 02/14] Use PackageSpec's to_string in formatting. --- include/vcpkg/base/graphs.h | 1 - src/vcpkg/binaryparagraph.cpp | 8 ++------ src/vcpkg/commands.ci.cpp | 10 +++++----- src/vcpkg/commands.export.cpp | 3 +-- src/vcpkg/commands.list.cpp | 9 +++++---- src/vcpkg/commands.set-installed.cpp | 25 +++++++++++++------------ src/vcpkg/dependencies.cpp | 5 ----- 7 files changed, 26 insertions(+), 35 deletions(-) diff --git a/include/vcpkg/base/graphs.h b/include/vcpkg/base/graphs.h index 8c10236efd..311a52bc28 100644 --- a/include/vcpkg/base/graphs.h +++ b/include/vcpkg/base/graphs.h @@ -16,7 +16,6 @@ namespace vcpkg struct AdjacencyProvider { virtual std::vector adjacency_list(const U& vertex) const = 0; - virtual std::string to_string(const V& vertex) const = 0; virtual U load_vertex_data(const V& vertex) const = 0; }; diff --git a/src/vcpkg/binaryparagraph.cpp b/src/vcpkg/binaryparagraph.cpp index 47b637bd0d..66a20b3acd 100644 --- a/src/vcpkg/binaryparagraph.cpp +++ b/src/vcpkg/binaryparagraph.cpp @@ -12,6 +12,7 @@ using namespace vcpkg::Paragraphs; namespace vcpkg { BinaryParagraph::BinaryParagraph(StringView origin, Paragraph&& fields) + : spec(), version(), description(), maintainers(), feature(), default_features(), dependencies(), abi() { ParagraphParser parser(origin, std::move(fields)); this->spec = PackageSpec(parser.required_field(ParagraphIdPackage), @@ -217,7 +218,6 @@ namespace vcpkg const size_t initial_end = out_str.size(); append_paragraph_field(ParagraphIdPackage, pgh.spec.name(), out_str); - append_paragraph_field(ParagraphIdVersion, pgh.version.text, out_str); if (pgh.version.port_version != 0) { @@ -237,13 +237,9 @@ namespace vcpkg append_paragraph_field(ParagraphIdArchitecture, pgh.spec.triplet().to_string(), out_str); append_paragraph_field(ParagraphIdMultiArch, "same", out_str); - serialize_paragraph(ParagraphIdMaintainer, pgh.maintainers, out_str); - append_paragraph_field(ParagraphIdAbi, pgh.abi, out_str); - serialize_paragraph(ParagraphIdDescription, pgh.description, out_str); - serialize_array(ParagraphIdDefaultFeatures, pgh.default_features, out_str); // sanity check the serialized data @@ -278,7 +274,7 @@ namespace vcpkg return fmt::format( "\nspec: \"{}\"\nversion: \"{}\"\nport_version: {}\ndescription: [\"{}\"]\nmaintainers: [\"{}\"]\nfeature: " "\"{}\"\ndefault_features: [\"{}\"]\ndependencies: [\"{}\"]\nabi: \"{}\"", - paragraph.spec.to_string(), + paragraph.spec, paragraph.version.text, paragraph.version.port_version, Strings::join(join_str, paragraph.description), diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 115964d7ca..24581927f5 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -52,11 +52,11 @@ namespace (void)filesystem.create_directory(target_path, VCPKG_LINE_INFO); if (children.empty()) { - std::string message = - "There are no build logs for " + spec.to_string() + - " build.\n" - "This is usually because the build failed early and outside of a task that is logged.\n" - "See the console output logs from vcpkg for more information on the failure.\n"; + auto message = + fmt::format("There are no build logs for {} build.\n" + "This is usually because the build failed early and outside of a task that is logged.\n" + "See the console output logs from vcpkg for more information on the failure.\n", + spec); filesystem.write_contents(std::move(target_path) / "readme.log", message, VCPKG_LINE_INFO); } else diff --git a/src/vcpkg/commands.export.cpp b/src/vcpkg/commands.export.cpp index a8a9a26330..4032519205 100644 --- a/src/vcpkg/commands.export.cpp +++ b/src/vcpkg/commands.export.cpp @@ -472,8 +472,7 @@ namespace Checks::unreachable(VCPKG_LINE_INFO); } - const std::string display_name = action.spec.to_string(); - msg::println(msgExportingPackage, msg::package_name = display_name); + msg::println(msgExportingPackage, msg::package_name = action.spec); const BinaryParagraph& binary_paragraph = action.core_paragraph().value_or_exit(VCPKG_LINE_INFO); diff --git a/src/vcpkg/commands.list.cpp b/src/vcpkg/commands.list.cpp index ea6373a558..a7ff89e78b 100644 --- a/src/vcpkg/commands.list.cpp +++ b/src/vcpkg/commands.list.cpp @@ -17,19 +17,20 @@ namespace Json::Object obj; for (const StatusParagraph* status_paragraph : installed_packages) { - auto current_spec = status_paragraph->package.spec; - if (obj.contains(current_spec.to_string())) + const auto& current_spec = status_paragraph->package.spec; + const auto current_spec_string = current_spec.to_string(); + if (obj.contains(current_spec_string)) { if (status_paragraph->package.is_feature()) { - Json::Value* value_obj = obj.get(current_spec.to_string()); + Json::Value* value_obj = obj.get(current_spec_string); auto& feature_list = value_obj->object(VCPKG_LINE_INFO)[JsonIdFeatures].array(VCPKG_LINE_INFO); feature_list.push_back(Json::Value::string(status_paragraph->package.feature)); } } else { - Json::Object& library_obj = obj.insert(current_spec.to_string(), Json::Object()); + Json::Object& library_obj = obj.insert(current_spec_string, Json::Object()); library_obj.insert(JsonIdPackageUnderscoreName, Json::Value::string(current_spec.name())); library_obj.insert(JsonIdTriplet, Json::Value::string(current_spec.triplet().to_string())); library_obj.insert(JsonIdVersion, Json::Value::string(status_paragraph->package.version.text)); diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 717858d3c4..f63d9f1bde 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -77,44 +77,45 @@ namespace vcpkg std::unordered_map map; for (auto&& action : action_plan.install_actions) { - if (!action.source_control_file_and_location.has_value()) + const auto scfl = action.source_control_file_and_location.get(); + if (!scfl) { return nullopt; } - const auto& scf = *action.source_control_file_and_location.get(); - auto version = scf.to_version().to_string(); - auto s = action.spec.to_string(); - auto pkg_url = Strings::concat("pkg:github/vcpkg/", s, "@", version); - map.insert({s, pkg_url}); + auto spec = action.spec.to_string(); + map.insert( + {spec, fmt::format("pkg:github/vcpkg/{}@{}", spec, scfl->source_control_file->to_version())}); } Json::Object resolved; for (auto&& action : action_plan.install_actions) { Json::Object resolved_item; - if (map.find(action.spec.to_string()) != map.end()) + auto spec = action.spec.to_string(); + const auto found = map.find(spec); + if (found != map.end()) { - auto pkg_url = map.at(action.spec.to_string()); + const auto& pkg_url = found->second; resolved_item.insert(JsonIdPackageUnderscoreUrl, pkg_url); resolved_item.insert(JsonIdRelationship, Json::Value::string(JsonIdDirect)); Json::Array deps_list; for (auto&& dep : action.package_dependencies) { - if (map.find(dep.to_string()) != map.end()) + const auto found_dep = map.find(dep.to_string()); + if (found_dep != map.end()) { - auto dep_pkg_url = map.at(dep.to_string()); - deps_list.push_back(dep_pkg_url); + deps_list.push_back(found_dep->second); } } resolved_item.insert(JsonIdDependencies, deps_list); resolved.insert(pkg_url, resolved_item); } } + manifest.insert(JsonIdResolved, resolved); Json::Object manifests; manifests.insert(JsonIdVcpkgDotJson, manifest); snapshot.insert(JsonIdManifests, manifests); - Debug::print(Json::stringify(snapshot)); return snapshot; } diff --git a/src/vcpkg/dependencies.cpp b/src/vcpkg/dependencies.cpp index 7142a61067..bfdd903138 100644 --- a/src/vcpkg/dependencies.cpp +++ b/src/vcpkg/dependencies.cpp @@ -680,8 +680,6 @@ namespace vcpkg } PackageSpec load_vertex_data(const PackageSpec& s) const override { return s; } - - std::string to_string(const PackageSpec& spec) const override { return spec.to_string(); } }; RemoveAdjacencyProvider p; @@ -746,8 +744,6 @@ namespace vcpkg return ExportPlanAction{spec, request_type}; } - - std::string to_string(const PackageSpec& spec) const override { return spec.to_string(); } }; const std::unordered_set specs_as_set(specs.cbegin(), specs.cend()); @@ -997,7 +993,6 @@ namespace vcpkg { BaseEdgeProvider(const ClusterGraph& parent) : m_parent(parent) { } - std::string to_string(const PackageSpec& spec) const override { return spec.to_string(); } const Cluster* load_vertex_data(const PackageSpec& spec) const override { return &m_parent.find_or_exit(spec, VCPKG_LINE_INFO); From 0da13666e92ebdf07ec6094301144b1a504e0374 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 27 Sep 2024 19:12:07 -0700 Subject: [PATCH 03/14] Avoid potential null derefs in create_dependency_graph_snapshot. --- src/vcpkg/commands.set-installed.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index f63d9f1bde..7bc701e519 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -50,8 +50,12 @@ namespace vcpkg Optional create_dependency_graph_snapshot(const VcpkgCmdArguments& args, const ActionPlan& action_plan) { - if (args.github_ref.has_value() && args.github_sha.has_value() && args.github_job.has_value() && - args.github_workflow.has_value() && args.github_run_id.has_value()) + const auto github_ref = args.github_ref.get(); + const auto github_sha = args.github_sha.get(); + const auto github_job = args.github_job.get(); + const auto github_workflow = args.github_workflow.get(); + const auto github_run_id = args.github_run_id.get(); + if (github_ref && github_sha && github_job && github_workflow && github_run_id) { Json::Object detector; detector.insert(JsonIdName, Json::Value::string("vcpkg")); @@ -59,15 +63,14 @@ namespace vcpkg detector.insert(JsonIdVersion, Json::Value::string("1.0.0")); Json::Object job; - job.insert(JsonIdId, Json::Value::string(*args.github_run_id.get())); - job.insert(JsonIdCorrelator, - Json::Value::string(*args.github_workflow.get() + "-" + *args.github_job.get())); + job.insert(JsonIdId, Json::Value::string(*github_run_id)); + job.insert(JsonIdCorrelator, Json::Value::string(fmt::format("{}-{}", *github_workflow, *github_run_id))); Json::Object snapshot; snapshot.insert(JsonIdJob, job); snapshot.insert(JsonIdVersion, Json::Value::integer(0)); - snapshot.insert(JsonIdSha, Json::Value::string(*args.github_sha.get())); - snapshot.insert(JsonIdRef, Json::Value::string(*args.github_ref.get())); + snapshot.insert(JsonIdSha, Json::Value::string(*github_sha)); + snapshot.insert(JsonIdRef, Json::Value::string(*github_ref)); snapshot.insert(JsonIdScanned, Json::Value::string(CTime::now_string())); snapshot.insert(JsonIdDetector, detector); From ee6dc8e8969ac790c250a8d1dbea0002c701af27 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 6 Sep 2024 16:54:25 -0700 Subject: [PATCH 04/14] Fix read operations like list on readonly filesystems. Resolves https://github.com/microsoft/vcpkg/issues/10812 --- include/vcpkg/vcpkglib.h | 8 +- src/vcpkg/commands.build.cpp | 2 +- src/vcpkg/commands.ci.cpp | 2 +- src/vcpkg/commands.export.cpp | 2 +- src/vcpkg/commands.install.cpp | 5 +- src/vcpkg/commands.list.cpp | 3 +- src/vcpkg/commands.owns.cpp | 6 +- src/vcpkg/commands.package-info.cpp | 2 +- src/vcpkg/commands.remove.cpp | 5 +- src/vcpkg/commands.set-installed.cpp | 2 +- src/vcpkg/commands.update.cpp | 2 +- src/vcpkg/commands.upgrade.cpp | 2 +- src/vcpkg/vcpkglib.cpp | 189 ++++++++++++++++++--------- 13 files changed, 149 insertions(+), 81 deletions(-) diff --git a/include/vcpkg/vcpkglib.h b/include/vcpkg/vcpkglib.h index 7621d8526b..6eeffbf4c3 100644 --- a/include/vcpkg/vcpkglib.h +++ b/include/vcpkg/vcpkglib.h @@ -13,7 +13,8 @@ namespace vcpkg { - StatusParagraphs database_load_check(const Filesystem& fs, const InstalledPaths& installed); + StatusParagraphs database_load(const ReadOnlyFilesystem& fs, const InstalledPaths& installed); + StatusParagraphs database_load_collapse(const Filesystem& fs, const InstalledPaths& installed); void write_update(const Filesystem& fs, const InstalledPaths& installed, const StatusParagraph& p); @@ -24,9 +25,12 @@ namespace vcpkg }; std::vector get_installed_ports(const StatusParagraphs& status_db); - std::vector get_installed_files(const Filesystem& fs, + std::vector get_installed_files(const ReadOnlyFilesystem& fs, const InstalledPaths& installed, const StatusParagraphs& status_db); + std::vector get_installed_files_and_upgrade(const Filesystem& fs, + const InstalledPaths& installed, + const StatusParagraphs& status_db); std::string shorten_text(StringView desc, const size_t length); } // namespace vcpkg diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index d1a8a8d494..5c6c60119a 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -132,7 +132,7 @@ namespace vcpkg auto& var_provider = *var_provider_storage; var_provider.load_dep_info_vars({{spec}}, host_triplet); - StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + StatusParagraphs status_db = database_load_collapse(paths.get_filesystem(), paths.installed()); auto action_plan = create_feature_install_plan( provider, var_provider, diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 24581927f5..e000d5dde7 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -502,7 +502,7 @@ namespace vcpkg } else { - StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + StatusParagraphs status_db = database_load_collapse(paths.get_filesystem(), paths.installed()); auto already_installed = adjust_action_plan_to_status_db(action_plan, status_db); Util::erase_if(already_installed, [&](auto& spec) { return Util::Sets::contains(split_specs->known, spec); }); diff --git a/src/vcpkg/commands.export.cpp b/src/vcpkg/commands.export.cpp index 4032519205..8546687ae1 100644 --- a/src/vcpkg/commands.export.cpp +++ b/src/vcpkg/commands.export.cpp @@ -596,7 +596,7 @@ namespace vcpkg Triplet host_triplet) { (void)host_triplet; - const StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + const StatusParagraphs status_db = database_load(paths.get_filesystem(), paths.installed()); const auto opts = handle_export_command_arguments(paths, args, default_triplet, status_db); // Load ports from ports dirs diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 82c91158d8..dd714e1b3b 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -224,7 +224,7 @@ namespace vcpkg const auto package_dir = paths.package_dir(bcf.core_paragraph.spec); Triplet triplet = bcf.core_paragraph.spec.triplet(); const std::vector pgh_and_files = - get_installed_files(fs, installed, *status_db); + get_installed_files_and_upgrade(fs, installed, *status_db); const SortedVector package_files = build_list_of_package_files(fs, package_dir); const SortedVector installed_files = build_list_of_installed_files(pgh_and_files, triplet); @@ -617,6 +617,7 @@ namespace vcpkg this_install.current_summary.build_result.emplace(std::move(result)); } + database_load_collapse(fs, paths.installed()); msg::println(msgTotalInstallTime, msg::elapsed = timer.to_string()); return InstallSummary{std::move(results)}; } @@ -1286,7 +1287,7 @@ namespace vcpkg // create the plan msg::println(msgComputingInstallPlan); - StatusParagraphs status_db = database_load_check(fs, paths.installed()); + StatusParagraphs status_db = database_load_collapse(fs, paths.installed()); // Note: action_plan will hold raw pointers to SourceControlFileLocations from this map auto action_plan = create_feature_install_plan(provider, var_provider, specs, status_db, create_options); diff --git a/src/vcpkg/commands.list.cpp b/src/vcpkg/commands.list.cpp index a7ff89e78b..0d4f4956b2 100644 --- a/src/vcpkg/commands.list.cpp +++ b/src/vcpkg/commands.list.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -103,7 +104,7 @@ namespace vcpkg msg::default_output_stream = OutputStream::StdErr; const ParsedArguments options = args.parse_arguments(CommandListMetadata); - const StatusParagraphs status_paragraphs = database_load_check(paths.get_filesystem(), paths.installed()); + const StatusParagraphs status_paragraphs = database_load(paths.get_filesystem(), paths.installed()); auto installed_ipv = get_installed_ports(status_paragraphs); const auto output_json = Util::Sets::contains(options.switches, SwitchXJson); diff --git a/src/vcpkg/commands.owns.cpp b/src/vcpkg/commands.owns.cpp index f960371c84..ca3e47c5ea 100644 --- a/src/vcpkg/commands.owns.cpp +++ b/src/vcpkg/commands.owns.cpp @@ -1,3 +1,5 @@ +#include + #include #include #include @@ -8,7 +10,7 @@ using namespace vcpkg; namespace { - void search_file(const Filesystem& fs, + void search_file(const ReadOnlyFilesystem& fs, const InstalledPaths& installed, const std::string& file_substr, const StatusParagraphs& status_db) @@ -46,7 +48,7 @@ namespace vcpkg void command_owns_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) { const auto parsed = args.parse_arguments(CommandOwnsMetadata); - const StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + const StatusParagraphs status_db = database_load(paths.get_filesystem(), paths.installed()); search_file(paths.get_filesystem(), paths.installed(), parsed.command_arguments[0], status_db); Checks::exit_success(VCPKG_LINE_INFO); } diff --git a/src/vcpkg/commands.package-info.cpp b/src/vcpkg/commands.package-info.cpp index 70608eb521..f7c9597905 100644 --- a/src/vcpkg/commands.package-info.cpp +++ b/src/vcpkg/commands.package-info.cpp @@ -61,7 +61,7 @@ namespace vcpkg auto& fs = paths.get_filesystem(); if (installed) { - const StatusParagraphs status_paragraphs = database_load_check(fs, paths.installed()); + const StatusParagraphs status_paragraphs = database_load(fs, paths.installed()); std::set specs_written; std::vector specs_to_write; for (auto&& arg : options.command_arguments) diff --git a/src/vcpkg/commands.remove.cpp b/src/vcpkg/commands.remove.cpp index 9d3dbe6e5f..835b9bdf9e 100644 --- a/src/vcpkg/commands.remove.cpp +++ b/src/vcpkg/commands.remove.cpp @@ -148,7 +148,7 @@ namespace std::vector valid_arguments(const VcpkgPaths& paths) { - const StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + const StatusParagraphs status_db = database_load(paths.get_filesystem(), paths.installed()); auto installed_packages = get_installed_ports(status_db); return Util::fmap(installed_packages, [](auto&& pgh) -> std::string { return pgh.spec().to_string(); }); @@ -181,7 +181,7 @@ namespace vcpkg } const ParsedArguments options = args.parse_arguments(CommandRemoveMetadata); - StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + StatusParagraphs status_db = database_load_collapse(paths.get_filesystem(), paths.installed()); std::vector specs; if (Util::Sets::contains(options.switches, SwitchOutdated)) { @@ -299,6 +299,7 @@ namespace vcpkg } } + database_load_collapse(fs, paths.installed()); Checks::exit_success(VCPKG_LINE_INFO); } } diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 7bc701e519..0d38a0f171 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -211,7 +211,7 @@ namespace vcpkg } // currently (or once) installed specifications - auto status_db = database_load_check(fs, paths.installed()); + auto status_db = database_load_collapse(fs, paths.installed()); adjust_action_plan_to_status_db(action_plan, status_db); print_plan(action_plan, paths.builtin_ports_directory()); diff --git a/src/vcpkg/commands.update.cpp b/src/vcpkg/commands.update.cpp index 3aec063456..e2b7b44109 100644 --- a/src/vcpkg/commands.update.cpp +++ b/src/vcpkg/commands.update.cpp @@ -65,7 +65,7 @@ namespace vcpkg msg::println(msgLocalPortfileVersion); auto& fs = paths.get_filesystem(); - const StatusParagraphs status_db = database_load_check(fs, paths.installed()); + const StatusParagraphs status_db = database_load(fs, paths.installed()); auto registry_set = paths.make_registry_set(); PathsPortFileProvider provider(*registry_set, diff --git a/src/vcpkg/commands.upgrade.cpp b/src/vcpkg/commands.upgrade.cpp index a113231a56..a59ff797d5 100644 --- a/src/vcpkg/commands.upgrade.cpp +++ b/src/vcpkg/commands.upgrade.cpp @@ -76,7 +76,7 @@ namespace vcpkg const CreateUpgradePlanOptions create_upgrade_plan_options{ nullptr, host_triplet, paths.packages(), unsupported_port_action}; - StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); + StatusParagraphs status_db = database_load_collapse(paths.get_filesystem(), paths.installed()); // Load ports from ports dirs auto& fs = paths.get_filesystem(); diff --git a/src/vcpkg/vcpkglib.cpp b/src/vcpkg/vcpkglib.cpp index eda0c1f641..339a518ed5 100644 --- a/src/vcpkg/vcpkglib.cpp +++ b/src/vcpkg/vcpkglib.cpp @@ -10,21 +10,8 @@ namespace vcpkg { - static StatusParagraphs load_current_database(const Filesystem& fs, - const Path& vcpkg_dir_status_file, - const Path& vcpkg_dir_status_file_old) + static StatusParagraphs load_current_database(const ReadOnlyFilesystem& fs, const Path& vcpkg_dir_status_file) { - if (!fs.exists(vcpkg_dir_status_file, IgnoreErrors{})) - { - if (!fs.exists(vcpkg_dir_status_file_old, IgnoreErrors{})) - { - // no status file, use empty db - return StatusParagraphs(); - } - - fs.rename(vcpkg_dir_status_file_old, vcpkg_dir_status_file, VCPKG_LINE_INFO); - } - auto pghs = Paragraphs::get_paragraphs(fs, vcpkg_dir_status_file).value_or_exit(VCPKG_LINE_INFO); std::vector> status_pghs; @@ -37,7 +24,76 @@ namespace vcpkg return StatusParagraphs(std::move(status_pghs)); } - StatusParagraphs database_load_check(const Filesystem& fs, const InstalledPaths& installed) + static std::vector apply_database_updates(const ReadOnlyFilesystem& fs, + StatusParagraphs& current_status_db, + const Path& updates_dir) + { + auto update_files = fs.get_regular_files_non_recursive(updates_dir, VCPKG_LINE_INFO); + Util::sort(update_files); + if (!update_files.empty()) + { + for (auto&& file : update_files) + { + if (file.filename() == "incomplete") continue; + + auto pghs = Paragraphs::get_paragraphs(fs, file).value_or_exit(VCPKG_LINE_INFO); + for (auto&& p : pghs) + { + current_status_db.insert(std::make_unique(file, std::move(p))); + } + } + } + + return update_files; + } + + static void apply_database_updates_on_disk(const Filesystem& fs, + const InstalledPaths& installed, + StatusParagraphs& current_status_db) + { + auto update_files = apply_database_updates(fs, current_status_db, installed.vcpkg_dir_updates()); + if (!update_files.empty()) + { + const auto status_file = installed.vcpkg_dir_status_file(); + const auto status_file_new = Path(status_file.parent_path()) / "status-new"; + fs.write_contents(status_file_new, Strings::serialize(current_status_db), VCPKG_LINE_INFO); + + fs.rename(status_file_new, status_file, VCPKG_LINE_INFO); + + for (auto&& file : update_files) + { + fs.remove(file, VCPKG_LINE_INFO); + } + } + } + + StatusParagraphs database_load(const ReadOnlyFilesystem& fs, const InstalledPaths& installed) + { + const auto maybe_status_file = installed.vcpkg_dir_status_file(); + const auto status_parent = Path(maybe_status_file.parent_path()); + const auto status_file_old = status_parent / "status-old"; + + auto status_file = &maybe_status_file; + + if (!fs.exists(maybe_status_file, IgnoreErrors{})) + { + if (!fs.exists(status_file_old, IgnoreErrors{})) + { + // no status file, use empty db + StatusParagraphs current_status_db; + (void)apply_database_updates(fs, current_status_db, installed.vcpkg_dir_updates()); + return current_status_db; + } + + status_file = &status_file_old; + } + + StatusParagraphs current_status_db = load_current_database(fs, *status_file); + (void)apply_database_updates(fs, current_status_db, installed.vcpkg_dir_updates()); + return current_status_db; + } + + StatusParagraphs database_load_collapse(const Filesystem& fs, const InstalledPaths& installed) { const auto updates_dir = installed.vcpkg_dir_updates(); @@ -49,37 +105,22 @@ namespace vcpkg const auto status_file = installed.vcpkg_dir_status_file(); const auto status_parent = Path(status_file.parent_path()); const auto status_file_old = status_parent / "status-old"; - const auto status_file_new = status_parent / "status-new"; - - StatusParagraphs current_status_db = load_current_database(fs, status_file, status_file_old); - auto update_files = fs.get_regular_files_non_recursive(updates_dir, VCPKG_LINE_INFO); - Util::sort(update_files); - if (update_files.empty()) - { - // updates directory is empty, control file is up-to-date. - return current_status_db; - } - for (auto&& file : update_files) + if (!fs.exists(status_file, IgnoreErrors{})) { - if (file.filename() == "incomplete") continue; - - auto pghs = Paragraphs::get_paragraphs(fs, file).value_or_exit(VCPKG_LINE_INFO); - for (auto&& p : pghs) + if (!fs.exists(status_file_old, IgnoreErrors{})) { - current_status_db.insert(std::make_unique(file, std::move(p))); + // no status file, use empty db + StatusParagraphs current_status_db; + apply_database_updates_on_disk(fs, installed, current_status_db); + return current_status_db; } - } - - fs.write_contents(status_file_new, Strings::serialize(current_status_db), VCPKG_LINE_INFO); - fs.rename(status_file_new, status_file, VCPKG_LINE_INFO); - - for (auto&& file : update_files) - { - fs.remove(file, VCPKG_LINE_INFO); + fs.rename(status_file_old, status_file, VCPKG_LINE_INFO); } + StatusParagraphs current_status_db = load_current_database(fs, status_file); + apply_database_updates_on_disk(fs, installed, current_status_db); return current_status_db; } @@ -93,25 +134,22 @@ namespace vcpkg fs.write_rename_contents(update_path, "incomplete", Strings::serialize(p), VCPKG_LINE_INFO); } - static void upgrade_to_slash_terminated_sorted_format(const Filesystem& fs, - std::vector* lines, - const Path& listfile_path) + static bool upgrade_to_slash_terminated_sorted_format(std::vector& lines) { - static bool was_tracked = false; + static std::atomic was_tracked = false; - if (lines->empty()) + if (lines.empty()) { - return; + return false; } - if (lines->at(0).back() == '/') + if (lines.front().back() == '/') { - return; // File already in the new format + return false; // File already in the new format } - if (!was_tracked) + if (was_tracked.exchange(true)) { - was_tracked = true; get_global_metrics_collector().track_string(StringMetric::ListFile, "update to new format"); } @@ -119,14 +157,15 @@ namespace vcpkg // (They are not necessarily sorted alphabetically, e.g. libflac) // Therefore we can detect the entries that represent directories by comparing every element with the next one // and checking if the next has a slash immediately after the current one's length - for (size_t i = 0; i < lines->size() - 1; i++) + const size_t end = lines.size() - 1; + for (size_t i = 0; i < end; i++) { - std::string& current_string = lines->at(i); - const std::string& next_string = lines->at(i + 1); - + std::string& current_string = lines[i]; + const std::string& next_string = lines[i + 1]; + // check if the next line is the same as this one with a slash after; that indicates that this one + // represents a directory const size_t potential_slash_char_index = current_string.length(); - // Make sure the index exists first - if (next_string.size() > potential_slash_char_index && next_string.at(potential_slash_char_index) == '/') + if (next_string.size() > potential_slash_char_index && next_string[potential_slash_char_index] == '/') { current_string += '/'; // Mark as a directory } @@ -155,12 +194,8 @@ namespace vcpkg */ // Note that after sorting, the FLAC++/ group will be placed before the FLAC/ group // The new format is lexicographically sorted - std::sort(lines->begin(), lines->end()); - - // Replace the listfile on disk - const auto updated_listfile_path = listfile_path + "_updated"; - fs.write_lines(updated_listfile_path, *lines, VCPKG_LINE_INFO); - fs.rename(updated_listfile_path, listfile_path, VCPKG_LINE_INFO); + Util::sort(lines); + return true; } std::vector get_installed_ports(const StatusParagraphs& status_db) @@ -189,9 +224,10 @@ namespace vcpkg return Util::fmap(ipv_map, [](auto&& p) -> InstalledPackageView { return std::move(p.second); }); } - std::vector get_installed_files(const Filesystem& fs, - const InstalledPaths& installed, - const StatusParagraphs& status_db) + template + static std::vector get_installed_files_impl(const FilesystemLike& fs, + const InstalledPaths& installed, + const StatusParagraphs& status_db) { std::vector installed_files; @@ -206,7 +242,16 @@ namespace vcpkg std::vector installed_files_of_current_pgh = fs.read_lines(listfile_path).value_or_exit(VCPKG_LINE_INFO); Strings::inplace_trim_all_and_remove_whitespace_strings(installed_files_of_current_pgh); - upgrade_to_slash_terminated_sorted_format(fs, &installed_files_of_current_pgh, listfile_path); + if (upgrade_to_slash_terminated_sorted_format(installed_files_of_current_pgh)) + { + if constexpr (AndUpdate) + { + // Replace the listfile on disk + const auto updated_listfile_path = listfile_path + "_updated"; + fs.write_lines(updated_listfile_path, installed_files_of_current_pgh, VCPKG_LINE_INFO); + fs.rename(updated_listfile_path, listfile_path, VCPKG_LINE_INFO); + } + } // Remove the directories Util::erase_remove_if(installed_files_of_current_pgh, @@ -220,6 +265,20 @@ namespace vcpkg return installed_files; } + std::vector get_installed_files(const ReadOnlyFilesystem& fs, + const InstalledPaths& installed, + const StatusParagraphs& status_db) + { + return get_installed_files_impl(fs, installed, status_db); + } + + std::vector get_installed_files_and_upgrade(const Filesystem& fs, + const InstalledPaths& installed, + const StatusParagraphs& status_db) + { + return get_installed_files_impl(fs, installed, status_db); + } + std::string shorten_text(StringView desc, const size_t length) { Checks::check_exit(VCPKG_LINE_INFO, length >= 3); From a48c10d6dc54dad8dd701266f5fc8dbf32ca94a7 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 30 Sep 2024 18:02:31 -0700 Subject: [PATCH 05/14] Extract some more contractual-constants file names. --- include/vcpkg/base/contractual-constants.h | 14 ++++++++++++++ include/vcpkg/installedpaths.h | 18 ++++++++++-------- src/vcpkg/commands.build.cpp | 4 ++-- src/vcpkg/export.prefab.cpp | 4 ++-- src/vcpkg/postbuildlint.cpp | 12 ++++++------ src/vcpkg/vcpkglib.cpp | 12 ++++++------ 6 files changed, 40 insertions(+), 24 deletions(-) diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h index 0b86b23290..72a95437db 100644 --- a/include/vcpkg/base/contractual-constants.h +++ b/include/vcpkg/base/contractual-constants.h @@ -330,18 +330,32 @@ namespace vcpkg // File names inline constexpr StringLiteral FileBaselineDotJson = "baseline.json"; inline constexpr StringLiteral FileBin = "bin"; + inline constexpr StringLiteral FileCopyright = "copyright"; + inline constexpr StringLiteral FileLicense = "LICENSE"; inline constexpr StringLiteral FileControl = "CONTROL"; inline constexpr StringLiteral FileDebug = "debug"; inline constexpr StringLiteral FileDetectCompiler = "detect_compiler"; inline constexpr StringLiteral FileDotDsStore = ".DS_Store"; inline constexpr StringLiteral FileInclude = "include"; + inline constexpr StringLiteral FileIncomplete = "incomplete"; + inline constexpr StringLiteral FileInfo = "info"; inline constexpr StringLiteral FilePortfileDotCMake = "portfile.cmake"; + inline constexpr StringLiteral FileShare = "share"; + inline constexpr StringLiteral FileStatus = "status"; + inline constexpr StringLiteral FileStatusNew = "status-new"; + inline constexpr StringLiteral FileStatusOld = "status-old"; inline constexpr StringLiteral FileTools = "tools"; + inline constexpr StringLiteral FileUpdates = "updates"; + inline constexpr StringLiteral FileUsage = "usage"; + inline constexpr StringLiteral FileVcpkg = "vcpkg"; inline constexpr StringLiteral FileVcpkgAbiInfo = "vcpkg_abi_info.txt"; inline constexpr StringLiteral FileVcpkgBundleDotJson = "vcpkg-bundle.json"; inline constexpr StringLiteral FileVcpkgConfigurationDotJson = "vcpkg-configuration.json"; inline constexpr StringLiteral FileVcpkgDotJson = "vcpkg.json"; + inline constexpr StringLiteral FileVcpkgLock = "vcpkg-lock.json"; inline constexpr StringLiteral FileVcpkgPathTxt = "vcpkg.path.txt"; + inline constexpr StringLiteral FileVcpkgPortConfig = "vcpkg-port-config.cmake"; + inline constexpr StringLiteral FileVcpkgSpdxJson = "vcpkg.spdx.json"; inline constexpr StringLiteral FileVcpkgUserProps = "vcpkg.user.props"; inline constexpr StringLiteral FileVcpkgUserTargets = "vcpkg.user.targets"; inline constexpr StringLiteral FileVersions = "versions"; diff --git a/include/vcpkg/installedpaths.h b/include/vcpkg/installedpaths.h index 101d8793d0..08349eb24f 100644 --- a/include/vcpkg/installedpaths.h +++ b/include/vcpkg/installedpaths.h @@ -2,6 +2,7 @@ #include +#include #include #include @@ -16,15 +17,16 @@ namespace vcpkg const Path& root() const { return m_root; } Path listfile_path(const BinaryParagraph& pgh) const; - Path vcpkg_dir() const { return m_root / "vcpkg"; } - Path vcpkg_dir_status_file() const { return vcpkg_dir() / "status"; } - Path vcpkg_dir_info() const { return vcpkg_dir() / "info"; } - Path vcpkg_dir_updates() const { return vcpkg_dir() / "updates"; } - Path lockfile_path() const { return vcpkg_dir() / "vcpkg-lock.json"; } + Path vcpkg_dir() const { return m_root / FileVcpkg; } + Path vcpkg_dir_status_file() const { return vcpkg_dir() / FileStatus; } + Path vcpkg_dir_info() const { return vcpkg_dir() / FileInfo; } + Path vcpkg_dir_updates() const { return vcpkg_dir() / FileUpdates; } + Path lockfile_path() const { return vcpkg_dir() / FileVcpkgLock; } Path triplet_dir(Triplet t) const { return m_root / t.canonical_name(); } - Path share_dir(const PackageSpec& p) const { return triplet_dir(p.triplet()) / "share" / p.name(); } - Path usage_file(const PackageSpec& p) const { return share_dir(p) / "usage"; } - Path vcpkg_port_config_cmake(const PackageSpec& p) const { return share_dir(p) / "vcpkg-port-config.cmake"; } + Path share_dir(const PackageSpec& p) const { return triplet_dir(p.triplet()) / FileShare / p.name(); } + Path usage_file(const PackageSpec& p) const { return share_dir(p) / FileUsage; } + Path spdx_file(const PackageSpec& p) const { return share_dir(p) / FileVcpkgSpdxJson; } + Path vcpkg_port_config_cmake(const PackageSpec& p) const { return share_dir(p) / FileVcpkgPortConfig; } private: Path m_root; diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 5c6c60119a..6d525a5e78 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -941,7 +941,7 @@ namespace vcpkg const auto& abi = action.abi_info.value_or_exit(VCPKG_LINE_INFO); const auto json_path = - action.package_dir.value_or_exit(VCPKG_LINE_INFO) / "share" / action.spec.name() / "vcpkg.spdx.json"; + action.package_dir.value_or_exit(VCPKG_LINE_INFO) / FileShare / action.spec.name() / FileVcpkgSpdxJson; fs.write_contents_and_dirs( json_path, create_spdx_sbom( @@ -1446,7 +1446,7 @@ namespace vcpkg if (abi_info.abi_tag_file) { auto& abi_file = *abi_info.abi_tag_file.get(); - const auto abi_package_dir = action.package_dir.value_or_exit(VCPKG_LINE_INFO) / "share" / spec.name(); + const auto abi_package_dir = action.package_dir.value_or_exit(VCPKG_LINE_INFO) / FileShare / spec.name(); const auto abi_file_in_package = abi_package_dir / FileVcpkgAbiInfo; build_logs_recorder.record_build_result(paths, spec, result.code); filesystem.create_directories(abi_package_dir, VCPKG_LINE_INFO); diff --git a/src/vcpkg/export.prefab.cpp b/src/vcpkg/export.prefab.cpp index 22f9fcad01..fe9ca1e044 100644 --- a/src/vcpkg/export.prefab.cpp +++ b/src/vcpkg/export.prefab.cpp @@ -433,8 +433,8 @@ namespace vcpkg::Prefab const auto share_root = paths.packages() / fmt::format("{}_{}", name, action.spec.triplet()); - fs.copy_file(share_root / "share" / name / "copyright", - meta_dir / "LICENSE", + fs.copy_file(share_root / FileVcpkgPortConfig / name / FileCopyright, + meta_dir / FileLicense, CopyOptions::overwrite_existing, IgnoreErrors{}); diff --git a/src/vcpkg/postbuildlint.cpp b/src/vcpkg/postbuildlint.cpp index 968e7c96b3..ce085bd416 100644 --- a/src/vcpkg/postbuildlint.cpp +++ b/src/vcpkg/postbuildlint.cpp @@ -282,7 +282,7 @@ namespace vcpkg const Path& portfile_cmake, MessageSink& msg_sink) { - const auto debug_share = package_dir / "debug" VCPKG_PREFERRED_SEPARATOR "share"; + const auto debug_share = package_dir / FileDebug / FileShare; if (fs.exists(debug_share, IgnoreErrors{})) { msg_sink.print(Color::warning, @@ -303,7 +303,7 @@ namespace vcpkg const Path& portfile_cmake, MessageSink& msg_sink) { - if (!fs.exists(package_dir / "share" / package_name / "vcpkg-port-config.cmake", IgnoreErrors{})) + if (!fs.exists(package_dir / FileShare / package_name / FileVcpkgPortConfig, IgnoreErrors{})) { msg_sink.print(Color::warning, LocalizedString::from_raw(portfile_cmake) @@ -327,8 +327,8 @@ namespace vcpkg static constexpr StringLiteral STANDARD_INSTALL_USAGE = R"###(file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}"))###"; - auto usage_path_from = port_dir / "usage"; - auto usage_path_to = package_dir / "share" / package_name / "usage"; + auto usage_path_from = port_dir / FileUsage; + auto usage_path_to = package_dir / FileShare / package_name / FileUsage; if (fs.is_regular_file(usage_path_from) && !fs.is_regular_file(usage_path_to)) { @@ -467,8 +467,8 @@ namespace vcpkg const Path& portfile_cmake, MessageSink& msg_sink) { - static constexpr StringLiteral copyright_filenames[] = {"COPYING", "LICENSE", "LICENSE.txt"}; - const auto copyright_file = package_dir / "share" / spec_name / "copyright"; + static constexpr StringLiteral copyright_filenames[] = {"COPYING", FileLicense, "LICENSE.txt"}; + const auto copyright_file = package_dir / FileShare / spec_name / FileCopyright; switch (fs.status(copyright_file, IgnoreErrors{})) { diff --git a/src/vcpkg/vcpkglib.cpp b/src/vcpkg/vcpkglib.cpp index 339a518ed5..38923d69bd 100644 --- a/src/vcpkg/vcpkglib.cpp +++ b/src/vcpkg/vcpkglib.cpp @@ -34,7 +34,7 @@ namespace vcpkg { for (auto&& file : update_files) { - if (file.filename() == "incomplete") continue; + if (file.filename() == FileIncomplete) continue; auto pghs = Paragraphs::get_paragraphs(fs, file).value_or_exit(VCPKG_LINE_INFO); for (auto&& p : pghs) @@ -46,7 +46,7 @@ namespace vcpkg return update_files; } - + static void apply_database_updates_on_disk(const Filesystem& fs, const InstalledPaths& installed, StatusParagraphs& current_status_db) @@ -55,7 +55,7 @@ namespace vcpkg if (!update_files.empty()) { const auto status_file = installed.vcpkg_dir_status_file(); - const auto status_file_new = Path(status_file.parent_path()) / "status-new"; + const auto status_file_new = Path(status_file.parent_path()) / FileStatusNew; fs.write_contents(status_file_new, Strings::serialize(current_status_db), VCPKG_LINE_INFO); fs.rename(status_file_new, status_file, VCPKG_LINE_INFO); @@ -71,7 +71,7 @@ namespace vcpkg { const auto maybe_status_file = installed.vcpkg_dir_status_file(); const auto status_parent = Path(maybe_status_file.parent_path()); - const auto status_file_old = status_parent / "status-old"; + const auto status_file_old = status_parent / FileStatusOld; auto status_file = &maybe_status_file; @@ -104,7 +104,7 @@ namespace vcpkg const auto status_file = installed.vcpkg_dir_status_file(); const auto status_parent = Path(status_file.parent_path()); - const auto status_file_old = status_parent / "status-old"; + const auto status_file_old = status_parent / FileStatusOld; if (!fs.exists(status_file, IgnoreErrors{})) { @@ -131,7 +131,7 @@ namespace vcpkg const auto my_update_id = update_id++; const auto update_path = installed.vcpkg_dir_updates() / fmt::format("{:010}", my_update_id); - fs.write_rename_contents(update_path, "incomplete", Strings::serialize(p), VCPKG_LINE_INFO); + fs.write_rename_contents(update_path, FileIncomplete, Strings::serialize(p), VCPKG_LINE_INFO); } static bool upgrade_to_slash_terminated_sorted_format(std::vector& lines) From ddfcadde5996059dd2bedddb664081dfa1312f1b Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 30 Sep 2024 18:04:39 -0700 Subject: [PATCH 06/14] Add "maybe_" operations to JSON to avoid assert anti-patterns. Drive by fix debug printing claiming that telemetry was not a string and that malformed git commit shas were not strings. --- include/vcpkg/base/json.h | 8 ++++ src/vcpkg/base/json.cpp | 64 ++++++++++++++++++++++++++++- src/vcpkg/binarycaching.cpp | 8 ++-- src/vcpkg/bundlesettings.cpp | 13 +++--- src/vcpkg/configuration.cpp | 30 +++++++------- src/vcpkg/configure-environment.cpp | 26 ++++++------ src/vcpkg/sourceparagraph.cpp | 4 +- src/vcpkg/vcpkgpaths.cpp | 38 +++++++++-------- 8 files changed, 133 insertions(+), 58 deletions(-) diff --git a/include/vcpkg/base/json.h b/include/vcpkg/base/json.h index 24cbceb80e..9b33d870ff 100644 --- a/include/vcpkg/base/json.h +++ b/include/vcpkg/base/json.h @@ -108,15 +108,23 @@ namespace vcpkg::Json int64_t integer(LineInfo li) const noexcept; double number(LineInfo li) const noexcept; StringView string(LineInfo li) const noexcept; + std::string* maybe_string() noexcept; + const std::string* maybe_string() const noexcept; const Array& array(LineInfo li) const& noexcept; Array& array(LineInfo li) & noexcept; Array&& array(LineInfo li) && noexcept; + Array* maybe_array() noexcept; + const Array* maybe_array() const noexcept; + const Object& object(LineInfo li) const& noexcept; Object& object(LineInfo li) & noexcept; Object&& object(LineInfo li) && noexcept; + Object* maybe_object() noexcept; + const Object* maybe_object() const noexcept; + static Value null(std::nullptr_t) noexcept; static Value boolean(bool) noexcept; static Value integer(int64_t i) noexcept; diff --git a/src/vcpkg/base/json.cpp b/src/vcpkg/base/json.cpp index e65c51e65a..2c7116d117 100644 --- a/src/vcpkg/base/json.cpp +++ b/src/vcpkg/base/json.cpp @@ -157,6 +157,26 @@ namespace vcpkg::Json return underlying_->string; } + std::string* Value::maybe_string() noexcept + { + if (underlying_ && underlying_->tag == VK::String) + { + return &underlying_->string; + } + + return nullptr; + } + + const std::string* Value::maybe_string() const noexcept + { + if (underlying_ && underlying_->tag == VK::String) + { + return &underlying_->string; + } + + return nullptr; + } + const Array& Value::array(LineInfo li) const& noexcept { vcpkg::Checks::msg_check_exit(li, is_array(), msgJsonValueNotArray); @@ -169,6 +189,26 @@ namespace vcpkg::Json } Array&& Value::array(LineInfo li) && noexcept { return std::move(this->array(li)); } + Array* Value::maybe_array() noexcept + { + if (underlying_ && underlying_->tag == VK::Array) + { + return &underlying_->array; + } + + return nullptr; + } + + const Array* Value::maybe_array() const noexcept + { + if (underlying_ && underlying_->tag == VK::Array) + { + return &underlying_->array; + } + + return nullptr; + } + const Object& Value::object(LineInfo li) const& noexcept { vcpkg::Checks::msg_check_exit(li, is_object(), msgJsonValueNotObject); @@ -181,6 +221,26 @@ namespace vcpkg::Json } Object&& Value::object(LineInfo li) && noexcept { return std::move(this->object(li)); } + Object* Value::maybe_object() noexcept + { + if (underlying_ && underlying_->tag == VK::Object) + { + return &underlying_->object; + } + + return nullptr; + } + + const Object* Value::maybe_object() const noexcept + { + if (underlying_ && underlying_->tag == VK::Object) + { + return &underlying_->object; + } + + return nullptr; + } + Value::Value() noexcept = default; Value::Value(Value&&) noexcept = default; Value& Value::operator=(Value&&) noexcept = default; @@ -1120,9 +1180,9 @@ namespace vcpkg::Json { return parse(text, origin).then([&](ParsedJson&& mabeValueIsh) -> ExpectedL { auto& asValue = mabeValueIsh.value; - if (asValue.is_object()) + if (auto as_object = asValue.maybe_object()) { - return std::move(asValue).object(VCPKG_LINE_INFO); + return std::move(*as_object); } return msg::format(msgJsonErrorMustBeAnObject, msg::path = origin); diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 1b093139cb..d28ebc20bb 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -812,10 +812,12 @@ namespace auto maybe_json = Json::parse_object(*p, m_url); if (auto json = maybe_json.get()) { - auto archive_location = json->get(JsonIdArchiveCapitalLocation); - if (archive_location && archive_location->is_string()) + if (auto archive_location = json->get(JsonIdArchiveCapitalLocation)) { - return archive_location->string(VCPKG_LINE_INFO).to_string(); + if (auto archive_location_string = archive_location->maybe_string()) + { + return *archive_location_string; + } } } } diff --git a/src/vcpkg/bundlesettings.cpp b/src/vcpkg/bundlesettings.cpp index 3d5fca9a1f..dfaebfb771 100644 --- a/src/vcpkg/bundlesettings.cpp +++ b/src/vcpkg/bundlesettings.cpp @@ -32,18 +32,17 @@ namespace bool parse_optional_json_string(const Json::Object& doc, StringLiteral field_name, Optional& output) { - auto value = doc.get(field_name); - if (!value) + if (auto value = doc.get(field_name)) { - return true; - } + if (auto value_string = value->maybe_string()) + { + output = *value_string; + return true; + } - if (!value->is_string()) - { return false; } - output = value->string(VCPKG_LINE_INFO).to_string(); return true; } } diff --git a/src/vcpkg/configuration.cpp b/src/vcpkg/configuration.cpp index 3c7f7155fa..9aa32cd370 100644 --- a/src/vcpkg/configuration.cpp +++ b/src/vcpkg/configuration.cpp @@ -441,20 +441,20 @@ namespace continue; } - if (!el.second.is_object()) + auto maybe_demand_obj = el.second.maybe_object(); + if (!maybe_demand_obj) { r.add_generic_error(type_name(), msg::format(msgJsonFieldNotObject, msg::json_field = key)); continue; } - const auto& demand_obj = el.second.object(VCPKG_LINE_INFO); - if (demand_obj.contains(JsonIdDemands)) + if (maybe_demand_obj->contains(JsonIdDemands)) { r.add_generic_error(type_name(), msg::format(msgConfigurationNestedDemands, msg::json_field = el.first)); } - auto maybe_demand = r.visit(demand_obj, CeMetadataDeserializer::instance); + auto maybe_demand = r.visit(*maybe_demand_obj, CeMetadataDeserializer::instance); if (maybe_demand.has_value()) { ret.insert_or_replace(key, maybe_demand.value_or_exit(VCPKG_LINE_INFO)); @@ -602,13 +602,14 @@ namespace auto serialize_demands = [](const Json::Object& obj, Json::Object& put_into) { if (auto demands = obj.get(JsonIdDemands)) { - if (!demands->is_object()) + auto demands_obj = demands->maybe_object(); + if (!demands_obj) { return; } Json::Object serialized_demands; - for (const auto& el : demands->object(VCPKG_LINE_INFO)) + for (const auto& el : *demands_obj) { auto key = el.first; if (Strings::starts_with(key, "$")) @@ -617,10 +618,10 @@ namespace continue; } - if (el.second.is_object()) + if (auto demand_obj = el.second.maybe_object()) { auto& inserted = serialized_demands.insert_or_replace(key, Json::Object{}); - serialize_ce_metadata(el.second.object(VCPKG_LINE_INFO), inserted); + serialize_ce_metadata(*demand_obj, inserted); } } put_into.insert_or_replace(JsonIdDemands, serialized_demands); @@ -667,12 +668,13 @@ namespace if (el.first == JsonIdDemands) { - if (!el.second.is_object()) + auto maybe_demands_object = el.second.maybe_object(); + if (!maybe_demands_object) { continue; } - for (const auto& demand : el.second.object(VCPKG_LINE_INFO)) + for (const auto& demand : *maybe_demands_object) { if (Strings::starts_with(demand.first, "$")) { @@ -840,13 +842,13 @@ namespace vcpkg } auto conf_value = std::move(conf).value(VCPKG_LINE_INFO).value; - if (!conf_value.is_object()) + if (auto conf_value_object = conf_value.maybe_object()) { - messageSink.println(msgFailedToParseNoTopLevelObj, msg::path = origin); - return nullopt; + return parse_configuration(std::move(*conf_value_object), origin, messageSink); } - return parse_configuration(std::move(conf_value).object(VCPKG_LINE_INFO), origin, messageSink); + messageSink.println(msgFailedToParseNoTopLevelObj, msg::path = origin); + return nullopt; } Optional parse_configuration(const Json::Object& obj, StringView origin, MessageSink& messageSink) diff --git a/src/vcpkg/configure-environment.cpp b/src/vcpkg/configure-environment.cpp index 75f65ec7b3..f36a3d2e6f 100644 --- a/src/vcpkg/configure-environment.cpp +++ b/src/vcpkg/configure-environment.cpp @@ -42,30 +42,32 @@ namespace return; } - auto acquired_artifacts = pparsed->get(JsonIdAcquiredArtifacts); - if (acquired_artifacts) + if (auto acquired_artifacts = pparsed->get(JsonIdAcquiredArtifacts)) { - if (acquired_artifacts->is_string()) + if (auto maybe_acquired_string = acquired_artifacts->maybe_string()) { - get_global_metrics_collector().track_string(StringMetric::AcquiredArtifacts, - acquired_artifacts->string(VCPKG_LINE_INFO)); + get_global_metrics_collector().track_string(StringMetric::AcquiredArtifacts, *maybe_acquired_string); + } + else + { + Debug::println("Acquired artifacts was not a string."); } - Debug::println("Acquired artifacts was not a string."); } else { Debug::println("No artifacts acquired."); } - auto activated_artifacts = pparsed->get(JsonIdActivatedArtifacts); - if (activated_artifacts) + if (auto activated_artifacts = pparsed->get(JsonIdActivatedArtifacts)) { - if (activated_artifacts->is_string()) + if (auto maybe_activated_string = activated_artifacts->maybe_string()) + { + get_global_metrics_collector().track_string(StringMetric::ActivatedArtifacts, *maybe_activated_string); + } + else { - get_global_metrics_collector().track_string(StringMetric::ActivatedArtifacts, - activated_artifacts->string(VCPKG_LINE_INFO)); + Debug::println("Activated artifacts was not a string."); } - Debug::println("Activated artifacts was not a string."); } else { diff --git a/src/vcpkg/sourceparagraph.cpp b/src/vcpkg/sourceparagraph.cpp index 52ecabaf94..1a581431ab 100644 --- a/src/vcpkg/sourceparagraph.cpp +++ b/src/vcpkg/sourceparagraph.cpp @@ -1194,9 +1194,9 @@ namespace vcpkg if (auto configuration = obj.get(JsonIdVcpkgConfiguration)) { - if (configuration->is_object()) + if (auto configuration_object = configuration->maybe_object()) { - spgh.vcpkg_configuration.emplace(configuration->object(VCPKG_LINE_INFO)); + spgh.vcpkg_configuration.emplace(*configuration_object); } else { diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index 8635927c72..9ca9881240 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -435,30 +435,32 @@ namespace { auto repo = repo_to_ref_info_value.first; const auto& ref_info_value = repo_to_ref_info_value.second; - - if (!ref_info_value.is_object()) + if (auto ref_info_value_object = ref_info_value.maybe_object()) { - Debug::print("Lockfile value for key '", repo, "' was not an object\n"); - return ret; - } + for (auto&& reference_to_commit : *ref_info_value_object) + { + auto reference = reference_to_commit.first; + const auto& commit = reference_to_commit.second; + if (auto commit_string = commit.maybe_string()) + { + if (!is_git_commit_sha(*commit_string)) + { + Debug::print("Lockfile value for key '", reference, "' was not a commit sha\n"); + return ret; + } - for (auto&& reference_to_commit : ref_info_value.object(VCPKG_LINE_INFO)) - { - auto reference = reference_to_commit.first; - const auto& commit = reference_to_commit.second; + ret.emplace(repo.to_string(), LockFile::EntryData{reference.to_string(), *commit_string, true}); + continue; + } - if (!commit.is_string()) - { - Debug::print("Lockfile value for key '", reference, "' was not a string\n"); - return ret; - } - auto sv = commit.string(VCPKG_LINE_INFO); - if (!is_git_commit_sha(sv)) - { Debug::print("Lockfile value for key '", reference, "' was not a string\n"); return ret; } - ret.emplace(repo.to_string(), LockFile::EntryData{reference.to_string(), sv.to_string(), true}); + } + else + { + Debug::print("Lockfile value for key '", repo, "' was not an object\n"); + return ret; } } return ret; From 12371d99c510e153025df2b8a74b4a37be2a3656 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 30 Sep 2024 18:06:15 -0700 Subject: [PATCH 07/14] Use Json::Value::maybe_* and fix some string/stringview ping pong in SPDX processing. --- include/vcpkg/commands.build.h | 2 +- include/vcpkg/spdx.h | 4 +-- src/vcpkg-test/spdx.cpp | 4 +-- src/vcpkg/commands.build.cpp | 2 +- src/vcpkg/spdx.cpp | 58 +++++++++++++++++++--------------- 5 files changed, 38 insertions(+), 32 deletions(-) diff --git a/include/vcpkg/commands.build.h b/include/vcpkg/commands.build.h index 747c130141..7a9a47e355 100644 --- a/include/vcpkg/commands.build.h +++ b/include/vcpkg/commands.build.h @@ -246,7 +246,7 @@ namespace vcpkg Optional abi_tag_file; std::vector relative_port_files; std::vector relative_port_hashes; - std::vector heuristic_resources; + std::vector heuristic_resources; }; void compute_all_abis(const VcpkgPaths& paths, diff --git a/include/vcpkg/spdx.h b/include/vcpkg/spdx.h index 902b731872..09d9bf5749 100644 --- a/include/vcpkg/spdx.h +++ b/include/vcpkg/spdx.h @@ -27,7 +27,7 @@ namespace vcpkg View hashes, std::string created_time, std::string document_namespace, - std::vector&& resource_docs); + std::vector&& resource_docs); - Json::Value run_resource_heuristics(StringView contents, StringView portRawVersion); + Json::Object run_resource_heuristics(StringView contents, StringView portRawVersion); } diff --git a/src/vcpkg-test/spdx.cpp b/src/vcpkg-test/spdx.cpp index 743e0207f2..efe68a3c71 100644 --- a/src/vcpkg-test/spdx.cpp +++ b/src/vcpkg-test/spdx.cpp @@ -316,7 +316,7 @@ TEST_CASE ("spdx concat resources", "[spdx]") })json", "test") .value(VCPKG_LINE_INFO) - .value; + .value.object(VCPKG_LINE_INFO); auto doc2 = Json::parse(R"json( { "packages": [ "p1", "p2", "p3" ], @@ -324,7 +324,7 @@ TEST_CASE ("spdx concat resources", "[spdx]") })json", "test") .value(VCPKG_LINE_INFO) - .value; + .value.object(VCPKG_LINE_INFO); const auto sbom = create_spdx_sbom(ipa, {}, {}, "now+1", "ns", {std::move(doc1), std::move(doc2)}); diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 6d525a5e78..9136d3e8d9 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -922,7 +922,7 @@ namespace vcpkg static void write_sbom(const VcpkgPaths& paths, const InstallPlanAction& action, - std::vector heuristic_resources) + std::vector heuristic_resources) { auto& fs = paths.get_filesystem(); const auto& scfl = action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO); diff --git a/src/vcpkg/spdx.cpp b/src/vcpkg/spdx.cpp index bcaba964b6..accf6cd834 100644 --- a/src/vcpkg/spdx.cpp +++ b/src/vcpkg/spdx.cpp @@ -14,19 +14,26 @@ static std::string fix_ref_version(StringView ref, StringView version) return Strings::replace_all(ref, "${VERSION}", version); } -static std::string conclude_license(const std::string& license) +static StringView conclude_license(const Optional& maybe_license) { - if (license.empty()) return SpdxNoAssertion.to_string(); - return license; + if (auto license = maybe_license.get()) + { + if (!license->empty()) + { + return *license; + } + } + + return SpdxNoAssertion; } static void append_move_if_exists_and_array(Json::Array& out, Json::Object& obj, StringView property) { if (auto p = obj.get(property)) { - if (p->is_array()) + if (auto arr = p->maybe_array()) { - for (auto& e : p->array(VCPKG_LINE_INFO)) + for (auto& e : *arr) { out.push_back(std::move(e)); } @@ -59,11 +66,11 @@ static StringView extract_cmake_invocation_argument(StringView command, StringVi } static Json::Object make_resource( - std::string spdxid, std::string name, std::string downloadLocation, StringView sha512, StringView filename) + std::string spdxid, StringView name, std::string downloadLocation, StringView sha512, StringView filename) { Json::Object obj; obj.insert(SpdxSpdxId, std::move(spdxid)); - obj.insert(JsonIdName, std::move(name)); + obj.insert(JsonIdName, name.to_string()); if (!filename.empty()) { obj.insert(SpdxPackageFileName, filename); @@ -82,7 +89,7 @@ static Json::Object make_resource( return obj; } -Json::Value vcpkg::run_resource_heuristics(StringView contents, StringView version_text) +Json::Object vcpkg::run_resource_heuristics(StringView contents, StringView version_text) { // These are a sequence of heuristics to enable proof-of-concept extraction of remote resources for SPDX SBOM // inclusion @@ -97,7 +104,7 @@ Json::Value vcpkg::run_resource_heuristics(StringView contents, StringView versi auto sha = extract_cmake_invocation_argument(github, CMakeVariableSHA512); packages.push_back(make_resource(fmt::format("SPDXRef-resource-{}", ++n), - repo.to_string(), + repo, fmt::format("git+https://github.com/{}@{}", repo, ref), sha, {})); @@ -107,8 +114,8 @@ Json::Value vcpkg::run_resource_heuristics(StringView contents, StringView versi { auto url = extract_cmake_invocation_argument(github, CMakeVariableUrl); auto ref = fix_ref_version(extract_cmake_invocation_argument(github, CMakeVariableRef), version_text); - packages.push_back(make_resource( - fmt::format("SPDXRef-resource-{}", ++n), url.to_string(), fmt::format("git+{}@{}", url, ref), {}, {})); + packages.push_back( + make_resource(fmt::format("SPDXRef-resource-{}", ++n), url, fmt::format("git+{}@{}", url, ref), {}, {})); } auto distfile = find_cmake_invocation(contents, "vcpkg_download_distfile"); if (!distfile.empty()) @@ -116,8 +123,8 @@ Json::Value vcpkg::run_resource_heuristics(StringView contents, StringView versi auto url = extract_cmake_invocation_argument(distfile, CMakeVariableUrls); auto filename = extract_cmake_invocation_argument(distfile, CMakeVariableFilename); auto sha = extract_cmake_invocation_argument(distfile, CMakeVariableSHA512); - packages.push_back(make_resource( - fmt::format("SPDXRef-resource-{}", ++n), filename.to_string(), url.to_string(), sha, filename)); + packages.push_back( + make_resource(fmt::format("SPDXRef-resource-{}", ++n), filename, url.to_string(), sha, filename)); } auto sfg = find_cmake_invocation(contents, "vcpkg_from_sourceforge"); if (!sfg.empty()) @@ -126,11 +133,12 @@ Json::Value vcpkg::run_resource_heuristics(StringView contents, StringView versi auto ref = fix_ref_version(extract_cmake_invocation_argument(sfg, CMakeVariableRef), version_text); auto filename = extract_cmake_invocation_argument(sfg, CMakeVariableFilename); auto sha = extract_cmake_invocation_argument(sfg, CMakeVariableSHA512); - auto url = Strings::concat("https://sourceforge.net/projects/", repo, "/files/", ref, '/', filename); - packages.push_back(make_resource( - fmt::format("SPDXRef-resource-{}", ++n), filename.to_string(), std::move(url), sha, filename)); + auto url = fmt::format("https://sourceforge.net/projects/{}/files/{}/{}", repo, ref, filename); + packages.push_back( + make_resource(fmt::format("SPDXRef-resource-{}", ++n), filename, std::move(url), sha, filename)); } - return Json::Value::object(std::move(ret)); + + return ret; } std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, @@ -138,7 +146,7 @@ std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, View hashes, std::string created_time, std::string document_namespace, - std::vector&& resource_docs) + std::vector&& resource_docs) { Checks::check_exit(VCPKG_LINE_INFO, relative_paths.size() == hashes.size()); @@ -156,7 +164,7 @@ std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, doc.insert(SpdxDataLicense, SpdxCCZero); doc.insert(SpdxSpdxId, SpdxRefDocument); doc.insert(SpdxDocumentNamespace, std::move(document_namespace)); - doc.insert(JsonIdName, Strings::concat(action.spec.to_string(), '@', cpgh.version.to_string(), ' ', abi)); + doc.insert(JsonIdName, fmt::format("{}@{} {}", action.spec, cpgh.version, abi)); { auto& cinfo = doc.insert(SpdxCreationInfo, Json::Object()); auto& creators = cinfo.insert(JsonIdCreators, Json::Array()); @@ -176,7 +184,7 @@ std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, { obj.insert(JsonIdHomepage, cpgh.homepage); } - obj.insert(SpdxLicenseConcluded, conclude_license(cpgh.license.value_or(""))); + obj.insert(SpdxLicenseConcluded, conclude_license(cpgh.license)); obj.insert(SpdxLicenseDeclared, SpdxNoAssertion); obj.insert(SpdxCopyrightText, SpdxNoAssertion); if (!cpgh.summary.empty()) obj.insert(JsonIdSummary, Strings::join("\n", cpgh.summary)); @@ -202,7 +210,7 @@ std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, obj.insert(SpdxSpdxId, SpdxRefBinary); obj.insert(SpdxVersionInfo, abi); obj.insert(SpdxDownloadLocation, SpdxNone); - obj.insert(SpdxLicenseConcluded, conclude_license(cpgh.license.value_or(""))); + obj.insert(SpdxLicenseConcluded, conclude_license(cpgh.license)); obj.insert(SpdxLicenseDeclared, SpdxNoAssertion); obj.insert(SpdxCopyrightText, SpdxNoAssertion); obj.insert(JsonIdComment, "This is a binary package built by vcpkg."); @@ -249,11 +257,9 @@ std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, for (auto&& rdoc : resource_docs) { - if (!rdoc.is_object()) continue; - auto robj = std::move(rdoc).object(VCPKG_LINE_INFO); - append_move_if_exists_and_array(rels, robj, JsonIdRelationships); - append_move_if_exists_and_array(files, robj, JsonIdFiles); - append_move_if_exists_and_array(packages, robj, JsonIdPackages); + append_move_if_exists_and_array(rels, rdoc, JsonIdRelationships); + append_move_if_exists_and_array(files, rdoc, JsonIdFiles); + append_move_if_exists_and_array(packages, rdoc, JsonIdPackages); } return Json::stringify(doc); From 721175c60072bf6ae474cf60999bd71a95748177 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 7 Oct 2024 15:06:11 -0700 Subject: [PATCH 08/14] Consistently use {}s in statusparagraphs.cpp --- src/vcpkg/statusparagraphs.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/vcpkg/statusparagraphs.cpp b/src/vcpkg/statusparagraphs.cpp index 414a86dde9..6b63d54535 100644 --- a/src/vcpkg/statusparagraphs.cpp +++ b/src/vcpkg/statusparagraphs.cpp @@ -19,11 +19,16 @@ namespace vcpkg if (p->package.spec.name() == name && p->package.spec.triplet() == triplet) { if (p->package.is_feature()) + { spghs.emplace_back(&p); + } else + { spghs.emplace(spghs.begin(), &p); + } } } + return spghs; } @@ -46,10 +51,13 @@ namespace vcpkg } } } + if (ipv.core != nullptr) + { return ipv; - else - return nullopt; + } + + return nullopt; } StatusParagraphs::iterator StatusParagraphs::find(const std::string& name, @@ -76,6 +84,7 @@ namespace vcpkg // The core feature maps to .feature == "" return find(name, triplet, ""); } + return std::find_if(begin(), end(), [&](const std::unique_ptr& pgh) { const PackageSpec& spec = pgh->package.spec; return spec.name() == name && spec.triplet() == triplet && pgh->package.feature == feature; @@ -89,10 +98,8 @@ namespace vcpkg { return it; } - else - { - return end(); - } + + return end(); } StatusParagraphs::const_iterator StatusParagraphs::find_installed(const FeatureSpec& spec) const @@ -102,10 +109,8 @@ namespace vcpkg { return it; } - else - { - return end(); - } + + return end(); } bool vcpkg::StatusParagraphs::is_installed(const PackageSpec& spec) const From db320df023e9f1650867e4f48aa6bba01b1b3716 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 7 Oct 2024 15:12:38 -0700 Subject: [PATCH 09/14] Implement re-parsing the license out of the installed SBOM. --- include/vcpkg/spdx.h | 3 + src/vcpkg-test/spdx.cpp | 636 +++++++++++++++++++++++++++++++++++++++- src/vcpkg/spdx.cpp | 344 +++++++++++++--------- 3 files changed, 830 insertions(+), 153 deletions(-) diff --git a/include/vcpkg/spdx.h b/include/vcpkg/spdx.h index 09d9bf5749..480bf4374a 100644 --- a/include/vcpkg/spdx.h +++ b/include/vcpkg/spdx.h @@ -6,6 +6,7 @@ #include +#include #include #include @@ -29,5 +30,7 @@ namespace vcpkg std::string document_namespace, std::vector&& resource_docs); + Optional read_spdx_license(StringView text, StringView origin); + Json::Object run_resource_heuristics(StringView contents, StringView portRawVersion); } diff --git a/src/vcpkg-test/spdx.cpp b/src/vcpkg-test/spdx.cpp index efe68a3c71..e691fa0170 100644 --- a/src/vcpkg-test/spdx.cpp +++ b/src/vcpkg-test/spdx.cpp @@ -33,7 +33,7 @@ TEST_CASE ("spdx maximum serialization", "[spdx]") "https://test-document-namespace", {}); - auto expected = Json::parse(R"json( + static constexpr StringLiteral expected_text = R"json( { "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", "spdxVersion": "SPDX-2.2", @@ -157,12 +157,13 @@ TEST_CASE ("spdx maximum serialization", "[spdx]") "copyrightText": "NOASSERTION" } ] -})json", - "test") - .value(VCPKG_LINE_INFO); +})json"; + auto expected = Json::parse(expected_text, "test").value(VCPKG_LINE_INFO); auto doc = Json::parse(sbom, "test").value(VCPKG_LINE_INFO); Test::check_json_eq(expected.value, doc.value); + + CHECK(read_spdx_license(expected_text, "test") == "MIT"); } TEST_CASE ("spdx minimum serialization", "[spdx]") @@ -187,7 +188,7 @@ TEST_CASE ("spdx minimum serialization", "[spdx]") "https://test-document-namespace-2", {}); - auto expected = Json::parse(R"json( + static constexpr StringLiteral expected_text = R"json( { "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", "spdxVersion": "SPDX-2.2", @@ -286,12 +287,12 @@ TEST_CASE ("spdx minimum serialization", "[spdx]") "copyrightText": "NOASSERTION" } ] -})json", - "test") - .value(VCPKG_LINE_INFO); +})json"; + auto expected = Json::parse(expected_text, "test").value(VCPKG_LINE_INFO); auto doc = Json::parse(sbom, "test").value(VCPKG_LINE_INFO); Test::check_json_eq(expected.value, doc.value); + CHECK(!read_spdx_license(expected_text, "test").has_value()); } TEST_CASE ("spdx concat resources", "[spdx]") @@ -396,3 +397,622 @@ TEST_CASE ("spdx concat resources", "[spdx]") auto doc = Json::parse(sbom, "test").value(VCPKG_LINE_INFO); Test::check_json_eq(expected.value, doc.value); } + +TEST_CASE ("spdx license parse edge cases", "[spdx]") +{ + CHECK(!read_spdx_license("this is not json", "test").has_value()); + + static constexpr StringLiteral missing_packages = R"json( +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://test-document-namespace-2", + "name": "zlib:arm-uwp@1.0 deadbeef", + "creationInfo": { + "creators": [ + "Tool: vcpkg-2999-12-31-unknownhash" + ], + "created": "now+1" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "files": [ + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-vcpkg.json" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-portfile.cmake" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +})json"; + + CHECK(!read_spdx_license(missing_packages, "test").has_value()); + + static constexpr StringLiteral empty_packages = R"json( +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://test-document-namespace-2", + "name": "zlib:arm-uwp@1.0 deadbeef", + "creationInfo": { + "creators": [ + "Tool: vcpkg-2999-12-31-unknownhash" + ], + "created": "now+1" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [], + "files": [ + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-vcpkg.json" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-portfile.cmake" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +})json"; + + CHECK(!read_spdx_license(empty_packages, "test").has_value()); + + static constexpr StringLiteral wrong_packages_type = R"json( +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://test-document-namespace-2", + "name": "zlib:arm-uwp@1.0 deadbeef", + "creationInfo": { + "creators": [ + "Tool: vcpkg-2999-12-31-unknownhash" + ], + "created": "now+1" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": {}, + "files": [ + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-vcpkg.json" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-portfile.cmake" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +})json"; + + CHECK(!read_spdx_license(wrong_packages_type, "test").has_value()); + + static constexpr StringLiteral wrong_packages_zero_type = R"json( +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://test-document-namespace-2", + "name": "zlib:arm-uwp@1.0 deadbeef", + "creationInfo": { + "creators": [ + "Tool: vcpkg-2999-12-31-unknownhash" + ], + "created": "now+1" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [42], + "files": [ + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-vcpkg.json" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-portfile.cmake" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +})json"; + + CHECK(!read_spdx_license(wrong_packages_type, "test").has_value()); + + static constexpr StringLiteral missing_license_block = R"json( +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://test-document-namespace-2", + "name": "zlib:arm-uwp@1.0 deadbeef", + "creationInfo": { + "creators": [ + "Tool: vcpkg-2999-12-31-unknownhash" + ], + "created": "now+1" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [ + { + "name": "zlib", + "SPDXID": "SPDXRef-port", + "versionInfo": "1.0", + "downloadLocation": "NOASSERTION", + "copyrightText": "NOASSERTION", + "comment": "This is the port (recipe) consumed by vcpkg." + } + ], + "files": [ + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-vcpkg.json" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-portfile.cmake" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +})json"; + + CHECK(!read_spdx_license(missing_license_block, "test").has_value()); + + static constexpr StringLiteral wrong_license_type = R"json( +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://test-document-namespace-2", + "name": "zlib:arm-uwp@1.0 deadbeef", + "creationInfo": { + "creators": [ + "Tool: vcpkg-2999-12-31-unknownhash" + ], + "created": "now+1" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [ + { + "name": "zlib", + "SPDXID": "SPDXRef-port", + "versionInfo": "1.0", + "downloadLocation": "NOASSERTION", + "licenseConcluded": 42, + "licenseDeclared": 42, + "copyrightText": "NOASSERTION", + "comment": "This is the port (recipe) consumed by vcpkg." + } + ], + "files": [ + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-vcpkg.json" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-portfile.cmake" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +})json"; + + CHECK(!read_spdx_license(wrong_license_type, "test").has_value()); + + static constexpr StringLiteral empty_license = R"json( +{ + "$schema": "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json", + "spdxVersion": "SPDX-2.2", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "documentNamespace": "https://test-document-namespace-2", + "name": "zlib:arm-uwp@1.0 deadbeef", + "creationInfo": { + "creators": [ + "Tool: vcpkg-2999-12-31-unknownhash" + ], + "created": "now+1" + }, + "relationships": [ + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "GENERATES", + "relatedSpdxElement": "SPDXRef-binary" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-0" + }, + { + "spdxElementId": "SPDXRef-port", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-file-1" + }, + { + "spdxElementId": "SPDXRef-binary", + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-0", + "relationshipType": "DEPENDENCY_MANIFEST_OF", + "relatedSpdxElement": "SPDXRef-port" + }, + { + "spdxElementId": "SPDXRef-file-1", + "relationshipType": "CONTAINED_BY", + "relatedSpdxElement": "SPDXRef-port" + } + ], + "packages": [ + { + "name": "zlib", + "SPDXID": "SPDXRef-port", + "versionInfo": "1.0", + "downloadLocation": "NOASSERTION", + "licenseConcluded": "", + "licenseDeclared": "", + "copyrightText": "NOASSERTION", + "comment": "This is the port (recipe) consumed by vcpkg." + }, + { + "name": "zlib:arm-uwp", + "SPDXID": "SPDXRef-binary", + "versionInfo": "deadbeef", + "downloadLocation": "NONE", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "comment": "This is a binary package built by vcpkg." + } + ], + "files": [ + { + "fileName": "./vcpkg.json", + "SPDXID": "SPDXRef-file-0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-vcpkg.json" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + }, + { + "fileName": "./portfile.cmake", + "SPDXID": "SPDXRef-file-1", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "hash-portfile.cmake" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "NOASSERTION" + } + ] +})json"; + + CHECK(!read_spdx_license(empty_license, "test").has_value()); +} diff --git a/src/vcpkg/spdx.cpp b/src/vcpkg/spdx.cpp index accf6cd834..8906599a6f 100644 --- a/src/vcpkg/spdx.cpp +++ b/src/vcpkg/spdx.cpp @@ -89,178 +89,232 @@ static Json::Object make_resource( return obj; } -Json::Object vcpkg::run_resource_heuristics(StringView contents, StringView version_text) +namespace vcpkg { - // These are a sequence of heuristics to enable proof-of-concept extraction of remote resources for SPDX SBOM - // inclusion - size_t n = 0; - Json::Object ret; - auto& packages = ret.insert(JsonIdPackages, Json::Array{}); - auto github = find_cmake_invocation(contents, "vcpkg_from_github"); - if (!github.empty()) + Json::Object run_resource_heuristics(StringView contents, StringView version_text) { - auto repo = extract_cmake_invocation_argument(github, CMakeVariableRepo); - auto ref = fix_ref_version(extract_cmake_invocation_argument(github, CMakeVariableRef), version_text); - auto sha = extract_cmake_invocation_argument(github, CMakeVariableSHA512); - - packages.push_back(make_resource(fmt::format("SPDXRef-resource-{}", ++n), - repo, - fmt::format("git+https://github.com/{}@{}", repo, ref), - sha, - {})); - } - auto git = find_cmake_invocation(contents, "vcpkg_from_git"); - if (!git.empty()) - { - auto url = extract_cmake_invocation_argument(github, CMakeVariableUrl); - auto ref = fix_ref_version(extract_cmake_invocation_argument(github, CMakeVariableRef), version_text); - packages.push_back( - make_resource(fmt::format("SPDXRef-resource-{}", ++n), url, fmt::format("git+{}@{}", url, ref), {}, {})); - } - auto distfile = find_cmake_invocation(contents, "vcpkg_download_distfile"); - if (!distfile.empty()) - { - auto url = extract_cmake_invocation_argument(distfile, CMakeVariableUrls); - auto filename = extract_cmake_invocation_argument(distfile, CMakeVariableFilename); - auto sha = extract_cmake_invocation_argument(distfile, CMakeVariableSHA512); - packages.push_back( - make_resource(fmt::format("SPDXRef-resource-{}", ++n), filename, url.to_string(), sha, filename)); - } - auto sfg = find_cmake_invocation(contents, "vcpkg_from_sourceforge"); - if (!sfg.empty()) - { - auto repo = extract_cmake_invocation_argument(sfg, CMakeVariableRepo); - auto ref = fix_ref_version(extract_cmake_invocation_argument(sfg, CMakeVariableRef), version_text); - auto filename = extract_cmake_invocation_argument(sfg, CMakeVariableFilename); - auto sha = extract_cmake_invocation_argument(sfg, CMakeVariableSHA512); - auto url = fmt::format("https://sourceforge.net/projects/{}/files/{}/{}", repo, ref, filename); - packages.push_back( - make_resource(fmt::format("SPDXRef-resource-{}", ++n), filename, std::move(url), sha, filename)); - } - - return ret; -} - -std::string vcpkg::create_spdx_sbom(const InstallPlanAction& action, - View relative_paths, - View hashes, - std::string created_time, - std::string document_namespace, - std::vector&& resource_docs) -{ - Checks::check_exit(VCPKG_LINE_INFO, relative_paths.size() == hashes.size()); - - const auto& scfl = action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO); - const auto& cpgh = *scfl.source_control_file->core_paragraph; - StringView abi{SpdxNone}; - if (auto package_abi = action.package_abi().get()) - { - abi = *package_abi; - } - - Json::Object doc; - doc.insert(JsonIdDollarSchema, "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json"); - doc.insert(SpdxVersion, SpdxTwoTwo); - doc.insert(SpdxDataLicense, SpdxCCZero); - doc.insert(SpdxSpdxId, SpdxRefDocument); - doc.insert(SpdxDocumentNamespace, std::move(document_namespace)); - doc.insert(JsonIdName, fmt::format("{}@{} {}", action.spec, cpgh.version, abi)); - { - auto& cinfo = doc.insert(SpdxCreationInfo, Json::Object()); - auto& creators = cinfo.insert(JsonIdCreators, Json::Array()); - creators.push_back(Strings::concat("Tool: vcpkg-", VCPKG_BASE_VERSION_AS_STRING, '-', VCPKG_VERSION_AS_STRING)); - cinfo.insert(JsonIdCreated, std::move(created_time)); - } + // These are a sequence of heuristics to enable proof-of-concept extraction of remote resources for SPDX SBOM + // inclusion + size_t n = 0; + Json::Object ret; + auto& packages = ret.insert(JsonIdPackages, Json::Array{}); + auto github = find_cmake_invocation(contents, "vcpkg_from_github"); + if (!github.empty()) + { + auto repo = extract_cmake_invocation_argument(github, CMakeVariableRepo); + auto ref = fix_ref_version(extract_cmake_invocation_argument(github, CMakeVariableRef), version_text); + auto sha = extract_cmake_invocation_argument(github, CMakeVariableSHA512); - auto& rels = doc.insert(JsonIdRelationships, Json::Array()); - auto& packages = doc.insert(JsonIdPackages, Json::Array()); - { - auto& obj = packages.push_back(Json::Object()); - obj.insert(JsonIdName, action.spec.name()); - obj.insert(SpdxSpdxId, SpdxRefPort); - obj.insert(SpdxVersionInfo, cpgh.version.to_string()); - obj.insert(SpdxDownloadLocation, scfl.spdx_location.empty() ? StringView{SpdxNoAssertion} : scfl.spdx_location); - if (!cpgh.homepage.empty()) + packages.push_back(make_resource(fmt::format("SPDXRef-resource-{}", ++n), + repo, + fmt::format("git+https://github.com/{}@{}", repo, ref), + sha, + {})); + } + auto git = find_cmake_invocation(contents, "vcpkg_from_git"); + if (!git.empty()) { - obj.insert(JsonIdHomepage, cpgh.homepage); + auto url = extract_cmake_invocation_argument(github, CMakeVariableUrl); + auto ref = fix_ref_version(extract_cmake_invocation_argument(github, CMakeVariableRef), version_text); + packages.push_back(make_resource( + fmt::format("SPDXRef-resource-{}", ++n), url, fmt::format("git+{}@{}", url, ref), {}, {})); } - obj.insert(SpdxLicenseConcluded, conclude_license(cpgh.license)); - obj.insert(SpdxLicenseDeclared, SpdxNoAssertion); - obj.insert(SpdxCopyrightText, SpdxNoAssertion); - if (!cpgh.summary.empty()) obj.insert(JsonIdSummary, Strings::join("\n", cpgh.summary)); - if (!cpgh.description.empty()) obj.insert(JsonIdDescription, Strings::join("\n", cpgh.description)); - obj.insert(JsonIdComment, "This is the port (recipe) consumed by vcpkg."); + auto distfile = find_cmake_invocation(contents, "vcpkg_download_distfile"); + if (!distfile.empty()) { - auto& rel = rels.push_back(Json::Object()); - rel.insert(SpdxElementId, SpdxRefPort); - rel.insert(SpdxRelationshipType, SpdxGenerates); - rel.insert(SpdxRelatedSpdxElement, SpdxRefBinary); + auto url = extract_cmake_invocation_argument(distfile, CMakeVariableUrls); + auto filename = extract_cmake_invocation_argument(distfile, CMakeVariableFilename); + auto sha = extract_cmake_invocation_argument(distfile, CMakeVariableSHA512); + packages.push_back( + make_resource(fmt::format("SPDXRef-resource-{}", ++n), filename, url.to_string(), sha, filename)); } - for (size_t i = 0; i < relative_paths.size(); ++i) + auto sfg = find_cmake_invocation(contents, "vcpkg_from_sourceforge"); + if (!sfg.empty()) { - auto& rel = rels.push_back(Json::Object()); - rel.insert(SpdxElementId, SpdxRefPort); - rel.insert(SpdxRelationshipType, SpdxContains); - rel.insert(SpdxRelatedSpdxElement, fmt::format("SPDXRef-file-{}", i)); + auto repo = extract_cmake_invocation_argument(sfg, CMakeVariableRepo); + auto ref = fix_ref_version(extract_cmake_invocation_argument(sfg, CMakeVariableRef), version_text); + auto filename = extract_cmake_invocation_argument(sfg, CMakeVariableFilename); + auto sha = extract_cmake_invocation_argument(sfg, CMakeVariableSHA512); + auto url = fmt::format("https://sourceforge.net/projects/{}/files/{}/{}", repo, ref, filename); + packages.push_back( + make_resource(fmt::format("SPDXRef-resource-{}", ++n), filename, std::move(url), sha, filename)); } + + return ret; } + + std::string create_spdx_sbom(const InstallPlanAction& action, + View relative_paths, + View hashes, + std::string created_time, + std::string document_namespace, + std::vector&& resource_docs) { - auto& obj = packages.push_back(Json::Object()); - obj.insert(JsonIdName, action.spec.to_string()); - obj.insert(SpdxSpdxId, SpdxRefBinary); - obj.insert(SpdxVersionInfo, abi); - obj.insert(SpdxDownloadLocation, SpdxNone); - obj.insert(SpdxLicenseConcluded, conclude_license(cpgh.license)); - obj.insert(SpdxLicenseDeclared, SpdxNoAssertion); - obj.insert(SpdxCopyrightText, SpdxNoAssertion); - obj.insert(JsonIdComment, "This is a binary package built by vcpkg."); + Checks::check_exit(VCPKG_LINE_INFO, relative_paths.size() == hashes.size()); + + const auto& scfl = action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO); + const auto& cpgh = *scfl.source_control_file->core_paragraph; + StringView abi{SpdxNone}; + if (auto package_abi = action.package_abi().get()) { - auto& rel = rels.push_back(Json::Object()); - rel.insert(SpdxElementId, SpdxRefBinary); - rel.insert(SpdxRelationshipType, SpdxGeneratedFrom); - rel.insert(SpdxRelatedSpdxElement, SpdxRefPort); + abi = *package_abi; } - } - auto& files = doc.insert(JsonIdFiles, Json::Array()); - { - for (size_t i = 0; i < relative_paths.size(); ++i) + Json::Object doc; + doc.insert(JsonIdDollarSchema, + "https://raw.githubusercontent.com/spdx/spdx-spec/v2.2.1/schemas/spdx-schema.json"); + doc.insert(SpdxVersion, SpdxTwoTwo); + doc.insert(SpdxDataLicense, SpdxCCZero); + doc.insert(SpdxSpdxId, SpdxRefDocument); + doc.insert(SpdxDocumentNamespace, std::move(document_namespace)); + doc.insert(JsonIdName, fmt::format("{}@{} {}", action.spec, cpgh.version, abi)); { - const auto& path = relative_paths[i]; - const auto& hash = hashes[i]; + auto& cinfo = doc.insert(SpdxCreationInfo, Json::Object()); + auto& creators = cinfo.insert(JsonIdCreators, Json::Array()); + creators.push_back( + Strings::concat("Tool: vcpkg-", VCPKG_BASE_VERSION_AS_STRING, '-', VCPKG_VERSION_AS_STRING)); + cinfo.insert(JsonIdCreated, std::move(created_time)); + } - auto& obj = files.push_back(Json::Object()); - obj.insert(SpdxFileName, "./" + path.generic_u8string()); - const auto ref = fmt::format("SPDXRef-file-{}", i); - obj.insert(SpdxSpdxId, ref); - auto& checksum = obj.insert(JsonIdChecksums, Json::Array()); - auto& checksum1 = checksum.push_back(Json::Object()); - checksum1.insert(JsonIdAlgorithm, JsonIdAllCapsSHA256); - checksum1.insert(SpdxChecksumValue, hash); - obj.insert(SpdxLicenseConcluded, SpdxNoAssertion); + auto& rels = doc.insert(JsonIdRelationships, Json::Array()); + auto& packages = doc.insert(JsonIdPackages, Json::Array()); + { + auto& obj = packages.push_back(Json::Object()); + obj.insert(JsonIdName, action.spec.name()); + obj.insert(SpdxSpdxId, SpdxRefPort); + obj.insert(SpdxVersionInfo, cpgh.version.to_string()); + obj.insert(SpdxDownloadLocation, + scfl.spdx_location.empty() ? StringView{SpdxNoAssertion} : scfl.spdx_location); + if (!cpgh.homepage.empty()) + { + obj.insert(JsonIdHomepage, cpgh.homepage); + } + obj.insert(SpdxLicenseConcluded, conclude_license(cpgh.license)); + obj.insert(SpdxLicenseDeclared, SpdxNoAssertion); obj.insert(SpdxCopyrightText, SpdxNoAssertion); + if (!cpgh.summary.empty()) obj.insert(JsonIdSummary, Strings::join("\n", cpgh.summary)); + if (!cpgh.description.empty()) obj.insert(JsonIdDescription, Strings::join("\n", cpgh.description)); + obj.insert(JsonIdComment, "This is the port (recipe) consumed by vcpkg."); { auto& rel = rels.push_back(Json::Object()); - rel.insert(SpdxElementId, ref); - rel.insert(SpdxRelationshipType, SpdxContainedBy); - rel.insert(SpdxRelatedSpdxElement, SpdxRefPort); + rel.insert(SpdxElementId, SpdxRefPort); + rel.insert(SpdxRelationshipType, SpdxGenerates); + rel.insert(SpdxRelatedSpdxElement, SpdxRefBinary); } - if (path == FileVcpkgDotJson) + for (size_t i = 0; i < relative_paths.size(); ++i) { auto& rel = rels.push_back(Json::Object()); - rel.insert(SpdxElementId, ref); - rel.insert(SpdxRelationshipType, SpdxDependencyManifestOf); + rel.insert(SpdxElementId, SpdxRefPort); + rel.insert(SpdxRelationshipType, SpdxContains); + rel.insert(SpdxRelatedSpdxElement, fmt::format("SPDXRef-file-{}", i)); + } + } + { + auto& obj = packages.push_back(Json::Object()); + obj.insert(JsonIdName, action.spec.to_string()); + obj.insert(SpdxSpdxId, SpdxRefBinary); + obj.insert(SpdxVersionInfo, abi); + obj.insert(SpdxDownloadLocation, SpdxNone); + obj.insert(SpdxLicenseConcluded, conclude_license(cpgh.license)); + obj.insert(SpdxLicenseDeclared, SpdxNoAssertion); + obj.insert(SpdxCopyrightText, SpdxNoAssertion); + obj.insert(JsonIdComment, "This is a binary package built by vcpkg."); + { + auto& rel = rels.push_back(Json::Object()); + rel.insert(SpdxElementId, SpdxRefBinary); + rel.insert(SpdxRelationshipType, SpdxGeneratedFrom); rel.insert(SpdxRelatedSpdxElement, SpdxRefPort); } } + + auto& files = doc.insert(JsonIdFiles, Json::Array()); + { + for (size_t i = 0; i < relative_paths.size(); ++i) + { + const auto& path = relative_paths[i]; + const auto& hash = hashes[i]; + + auto& obj = files.push_back(Json::Object()); + obj.insert(SpdxFileName, "./" + path.generic_u8string()); + const auto ref = fmt::format("SPDXRef-file-{}", i); + obj.insert(SpdxSpdxId, ref); + auto& checksum = obj.insert(JsonIdChecksums, Json::Array()); + auto& checksum1 = checksum.push_back(Json::Object()); + checksum1.insert(JsonIdAlgorithm, JsonIdAllCapsSHA256); + checksum1.insert(SpdxChecksumValue, hash); + obj.insert(SpdxLicenseConcluded, SpdxNoAssertion); + obj.insert(SpdxCopyrightText, SpdxNoAssertion); + { + auto& rel = rels.push_back(Json::Object()); + rel.insert(SpdxElementId, ref); + rel.insert(SpdxRelationshipType, SpdxContainedBy); + rel.insert(SpdxRelatedSpdxElement, SpdxRefPort); + } + if (path == FileVcpkgDotJson) + { + auto& rel = rels.push_back(Json::Object()); + rel.insert(SpdxElementId, ref); + rel.insert(SpdxRelationshipType, SpdxDependencyManifestOf); + rel.insert(SpdxRelatedSpdxElement, SpdxRefPort); + } + } + } + + for (auto&& rdoc : resource_docs) + { + append_move_if_exists_and_array(rels, rdoc, JsonIdRelationships); + append_move_if_exists_and_array(files, rdoc, JsonIdFiles); + append_move_if_exists_and_array(packages, rdoc, JsonIdPackages); + } + + return Json::stringify(doc); } - for (auto&& rdoc : resource_docs) + Optional read_spdx_license(StringView text, StringView origin) { - append_move_if_exists_and_array(rels, rdoc, JsonIdRelationships); - append_move_if_exists_and_array(files, rdoc, JsonIdFiles); - append_move_if_exists_and_array(packages, rdoc, JsonIdPackages); - } + // JsonIdPackages[0]/SpdxLicenseConcluded + auto maybe_parsed = Json::parse_object(text, origin); + auto parsed = maybe_parsed.get(); + if (!parsed) + { + return nullopt; + } + + auto maybe_packages_value = parsed->get(JsonIdPackages); + if (!maybe_packages_value) + { + return nullopt; + } - return Json::stringify(doc); + auto maybe_packages_array = maybe_packages_value->maybe_array(); + if (!maybe_packages_array || maybe_packages_array->size() == 0) + { + return nullopt; + } + + auto maybe_first_package_object = maybe_packages_array->operator[](0).maybe_object(); + if (!maybe_first_package_object) + { + return nullopt; + } + + auto maybe_license_concluded_value = maybe_first_package_object->get(SpdxLicenseConcluded); + if (!maybe_license_concluded_value) + { + return nullopt; + } + + auto maybe_license_concluded = maybe_license_concluded_value->maybe_string(); + if (!maybe_license_concluded) + { + return nullopt; + } + + if (maybe_license_concluded->empty() || *maybe_license_concluded == SpdxNoAssertion) + { + return nullopt; + } + + return std::move(*maybe_license_concluded); + } } From e3bd5572d5ac7a0970c89941a075f209973d4625 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 7 Oct 2024 15:29:18 -0700 Subject: [PATCH 10/14] Remove formatting concern PrintUsage from BuildPackageOptions because building a package never prints usage. --- include/vcpkg/commands.build.h | 1 - include/vcpkg/commands.set-installed.h | 1 + src/vcpkg/commands.build-external.cpp | 1 - src/vcpkg/commands.build.cpp | 1 - src/vcpkg/commands.ci.cpp | 1 - src/vcpkg/commands.install.cpp | 7 +++---- src/vcpkg/commands.set-installed.cpp | 28 +++++++++++++------------- src/vcpkg/commands.upgrade.cpp | 1 - 8 files changed, 18 insertions(+), 23 deletions(-) diff --git a/include/vcpkg/commands.build.h b/include/vcpkg/commands.build.h index 7a9a47e355..7dd6481a0c 100644 --- a/include/vcpkg/commands.build.h +++ b/include/vcpkg/commands.build.h @@ -72,7 +72,6 @@ namespace vcpkg CleanDownloads clean_downloads; DownloadTool download_tool; BackcompatFeatures backcompat_features; - PrintUsage print_usage; KeepGoing keep_going; }; diff --git a/include/vcpkg/commands.set-installed.h b/include/vcpkg/commands.set-installed.h index cec186a9e8..1db780b9c8 100644 --- a/include/vcpkg/commands.set-installed.h +++ b/include/vcpkg/commands.set-installed.h @@ -40,6 +40,7 @@ namespace vcpkg const CMakeVars::CMakeVarProvider& cmake_vars, ActionPlan action_plan, DryRun dry_run, + PrintUsage print_usage, const Optional& maybe_pkgconfig, bool include_manifest_in_github_issue); void command_set_installed_and_exit(const VcpkgCmdArguments& args, diff --git a/src/vcpkg/commands.build-external.cpp b/src/vcpkg/commands.build-external.cpp index 9ef71a66a6..2cc4a64316 100644 --- a/src/vcpkg/commands.build-external.cpp +++ b/src/vcpkg/commands.build-external.cpp @@ -36,7 +36,6 @@ namespace vcpkg CleanDownloads::No, DownloadTool::Builtin, BackcompatFeatures::Allow, - PrintUsage::Yes, }; const FullPackageSpec spec = diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 9136d3e8d9..eaf6ad8d6f 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -98,7 +98,6 @@ namespace vcpkg CleanDownloads::No, DownloadTool::Builtin, BackcompatFeatures::Allow, - PrintUsage::Yes, }; const FullPackageSpec spec = diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index e000d5dde7..03e71dfbd5 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -327,7 +327,6 @@ namespace vcpkg CleanDownloads::No, DownloadTool::Builtin, BackcompatFeatures::Prohibit, - PrintUsage::Yes, KeepGoing::Yes, }; diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index dd714e1b3b..5981857fd3 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -1054,8 +1054,7 @@ namespace vcpkg const auto unsupported_port_action = Util::Sets::contains(options.switches, SwitchAllowUnsupported) ? UnsupportedPortAction::Warn : UnsupportedPortAction::Error; - const PrintUsage print_cmake_usage = - Util::Sets::contains(options.switches, SwitchNoPrintUsage) ? PrintUsage::No : PrintUsage::Yes; + const bool print_cmake_usage = Util::Sets::contains(options.switches, SwitchNoPrintUsage); get_global_metrics_collector().track_bool(BoolMetric::InstallManifestMode, paths.manifest_mode_enabled()); @@ -1124,7 +1123,6 @@ namespace vcpkg Util::Enum::to_enum(clean_after_build || clean_downloads_after_build), download_tool, prohibit_backcompat_features ? BackcompatFeatures::Prohibit : BackcompatFeatures::Allow, - print_cmake_usage, keep_going, }; @@ -1272,6 +1270,7 @@ namespace vcpkg var_provider, std::move(install_plan), dry_run ? DryRun::Yes : DryRun::No, + print_cmake_usage ? PrintUsage::No : PrintUsage::Yes, pkgsconfig, true); } @@ -1397,7 +1396,7 @@ namespace vcpkg fs.write_contents(it_xunit->second, xwriter.build_xml(default_triplet), VCPKG_LINE_INFO); } - if (build_package_options.print_usage == PrintUsage::Yes) + if (print_cmake_usage) { std::set printed_usages; for (auto&& result : summary.results) diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 0d38a0f171..0cbaae8c50 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -172,6 +172,7 @@ namespace vcpkg const CMakeVars::CMakeVarProvider& cmake_vars, ActionPlan action_plan, DryRun dry_run, + PrintUsage print_usage, const Optional& maybe_pkgconfig, bool include_manifest_in_github_issue) { @@ -257,7 +258,7 @@ namespace vcpkg } } - if (build_options.print_usage == PrintUsage::Yes) + if (print_usage == PrintUsage::Yes) { // Note that this differs from the behavior of `vcpkg install` in that it will print usage information for // packages named but not installed here @@ -287,15 +288,12 @@ namespace vcpkg .value_or_exit(VCPKG_LINE_INFO); }); - const bool dry_run = Util::Sets::contains(options.switches, SwitchDryRun); const auto only_downloads = Util::Sets::contains(options.switches, SwitchOnlyDownloads) ? OnlyDownloads::Yes : OnlyDownloads::No; const auto keep_going = Util::Sets::contains(options.switches, SwitchKeepGoing) || only_downloads == OnlyDownloads::Yes ? KeepGoing::Yes : KeepGoing::No; - const auto print_usage = - Util::Sets::contains(options.switches, SwitchNoPrintUsage) ? PrintUsage::No : PrintUsage::Yes; const auto unsupported_port_action = Util::Sets::contains(options.switches, SwitchAllowUnsupported) ? UnsupportedPortAction::Warn : UnsupportedPortAction::Error; @@ -312,7 +310,6 @@ namespace vcpkg CleanDownloads::No, DownloadTool::Builtin, prohibit_backcompat_features, - print_usage, keep_going, }; @@ -339,14 +336,17 @@ namespace vcpkg specs, {}, {nullptr, host_triplet, paths.packages(), unsupported_port_action, UseHeadVersion::No, Editable::No}); - command_set_installed_and_exit_ex(args, - paths, - host_triplet, - build_options, - *cmake_vars, - std::move(action_plan), - dry_run ? DryRun::Yes : DryRun::No, - pkgsconfig, - false); + + command_set_installed_and_exit_ex( + args, + paths, + host_triplet, + build_options, + *cmake_vars, + std::move(action_plan), + Util::Sets::contains(options.switches, SwitchDryRun) ? DryRun::Yes : DryRun::No, + Util::Sets::contains(options.switches, SwitchNoPrintUsage) ? PrintUsage::No : PrintUsage::Yes, + pkgsconfig, + false); } } // namespace vcpkg diff --git a/src/vcpkg/commands.upgrade.cpp b/src/vcpkg/commands.upgrade.cpp index a59ff797d5..f70163f058 100644 --- a/src/vcpkg/commands.upgrade.cpp +++ b/src/vcpkg/commands.upgrade.cpp @@ -69,7 +69,6 @@ namespace vcpkg CleanDownloads::No, DownloadTool::Builtin, BackcompatFeatures::Allow, - PrintUsage::Yes, keep_going, }; From 0875ad6f141802f775045dbe1e2e89cdec16745a Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 11 Oct 2024 15:19:34 -0700 Subject: [PATCH 11/14] Improve console output when installing packages by deduplicating 'already installed' message, --head warning, and merging the 'green success' message into the total elapsed time part. --- include/vcpkg/base/message-data.inc.h | 16 +++-- include/vcpkg/commands.install.h | 7 +- include/vcpkg/dependencies.h | 5 +- src/vcpkg-test/dependencies.cpp | 18 ++--- src/vcpkg/commands.ci.cpp | 2 +- src/vcpkg/commands.install.cpp | 96 +++++++++++++++------------ src/vcpkg/commands.set-installed.cpp | 17 ++++- src/vcpkg/commands.upgrade.cpp | 3 +- src/vcpkg/dependencies.cpp | 82 +++++++++++++++++------ 9 files changed, 157 insertions(+), 89 deletions(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 3545690ee1..cf90643636 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -137,12 +137,7 @@ DECLARE_MESSAGE(AllFormatArgsUnbalancedBraces, (msg::value), "example of {value} is 'foo bar {'", "unbalanced brace in format string \"{value}\"") -DECLARE_MESSAGE(AllPackagesAreUpdated, (), "", "All installed packages are up-to-date.") -DECLARE_MESSAGE(AlreadyInstalled, (msg::spec), "", "{spec} is already installed") -DECLARE_MESSAGE(AlreadyInstalledNotHead, - (msg::spec), - "'HEAD' means the most recent version of source code", - "{spec} is already installed -- not building from HEAD") +DECLARE_MESSAGE(AllPackagesAreUpdated, (), "", "No action taken because all installed packages are up-to-date.") DECLARE_MESSAGE(AManifest, (), "", "a manifest") DECLARE_MESSAGE(AMaximumOfOneAssetReadUrlCanBeSpecified, (), "", "a maximum of one asset read url can be specified.") DECLARE_MESSAGE(AMaximumOfOneAssetWriteUrlCanBeSpecified, (), "", "a maximum of one asset write url can be specified.") @@ -1728,6 +1723,11 @@ DECLARE_MESSAGE(InstallCopiedFile, "{path_source} -> {path_destination} done") DECLARE_MESSAGE(InstalledBy, (msg::path), "", "Installed by {path}") DECLARE_MESSAGE(InstalledPackages, (), "", "The following packages are already installed:") +DECLARE_MESSAGE(InstalledPackagesHead, + (), + "", + "The following packages are already installed, but were requested at --head version. Their installed " + "contents will not be changed. To get updated versions, remove these packages first:") DECLARE_MESSAGE(InstalledRequestedPackages, (), "", "All requested packages are currently installed.") DECLARE_MESSAGE(InstallFailed, (msg::path, msg::error_msg), "", "failed: {path}: {error_msg}") DECLARE_MESSAGE(InstallingMavenFileFailure, @@ -2700,6 +2700,10 @@ DECLARE_MESSAGE(ToRemovePackages, "", "To only remove outdated packages, run\n{command_name} remove --outdated") DECLARE_MESSAGE(TotalInstallTime, (msg::elapsed), "", "Total install time: {elapsed}") +DECLARE_MESSAGE(TotalInstallTimeSuccess, + (msg::elapsed), + "", + "All requested installations completed successfully in: {elapsed}") DECLARE_MESSAGE(ToUpdatePackages, (msg::command_name), "", diff --git a/include/vcpkg/commands.install.h b/include/vcpkg/commands.install.h index 22d7f8f394..1164068407 100644 --- a/include/vcpkg/commands.install.h +++ b/include/vcpkg/commands.install.h @@ -40,11 +40,12 @@ namespace vcpkg struct InstallSummary { std::vector results; + ElapsedTime timing; + bool failed = false; - LocalizedString format() const; + LocalizedString format_results() const; void print_failed() const; - std::string xunit_results() const; - bool failed() const; + void print_complete_message() const; }; struct InstallDir diff --git a/include/vcpkg/dependencies.h b/include/vcpkg/dependencies.h index 994952581f..cba7ec3821 100644 --- a/include/vcpkg/dependencies.h +++ b/include/vcpkg/dependencies.h @@ -209,7 +209,10 @@ namespace vcpkg struct FormattedPlan { bool has_removals = false; - LocalizedString text; + LocalizedString warning_text; + LocalizedString normal_text; + + LocalizedString all_text() const; }; FormattedPlan format_plan(const ActionPlan& action_plan, const Path& builtin_ports_dir); diff --git a/src/vcpkg-test/dependencies.cpp b/src/vcpkg-test/dependencies.cpp index 4a5e3457e1..d72a364e88 100644 --- a/src/vcpkg-test/dependencies.cpp +++ b/src/vcpkg-test/dependencies.cpp @@ -2406,25 +2406,25 @@ TEST_CASE ("formatting plan 1", "[dependencies]") { auto formatted = format_plan(plan, "/builtin"); CHECK_FALSE(formatted.has_removals); - CHECK(formatted.text == "All requested packages are currently installed.\n"); + CHECK(formatted.all_text() == "All requested packages are currently installed.\n"); } plan.remove_actions.push_back(remove_b); { auto formatted = format_plan(plan, "/builtin"); CHECK(formatted.has_removals); - CHECK(formatted.text == "The following packages will be removed:\n" - " b:x64-osx\n"); + CHECK(formatted.all_text() == "The following packages will be removed:\n" + " b:x64-osx\n"); } plan.remove_actions.push_back(remove_a); - REQUIRE_LINES(format_plan(plan, "/builtin").text, + REQUIRE_LINES(format_plan(plan, "/builtin").all_text(), "The following packages will be removed:\n" " a:x64-osx\n" " b:x64-osx\n"); plan.install_actions.push_back(std::move(install_c)); - REQUIRE_LINES(format_plan(plan, "/builtin").text, + REQUIRE_LINES(format_plan(plan, "/builtin").all_text(), "The following packages will be removed:\n" " a:x64-osx\n" " b:x64-osx\n" @@ -2432,7 +2432,7 @@ TEST_CASE ("formatting plan 1", "[dependencies]") " c:x64-osx@1 -- c\n"); plan.remove_actions.push_back(remove_c); - REQUIRE_LINES(format_plan(plan, "c").text, + REQUIRE_LINES(format_plan(plan, "c").all_text(), "The following packages will be removed:\n" " a:x64-osx\n" " b:x64-osx\n" @@ -2440,7 +2440,7 @@ TEST_CASE ("formatting plan 1", "[dependencies]") " c:x64-osx@1\n"); plan.install_actions.push_back(std::move(install_b)); - REQUIRE_LINES(format_plan(plan, "c").text, + REQUIRE_LINES(format_plan(plan, "c").all_text(), "The following packages will be removed:\n" " a:x64-osx\n" "The following packages will be rebuilt:\n" @@ -2454,7 +2454,7 @@ TEST_CASE ("formatting plan 1", "[dependencies]") { auto formatted = format_plan(plan, "b"); CHECK(formatted.has_removals); - REQUIRE_LINES(formatted.text, + REQUIRE_LINES(formatted.all_text(), "The following packages are already installed:\n" " * d:x86-windows@1\n" " e:x86-windows@1\n" @@ -2466,7 +2466,7 @@ TEST_CASE ("formatting plan 1", "[dependencies]") } plan.install_actions.push_back(std::move(install_f)); - REQUIRE_LINES(format_plan(plan, "b").text, + REQUIRE_LINES(format_plan(plan, "b").all_text(), "The following packages are excluded:\n" " f:x64-osx@1 -- f\n" "The following packages are already installed:\n" diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 03e71dfbd5..ded61191af 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -531,7 +531,7 @@ namespace vcpkg .append_raw(' ') .append_raw(target_triplet) .append_raw('\n') - .append(summary.format())); + .append(summary.format_results())); const bool any_regressions = print_regressions(summary.results, split_specs->known, cidata, diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 5981857fd3..e07507b28c 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -328,10 +328,6 @@ namespace vcpkg const InstallPlanType& plan_type = action.plan_type; if (plan_type == InstallPlanType::ALREADY_INSTALLED) { - if (action.use_head_version == UseHeadVersion::Yes) - msg::println(Color::warning, msgAlreadyInstalledNotHead, msg::spec = action.spec); - else - msg::println(Color::success, msgAlreadyInstalled, msg::spec = action.spec); return ExtendedBuildResult{BuildResult::Succeeded}; } @@ -435,7 +431,7 @@ namespace vcpkg .append_raw(result.timing.to_string()); } - LocalizedString InstallSummary::format() const + LocalizedString InstallSummary::format_results() const { LocalizedString to_print; to_print.append(msgResultsHeader).append_raw('\n'); @@ -475,26 +471,16 @@ namespace vcpkg msg::print(output); } - bool InstallSummary::failed() const + void InstallSummary::print_complete_message() const { - for (const auto& result : this->results) + if (failed) { - switch (result.build_result.value_or_exit(VCPKG_LINE_INFO).code) - { - case BuildResult::Succeeded: - case BuildResult::Removed: - case BuildResult::Downloaded: - case BuildResult::Excluded: continue; - case BuildResult::BuildFailed: - case BuildResult::PostBuildChecksFailed: - case BuildResult::FileConflicts: - case BuildResult::CascadedDueToMissingDependencies: - case BuildResult::CacheMissing: return true; - default: Checks::unreachable(VCPKG_LINE_INFO); - } + msg::println(msgTotalInstallTime, msg::elapsed = timing); + } + else + { + msg::println(Color::success, msgTotalInstallTimeSuccess, msg::elapsed = timing); } - - return false; } struct TrackedPackageInstallGuard @@ -576,50 +562,71 @@ namespace vcpkg const IBuildLogsRecorder& build_logs_recorder, bool include_manifest_in_github_issue) { - const ElapsedTimer timer; - std::vector results; + ElapsedTimer timer; + InstallSummary summary; const size_t action_count = action_plan.remove_actions.size() + action_plan.install_actions.size(); size_t action_index = 1; auto& fs = paths.get_filesystem(); for (auto&& action : action_plan.remove_actions) { - TrackedPackageInstallGuard this_install(action_index++, action_count, results, action); + TrackedPackageInstallGuard this_install(action_index++, action_count, summary.results, action); remove_package(fs, paths.installed(), action.spec, status_db); - results.back().build_result.emplace(BuildResult::Removed); + summary.results.back().build_result.emplace(BuildResult::Removed); } for (auto&& action : action_plan.already_installed) { - results.emplace_back(action).build_result.emplace(perform_install_plan_action( + summary.results.emplace_back(action).build_result.emplace(perform_install_plan_action( args, paths, host_triplet, build_options, action, status_db, binary_cache, build_logs_recorder)); } for (auto&& action : action_plan.install_actions) { - TrackedPackageInstallGuard this_install(action_index++, action_count, results, action); + TrackedPackageInstallGuard this_install(action_index++, action_count, summary.results, action); auto result = perform_install_plan_action( args, paths, host_triplet, build_options, action, status_db, binary_cache, build_logs_recorder); - if (result.code != BuildResult::Succeeded && build_options.keep_going == KeepGoing::No) + if (result.code == BuildResult::Succeeded) { - this_install.print_elapsed_time(); - print_user_troubleshooting_message(action, paths, result.stdoutlog.then([&](auto&) -> Optional { - auto issue_body_path = paths.installed().root() / "vcpkg" / "issue_body.md"; - paths.get_filesystem().write_contents( - issue_body_path, - create_github_issue(args, result, paths, action, include_manifest_in_github_issue), - VCPKG_LINE_INFO); - return issue_body_path; - })); - Checks::exit_fail(VCPKG_LINE_INFO); + } + else + { + if (build_options.keep_going == KeepGoing::No) + { + this_install.print_elapsed_time(); + print_user_troubleshooting_message( + action, paths, result.stdoutlog.then([&](auto&) -> Optional { + auto issue_body_path = paths.installed().root() / "vcpkg" / "issue_body.md"; + paths.get_filesystem().write_contents( + issue_body_path, + create_github_issue(args, result, paths, action, include_manifest_in_github_issue), + VCPKG_LINE_INFO); + return issue_body_path; + })); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + switch (result.code) + { + case BuildResult::Succeeded: + case BuildResult::Removed: + case BuildResult::Downloaded: + case BuildResult::Excluded: break; + case BuildResult::BuildFailed: + case BuildResult::PostBuildChecksFailed: + case BuildResult::FileConflicts: + case BuildResult::CascadedDueToMissingDependencies: + case BuildResult::CacheMissing: summary.failed = true; break; + default: Checks::unreachable(VCPKG_LINE_INFO); + } } this_install.current_summary.build_result.emplace(std::move(result)); } database_load_collapse(fs, paths.installed()); - msg::println(msgTotalInstallTime, msg::elapsed = timer.to_string()); - return InstallSummary{std::move(results)}; + summary.timing = timer.elapsed(); + return summary; } static constexpr CommandSwitch INSTALL_SWITCHES[] = { @@ -1054,7 +1061,7 @@ namespace vcpkg const auto unsupported_port_action = Util::Sets::contains(options.switches, SwitchAllowUnsupported) ? UnsupportedPortAction::Warn : UnsupportedPortAction::Error; - const bool print_cmake_usage = Util::Sets::contains(options.switches, SwitchNoPrintUsage); + const bool print_cmake_usage = !Util::Sets::contains(options.switches, SwitchNoPrintUsage); get_global_metrics_collector().track_bool(BoolMetric::InstallManifestMode, paths.manifest_mode_enabled()); @@ -1375,7 +1382,7 @@ namespace vcpkg // success. if (keep_going == KeepGoing::Yes) { - msg::print(summary.format()); + msg::print(summary.format_results()); } auto it_xunit = options.settings.find(SwitchXXUnit); @@ -1410,7 +1417,8 @@ namespace vcpkg } } - Checks::exit_with_code(VCPKG_LINE_INFO, summary.failed()); + summary.print_complete_message(); + Checks::exit_with_code(VCPKG_LINE_INFO, summary.failed); } SpecSummary::SpecSummary(const InstallPlanAction& action) diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 0cbaae8c50..c8b0f7114f 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -159,9 +159,19 @@ namespace vcpkg specs_installed.erase(action.spec); } - Util::erase_remove_if(action_plan.install_actions, [&](const InstallPlanAction& ipa) { - return Util::Sets::contains(specs_installed, ipa.spec); + Util::erase_remove_if(action_plan.install_actions, [&](InstallPlanAction& ipa) { + if (Util::Sets::contains(specs_installed, ipa.spec)) + { + // convert the 'to install' entry to an already installed entry + ipa.installed_package = status_db.get_installed_package_view(ipa.spec); + ipa.plan_type = InstallPlanType::ALREADY_INSTALLED; + action_plan.already_installed.push_back(std::move(ipa)); + return true; + } + + return false; }); + return specs_installed; } @@ -249,7 +259,7 @@ namespace vcpkg null_build_logs_recorder(), include_manifest_in_github_issue); - if (build_options.keep_going == KeepGoing::Yes && summary.failed()) + if (build_options.keep_going == KeepGoing::Yes && summary.failed) { summary.print_failed(); if (build_options.only_downloads == OnlyDownloads::No) @@ -273,6 +283,7 @@ namespace vcpkg } } + summary.print_complete_message(); Checks::exit_success(VCPKG_LINE_INFO); } diff --git a/src/vcpkg/commands.upgrade.cpp b/src/vcpkg/commands.upgrade.cpp index f70163f058..26b980ae5c 100644 --- a/src/vcpkg/commands.upgrade.cpp +++ b/src/vcpkg/commands.upgrade.cpp @@ -214,9 +214,10 @@ namespace vcpkg // success. if (keep_going == KeepGoing::Yes) { - msg::print(summary.format()); + msg::print(summary.format_results()); } + summary.print_complete_message(); Checks::exit_success(VCPKG_LINE_INFO); } } // namespace vcpkg diff --git a/src/vcpkg/dependencies.cpp b/src/vcpkg/dependencies.cpp index bfdd903138..0ef5e6d8d1 100644 --- a/src/vcpkg/dependencies.cpp +++ b/src/vcpkg/dependencies.cpp @@ -1226,14 +1226,21 @@ namespace vcpkg } } + LocalizedString FormattedPlan::all_text() const + { + auto result = warning_text; + result.append(normal_text); + return result; + } + FormattedPlan format_plan(const ActionPlan& action_plan, const Path& builtin_ports_dir) { FormattedPlan ret; if (action_plan.remove_actions.empty() && action_plan.already_installed.empty() && action_plan.install_actions.empty()) { - ret.text = msg::format(msgInstalledRequestedPackages); - ret.text.append_raw('\n'); + ret.normal_text = msg::format(msgInstalledRequestedPackages); + ret.normal_text.append_raw('\n'); return ret; } @@ -1241,6 +1248,7 @@ namespace vcpkg std::vector rebuilt_plans; std::vector new_plans; std::vector already_installed_plans; + std::vector already_installed_head_plans; std::vector excluded; const bool has_non_user_requested_packages = @@ -1248,6 +1256,20 @@ namespace vcpkg return action.request_type != RequestType::USER_REQUESTED; }) != action_plan.install_actions.cend(); + for (auto&& already_installed_action : action_plan.already_installed) + { + std::vector* to_add; + if (already_installed_action.use_head_version == UseHeadVersion::Yes) + { + to_add = &already_installed_head_plans; + } + else + { + to_add = &already_installed_plans; + } + + to_add->push_back(&already_installed_action); + } for (auto&& remove_action : action_plan.remove_actions) { remove_specs.emplace(remove_action.spec); @@ -1257,54 +1279,67 @@ namespace vcpkg // remove plans are guaranteed to come before install plans, so we know the plan will be contained // if at all. auto it = remove_specs.find(install_action.spec); - if (it != remove_specs.end()) + if (it == remove_specs.end()) { - remove_specs.erase(it); - rebuilt_plans.push_back(&install_action); + std::vector* to_add; + if (install_action.plan_type == InstallPlanType::EXCLUDED) + { + to_add = &excluded; + } + else + { + to_add = &new_plans; + } + + to_add->push_back(&install_action); } else { - if (install_action.plan_type == InstallPlanType::EXCLUDED) - excluded.push_back(&install_action); - else - new_plans.push_back(&install_action); + remove_specs.erase(it); + rebuilt_plans.push_back(&install_action); } } - already_installed_plans = Util::fmap(action_plan.already_installed, [](auto&& action) { return &action; }); - std::sort(rebuilt_plans.begin(), rebuilt_plans.end(), &InstallPlanAction::compare_by_name); - std::sort(new_plans.begin(), new_plans.end(), &InstallPlanAction::compare_by_name); - std::sort(already_installed_plans.begin(), already_installed_plans.end(), &InstallPlanAction::compare_by_name); - std::sort(excluded.begin(), excluded.end(), &InstallPlanAction::compare_by_name); + Util::sort(rebuilt_plans, &InstallPlanAction::compare_by_name); + Util::sort(new_plans, &InstallPlanAction::compare_by_name); + Util::sort(already_installed_plans, &InstallPlanAction::compare_by_name); + Util::sort(already_installed_head_plans, &InstallPlanAction::compare_by_name); + Util::sort(excluded, &InstallPlanAction::compare_by_name); if (!excluded.empty()) { - format_plan_block(ret.text, msgExcludedPackages, false, excluded, builtin_ports_dir); + format_plan_block(ret.warning_text, msgExcludedPackages, false, excluded, builtin_ports_dir); + } + + if (!already_installed_head_plans.empty()) + { + format_plan_block( + ret.warning_text, msgInstalledPackagesHead, false, already_installed_head_plans, builtin_ports_dir); } if (!already_installed_plans.empty()) { - format_plan_block(ret.text, msgInstalledPackages, false, already_installed_plans, builtin_ports_dir); + format_plan_block(ret.normal_text, msgInstalledPackages, false, already_installed_plans, builtin_ports_dir); } if (!remove_specs.empty()) { - format_plan_block(ret.text, msgPackagesToRemove, remove_specs); + format_plan_block(ret.normal_text, msgPackagesToRemove, remove_specs); } if (!rebuilt_plans.empty()) { - format_plan_block(ret.text, msgPackagesToRebuild, true, rebuilt_plans, builtin_ports_dir); + format_plan_block(ret.normal_text, msgPackagesToRebuild, true, rebuilt_plans, builtin_ports_dir); } if (!new_plans.empty()) { - format_plan_block(ret.text, msgPackagesToInstall, true, new_plans, builtin_ports_dir); + format_plan_block(ret.normal_text, msgPackagesToInstall, true, new_plans, builtin_ports_dir); } if (has_non_user_requested_packages) { - ret.text.append(msgPackagesToModify).append_raw('\n'); + ret.normal_text.append(msgPackagesToModify).append_raw('\n'); } ret.has_removals = !remove_specs.empty() || !rebuilt_plans.empty(); @@ -1314,7 +1349,12 @@ namespace vcpkg FormattedPlan print_plan(const ActionPlan& action_plan, const Path& builtin_ports_dir) { auto formatted = format_plan(action_plan, builtin_ports_dir); - msg::print(formatted.text); + if (!formatted.warning_text.empty()) + { + msg::print(Color::warning, formatted.warning_text); + } + + msg::print(formatted.normal_text); return formatted; } From 4d1595607297071386fd489c7aa9e60b2b2deae0 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 11 Oct 2024 15:26:53 -0700 Subject: [PATCH 12/14] Add a message to `vcpkg install` where the licenses of the packages installed in this invocation are printed, and a separate license-report command that prints all information known in the installed tree. --- .../vcpkg-license-bsd-on-mit/portfile.cmake | 1 + .../vcpkg-license-bsd-on-mit/vcpkg.json | 8 ++ .../vcpkg-license-bsd/portfile.cmake | 1 + .../e2e-ports/vcpkg-license-bsd/vcpkg.json | 5 + .../vcpkg-license-mit/portfile.cmake | 1 + .../e2e-ports/vcpkg-license-mit/vcpkg.json | 5 + .../vcpkg-license-null/portfile.cmake | 1 + .../e2e-ports/vcpkg-license-null/vcpkg.json | 5 + .../end-to-end-tests-dir/build-test-ports.ps1 | 11 +- .../ci-verify-versions.ps1 | 12 +- azure-pipelines/end-to-end-tests-dir/cli.ps1 | 16 +-- .../end-to-end-tests-dir/env-passthrough.ps1 | 4 +- .../end-to-end-tests-dir/license-report.ps1 | 90 +++++++++++++ .../post-build-checks.ps1 | 126 +++++++++--------- .../end-to-end-tests-dir/usage.ps1 | 2 - .../end-to-end-tests-dir/versions.ps1 | 12 +- azure-pipelines/end-to-end-tests-prelude.ps1 | 54 +++++++- include/vcpkg/base/message-data.inc.h | 21 +++ include/vcpkg/commands.install.h | 8 ++ include/vcpkg/commands.license-report.h | 10 ++ locales/messages.json | 15 ++- src/vcpkg/commands.cpp | 2 + src/vcpkg/commands.install.cpp | 34 +++++ src/vcpkg/commands.license-report.cpp | 64 +++++++++ src/vcpkg/commands.set-installed.cpp | 2 + 25 files changed, 409 insertions(+), 101 deletions(-) create mode 100644 azure-pipelines/e2e-ports/vcpkg-license-bsd-on-mit/portfile.cmake create mode 100644 azure-pipelines/e2e-ports/vcpkg-license-bsd-on-mit/vcpkg.json create mode 100644 azure-pipelines/e2e-ports/vcpkg-license-bsd/portfile.cmake create mode 100644 azure-pipelines/e2e-ports/vcpkg-license-bsd/vcpkg.json create mode 100644 azure-pipelines/e2e-ports/vcpkg-license-mit/portfile.cmake create mode 100644 azure-pipelines/e2e-ports/vcpkg-license-mit/vcpkg.json create mode 100644 azure-pipelines/e2e-ports/vcpkg-license-null/portfile.cmake create mode 100644 azure-pipelines/e2e-ports/vcpkg-license-null/vcpkg.json create mode 100644 azure-pipelines/end-to-end-tests-dir/license-report.ps1 create mode 100644 include/vcpkg/commands.license-report.h create mode 100644 src/vcpkg/commands.license-report.cpp diff --git a/azure-pipelines/e2e-ports/vcpkg-license-bsd-on-mit/portfile.cmake b/azure-pipelines/e2e-ports/vcpkg-license-bsd-on-mit/portfile.cmake new file mode 100644 index 0000000000..065116c276 --- /dev/null +++ b/azure-pipelines/e2e-ports/vcpkg-license-bsd-on-mit/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-ports/vcpkg-license-bsd-on-mit/vcpkg.json b/azure-pipelines/e2e-ports/vcpkg-license-bsd-on-mit/vcpkg.json new file mode 100644 index 0000000000..f1e5cce8ed --- /dev/null +++ b/azure-pipelines/e2e-ports/vcpkg-license-bsd-on-mit/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "vcpkg-license-bsd-on-mit", + "version": "0", + "license": "BSD-3-Clause", + "dependencies": [ + "vcpkg-license-mit" + ] +} diff --git a/azure-pipelines/e2e-ports/vcpkg-license-bsd/portfile.cmake b/azure-pipelines/e2e-ports/vcpkg-license-bsd/portfile.cmake new file mode 100644 index 0000000000..065116c276 --- /dev/null +++ b/azure-pipelines/e2e-ports/vcpkg-license-bsd/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-ports/vcpkg-license-bsd/vcpkg.json b/azure-pipelines/e2e-ports/vcpkg-license-bsd/vcpkg.json new file mode 100644 index 0000000000..8b0d5d6e55 --- /dev/null +++ b/azure-pipelines/e2e-ports/vcpkg-license-bsd/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "vcpkg-license-bsd", + "version": "0", + "license": "BSD-3-Clause" +} diff --git a/azure-pipelines/e2e-ports/vcpkg-license-mit/portfile.cmake b/azure-pipelines/e2e-ports/vcpkg-license-mit/portfile.cmake new file mode 100644 index 0000000000..065116c276 --- /dev/null +++ b/azure-pipelines/e2e-ports/vcpkg-license-mit/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-ports/vcpkg-license-mit/vcpkg.json b/azure-pipelines/e2e-ports/vcpkg-license-mit/vcpkg.json new file mode 100644 index 0000000000..814573e267 --- /dev/null +++ b/azure-pipelines/e2e-ports/vcpkg-license-mit/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "vcpkg-license-mit", + "version": "0", + "license": "MIT" +} diff --git a/azure-pipelines/e2e-ports/vcpkg-license-null/portfile.cmake b/azure-pipelines/e2e-ports/vcpkg-license-null/portfile.cmake new file mode 100644 index 0000000000..065116c276 --- /dev/null +++ b/azure-pipelines/e2e-ports/vcpkg-license-null/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-ports/vcpkg-license-null/vcpkg.json b/azure-pipelines/e2e-ports/vcpkg-license-null/vcpkg.json new file mode 100644 index 0000000000..54c73ca7bd --- /dev/null +++ b/azure-pipelines/e2e-ports/vcpkg-license-null/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "vcpkg-license-null", + "version": "0", + "license": null +} diff --git a/azure-pipelines/end-to-end-tests-dir/build-test-ports.ps1 b/azure-pipelines/end-to-end-tests-dir/build-test-ports.ps1 index 127a788491..7fde9f0459 100644 --- a/azure-pipelines/end-to-end-tests-dir/build-test-ports.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/build-test-ports.ps1 @@ -51,20 +51,25 @@ if ($output -notmatch 'Trailing comma') { # Check for msgAlreadyInstalled vs. msgAlreadyInstalledNotHead $output = Run-VcpkgAndCaptureOutput @commonArgs --overlay-ports="$PSScriptRoot/../e2e-ports" install vcpkg-internal-e2e-test-port3 Throw-IfFailed -if ($output -notmatch 'vcpkg-internal-e2e-test-port3:[^ ]+ is already installed') { +if (-not $output -contains @" +The following packages are already installed: + vcpkg-internal-e2e-test-port3: +"@) { throw 'Wrong already installed message' } $output = Run-VcpkgAndCaptureOutput @commonArgs --overlay-ports="$PSScriptRoot/../e2e-ports" install vcpkg-internal-e2e-test-port3 --head Throw-IfFailed -if ($output -notmatch 'vcpkg-internal-e2e-test-port3:[^ ]+ is already installed -- not building from HEAD') { +if (-not $output -contains @" +The following packages are already installed, but were requested at --head version. Their installed contents will not be changed. To get updated versions, remove these packages first: + vcpkg-internal-e2e-test-port3: +"@) { throw 'Wrong already installed message for --head' } Refresh-TestRoot $output = Run-VcpkgAndCaptureOutput @commonArgs --x-builtin-ports-root="$PSScriptRoot/../e2e-ports" install vcpkg-bad-spdx-license Throw-IfFailed -$output = $output.Replace("`r`n", "`n") $expected = @" vcpkg.json: warning: $.license (an SPDX license expression): warning: Unknown license identifier 'BSD-new'. Known values are listed at https://spdx.org/licenses/ on expression: BSD-new diff --git a/azure-pipelines/end-to-end-tests-dir/ci-verify-versions.ps1 b/azure-pipelines/end-to-end-tests-dir/ci-verify-versions.ps1 index 6ff05dc086..f18c8dec7d 100644 --- a/azure-pipelines/end-to-end-tests-dir/ci-verify-versions.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/ci-verify-versions.ps1 @@ -1,6 +1,5 @@ . "$PSScriptRoot/../end-to-end-tests-prelude.ps1" -Refresh-TestRoot Copy-Item -Recurse "$PSScriptRoot/../e2e-assets/ci-verify-versions-registry" "$TestingRoot/ci-verify-versions-registry" git -C "$TestingRoot/ci-verify-versions-registry" @gitConfigOptions init @@ -160,17 +159,12 @@ Throw-IfNotFailed function Sanitize() { Param([string]$text) - $workTreeRegex = 'error: failed to execute:[^\r\n]+' # Git command line has an unpredictable PID inside - $text = $text.Replace('\', '/').Replace("`r`n", "`n").Trim() + $workTreeRegex = 'error: failed to execute:[^\n]+' # Git command line has an unpredictable PID inside + $text = $text.Replace('\', '/').Trim() $text = [System.Text.RegularExpressions.Regex]::Replace($text, $workTreeRegex, '') return $text } $expected = Sanitize $expected $actual = Sanitize $actual -if ($actual -ne $expected) { - Set-Content -Value $expected -LiteralPath "$TestingRoot/expected.txt" - Set-Content -Value $actual -LiteralPath "$TestingRoot/actual.txt" - git diff --no-index -- "$TestingRoot/expected.txt" "$TestingRoot/actual.txt" - throw "Bad x-ci-verify-versions output." -} +Throw-IfNonEqual -Expected $expected -Actual $actual diff --git a/azure-pipelines/end-to-end-tests-dir/cli.ps1 b/azure-pipelines/end-to-end-tests-dir/cli.ps1 index 1e7268b238..4784d2a5dd 100644 --- a/azure-pipelines/end-to-end-tests-dir/cli.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/cli.ps1 @@ -56,7 +56,7 @@ error: expected the end of input parsing a package spec; this usually means the "@ -if (-Not ($out.Replace("`r`n", "`n").EndsWith($expected))) +if (-Not ($out.EndsWith($expected))) { throw 'Bad malformed port name output; it was: ' + $out } @@ -71,7 +71,7 @@ error: unknown binary provider type: valid providers are 'clear', 'default', 'nu "@ -if (-Not ($out.Replace("`r`n", "`n").EndsWith($expected))) +if (-Not ($out.EndsWith($expected))) { throw 'Bad malformed --binarysource output; it was: ' + $out } @@ -86,7 +86,7 @@ error: Invalid triplet name. Triplet names are all lowercase alphanumeric+hyphen Built-in Triplets: "@ -if (-Not ($out.Replace("`r`n", "`n").StartsWith($expected))) +if (-Not ($out.StartsWith($expected))) { throw 'Bad malformed triplet output. It was: ' + $out } @@ -99,7 +99,7 @@ error: expected an explicit triplet ^ "@ -if (-Not ($out.Replace("`r`n", "`n").EndsWith($expected))) +if (-Not ($out.EndsWith($expected))) { throw ('Bad error output: ' + $out) } @@ -112,7 +112,7 @@ error: expected an explicit triplet ^ "@ -if (-Not ($out.Replace("`r`n", "`n").EndsWith($expected))) +if (-Not ($out.EndsWith($expected))) { throw ('Bad error output: ' + $out) } @@ -125,7 +125,7 @@ error: expected the end of input parsing a package spec; did you mean zlib[core] ^ "@ -if (-Not ($out.Replace("`r`n", "`n").EndsWith($expected))) +if (-Not ($out.EndsWith($expected))) { throw ('Bad error output: ' + $out) } @@ -138,7 +138,7 @@ error: List of features is not allowed in this context ^ "@ -if (-Not ($out.Replace("`r`n", "`n").EndsWith($expected))) +if (-Not ($out.EndsWith($expected))) { throw ('Bad error output: ' + $out) } @@ -151,7 +151,7 @@ error: Platform qualifier is not allowed in this context ^ "@ -if (-Not ($out.Replace("`r`n", "`n").EndsWith($expected))) +if (-Not ($out.EndsWith($expected))) { throw ('Bad error output: ' + $out) } diff --git a/azure-pipelines/end-to-end-tests-dir/env-passthrough.ps1 b/azure-pipelines/end-to-end-tests-dir/env-passthrough.ps1 index 2e6fe7b253..3201018549 100644 --- a/azure-pipelines/end-to-end-tests-dir/env-passthrough.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/env-passthrough.ps1 @@ -5,13 +5,13 @@ if (-not $IsLinux -and -not $IsMacOS) { $env:_VCPKG_TEST_UNTRACKED = "b" $x = Run-VcpkgAndCaptureOutput "--overlay-triplets=$PSScriptRoot/../e2e-ports/env-passthrough" env "echo %_VCPKG_TEST_TRACKED% %_VCPKG_TEST_TRACKED2% %_VCPKG_TEST_UNTRACKED% %_VCPKG_TEST_UNTRACKED2%" - if ($x -ne "%_VCPKG_TEST_TRACKED% %_VCPKG_TEST_TRACKED2% %_VCPKG_TEST_UNTRACKED% %_VCPKG_TEST_UNTRACKED2%`r`n") + if ($x -ne "%_VCPKG_TEST_TRACKED% %_VCPKG_TEST_TRACKED2% %_VCPKG_TEST_UNTRACKED% %_VCPKG_TEST_UNTRACKED2%`n") { throw "env should have cleaned the environment ($x)" } $y = Run-VcpkgAndCaptureOutput "--overlay-triplets=$PSScriptRoot/../e2e-ports/env-passthrough" env --triplet passthrough "echo %_VCPKG_TEST_TRACKED% %_VCPKG_TEST_TRACKED2% %_VCPKG_TEST_UNTRACKED% %_VCPKG_TEST_UNTRACKED2%" - if ($y -ne "a %_VCPKG_TEST_TRACKED2% b %_VCPKG_TEST_UNTRACKED2%`r`n") + if ($y -ne "a %_VCPKG_TEST_TRACKED2% b %_VCPKG_TEST_UNTRACKED2%`n") { throw "env should have kept the environment ($y)" } diff --git a/azure-pipelines/end-to-end-tests-dir/license-report.ps1 b/azure-pipelines/end-to-end-tests-dir/license-report.ps1 new file mode 100644 index 0000000000..5be271c519 --- /dev/null +++ b/azure-pipelines/end-to-end-tests-dir/license-report.ps1 @@ -0,0 +1,90 @@ +. $PSScriptRoot/../end-to-end-tests-prelude.ps1 + +[string]$output = Run-VcpkgAndCaptureOutput @commonArgs license-report "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" +Throw-IfFailed +Throw-IfNonEqual -Actual $output -Expected @" +There are no installed packages, and thus no licenses of installed packages. Did you mean to install something first? + +"@ + +$output = Run-VcpkgAndCaptureOutput @commonArgs install "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" vcpkg-license-bsd vcpkg-license-mit +Throw-IfFailed +Throw-IfNonContains -Actual $output -Expected @" +Packages installed in this vcpkg installation declare the following licenses: +BSD-3-Clause +MIT +"@ + +$output = Run-VcpkgAndCaptureOutput @commonArgs license-report "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" +Throw-IfFailed +Throw-IfNonEqual -Actual $output -Expected @" +Installed contents are licensed to you by owners. Microsoft is not responsible for, nor does it grant any licenses to, third-party packages. +Installed packages declare the following licenses: +BSD-3-Clause +MIT + +"@ + +# Note that the MIT license already is not displayed +$output = Run-VcpkgAndCaptureOutput @commonArgs install "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" vcpkg-license-bsd-on-mit +Throw-IfFailed +Throw-IfNonContains -Actual $output -Expected @" +Packages installed in this vcpkg installation declare the following licenses: +BSD-3-Clause +"@ + +$output = Run-VcpkgAndCaptureOutput @commonArgs license-report "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" +Throw-IfFailed +Throw-IfNonEqual -Actual $output -Expected @" +Installed contents are licensed to you by owners. Microsoft is not responsible for, nor does it grant any licenses to, third-party packages. +Installed packages declare the following licenses: +BSD-3-Clause +MIT + +"@ + +# Empty port == no license field set at all +$output = Run-VcpkgAndCaptureOutput @commonArgs install "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" vcpkg-empty-port +Throw-IfFailed +Throw-IfNonContains -Actual $output -Expected @" +Installed contents are licensed to you by owners. Microsoft is not responsible for, nor does it grant any licenses to, third-party packages. +Some packages did not declare an SPDX license. Check the ``copyright`` file for each package for more information about their licensing. +"@ + +$output = Run-VcpkgAndCaptureOutput @commonArgs license-report "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" +Throw-IfFailed +Throw-IfNonEqual -Actual $output -Expected @" +Installed contents are licensed to you by owners. Microsoft is not responsible for, nor does it grant any licenses to, third-party packages. +Some packages did not declare an SPDX license. Check the ``copyright`` file for each package for more information about their licensing. +Installed packages declare the following licenses: +BSD-3-Clause +MIT + +"@ + +Run-Vcpkg @commonArgs remove "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" vcpkg-license-bsd +Throw-IfFailed + +# bsd-on-mit is still here so no change +$output = Run-VcpkgAndCaptureOutput @commonArgs license-report "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" +Throw-IfFailed +Throw-IfNonEqual -Actual $output -Expected @" +Installed contents are licensed to you by owners. Microsoft is not responsible for, nor does it grant any licenses to, third-party packages. +Some packages did not declare an SPDX license. Check the ``copyright`` file for each package for more information about their licensing. +Installed packages declare the following licenses: +BSD-3-Clause +MIT + +"@ + +Run-Vcpkg @commonArgs remove "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" vcpkg-license-bsd-on-mit vcpkg-license-mit +Throw-IfFailed + +# Only unknown left +$output = Run-VcpkgAndCaptureOutput @commonArgs license-report "--x-builtin-ports-root=$PSScriptRoot/../e2e-ports" +Throw-IfFailed +Throw-IfNonEqual -Actual $output -Expected @" +Installed contents are licensed to you by owners. Microsoft is not responsible for, nor does it grant any licenses to, third-party packages. +Some packages did not declare an SPDX license. Check the ``copyright`` file for each package for more information about their licensing. + +"@ diff --git a/azure-pipelines/end-to-end-tests-dir/post-build-checks.ps1 b/azure-pipelines/end-to-end-tests-dir/post-build-checks.ps1 index 2ce8fac406..bd190dcb71 100644 --- a/azure-pipelines/end-to-end-tests-dir/post-build-checks.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/post-build-checks.ps1 @@ -17,7 +17,7 @@ if (-not $dupOutput.Contains('The following files are already installed')) { Refresh-TestRoot [string]$buildOutput = Run-VcpkgAndCaptureStderr install @commonArgs --overlay-ports="$PSScriptRoot/../e2e-ports" vcpkg-policy-set-incorrectly Throw-IfNotFailed -if (-not $buildOutput.Replace("`r`n", "`n").EndsWith("error: Unknown setting of VCPKG_POLICY_EMPTY_PACKAGE: ON. Valid policy values are '', 'disabled', and 'enabled'.`n")) { +if (-not $buildOutput.EndsWith("error: Unknown setting of VCPKG_POLICY_EMPTY_PACKAGE: ON. Valid policy values are '', 'disabled', and 'enabled'.`n")) { throw ('Incorrect error message for incorrect policy value; output was ' + $buildOutput) } @@ -83,13 +83,13 @@ $($packagesRoot)$($NativeSlash)vcpkg-policy-include-folder_$($Triplet)$($NativeS note: "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect restricted header' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-include-folder[do-install-restricted,policy-allow-restricted-headers]' --no-binarycaching --enforce-port-checks Throw-IfFailed -if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_ALLOW_RESTRICTED_HEADERS didn''t allow' } @@ -100,13 +100,13 @@ $expected = @" $($PortfilePath): warning: `${CURRENT_PACKAGES_DIR}/debug/include should not exist. To suppress this message, add set(VCPKG_POLICY_ALLOW_DEBUG_INCLUDE enabled) note: If this directory was created by a build system that does not allow installing headers in debug to be disabled, delete the duplicate directory with file(REMOVE_RECURSE "`${CURRENT_PACKAGES_DIR}/debug/include") "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect debug headers' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-include-folder[do-install,do-install-debug,policy-allow-debug-include]' --no-binarycaching --enforce-port-checks Throw-IfFailed -if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_ALLOW_DEBUG_INCLUDE didn''t suppress' } @@ -116,13 +116,13 @@ Throw-IfNotFailed $expected = @" $($PortfilePath): warning: `${CURRENT_PACKAGES_DIR}/debug/share should not exist. Please reorganize any important files, then delete any remaining by adding ``file(REMOVE_RECURSE "`${CURRENT_PACKAGES_DIR}/debug/share")``. To suppress this message, add set(VCPKG_POLICY_ALLOW_DEBUG_SHARE enabled) "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect debug share' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-include-folder[do-install,do-install-debug-share,policy-allow-debug-share]' --no-binarycaching --enforce-port-checks Throw-IfFailed -if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_ALLOW_DEBUG_SHARE didn''t suppress' } @@ -144,7 +144,7 @@ note: debug/cmake/some_cmake.cmake if ($buildOutput.Contains("legitimate.cmake")) { throw 'Complained about legitimate CMake files' } -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad CMake files' } @@ -161,7 +161,7 @@ $($PortfilePath): warning: This port creates `${CURRENT_PACKAGES_DIR}/lib/cmake if ($buildOutput.Contains("legitimate.cmake")) { throw 'Complained about legitimate CMake files' } -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad CMake files' } @@ -180,7 +180,7 @@ $($PortfilePath): warning: This port creates `${CURRENT_PACKAGES_DIR}/lib/cmake if ($buildOutput.Contains("legitimate.cmake")) { throw 'Complained about legitimate CMake files' } -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad CMake files' } @@ -193,7 +193,7 @@ $($PortfilePath): warning: This port creates `${CURRENT_PACKAGES_DIR}/lib/cmake if ($buildOutput.Contains("legitimate.cmake")) { throw 'Complained about legitimate CMake files' } -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad CMake files' } @@ -211,7 +211,7 @@ note: debug/lib/cmake/some_cmake.cmake if ($buildOutput.Contains("legitimate.cmake")) { throw 'Complained about legitimate CMake files' } -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad CMake files' } @@ -248,7 +248,7 @@ $($PortfilePath): warning: the license is not installed to `${CURRENT_PACKAGES_D $buildOutput = Run-VcpkgAndCaptureOutput install @commonArgs --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-copyright' --no-binarycaching --enforce-port-checks Throw-IfNotFailed -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect missing copyright no source' } @@ -260,7 +260,7 @@ $($PortfilePath): note: Consider adding: vcpkg_install_copyright(FILE_LIST "`${S $buildOutput = Run-VcpkgAndCaptureOutput install @commonArgs --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-copyright[source]' --no-binarycaching --enforce-port-checks Throw-IfNotFailed -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect missing copyright source' } @@ -272,7 +272,7 @@ $($PortfilePath): note: Consider adding: vcpkg_install_copyright(FILE_LIST "`${S $buildOutput = Run-VcpkgAndCaptureOutput install @commonArgs --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-copyright[source2]' --no-binarycaching --enforce-port-checks Throw-IfNotFailed -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect missing copyright source2' } @@ -288,13 +288,13 @@ note: src/v1.3.1-2e5db616bf.clean/LICENSE.txt $buildOutput = Run-VcpkgAndCaptureOutput install @commonArgs --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-copyright[source,source2]' --no-binarycaching --enforce-port-checks Throw-IfNotFailed -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect missing copyright source + source2' } $buildOutput = Run-VcpkgAndCaptureOutput install @commonArgs --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-copyright[source,source2,policy]' --no-binarycaching --enforce-port-checks Throw-IfFailed -if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_SKIP_COPYRIGHT_CHECK didn''t suppress source + source2' } @@ -314,7 +314,7 @@ $($packagesRoot)$($NativeSlash)test-exe_x86-windows: note: the executables are r note: bin/test_exe.exe "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect EXE in bin.' } @@ -327,13 +327,13 @@ note: debug/bin/test_exe.exe note: bin/test_exe.exe "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect EXEs in bin.' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/exes-in-bin" "test-exe[policy-allow-exes-in-bin]" --no-binarycaching --enforce-port-checks Throw-IfFailed - if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_ALLOW_EXES_IN_BIN didn''t suppress' } } # windows @@ -349,12 +349,12 @@ $PSScriptRoot/../e2e-ports$($NativeSlash)vcpkg-policy-forgot-usage$($NativeSlash note: you can install the usage file with the following CMake note: file(INSTALL "`${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "`${CURRENT_PACKAGES_DIR}/share/`${PORT}") "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect forgotten usage' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-forgot-usage[policy]' --no-binarycaching --enforce-port-checks Throw-IfFailed -if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_SKIP_USAGE_INSTALL_CHECK didn''t suppress' } @@ -376,7 +376,7 @@ note: Release binaries were not found. "@ $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/debug-release-mismatch" 'test-dll[debug-only]' --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect debug only mismatch' } @@ -390,7 +390,7 @@ note: bin/test_dll.dll "@ $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/debug-release-mismatch" 'test-dll[bad-release-only]' --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect release only mismatch' } @@ -408,7 +408,7 @@ note: bin/test_dll.dll "@ $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/debug-release-mismatch" 'test-dll[extra-debug]' --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect extra debug mismatch' } @@ -426,13 +426,13 @@ note: bin/test_dll2.dll "@ $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/debug-release-mismatch" 'test-dll[extra-release]' --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect extra release mismatch' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/debug-release-mismatch" 'test-dll[extra-release,policy-mismatched-number-of-binaries]' --no-binarycaching --enforce-port-checks Throw-IfFailed - if ($buildOutput.Replace("`r`n", "`n").Contains('mismatching number of debug and release binaries')) { + if ($buildOutput.Contains('mismatching number of debug and release binaries')) { throw 'VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES didn''t suppress' } } @@ -454,13 +454,13 @@ note: debug/bin/test_dll.dll note: bin/test_dll.dll "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect Kernel32 from xbox.' } $buildOutput = Run-VcpkgAndCaptureOutput --triplet x64-xbox-xboxone "--x-buildtrees-root=$buildtreesRoot" "--x-install-root=$installRoot" "--x-packages-root=$packagesRoot" install --overlay-ports="$TestingRoot/kernel32-from-xbox" 'test-dll[policy-allow-kernel32-from-xbox]' --no-binarycaching --enforce-port-checks Throw-IfFailed - if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_ALLOW_KERNEL32_FROM_XBOX didn''t suppress' } } # windows @@ -479,19 +479,19 @@ $($PortfilePath): warning: Import libraries for installed DLLs appear to be miss $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/dlls-no-lib" "test-dll[install-no-lib-debug]" --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect DLLs with no import libraries debug' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/dlls-no-lib" "test-dll[install-no-lib-release]" --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect DLLs with no import libraries release' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/dlls-no-lib" "test-dll[install-no-lib-debug,install-no-lib-release]" --no-binarycaching --enforce-port-checks Throw-IfNotFailed - $buildOutput = $buildOutput.Replace("`r`n", "`n") + $buildOutput = $buildOutput $first = $buildOutput.IndexOf($expected) if ($first -lt 0) { throw 'Did not detect DLLs with no import libraries both' @@ -502,7 +502,7 @@ $($PortfilePath): warning: Import libraries for installed DLLs appear to be miss $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/dlls-in-lib" "test-dll[install-no-lib-debug,install-no-lib-release,policy-dlls-without-libs]" --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_DLLS_WITHOUT_LIBS didn''t suppress' } } # windows @@ -524,13 +524,13 @@ note: debug/lib/test_dll.dll note: lib/test_dll.dll "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect DLL in lib.' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$TestingRoot/dlls-in-lib" "test-dll[install-to-lib,policy-allow-dlls-in-lib]" --no-binarycaching --enforce-port-checks Throw-IfFailed - if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_ALLOW_DLLS_IN_LIB didn''t suppress' } } # windows @@ -548,14 +548,14 @@ note: debug/bin/no_exports.dll note: bin/no_exports.dll "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect DLLs with no exports' } Refresh-TestRoot $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-internal-dll-with-no-exports[policy]' --no-binarycaching --enforce-port-checks Throw-IfFailed - if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_DLLS_WITHOUT_EXPORTS didn''t suppress' } } # windows @@ -578,7 +578,7 @@ note: debug/bin/test_dll.dll is built for x64 note: bin/test_dll.dll is built for x64 "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect DLL with wrong architecture.' } @@ -606,7 +606,7 @@ note: debug/bin/test_dll.dll note: bin/test_dll.dll "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect DLL with wrong appcontainer.' } @@ -630,13 +630,13 @@ note: bin/test_dll.dll $buildOutput = Run-VcpkgAndCaptureOutput --triplet x86-windows "--x-buildtrees-root=$buildtreesRoot" "--x-install-root=$installRoot" "--x-packages-root=$packagesRoot" install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-msvc-2013' --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect obsolete CRT.' } $buildOutput = Run-VcpkgAndCaptureOutput --triplet x86-windows "--x-buildtrees-root=$buildtreesRoot" "--x-install-root=$installRoot" "--x-packages-root=$packagesRoot" install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-msvc-2013[policy]' --no-binarycaching --enforce-port-checks Throw-IfFailed - if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_ALLOW_OBSOLETE_MSVCRT did not suppress' } @@ -649,7 +649,7 @@ note: bin/test_dll.dll $buildOutput = Run-VcpkgAndCaptureOutput --triplet x86-windows "--x-buildtrees-root=$buildtreesRoot" "--x-install-root=$installRoot" "--x-packages-root=$packagesRoot" install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-msvc-2013[release-only]' --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect obsolete CRT.' } } # windows @@ -675,7 +675,7 @@ if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") endif() "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect DLL in static release-only.' } @@ -695,13 +695,13 @@ if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") endif() "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect DLL in static.' } $buildOutput = Run-VcpkgAndCaptureOutput @directoryArgs install --triplet x64-windows-static --overlay-ports="$TestingRoot/dlls-in-static" "test-dll[policy-dlls-in-static-library]" --no-binarycaching --enforce-port-checks Throw-IfFailed - if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_DLLS_IN_STATIC_LIBRARY didn''t suppress' } } # windows @@ -728,7 +728,7 @@ note: lib/both_lib.lib links with: Dynamic Release (/MD) note: lib/test_lib.lib links with: Dynamic Release (/MD) "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect lib with wrong CRT linkage.' } @@ -756,7 +756,7 @@ note: The following binaries should link with only: Static Release (/MT) note: lib/test_lib.lib links with: Dynamic Release (/MD) "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect lib with wrong CRT linkage release only.' } @@ -770,7 +770,7 @@ note: debug/lib/test_lib.lib links with: Dynamic Release (/MD) note: lib/test_lib.lib links with: Dynamic Release (/MD) "@ - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { + if (-not $buildOutput.Contains($expected)) { throw 'Did not detect lib with wrong CRT linkage release only.' } @@ -795,13 +795,13 @@ note: file(REMOVE_RECURSE "`${CURRENT_PACKAGES_DIR}/empty-directory" "`${CURRENT "@ $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-empty-folders' --no-binarycaching --enforce-port-checks Throw-IfNotFailed -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect empty directories' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-empty-folders[policy]' --no-binarycaching --enforce-port-checks Throw-IfFailed -if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_ALLOW_EMPTY_FOLDERS didn''t suppress' } @@ -816,12 +816,12 @@ $($packagesRoot)$($NativeSlash)vcpkg-policy-misplaced-regular-files_$($Triplet): note: debug/bad_debug_file.txt note: bad_file.txt "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad regular files' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-misplaced-regular-files[policy]' --no-binarycaching --enforce-port-checks Throw-IfFailed -if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_SKIP_MISPLACED_REGULAR_FILES_CHECK didn''t suppress' } @@ -842,7 +842,7 @@ file(RENAME "`${CURRENT_PACKAGES_DIR}/debug/bin/pkgconfig/zlib.pc" "`${CURRENT_P vcpkg_fixup_pkgconfig() file(REMOVE_RECURSE empty directories left by the above renames) "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad pkgconfig misplaced' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-misplaced-pkgconfig[install-arch-agnostic-bad-misplaced,install-arch-agnostic-empty-libs-bad-misplaced,install-arch-dependent-bad-misplaced,install-arch-dependent-bad-share]' --no-binarycaching --enforce-port-checks @@ -865,12 +865,12 @@ file(RENAME "`${CURRENT_PACKAGES_DIR}/share/pkgconfig/zlib.pc" "`${CURRENT_PACKA vcpkg_fixup_pkgconfig() file(REMOVE_RECURSE empty directories left by the above renames) "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad pkgconfig all bad' } $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" 'vcpkg-policy-misplaced-pkgconfig[install-arch-agnostic-bad-misplaced,install-arch-agnostic-empty-libs-bad-misplaced,install-arch-dependent-bad-misplaced,install-arch-dependent-bad-share,policy]' --no-binarycaching --enforce-port-checks Throw-IfFailed -if ($buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if ($buildOutput.Contains($expected)) { throw 'VCPKG_POLICY_SKIP_PKGCONFIG_CHECK didn''t suppress' } @@ -888,7 +888,7 @@ file(RENAME "`${CURRENT_PACKAGES_DIR}/bin/pkgconfig/zlib.pc" "`${CURRENT_PACKAGE vcpkg_fixup_pkgconfig() file(REMOVE_RECURSE empty directories left by the above renames) "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad pkgconfig release only' } @@ -904,7 +904,7 @@ file(RENAME "`${CURRENT_PACKAGES_DIR}/bin/pkgconfig/libmorton.pc" "`${CURRENT_PA vcpkg_fixup_pkgconfig() file(REMOVE_RECURSE empty directories left by the above renames) "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad pkgconfig arch agnostic' } @@ -920,7 +920,7 @@ file(RENAME "`${CURRENT_PACKAGES_DIR}/bin/pkgconfig/zlib-no-libs.pc" "`${CURRENT vcpkg_fixup_pkgconfig() file(REMOVE_RECURSE empty directories left by the above renames) "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad pkgconfig arch agnostic empty libs' } @@ -936,7 +936,7 @@ file(RENAME "`${CURRENT_PACKAGES_DIR}/share/pkgconfig/zlib.pc" "`${CURRENT_PACKA vcpkg_fixup_pkgconfig() file(REMOVE_RECURSE empty directories left by the above renames) "@ -if (-not $buildOutput.Replace("`r`n", "`n").Contains($expected)) { +if (-not $buildOutput.Contains($expected)) { throw 'Did not detect bad pkgconfig arch dependent share' } @@ -962,10 +962,10 @@ $($packagesRoot)$($NativeSlash)vcpkg-policy-absolute-paths_$($Triplet)$($NativeS foreach ($bad_dir in @('build-dir', 'downloads', 'installed-root', 'package-dir')) { $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" "vcpkg-policy-absolute-paths[$bad_dir]" --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expectedHeader)) { + if (-not $buildOutput.Contains($expectedHeader)) { throw 'Did not detect bad absolute paths header' } - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expectedFooter)) { + if (-not $buildOutput.Contains($expectedFooter)) { throw 'Did not detect bad absolute paths footer' } } @@ -980,10 +980,10 @@ $($packagesRoot)$($NativeSlash)vcpkg-policy-absolute-paths_$($Triplet)$($NativeS foreach ($bad_dir in @('build-dir', 'downloads', 'installed-root', 'package-dir')) { $buildOutput = Run-VcpkgAndCaptureOutput @commonArgs install --overlay-ports="$PSScriptRoot/../e2e-ports" "vcpkg-policy-absolute-paths[$bad_dir,pkgconfig]" --no-binarycaching --enforce-port-checks Throw-IfNotFailed - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expectedHeader)) { + if (-not $buildOutput.Contains($expectedHeader)) { throw 'Did not detect bad absolute paths header' } - if (-not $buildOutput.Replace("`r`n", "`n").Contains($expectedFooter)) { + if (-not $buildOutput.Contains($expectedFooter)) { throw 'Did not detect bad absolute paths pkgconfig footer' } } diff --git a/azure-pipelines/end-to-end-tests-dir/usage.ps1 b/azure-pipelines/end-to-end-tests-dir/usage.ps1 index f2d84316e3..266ee2c4f4 100644 --- a/azure-pipelines/end-to-end-tests-dir/usage.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/usage.ps1 @@ -59,8 +59,6 @@ vcpkg-header-only is header-only and can be used from CMake via: "--overlay-ports=$PSScriptRoot/../e2e-ports" )) -$usage = $usage.Replace("`r`n", "`n") - foreach ($requiredUsage in $requiredUsages) { if (-Not $usage.Contains($requiredUsage)) { throw "The usage text didn't contain the required entry:`n$requiredUsage" diff --git a/azure-pipelines/end-to-end-tests-dir/versions.ps1 b/azure-pipelines/end-to-end-tests-dir/versions.ps1 index 6f4ac057e0..31799da837 100644 --- a/azure-pipelines/end-to-end-tests-dir/versions.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/versions.ps1 @@ -86,10 +86,10 @@ git -C $versionFilesPath @gitConfigOptions add -A git -C $versionFilesPath @gitConfigOptions commit -m "add octopus 1.0#1" $output = Run-VcpkgAndCaptureOutput @portsRedirectArgsOK x-add-version octopus Throw-IfNotFailed -if ($output.Replace("`r`n", "`n") -notmatch @" +if ($output -notmatch @" warning: In octopus, 1.0 is completely new version, so the "port-version" field should be removed. Remove "port-version", commit that change, and try again. To skip this check, rerun with --skip-version-format-check . "@) { - throw "Expected detecting present port-version when a new version is added as bad" + throw "Expected detecting present port-version when a new version is added as bad" } Run-Vcpkg @portsRedirectArgsOK x-add-version octopus --skip-version-format-check @@ -102,10 +102,10 @@ git -C $versionFilesPath @gitConfigOptions add -A git -C $versionFilesPath @gitConfigOptions commit -m "add octopus 2.0#1" $output = Run-VcpkgAndCaptureOutput @portsRedirectArgsOK x-add-version octopus Throw-IfNotFailed -if ($output.Replace("`r`n", "`n") -notmatch @" +if ($output -notmatch @" warning: In octopus, 2.0 is completely new version, so the "port-version" field should be removed. Remove "port-version", commit that change, and try again. To skip this check, rerun with --skip-version-format-check . "@) { - throw "Expected detecting present port-version when a new version is added as bad" + throw "Expected detecting present port-version when a new version is added as bad" } Run-Vcpkg @portsRedirectArgsOK x-add-version octopus --skip-version-format-check @@ -118,10 +118,10 @@ git -C $versionFilesPath @gitConfigOptions add -A git -C $versionFilesPath @gitConfigOptions commit -m "add octopus 2.0#3" $output = Run-VcpkgAndCaptureOutput @portsRedirectArgsOK x-add-version octopus Throw-IfNotFailed -if ($output.Replace("`r`n", "`n") -notmatch @" +if ($output -notmatch @" warning: In octopus, the current "port-version" for 2.0 is 1, so the next added "port-version" should be 2, but the port declares "port-version" 3. Change "port-version" to 2, commit that change, and try again. To skip this check, rerun with --skip-version-format-check . "@) { - throw "Expected detecting present port-version when a new version is added as bad" + throw "Expected detecting present port-version when a new version is added as bad" } Run-Vcpkg @portsRedirectArgsOK x-add-version octopus --skip-version-format-check diff --git a/azure-pipelines/end-to-end-tests-prelude.ps1 b/azure-pipelines/end-to-end-tests-prelude.ps1 index 9d0eaa3c8d..7a11e196b6 100644 --- a/azure-pipelines/end-to-end-tests-prelude.ps1 +++ b/azure-pipelines/end-to-end-tests-prelude.ps1 @@ -129,7 +129,7 @@ function Run-VcpkgAndCaptureOutput { Write-Host -ForegroundColor red $Script:CurrentTest $result = (& "$thisVcpkg" @testArgs) | Out-String Write-Host -ForegroundColor Gray $result - $result + $result.Replace("`r`n", "`n") } function Run-VcpkgAndCaptureStdErr { @@ -152,7 +152,7 @@ function Run-VcpkgAndCaptureStdErr { if ($null -eq $result) { $result = [string]::Empty } - return $result + return $result.Replace("`r`n", "`n") } function Run-Vcpkg { @@ -201,7 +201,6 @@ function Set-EmptyTestPort { "version": "$Version" "@ - $json = $json.Replace("`r`n", "`n") if (-not $null -eq $PortVersion) { $json += ",`n `"port-version`": $PortVersion" @@ -216,4 +215,53 @@ function Set-EmptyTestPort { Set-Content -Value $json -LiteralPath (Join-Path $portDir 'vcpkg.json') -Encoding Ascii -NoNewline } +function Throw-IfNonEqual { + Param( + [string]$Actual, + [string]$Expected + ) + if ($Actual -ne $Expected) { + Set-Content -Value $Expected -LiteralPath "$TestingRoot/expected.txt" + Set-Content -Value $Actual -LiteralPath "$TestingRoot/actual.txt" + git diff --no-index -- "$TestingRoot/expected.txt" "$TestingRoot/actual.txt" + Write-Stack + throw "Expected '$Expected' but got '$Actual'" + } +} + +function Throw-IfNonEndsWith { + Param( + [string]$Actual, + [string]$Expected + ) + + [string]$actualSuffix = $actual + $actualLength = $Actual.Length + if ($actualLength -gt $expected.Length) { + $actualSuffix = $Actual.Substring($actualLength - $expected.Length, $expected.Length) + } + + if ($actualSuffix -ne $Expected) { + Set-Content -Value $Expected -LiteralPath "$TestingRoot/expected.txt" + Set-Content -Value $Actual -LiteralPath "$TestingRoot/actual.txt" + git diff --no-index -- "$TestingRoot/expected.txt" "$TestingRoot/actual.txt" + Write-Stack + throw "Expected '$Expected' but got '$actualSuffix'" + } +} + +function Throw-IfNonContains { + Param( + [string]$Actual, + [string]$Expected + ) + if (-not ($Actual.Contains($Expected))) { + Set-Content -Value $Expected -LiteralPath "$TestingRoot/expected.txt" + Set-Content -Value $Actual -LiteralPath "$TestingRoot/actual.txt" + git diff --no-index -- "$TestingRoot/expected.txt" "$TestingRoot/actual.txt" + Write-Stack + throw "Expected '$Expected' to be in '$Actual'" + } +} + Refresh-TestRoot diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index cf90643636..88393fa617 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -730,6 +730,7 @@ DECLARE_MESSAGE(CmdInstallExample1, "This is a command line, only the <> parts should be localized", "vcpkg install ...") DECLARE_MESSAGE(CmdIntegrateSynopsis, (), "", "Integrates vcpkg with machines, projects, or shells") +DECLARE_MESSAGE(CmdLicenseReportSynopsis, (), "", "Displays the declared licenses of all ports in the installed tree") DECLARE_MESSAGE(CmdListExample2, (), "This is a command line, only the part should be localized", @@ -2137,6 +2138,11 @@ DECLARE_MESSAGE(NoInstalledPackages, (), "The name 'search' is the name of a command that is not localized.", "No packages are installed. Did you mean `search`?") +DECLARE_MESSAGE(NoInstalledPackagesLicenseReport, + (), + "", + "There are no installed packages, and thus no licenses of installed packages. Did you mean to install " + "something first?") DECLARE_MESSAGE(NonExactlyArgs, (msg::command_name, msg::expected, msg::actual), "{expected} and {actual} are integers", @@ -2200,6 +2206,21 @@ DECLARE_MESSAGE(OverwritingFile, (msg::path), "", "File {path} was already prese DECLARE_MESSAGE(PackageAbi, (msg::spec, msg::package_abi), "", "{spec} package ABI: {package_abi}") DECLARE_MESSAGE(PackageAlreadyRemoved, (msg::spec), "", "unable to remove {spec}: already removed") DECLARE_MESSAGE(PackageDiscoveryHeader, (), "", "Package Discovery") +DECLARE_MESSAGE(PackageLicenseSpdx, (), "", "Installed packages declare the following licenses:") +DECLARE_MESSAGE(PackageLicenseSpdxThisInstall, + (), + "", + "Packages installed in this vcpkg installation declare the following licenses:") +DECLARE_MESSAGE(PackageLicenseUnknown, + (), + "", + "Some packages did not declare an SPDX license. Check the `copyright` file for each package for more " + "information about their licensing.") +DECLARE_MESSAGE(PackageLicenseWarning, + (), + "", + "Installed contents are licensed to you by owners. Microsoft is not responsible for, nor does it grant " + "any licenses to, third-party packages.") DECLARE_MESSAGE(PackageManipulationHeader, (), "", "Package Manipulation") DECLARE_MESSAGE(PackageInfoHelp, (), "", "Display detailed information on packages") DECLARE_MESSAGE(PackageFailedtWhileExtracting, diff --git a/include/vcpkg/commands.install.h b/include/vcpkg/commands.install.h index 1164068407..58c2ec4e70 100644 --- a/include/vcpkg/commands.install.h +++ b/include/vcpkg/commands.install.h @@ -37,10 +37,18 @@ namespace vcpkg PackageSpec m_spec; }; + struct LicenseReport + { + bool any_unknown_licenses = false; + std::set named_licenses; + void print_license_report(const msg::MessageT<>& named_license_heading) const; + }; + struct InstallSummary { std::vector results; ElapsedTime timing; + LicenseReport license_report; bool failed = false; LocalizedString format_results() const; diff --git a/include/vcpkg/commands.license-report.h b/include/vcpkg/commands.license-report.h new file mode 100644 index 0000000000..75011c37b3 --- /dev/null +++ b/include/vcpkg/commands.license-report.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +namespace vcpkg +{ + extern const CommandMetadata CommandLicenseReportMetadata; + void command_license_report_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths); +} diff --git a/locales/messages.json b/locales/messages.json index 1b4622a7d9..3c9e64762b 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -111,11 +111,7 @@ "_AllFormatArgsRawArgument.comment": "example of {value} is 'foo {} bar'", "AllFormatArgsUnbalancedBraces": "unbalanced brace in format string \"{value}\"", "_AllFormatArgsUnbalancedBraces.comment": "example of {value} is 'foo bar {'", - "AllPackagesAreUpdated": "All installed packages are up-to-date.", - "AlreadyInstalled": "{spec} is already installed", - "_AlreadyInstalled.comment": "An example of {spec} is zlib:x64-windows.", - "AlreadyInstalledNotHead": "{spec} is already installed -- not building from HEAD", - "_AlreadyInstalledNotHead.comment": "'HEAD' means the most recent version of source code An example of {spec} is zlib:x64-windows.", + "AllPackagesAreUpdated": "No action taken because all installed packages are up-to-date.", "AmbiguousConfigDeleteConfigFile": "Ambiguous vcpkg configuration provided by both manifest and configuration file.\n-- Delete configuration file {path}", "_AmbiguousConfigDeleteConfigFile.comment": "An example of {path} is /foo/bar.", "AnArrayOfDefaultFeatures": "an array of default features", @@ -430,6 +426,7 @@ "CmdInstallExample1": "vcpkg install ...", "_CmdInstallExample1.comment": "This is a command line, only the <> parts should be localized", "CmdIntegrateSynopsis": "Integrates vcpkg with machines, projects, or shells", + "CmdLicenseReportSynopsis": "Displays the declared licenses of all ports in the installed tree", "CmdListExample2": "vcpkg list ", "_CmdListExample2.comment": "This is a command line, only the part should be localized", "CmdNewExample1": "vcpkg new --name=example --version=1.0", @@ -970,6 +967,7 @@ "InstalledBy": "Installed by {path}", "_InstalledBy.comment": "An example of {path} is /foo/bar.", "InstalledPackages": "The following packages are already installed:", + "InstalledPackagesHead": "The following packages are already installed, but were requested at --head version. Their installed contents will not be changed. To get updated versions, remove these packages first:", "InstalledRequestedPackages": "All requested packages are currently installed.", "InstallingMavenFileFailure": "{path} installing Maven file, {command_line} failed with {exit_code}", "_InstallingMavenFileFailure.comment": "Printed after a maven install command fails An example of {path} is /foo/bar. An example of {command_line} is vcpkg install zlib. An example of {exit_code} is 127.", @@ -1181,6 +1179,7 @@ "NoError": "no error", "NoInstalledPackages": "No packages are installed. Did you mean `search`?", "_NoInstalledPackages.comment": "The name 'search' is the name of a command that is not localized.", + "NoInstalledPackagesLicenseReport": "There are no installed packages, and thus no licenses of installed packages. Did you mean to install something first?", "NoOutdatedPackages": "There are no outdated packages.", "NoRegistryForPort": "no registry configured for port {package_name}", "_NoRegistryForPort.comment": "An example of {package_name} is zlib.", @@ -1246,6 +1245,10 @@ "_PackageFailedtWhileExtracting.comment": "'{value}' is either a tool name or a package name. An example of {path} is /foo/bar.", "PackageInfoHelp": "Display detailed information on packages", "PackageInstallationHeader": "Package Installation", + "PackageLicenseSpdx": "Installed packages declare the following licenses:", + "PackageLicenseSpdxThisInstall": "Packages installed in this vcpkg installation declare the following licenses:", + "PackageLicenseUnknown": "Some packages did not declare an SPDX license. Check the `copyright` file for each package for more information about their licensing.", + "PackageLicenseWarning": "Installed contents are licensed to you by owners. Microsoft is not responsible for, nor does it grant any licenses to, third-party packages.", "PackageManipulationHeader": "Package Manipulation", "PackageRootDir": "Packages directory (experimental)", "PackagesToInstall": "The following packages will be built and installed:", @@ -1432,6 +1435,8 @@ "_ToolOfVersionXNotFound.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", "TotalInstallTime": "Total install time: {elapsed}", "_TotalInstallTime.comment": "An example of {elapsed} is 3.532 min.", + "TotalInstallTimeSuccess": "All requested installations completed successfully in: {elapsed}", + "_TotalInstallTimeSuccess.comment": "An example of {elapsed} is 3.532 min.", "TrailingCommaInArray": "Trailing comma in array", "TrailingCommaInObj": "Trailing comma in an object", "TripletFileNotFound": "Triplet file {triplet}.cmake not found", diff --git a/src/vcpkg/commands.cpp b/src/vcpkg/commands.cpp index d70ab16afb..8832bb5ff5 100644 --- a/src/vcpkg/commands.cpp +++ b/src/vcpkg/commands.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -92,6 +93,7 @@ namespace vcpkg {CommandFormatManifestMetadata, command_format_manifest_and_exit}, {CommandHelpMetadata, command_help_and_exit}, {CommandIntegrateMetadata, command_integrate_and_exit}, + {CommandLicenseReportMetadata, command_license_report_and_exit}, {CommandListMetadata, command_list_and_exit}, {CommandNewMetadata, command_new_and_exit}, {CommandOwnsMetadata, command_owns_and_exit}, diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index e07507b28c..d82f4376b8 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -315,6 +315,27 @@ namespace vcpkg return InstallResult::SUCCESS; } + void LicenseReport::print_license_report(const msg::MessageT<>& named_license_heading) const + { + if (any_unknown_licenses || !named_licenses.empty()) + { + msg::println(msgPackageLicenseWarning); + if (any_unknown_licenses) + { + msg::println(msgPackageLicenseUnknown); + } + + if (!named_licenses.empty()) + { + msg::println(named_license_heading); + for (auto&& license : named_licenses) + { + msg::print(LocalizedString::from_raw(license).append_raw('\n')); + } + } + } + } + static ExtendedBuildResult perform_install_plan_action(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet host_triplet, @@ -588,6 +609,17 @@ namespace vcpkg args, paths, host_triplet, build_options, action, status_db, binary_cache, build_logs_recorder); if (result.code == BuildResult::Succeeded) { + if (auto scfl = action.source_control_file_and_location.get()) + { + if (auto license = scfl->source_control_file->core_paragraph->license.get()) + { + summary.license_report.named_licenses.insert(*license); + } + else + { + summary.license_report.any_unknown_licenses = true; + } + } } else { @@ -1403,6 +1435,8 @@ namespace vcpkg fs.write_contents(it_xunit->second, xwriter.build_xml(default_triplet), VCPKG_LINE_INFO); } + summary.license_report.print_license_report(msgPackageLicenseSpdxThisInstall); + if (print_cmake_usage) { std::set printed_usages; diff --git a/src/vcpkg/commands.license-report.cpp b/src/vcpkg/commands.license-report.cpp new file mode 100644 index 0000000000..4add1032d7 --- /dev/null +++ b/src/vcpkg/commands.license-report.cpp @@ -0,0 +1,64 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace vcpkg; + +namespace vcpkg +{ + + constexpr CommandMetadata CommandLicenseReportMetadata{ + "license-report", + msgCmdLicenseReportSynopsis, + {"vcpkg license-report"}, + Undocumented, + AutocompletePriority::Public, + 0, + 0, + {}, + nullptr, + }; + + void command_license_report_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths) + { + (void)args.parse_arguments(CommandLicenseReportMetadata); + + auto&& fs = paths.get_filesystem(); + auto&& installed_paths = paths.installed(); + LicenseReport report; + auto status_paragraphs = database_load(fs, installed_paths); + auto installed_ipvs = get_installed_ports(status_paragraphs); + if (installed_ipvs.empty()) + { + msg::println(msgNoInstalledPackagesLicenseReport); + Checks::exit_success(VCPKG_LINE_INFO); + } + + for (auto&& installed_ipv : installed_ipvs) + { + auto spdx_file = installed_paths.spdx_file(installed_ipv.spec()); + auto maybe_spdx_content = fs.try_read_contents(spdx_file); + if (auto spdx_content = maybe_spdx_content.get()) + { + auto maybe_parsed_license = read_spdx_license(spdx_content->content, spdx_content->origin); + if (auto parsed_license = maybe_parsed_license.get()) + { + report.named_licenses.insert(*parsed_license); + continue; + } + } + + report.any_unknown_licenses = true; + } + + report.print_license_report(msgPackageLicenseSpdx); + Checks::exit_success(VCPKG_LINE_INFO); + } +} // namespace vcpkg diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index c8b0f7114f..ad607df092 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -268,6 +268,8 @@ namespace vcpkg } } + summary.license_report.print_license_report(msgPackageLicenseSpdxThisInstall); + if (print_usage == PrintUsage::Yes) { // Note that this differs from the behavior of `vcpkg install` in that it will print usage information for From a0ac8fe717ad268321f9dfcf1c3b225fb125fcdf Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 17 Oct 2024 17:43:11 -0700 Subject: [PATCH 13/14] Fix linux and macos build. --- src/vcpkg-test/spdx.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vcpkg-test/spdx.cpp b/src/vcpkg-test/spdx.cpp index e691fa0170..b6da7cfdce 100644 --- a/src/vcpkg-test/spdx.cpp +++ b/src/vcpkg-test/spdx.cpp @@ -727,7 +727,7 @@ TEST_CASE ("spdx license parse edge cases", "[spdx]") ] })json"; - CHECK(!read_spdx_license(wrong_packages_type, "test").has_value()); + CHECK(!read_spdx_license(wrong_packages_zero_type, "test").has_value()); static constexpr StringLiteral missing_license_block = R"json( { From 43bf3d13d7dd2da986f67f4dba754fd824347e8e Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 24 Oct 2024 12:59:01 -0700 Subject: [PATCH 14/14] Fix issues found in extractions. --- include/vcpkg/base/contractual-constants.h | 7 ++++--- src/vcpkg/export.prefab.cpp | 2 +- src/vcpkg/postbuildlint.cpp | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h index cc22195ffb..61f731ca11 100644 --- a/include/vcpkg/base/contractual-constants.h +++ b/include/vcpkg/base/contractual-constants.h @@ -330,15 +330,17 @@ namespace vcpkg // File names inline constexpr StringLiteral FileBaselineDotJson = "baseline.json"; inline constexpr StringLiteral FileBin = "bin"; - inline constexpr StringLiteral FileCopyright = "copyright"; - inline constexpr StringLiteral FileLicense = "LICENSE"; inline constexpr StringLiteral FileControl = "CONTROL"; + inline constexpr StringLiteral FileCopying = "COPYING"; + inline constexpr StringLiteral FileCopyright = "copyright"; inline constexpr StringLiteral FileDebug = "debug"; inline constexpr StringLiteral FileDetectCompiler = "detect_compiler"; inline constexpr StringLiteral FileDotDsStore = ".DS_Store"; inline constexpr StringLiteral FileInclude = "include"; inline constexpr StringLiteral FileIncomplete = "incomplete"; inline constexpr StringLiteral FileInfo = "info"; + inline constexpr StringLiteral FileLicense = "LICENSE"; + inline constexpr StringLiteral FileLicenseDotTxt = "LICENSE.txt"; inline constexpr StringLiteral FilePortfileDotCMake = "portfile.cmake"; inline constexpr StringLiteral FileShare = "share"; inline constexpr StringLiteral FileStatus = "status"; @@ -581,5 +583,4 @@ namespace vcpkg inline constexpr StringLiteral StatusInstalled = "installed"; inline constexpr StringLiteral StatusNotInstalled = "not-installed"; inline constexpr StringLiteral StatusPurge = "purge"; - inline constexpr StringLiteral StatusUnknown = "unknown"; } diff --git a/src/vcpkg/export.prefab.cpp b/src/vcpkg/export.prefab.cpp index fe9ca1e044..64fe9998a7 100644 --- a/src/vcpkg/export.prefab.cpp +++ b/src/vcpkg/export.prefab.cpp @@ -433,7 +433,7 @@ namespace vcpkg::Prefab const auto share_root = paths.packages() / fmt::format("{}_{}", name, action.spec.triplet()); - fs.copy_file(share_root / FileVcpkgPortConfig / name / FileCopyright, + fs.copy_file(share_root / FileShare / name / FileCopyright, meta_dir / FileLicense, CopyOptions::overwrite_existing, IgnoreErrors{}); diff --git a/src/vcpkg/postbuildlint.cpp b/src/vcpkg/postbuildlint.cpp index ce085bd416..a11e6131e7 100644 --- a/src/vcpkg/postbuildlint.cpp +++ b/src/vcpkg/postbuildlint.cpp @@ -467,7 +467,7 @@ namespace vcpkg const Path& portfile_cmake, MessageSink& msg_sink) { - static constexpr StringLiteral copyright_filenames[] = {"COPYING", FileLicense, "LICENSE.txt"}; + static constexpr StringLiteral copyright_filenames[] = {FileCopying, FileLicense, FileLicenseDotTxt}; const auto copyright_file = package_dir / FileShare / spec_name / FileCopyright; switch (fs.status(copyright_file, IgnoreErrors{}))