From 2c6129f3826aaeb9083c761f515df58ff441031b Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 10 Sep 2024 20:11:45 -0400 Subject: [PATCH 01/35] no fmt :/ --- CMakeLists.txt | 4 +++ wpilogcli/CMakeLists.txt | 32 +++++++++++++++++++ .../src/main/generate/WPILibVersion.cpp.in | 7 ++++ wpilogcli/src/main/native/cpp/main.cpp | 29 +++++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 wpilogcli/CMakeLists.txt create mode 100644 wpilogcli/src/main/generate/WPILibVersion.cpp.in create mode 100644 wpilogcli/src/main/native/cpp/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b88f9e010a..89dfa37c566 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -300,6 +300,10 @@ if(WITH_NTCORE) add_subdirectory(ntcore) endif() +add_subdirectory(wpilogcli) + +add_subdirectory(protoplugin) + if(WITH_WPIMATH) if(WITH_JAVA) set(WPIUNITS_DEP_REPLACE ${WPIUNITS_DEP_REPLACE_IMPL}) diff --git a/wpilogcli/CMakeLists.txt b/wpilogcli/CMakeLists.txt new file mode 100644 index 00000000000..b4c7ee8a746 --- /dev/null +++ b/wpilogcli/CMakeLists.txt @@ -0,0 +1,32 @@ +project(wpilogcli) + +include(CompileWarnings) +include(GenResources) + +configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) +generate_resources(src/main/native/resources generated/main/cpp DLT dlt wpilogcli_resources_src) + +file(GLOB wpilogcli_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) + +if(WIN32) + set(wpilogcli_rc src/main/native/win/wpilogcli.rc) +elseif(APPLE) + set(MACOSX_BUNDLE_ICON_FILE wpilogcli.icns) + set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") +endif() + +add_executable( + wpilogcli + ${wpilogcli_src} + ${wpilogcli_resources_src} + ${wpilogcli_rc} + ${APP_ICON_MACOSX} +) + +target_link_libraries(wpilogcli PRIVATE wpiutil) + +if(WIN32) + set_target_properties(wpilogcli PROPERTIES WIN32_EXECUTABLE YES) +elseif(APPLE) + set_target_properties(wpilogcli PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "wpilogcli") +endif() \ No newline at end of file diff --git a/wpilogcli/src/main/generate/WPILibVersion.cpp.in b/wpilogcli/src/main/generate/WPILibVersion.cpp.in new file mode 100644 index 00000000000..d4bc735e071 --- /dev/null +++ b/wpilogcli/src/main/generate/WPILibVersion.cpp.in @@ -0,0 +1,7 @@ +/** + * Autogenerated file! Do not manually edit this file. This version is regenerated + * any time the publish task is run, or when this file is deleted. + */ +const char* GetWPILibVersion() { + return "${wpilib_version}"; +} \ No newline at end of file diff --git a/wpilogcli/src/main/native/cpp/main.cpp b/wpilogcli/src/main/native/cpp/main.cpp new file mode 100644 index 00000000000..ccddaa68a9b --- /dev/null +++ b/wpilogcli/src/main/native/cpp/main.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + +void show(std::string_view message_type, bool raw) { + +} + +void list() { + +} + +int main(int argc, char *argv[]) { + if (argc > 1) { // + std::string_view command{argv[1]}; + std::string_view message_type{}; + + if (command.compare("json")) { // + + } else if (command.compare("csv")) { + + } else if (command.compare("extract")) { + + } + } else { + // no args given + // report error + } +} \ No newline at end of file From 1bd3ee337def2617b4287af98e4535e64226563a Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 10 Sep 2024 23:37:56 -0400 Subject: [PATCH 02/35] add argparse --- .../src/main/native/cpp/argparse/argparse.hpp | 2543 +++++++++++++++++ 1 file changed, 2543 insertions(+) create mode 100644 wpilogcli/src/main/native/cpp/argparse/argparse.hpp diff --git a/wpilogcli/src/main/native/cpp/argparse/argparse.hpp b/wpilogcli/src/main/native/cpp/argparse/argparse.hpp new file mode 100644 index 00000000000..0843613bab8 --- /dev/null +++ b/wpilogcli/src/main/native/cpp/argparse/argparse.hpp @@ -0,0 +1,2543 @@ +/* + __ _ _ __ __ _ _ __ __ _ _ __ ___ ___ + / _` | '__/ _` | '_ \ / _` | '__/ __|/ _ \ Argument Parser for Modern C++ +| (_| | | | (_| | |_) | (_| | | \__ \ __/ http://github.com/p-ranav/argparse + \__,_|_| \__, | .__/ \__,_|_| |___/\___| + |___/|_| + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2019-2022 Pranav Srinivas Kumar +and other contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ +#pragma once + +#include + +#ifndef ARGPARSE_MODULE_USE_STD_MODULE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#ifndef ARGPARSE_CUSTOM_STRTOF +#define ARGPARSE_CUSTOM_STRTOF strtof +#endif + +#ifndef ARGPARSE_CUSTOM_STRTOD +#define ARGPARSE_CUSTOM_STRTOD strtod +#endif + +#ifndef ARGPARSE_CUSTOM_STRTOLD +#define ARGPARSE_CUSTOM_STRTOLD strtold +#endif + +namespace argparse { + +namespace details { // namespace for helper methods + +template +struct HasContainerTraits : std::false_type {}; + +template <> struct HasContainerTraits : std::false_type {}; + +template <> struct HasContainerTraits : std::false_type {}; + +template +struct HasContainerTraits< + T, std::void_t().begin()), + decltype(std::declval().end()), + decltype(std::declval().size())>> : std::true_type {}; + +template +inline constexpr bool IsContainer = HasContainerTraits::value; + +template +struct HasStreamableTraits : std::false_type {}; + +template +struct HasStreamableTraits< + T, + std::void_t() << std::declval())>> + : std::true_type {}; + +template +inline constexpr bool IsStreamable = HasStreamableTraits::value; + +constexpr std::size_t repr_max_container_size = 5; + +template std::string repr(T const &val) { + if constexpr (std::is_same_v) { + return val ? "true" : "false"; + } else if constexpr (std::is_convertible_v) { + return '"' + std::string{std::string_view{val}} + '"'; + } else if constexpr (IsContainer) { + std::stringstream out; + out << "{"; + const auto size = val.size(); + if (size > 1) { + out << repr(*val.begin()); + std::for_each( + std::next(val.begin()), + std::next( + val.begin(), + static_cast( + std::min(size, repr_max_container_size) - 1)), + [&out](const auto &v) { out << " " << repr(v); }); + if (size <= repr_max_container_size) { + out << " "; + } else { + out << "..."; + } + } + if (size > 0) { + out << repr(*std::prev(val.end())); + } + out << "}"; + return out.str(); + } else if constexpr (IsStreamable) { + std::stringstream out; + out << val; + return out.str(); + } else { + return ""; + } +} + +namespace { + +template constexpr bool standard_signed_integer = false; +template <> constexpr bool standard_signed_integer = true; +template <> constexpr bool standard_signed_integer = true; +template <> constexpr bool standard_signed_integer = true; +template <> constexpr bool standard_signed_integer = true; +template <> constexpr bool standard_signed_integer = true; + +template constexpr bool standard_unsigned_integer = false; +template <> constexpr bool standard_unsigned_integer = true; +template <> constexpr bool standard_unsigned_integer = true; +template <> constexpr bool standard_unsigned_integer = true; +template <> constexpr bool standard_unsigned_integer = true; +template <> +constexpr bool standard_unsigned_integer = true; + +} // namespace + +constexpr int radix_2 = 2; +constexpr int radix_8 = 8; +constexpr int radix_10 = 10; +constexpr int radix_16 = 16; + +template +constexpr bool standard_integer = + standard_signed_integer || standard_unsigned_integer; + +template +constexpr decltype(auto) +apply_plus_one_impl(F &&f, Tuple &&t, Extra &&x, + std::index_sequence /*unused*/) { + return std::invoke(std::forward(f), std::get(std::forward(t))..., + std::forward(x)); +} + +template +constexpr decltype(auto) apply_plus_one(F &&f, Tuple &&t, Extra &&x) { + return details::apply_plus_one_impl( + std::forward(f), std::forward(t), std::forward(x), + std::make_index_sequence< + std::tuple_size_v>>{}); +} + +constexpr auto pointer_range(std::string_view s) noexcept { + return std::tuple(s.data(), s.data() + s.size()); +} + +template +constexpr bool starts_with(std::basic_string_view prefix, + std::basic_string_view s) noexcept { + return s.substr(0, prefix.size()) == prefix; +} + +enum class chars_format { + scientific = 0xf1, + fixed = 0xf2, + hex = 0xf4, + binary = 0xf8, + general = fixed | scientific +}; + +struct ConsumeBinaryPrefixResult { + bool is_binary; + std::string_view rest; +}; + +constexpr auto consume_binary_prefix(std::string_view s) + -> ConsumeBinaryPrefixResult { + if (starts_with(std::string_view{"0b"}, s) || + starts_with(std::string_view{"0B"}, s)) { + s.remove_prefix(2); + return {true, s}; + } + return {false, s}; +} + +struct ConsumeHexPrefixResult { + bool is_hexadecimal; + std::string_view rest; +}; + +using namespace std::literals; + +constexpr auto consume_hex_prefix(std::string_view s) + -> ConsumeHexPrefixResult { + if (starts_with("0x"sv, s) || starts_with("0X"sv, s)) { + s.remove_prefix(2); + return {true, s}; + } + return {false, s}; +} + +template +inline auto do_from_chars(std::string_view s) -> T { + T x{0}; + auto [first, last] = pointer_range(s); + auto [ptr, ec] = std::from_chars(first, last, x, Param); + if (ec == std::errc()) { + if (ptr == last) { + return x; + } + throw std::invalid_argument{"pattern '" + std::string(s) + + "' does not match to the end"}; + } + if (ec == std::errc::invalid_argument) { + throw std::invalid_argument{"pattern '" + std::string(s) + "' not found"}; + } + if (ec == std::errc::result_out_of_range) { + throw std::range_error{"'" + std::string(s) + "' not representable"}; + } + return x; // unreachable +} + +template struct parse_number { + auto operator()(std::string_view s) -> T { + return do_from_chars(s); + } +}; + +template struct parse_number { + auto operator()(std::string_view s) -> T { + if (auto [ok, rest] = consume_binary_prefix(s); ok) { + return do_from_chars(rest); + } + throw std::invalid_argument{"pattern not found"}; + } +}; + +template struct parse_number { + auto operator()(std::string_view s) -> T { + if (starts_with("0x"sv, s) || starts_with("0X"sv, s)) { + if (auto [ok, rest] = consume_hex_prefix(s); ok) { + try { + return do_from_chars(rest); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + std::string(s) + + "' as hexadecimal: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + std::string(s) + + "' as hexadecimal: " + err.what()); + } + } + } else { + // Allow passing hex numbers without prefix + // Shape 'x' already has to be specified + try { + return do_from_chars(s); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + std::string(s) + + "' as hexadecimal: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + std::string(s) + + "' as hexadecimal: " + err.what()); + } + } + + throw std::invalid_argument{"pattern '" + std::string(s) + + "' not identified as hexadecimal"}; + } +}; + +template struct parse_number { + auto operator()(std::string_view s) -> T { + auto [ok, rest] = consume_hex_prefix(s); + if (ok) { + try { + return do_from_chars(rest); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + std::string(s) + + "' as hexadecimal: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + std::string(s) + + "' as hexadecimal: " + err.what()); + } + } + + auto [ok_binary, rest_binary] = consume_binary_prefix(s); + if (ok_binary) { + try { + return do_from_chars(rest_binary); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + std::string(s) + + "' as binary: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + std::string(s) + + "' as binary: " + err.what()); + } + } + + if (starts_with("0"sv, s)) { + try { + return do_from_chars(rest); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + std::string(s) + + "' as octal: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + std::string(s) + + "' as octal: " + err.what()); + } + } + + try { + return do_from_chars(rest); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + std::string(s) + + "' as decimal integer: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + std::string(s) + + "' as decimal integer: " + err.what()); + } + } +}; + +namespace { + +template inline const auto generic_strtod = nullptr; +template <> inline const auto generic_strtod = ARGPARSE_CUSTOM_STRTOF; +template <> inline const auto generic_strtod = ARGPARSE_CUSTOM_STRTOD; +template <> +inline const auto generic_strtod = ARGPARSE_CUSTOM_STRTOLD; + +} // namespace + +template inline auto do_strtod(std::string const &s) -> T { + if (isspace(static_cast(s[0])) || s[0] == '+') { + throw std::invalid_argument{"pattern '" + s + "' not found"}; + } + + auto [first, last] = pointer_range(s); + char *ptr; + + errno = 0; + auto x = generic_strtod(first, &ptr); + if (errno == 0) { + if (ptr == last) { + return x; + } + throw std::invalid_argument{"pattern '" + s + + "' does not match to the end"}; + } + if (errno == ERANGE) { + throw std::range_error{"'" + s + "' not representable"}; + } + return x; // unreachable +} + +template struct parse_number { + auto operator()(std::string const &s) -> T { + if (auto r = consume_hex_prefix(s); r.is_hexadecimal) { + throw std::invalid_argument{ + "chars_format::general does not parse hexfloat"}; + } + if (auto r = consume_binary_prefix(s); r.is_binary) { + throw std::invalid_argument{ + "chars_format::general does not parse binfloat"}; + } + + try { + return do_strtod(s); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + s + + "' as number: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + s + + "' as number: " + err.what()); + } + } +}; + +template struct parse_number { + auto operator()(std::string const &s) -> T { + if (auto r = consume_hex_prefix(s); !r.is_hexadecimal) { + throw std::invalid_argument{"chars_format::hex parses hexfloat"}; + } + if (auto r = consume_binary_prefix(s); r.is_binary) { + throw std::invalid_argument{"chars_format::hex does not parse binfloat"}; + } + + try { + return do_strtod(s); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + s + + "' as hexadecimal: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + s + + "' as hexadecimal: " + err.what()); + } + } +}; + +template struct parse_number { + auto operator()(std::string const &s) -> T { + if (auto r = consume_hex_prefix(s); r.is_hexadecimal) { + throw std::invalid_argument{ + "chars_format::binary does not parse hexfloat"}; + } + if (auto r = consume_binary_prefix(s); !r.is_binary) { + throw std::invalid_argument{"chars_format::binary parses binfloat"}; + } + + return do_strtod(s); + } +}; + +template struct parse_number { + auto operator()(std::string const &s) -> T { + if (auto r = consume_hex_prefix(s); r.is_hexadecimal) { + throw std::invalid_argument{ + "chars_format::scientific does not parse hexfloat"}; + } + if (auto r = consume_binary_prefix(s); r.is_binary) { + throw std::invalid_argument{ + "chars_format::scientific does not parse binfloat"}; + } + if (s.find_first_of("eE") == std::string::npos) { + throw std::invalid_argument{ + "chars_format::scientific requires exponent part"}; + } + + try { + return do_strtod(s); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + s + + "' as scientific notation: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + s + + "' as scientific notation: " + err.what()); + } + } +}; + +template struct parse_number { + auto operator()(std::string const &s) -> T { + if (auto r = consume_hex_prefix(s); r.is_hexadecimal) { + throw std::invalid_argument{ + "chars_format::fixed does not parse hexfloat"}; + } + if (auto r = consume_binary_prefix(s); r.is_binary) { + throw std::invalid_argument{ + "chars_format::fixed does not parse binfloat"}; + } + if (s.find_first_of("eE") != std::string::npos) { + throw std::invalid_argument{ + "chars_format::fixed does not parse exponent part"}; + } + + try { + return do_strtod(s); + } catch (const std::invalid_argument &err) { + throw std::invalid_argument("Failed to parse '" + s + + "' as fixed notation: " + err.what()); + } catch (const std::range_error &err) { + throw std::range_error("Failed to parse '" + s + + "' as fixed notation: " + err.what()); + } + } +}; + +template +std::string join(StrIt first, StrIt last, const std::string &separator) { + if (first == last) { + return ""; + } + std::stringstream value; + value << *first; + ++first; + while (first != last) { + value << separator << *first; + ++first; + } + return value.str(); +} + +template struct can_invoke_to_string { + template + static auto test(int) + -> decltype(std::to_string(std::declval()), std::true_type{}); + + template static auto test(...) -> std::false_type; + + static constexpr bool value = decltype(test(0))::value; +}; + +template struct IsChoiceTypeSupported { + using CleanType = typename std::decay::type; + static const bool value = std::is_integral::value || + std::is_same::value || + std::is_same::value || + std::is_same::value; +}; + +template +std::size_t get_levenshtein_distance(const StringType &s1, + const StringType &s2) { + std::vector> dp( + s1.size() + 1, std::vector(s2.size() + 1, 0)); + + for (std::size_t i = 0; i <= s1.size(); ++i) { + for (std::size_t j = 0; j <= s2.size(); ++j) { + if (i == 0) { + dp[i][j] = j; + } else if (j == 0) { + dp[i][j] = i; + } else if (s1[i - 1] == s2[j - 1]) { + dp[i][j] = dp[i - 1][j - 1]; + } else { + dp[i][j] = 1 + std::min({dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]}); + } + } + } + + return dp[s1.size()][s2.size()]; +} + +template +std::string get_most_similar_string(const std::map &map, + const std::string &input) { + std::string most_similar{}; + std::size_t min_distance = (std::numeric_limits::max)(); + + for (const auto &entry : map) { + std::size_t distance = get_levenshtein_distance(entry.first, input); + if (distance < min_distance) { + min_distance = distance; + most_similar = entry.first; + } + } + + return most_similar; +} + +} // namespace details + +enum class nargs_pattern { optional, any, at_least_one }; + +enum class default_arguments : unsigned int { + none = 0, + help = 1, + version = 2, + all = help | version, +}; + +inline default_arguments operator&(const default_arguments &a, + const default_arguments &b) { + return static_cast( + static_cast::type>(a) & + static_cast::type>(b)); +} + +class ArgumentParser; + +class Argument { + friend class ArgumentParser; + friend auto operator<<(std::ostream &stream, const ArgumentParser &parser) + -> std::ostream &; + + template + explicit Argument(std::string_view prefix_chars, + std::array &&a, + std::index_sequence /*unused*/) + : m_accepts_optional_like_value(false), + m_is_optional((is_optional(a[I], prefix_chars) || ...)), + m_is_required(false), m_is_repeatable(false), m_is_used(false), + m_is_hidden(false), m_prefix_chars(prefix_chars) { + ((void)m_names.emplace_back(a[I]), ...); + std::sort( + m_names.begin(), m_names.end(), [](const auto &lhs, const auto &rhs) { + return lhs.size() == rhs.size() ? lhs < rhs : lhs.size() < rhs.size(); + }); + } + +public: + template + explicit Argument(std::string_view prefix_chars, + std::array &&a) + : Argument(prefix_chars, std::move(a), std::make_index_sequence{}) {} + + Argument &help(std::string help_text) { + m_help = std::move(help_text); + return *this; + } + + Argument &metavar(std::string metavar) { + m_metavar = std::move(metavar); + return *this; + } + + template Argument &default_value(T &&value) { + m_num_args_range = NArgsRange{0, m_num_args_range.get_max()}; + m_default_value_repr = details::repr(value); + + if constexpr (std::is_convertible_v) { + m_default_value_str = std::string{std::string_view{value}}; + } else if constexpr (details::can_invoke_to_string::value) { + m_default_value_str = std::to_string(value); + } + + m_default_value = std::forward(value); + return *this; + } + + Argument &default_value(const char *value) { + return default_value(std::string(value)); + } + + Argument &required() { + m_is_required = true; + return *this; + } + + Argument &implicit_value(std::any value) { + m_implicit_value = std::move(value); + m_num_args_range = NArgsRange{0, 0}; + return *this; + } + + // This is shorthand for: + // program.add_argument("foo") + // .default_value(false) + // .implicit_value(true) + Argument &flag() { + default_value(false); + implicit_value(true); + return *this; + } + + template + auto action(F &&callable, Args &&... bound_args) + -> std::enable_if_t, + Argument &> { + using action_type = std::conditional_t< + std::is_void_v>, + void_action, valued_action>; + if constexpr (sizeof...(Args) == 0) { + m_action.emplace(std::forward(callable)); + } else { + m_action.emplace( + [f = std::forward(callable), + tup = std::make_tuple(std::forward(bound_args)...)]( + std::string const &opt) mutable { + return details::apply_plus_one(f, tup, opt); + }); + } + return *this; + } + + auto &store_into(bool &var) { + flag(); + if (m_default_value.has_value()) { + var = std::any_cast(m_default_value); + } + action([&var](const auto & /*unused*/) { var = true; }); + return *this; + } + + template ::value>::type * = nullptr> + auto &store_into(T &var) { + if (m_default_value.has_value()) { + var = std::any_cast(m_default_value); + } + action([&var](const auto &s) { + var = details::parse_number()(s); + }); + return *this; + } + + auto &store_into(double &var) { + if (m_default_value.has_value()) { + var = std::any_cast(m_default_value); + } + action([&var](const auto &s) { + var = details::parse_number()(s); + }); + return *this; + } + + auto &store_into(std::string &var) { + if (m_default_value.has_value()) { + var = std::any_cast(m_default_value); + } + action([&var](const std::string &s) { var = s; }); + return *this; + } + + auto &store_into(std::vector &var) { + if (m_default_value.has_value()) { + var = std::any_cast>(m_default_value); + } + action([this, &var](const std::string &s) { + if (!m_is_used) { + var.clear(); + } + m_is_used = true; + var.push_back(s); + }); + return *this; + } + + auto &store_into(std::vector &var) { + if (m_default_value.has_value()) { + var = std::any_cast>(m_default_value); + } + action([this, &var](const std::string &s) { + if (!m_is_used) { + var.clear(); + } + m_is_used = true; + var.push_back(details::parse_number()(s)); + }); + return *this; + } + + auto &store_into(std::set &var) { + if (m_default_value.has_value()) { + var = std::any_cast>(m_default_value); + } + action([this, &var](const std::string &s) { + if (!m_is_used) { + var.clear(); + } + m_is_used = true; + var.insert(s); + }); + return *this; + } + + auto &store_into(std::set &var) { + if (m_default_value.has_value()) { + var = std::any_cast>(m_default_value); + } + action([this, &var](const std::string &s) { + if (!m_is_used) { + var.clear(); + } + m_is_used = true; + var.insert(details::parse_number()(s)); + }); + return *this; + } + + auto &append() { + m_is_repeatable = true; + return *this; + } + + // Cause the argument to be invisible in usage and help + auto &hidden() { + m_is_hidden = true; + return *this; + } + + template + auto scan() -> std::enable_if_t, Argument &> { + static_assert(!(std::is_const_v || std::is_volatile_v), + "T should not be cv-qualified"); + auto is_one_of = [](char c, auto... x) constexpr { + return ((c == x) || ...); + }; + + if constexpr (is_one_of(Shape, 'd') && details::standard_integer) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'i') && + details::standard_integer) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'u') && + details::standard_unsigned_integer) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'b') && + details::standard_unsigned_integer) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'o') && + details::standard_unsigned_integer) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'x', 'X') && + details::standard_unsigned_integer) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'a', 'A') && + std::is_floating_point_v) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'e', 'E') && + std::is_floating_point_v) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'f', 'F') && + std::is_floating_point_v) { + action(details::parse_number()); + } else if constexpr (is_one_of(Shape, 'g', 'G') && + std::is_floating_point_v) { + action(details::parse_number()); + } else { + static_assert(alignof(T) == 0, "No scan specification for T"); + } + + return *this; + } + + Argument &nargs(std::size_t num_args) { + m_num_args_range = NArgsRange{num_args, num_args}; + return *this; + } + + Argument &nargs(std::size_t num_args_min, std::size_t num_args_max) { + m_num_args_range = NArgsRange{num_args_min, num_args_max}; + return *this; + } + + Argument &nargs(nargs_pattern pattern) { + switch (pattern) { + case nargs_pattern::optional: + m_num_args_range = NArgsRange{0, 1}; + break; + case nargs_pattern::any: + m_num_args_range = + NArgsRange{0, (std::numeric_limits::max)()}; + break; + case nargs_pattern::at_least_one: + m_num_args_range = + NArgsRange{1, (std::numeric_limits::max)()}; + break; + } + return *this; + } + + Argument &remaining() { + m_accepts_optional_like_value = true; + return nargs(nargs_pattern::any); + } + + template void add_choice(T &&choice) { + static_assert(details::IsChoiceTypeSupported::value, + "Only string or integer type supported for choice"); + static_assert(std::is_convertible_v || + details::can_invoke_to_string::value, + "Choice is not convertible to string_type"); + if (!m_choices.has_value()) { + m_choices = std::vector{}; + } + + if constexpr (std::is_convertible_v) { + m_choices.value().push_back( + std::string{std::string_view{std::forward(choice)}}); + } else if constexpr (details::can_invoke_to_string::value) { + m_choices.value().push_back(std::to_string(std::forward(choice))); + } + } + + Argument &choices() { + if (!m_choices.has_value()) { + throw std::runtime_error("Zero choices provided"); + } + return *this; + } + + template + Argument &choices(T &&first, U &&... rest) { + add_choice(std::forward(first)); + choices(std::forward(rest)...); + return *this; + } + + void find_default_value_in_choices_or_throw() const { + + const auto &choices = m_choices.value(); + + if (m_default_value.has_value()) { + if (std::find(choices.begin(), choices.end(), m_default_value_str) == + choices.end()) { + // provided arg not in list of allowed choices + // report error + + std::string choices_as_csv = + std::accumulate(choices.begin(), choices.end(), std::string(), + [](const std::string &a, const std::string &b) { + return a + (a.empty() ? "" : ", ") + b; + }); + + throw std::runtime_error( + std::string{"Invalid default value "} + m_default_value_repr + + " - allowed options: {" + choices_as_csv + "}"); + } + } + } + + template + void find_value_in_choices_or_throw(Iterator it) const { + + const auto &choices = m_choices.value(); + + if (std::find(choices.begin(), choices.end(), *it) == choices.end()) { + // provided arg not in list of allowed choices + // report error + + std::string choices_as_csv = + std::accumulate(choices.begin(), choices.end(), std::string(), + [](const std::string &a, const std::string &b) { + return a + (a.empty() ? "" : ", ") + b; + }); + + throw std::runtime_error(std::string{"Invalid argument "} + + details::repr(*it) + " - allowed options: {" + + choices_as_csv + "}"); + } + } + + /* The dry_run parameter can be set to true to avoid running the actions, + * and setting m_is_used. This may be used by a pre-processing step to do + * a first iteration over arguments. + */ + template + Iterator consume(Iterator start, Iterator end, + std::string_view used_name = {}, bool dry_run = false) { + if (!m_is_repeatable && m_is_used) { + throw std::runtime_error( + std::string("Duplicate argument ").append(used_name)); + } + m_used_name = used_name; + + if (m_choices.has_value()) { + // Check each value in (start, end) and make sure + // it is in the list of allowed choices/options + std::size_t i = 0; + auto max_number_of_args = m_num_args_range.get_max(); + for (auto it = start; it != end; ++it) { + if (i == max_number_of_args) { + break; + } + find_value_in_choices_or_throw(it); + i += 1; + } + } + + const auto num_args_max = m_num_args_range.get_max(); + const auto num_args_min = m_num_args_range.get_min(); + std::size_t dist = 0; + if (num_args_max == 0) { + if (!dry_run) { + m_values.emplace_back(m_implicit_value); + std::visit([](const auto &f) { f({}); }, m_action); + m_is_used = true; + } + return start; + } + if ((dist = static_cast(std::distance(start, end))) >= + num_args_min) { + if (num_args_max < dist) { + end = std::next(start, static_cast( + num_args_max)); + } + if (!m_accepts_optional_like_value) { + end = std::find_if( + start, end, + std::bind(is_optional, std::placeholders::_1, m_prefix_chars)); + dist = static_cast(std::distance(start, end)); + if (dist < num_args_min) { + throw std::runtime_error("Too few arguments for '" + + std::string(m_used_name) + "'."); + } + } + + struct ActionApply { + void operator()(valued_action &f) { + std::transform(first, last, std::back_inserter(self.m_values), f); + } + + void operator()(void_action &f) { + std::for_each(first, last, f); + if (!self.m_default_value.has_value()) { + if (!self.m_accepts_optional_like_value) { + self.m_values.resize( + static_cast(std::distance(first, last))); + } + } + } + + Iterator first, last; + Argument &self; + }; + if (!dry_run) { + std::visit(ActionApply{start, end, *this}, m_action); + m_is_used = true; + } + return end; + } + if (m_default_value.has_value()) { + if (!dry_run) { + m_is_used = true; + } + return start; + } + throw std::runtime_error("Too few arguments for '" + + std::string(m_used_name) + "'."); + } + + /* + * @throws std::runtime_error if argument values are not valid + */ + void validate() const { + if (m_is_optional) { + // TODO: check if an implicit value was programmed for this argument + if (!m_is_used && !m_default_value.has_value() && m_is_required) { + throw_required_arg_not_used_error(); + } + if (m_is_used && m_is_required && m_values.empty()) { + throw_required_arg_no_value_provided_error(); + } + } else { + if (!m_num_args_range.contains(m_values.size()) && + !m_default_value.has_value()) { + throw_nargs_range_validation_error(); + } + } + + if (m_choices.has_value()) { + // Make sure the default value (if provided) + // is in the list of choices + find_default_value_in_choices_or_throw(); + } + } + + std::string get_names_csv(char separator = ',') const { + return std::accumulate( + m_names.begin(), m_names.end(), std::string{""}, + [&](const std::string &result, const std::string &name) { + return result.empty() ? name : result + separator + name; + }); + } + + std::string get_usage_full() const { + std::stringstream usage; + + usage << get_names_csv('/'); + const std::string metavar = !m_metavar.empty() ? m_metavar : "VAR"; + if (m_num_args_range.get_max() > 0) { + usage << " " << metavar; + if (m_num_args_range.get_max() > 1) { + usage << "..."; + } + } + return usage.str(); + } + + std::string get_inline_usage() const { + std::stringstream usage; + // Find the longest variant to show in the usage string + std::string longest_name = m_names.front(); + for (const auto &s : m_names) { + if (s.size() > longest_name.size()) { + longest_name = s; + } + } + if (!m_is_required) { + usage << "["; + } + usage << longest_name; + const std::string metavar = !m_metavar.empty() ? m_metavar : "VAR"; + if (m_num_args_range.get_max() > 0) { + usage << " " << metavar; + if (m_num_args_range.get_max() > 1 && + m_metavar.find("> <") == std::string::npos) { + usage << "..."; + } + } + if (!m_is_required) { + usage << "]"; + } + if (m_is_repeatable) { + usage << "..."; + } + return usage.str(); + } + + std::size_t get_arguments_length() const { + + std::size_t names_size = std::accumulate( + std::begin(m_names), std::end(m_names), std::size_t(0), + [](const auto &sum, const auto &s) { return sum + s.size(); }); + + if (is_positional(m_names.front(), m_prefix_chars)) { + // A set metavar means this replaces the names + if (!m_metavar.empty()) { + // Indent and metavar + return 2 + m_metavar.size(); + } + + // Indent and space-separated + return 2 + names_size + (m_names.size() - 1); + } + // Is an option - include both names _and_ metavar + // size = text + (", " between names) + std::size_t size = names_size + 2 * (m_names.size() - 1); + if (!m_metavar.empty() && m_num_args_range == NArgsRange{1, 1}) { + size += m_metavar.size() + 1; + } + return size + 2; // indent + } + + friend std::ostream &operator<<(std::ostream &stream, + const Argument &argument) { + std::stringstream name_stream; + name_stream << " "; // indent + if (argument.is_positional(argument.m_names.front(), + argument.m_prefix_chars)) { + if (!argument.m_metavar.empty()) { + name_stream << argument.m_metavar; + } else { + name_stream << details::join(argument.m_names.begin(), + argument.m_names.end(), " "); + } + } else { + name_stream << details::join(argument.m_names.begin(), + argument.m_names.end(), ", "); + // If we have a metavar, and one narg - print the metavar + if (!argument.m_metavar.empty() && + argument.m_num_args_range == NArgsRange{1, 1}) { + name_stream << " " << argument.m_metavar; + } + else if (!argument.m_metavar.empty() && + argument.m_num_args_range.get_min() == argument.m_num_args_range.get_max() && + argument.m_metavar.find("> <") != std::string::npos) { + name_stream << " " << argument.m_metavar; + } + } + + // align multiline help message + auto stream_width = stream.width(); + auto name_padding = std::string(name_stream.str().size(), ' '); + auto pos = std::string::size_type{}; + auto prev = std::string::size_type{}; + auto first_line = true; + auto hspace = " "; // minimal space between name and help message + stream << name_stream.str(); + std::string_view help_view(argument.m_help); + while ((pos = argument.m_help.find('\n', prev)) != std::string::npos) { + auto line = help_view.substr(prev, pos - prev + 1); + if (first_line) { + stream << hspace << line; + first_line = false; + } else { + stream.width(stream_width); + stream << name_padding << hspace << line; + } + prev += pos - prev + 1; + } + if (first_line) { + stream << hspace << argument.m_help; + } else { + auto leftover = help_view.substr(prev, argument.m_help.size() - prev); + if (!leftover.empty()) { + stream.width(stream_width); + stream << name_padding << hspace << leftover; + } + } + + // print nargs spec + if (!argument.m_help.empty()) { + stream << " "; + } + stream << argument.m_num_args_range; + + bool add_space = false; + if (argument.m_default_value.has_value() && + argument.m_num_args_range != NArgsRange{0, 0}) { + stream << "[default: " << argument.m_default_value_repr << "]"; + add_space = true; + } else if (argument.m_is_required) { + stream << "[required]"; + add_space = true; + } + if (argument.m_is_repeatable) { + if (add_space) { + stream << " "; + } + stream << "[may be repeated]"; + } + stream << "\n"; + return stream; + } + + template bool operator!=(const T &rhs) const { + return !(*this == rhs); + } + + /* + * Compare to an argument value of known type + * @throws std::logic_error in case of incompatible types + */ + template bool operator==(const T &rhs) const { + if constexpr (!details::IsContainer) { + return get() == rhs; + } else { + using ValueType = typename T::value_type; + auto lhs = get(); + return std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs), + std::end(rhs), [](const auto &a, const auto &b) { + return std::any_cast(a) == b; + }); + } + } + + /* + * positional: + * _empty_ + * '-' + * '-' decimal-literal + * !'-' anything + */ + static bool is_positional(std::string_view name, + std::string_view prefix_chars) { + auto first = lookahead(name); + + if (first == eof) { + return true; + } + if (prefix_chars.find(static_cast(first)) != + std::string_view::npos) { + name.remove_prefix(1); + if (name.empty()) { + return true; + } + return is_decimal_literal(name); + } + return true; + } + +private: + class NArgsRange { + std::size_t m_min; + std::size_t m_max; + + public: + NArgsRange(std::size_t minimum, std::size_t maximum) + : m_min(minimum), m_max(maximum) { + if (minimum > maximum) { + throw std::logic_error("Range of number of arguments is invalid"); + } + } + + bool contains(std::size_t value) const { + return value >= m_min && value <= m_max; + } + + bool is_exact() const { return m_min == m_max; } + + bool is_right_bounded() const { + return m_max < (std::numeric_limits::max)(); + } + + std::size_t get_min() const { return m_min; } + + std::size_t get_max() const { return m_max; } + + // Print help message + friend auto operator<<(std::ostream &stream, const NArgsRange &range) + -> std::ostream & { + if (range.m_min == range.m_max) { + if (range.m_min != 0 && range.m_min != 1) { + stream << "[nargs: " << range.m_min << "] "; + } + } else { + if (range.m_max == (std::numeric_limits::max)()) { + stream << "[nargs: " << range.m_min << " or more] "; + } else { + stream << "[nargs=" << range.m_min << ".." << range.m_max << "] "; + } + } + return stream; + } + + bool operator==(const NArgsRange &rhs) const { + return rhs.m_min == m_min && rhs.m_max == m_max; + } + + bool operator!=(const NArgsRange &rhs) const { return !(*this == rhs); } + }; + + void throw_nargs_range_validation_error() const { + std::stringstream stream; + if (!m_used_name.empty()) { + stream << m_used_name << ": "; + } else { + stream << m_names.front() << ": "; + } + if (m_num_args_range.is_exact()) { + stream << m_num_args_range.get_min(); + } else if (m_num_args_range.is_right_bounded()) { + stream << m_num_args_range.get_min() << " to " + << m_num_args_range.get_max(); + } else { + stream << m_num_args_range.get_min() << " or more"; + } + stream << " argument(s) expected. " << m_values.size() << " provided."; + throw std::runtime_error(stream.str()); + } + + void throw_required_arg_not_used_error() const { + std::stringstream stream; + stream << m_names.front() << ": required."; + throw std::runtime_error(stream.str()); + } + + void throw_required_arg_no_value_provided_error() const { + std::stringstream stream; + stream << m_used_name << ": no value provided."; + throw std::runtime_error(stream.str()); + } + + static constexpr int eof = std::char_traits::eof(); + + static auto lookahead(std::string_view s) -> int { + if (s.empty()) { + return eof; + } + return static_cast(static_cast(s[0])); + } + + /* + * decimal-literal: + * '0' + * nonzero-digit digit-sequence_opt + * integer-part fractional-part + * fractional-part + * integer-part '.' exponent-part_opt + * integer-part exponent-part + * + * integer-part: + * digit-sequence + * + * fractional-part: + * '.' post-decimal-point + * + * post-decimal-point: + * digit-sequence exponent-part_opt + * + * exponent-part: + * 'e' post-e + * 'E' post-e + * + * post-e: + * sign_opt digit-sequence + * + * sign: one of + * '+' '-' + */ + static bool is_decimal_literal(std::string_view s) { + auto is_digit = [](auto c) constexpr { + switch (c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return true; + default: + return false; + } + }; + + // precondition: we have consumed or will consume at least one digit + auto consume_digits = [=](std::string_view sd) { + // NOLINTNEXTLINE(readability-qualified-auto) + auto it = std::find_if_not(std::begin(sd), std::end(sd), is_digit); + return sd.substr(static_cast(it - std::begin(sd))); + }; + + switch (lookahead(s)) { + case '0': { + s.remove_prefix(1); + if (s.empty()) { + return true; + } + goto integer_part; + } + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + s = consume_digits(s); + if (s.empty()) { + return true; + } + goto integer_part_consumed; + } + case '.': { + s.remove_prefix(1); + goto post_decimal_point; + } + default: + return false; + } + + integer_part: + s = consume_digits(s); + integer_part_consumed: + switch (lookahead(s)) { + case '.': { + s.remove_prefix(1); + if (is_digit(lookahead(s))) { + goto post_decimal_point; + } else { + goto exponent_part_opt; + } + } + case 'e': + case 'E': { + s.remove_prefix(1); + goto post_e; + } + default: + return false; + } + + post_decimal_point: + if (is_digit(lookahead(s))) { + s = consume_digits(s); + goto exponent_part_opt; + } + return false; + + exponent_part_opt: + switch (lookahead(s)) { + case eof: + return true; + case 'e': + case 'E': { + s.remove_prefix(1); + goto post_e; + } + default: + return false; + } + + post_e: + switch (lookahead(s)) { + case '-': + case '+': + s.remove_prefix(1); + } + if (is_digit(lookahead(s))) { + s = consume_digits(s); + return s.empty(); + } + return false; + } + + static bool is_optional(std::string_view name, + std::string_view prefix_chars) { + return !is_positional(name, prefix_chars); + } + + /* + * Get argument value given a type + * @throws std::logic_error in case of incompatible types + */ + template T get() const { + if (!m_values.empty()) { + if constexpr (details::IsContainer) { + return any_cast_container(m_values); + } else { + return std::any_cast(m_values.front()); + } + } + if (m_default_value.has_value()) { + return std::any_cast(m_default_value); + } + if constexpr (details::IsContainer) { + if (!m_accepts_optional_like_value) { + return any_cast_container(m_values); + } + } + + throw std::logic_error("No value provided for '" + m_names.back() + "'."); + } + + /* + * Get argument value given a type. + * @pre The object has no default value. + * @returns The stored value if any, std::nullopt otherwise. + */ + template auto present() const -> std::optional { + if (m_default_value.has_value()) { + throw std::logic_error("Argument with default value always presents"); + } + if (m_values.empty()) { + return std::nullopt; + } + if constexpr (details::IsContainer) { + return any_cast_container(m_values); + } + return std::any_cast(m_values.front()); + } + + template + static auto any_cast_container(const std::vector &operand) -> T { + using ValueType = typename T::value_type; + + T result; + std::transform( + std::begin(operand), std::end(operand), std::back_inserter(result), + [](const auto &value) { return std::any_cast(value); }); + return result; + } + + void set_usage_newline_counter(int i) { m_usage_newline_counter = i; } + + void set_group_idx(std::size_t i) { m_group_idx = i; } + + std::vector m_names; + std::string_view m_used_name; + std::string m_help; + std::string m_metavar; + std::any m_default_value; + std::string m_default_value_repr; + std::optional + m_default_value_str; // used for checking default_value against choices + std::any m_implicit_value; + std::optional> m_choices{std::nullopt}; + using valued_action = std::function; + using void_action = std::function; + std::variant m_action{ + std::in_place_type, + [](const std::string &value) { return value; }}; + std::vector m_values; + NArgsRange m_num_args_range{1, 1}; + // Bit field of bool values. Set default value in ctor. + bool m_accepts_optional_like_value : 1; + bool m_is_optional : 1; + bool m_is_required : 1; + bool m_is_repeatable : 1; + bool m_is_used : 1; + bool m_is_hidden : 1; // if set, does not appear in usage or help + std::string_view m_prefix_chars; // ArgumentParser has the prefix_chars + int m_usage_newline_counter = 0; + std::size_t m_group_idx = 0; +}; + +class ArgumentParser { +public: + explicit ArgumentParser(std::string program_name = {}, + std::string version = "1.0", + default_arguments add_args = default_arguments::all, + bool exit_on_default_arguments = true, + std::ostream &os = std::cout) + : m_program_name(std::move(program_name)), m_version(std::move(version)), + m_exit_on_default_arguments(exit_on_default_arguments), + m_parser_path(m_program_name) { + if ((add_args & default_arguments::help) == default_arguments::help) { + add_argument("-h", "--help") + .action([&](const auto & /*unused*/) { + os << help().str(); + if (m_exit_on_default_arguments) { + std::exit(0); + } + }) + .default_value(false) + .help("shows help message and exits") + .implicit_value(true) + .nargs(0); + } + if ((add_args & default_arguments::version) == default_arguments::version) { + add_argument("-v", "--version") + .action([&](const auto & /*unused*/) { + os << m_version << std::endl; + if (m_exit_on_default_arguments) { + std::exit(0); + } + }) + .default_value(false) + .help("prints version information and exits") + .implicit_value(true) + .nargs(0); + } + } + + ~ArgumentParser() = default; + + // ArgumentParser is meant to be used in a single function. + // Setup everything and parse arguments in one place. + // + // ArgumentParser internally uses std::string_views, + // references, iterators, etc. + // Many of these elements become invalidated after a copy or move. + ArgumentParser(const ArgumentParser &other) = delete; + ArgumentParser &operator=(const ArgumentParser &other) = delete; + ArgumentParser(ArgumentParser &&) noexcept = delete; + ArgumentParser &operator=(ArgumentParser &&) = delete; + + explicit operator bool() const { + auto arg_used = std::any_of(m_argument_map.cbegin(), m_argument_map.cend(), + [](auto &it) { return it.second->m_is_used; }); + auto subparser_used = + std::any_of(m_subparser_used.cbegin(), m_subparser_used.cend(), + [](auto &it) { return it.second; }); + + return m_is_parsed && (arg_used || subparser_used); + } + + // Parameter packing + // Call add_argument with variadic number of string arguments + template Argument &add_argument(Targs... f_args) { + using array_of_sv = std::array; + auto argument = + m_optional_arguments.emplace(std::cend(m_optional_arguments), + m_prefix_chars, array_of_sv{f_args...}); + + if (!argument->m_is_optional) { + m_positional_arguments.splice(std::cend(m_positional_arguments), + m_optional_arguments, argument); + } + argument->set_usage_newline_counter(m_usage_newline_counter); + argument->set_group_idx(m_group_names.size()); + + index_argument(argument); + return *argument; + } + + class MutuallyExclusiveGroup { + friend class ArgumentParser; + + public: + MutuallyExclusiveGroup() = delete; + + explicit MutuallyExclusiveGroup(ArgumentParser &parent, + bool required = false) + : m_parent(parent), m_required(required), m_elements({}) {} + + MutuallyExclusiveGroup(const MutuallyExclusiveGroup &other) = delete; + MutuallyExclusiveGroup & + operator=(const MutuallyExclusiveGroup &other) = delete; + + MutuallyExclusiveGroup(MutuallyExclusiveGroup &&other) noexcept + : m_parent(other.m_parent), m_required(other.m_required), + m_elements(std::move(other.m_elements)) { + other.m_elements.clear(); + } + + template Argument &add_argument(Targs... f_args) { + auto &argument = m_parent.add_argument(std::forward(f_args)...); + m_elements.push_back(&argument); + argument.set_usage_newline_counter(m_parent.m_usage_newline_counter); + argument.set_group_idx(m_parent.m_group_names.size()); + return argument; + } + + private: + ArgumentParser &m_parent; + bool m_required{false}; + std::vector m_elements{}; + }; + + MutuallyExclusiveGroup &add_mutually_exclusive_group(bool required = false) { + m_mutually_exclusive_groups.emplace_back(*this, required); + return m_mutually_exclusive_groups.back(); + } + + // Parameter packed add_parents method + // Accepts a variadic number of ArgumentParser objects + template + ArgumentParser &add_parents(const Targs &... f_args) { + for (const ArgumentParser &parent_parser : {std::ref(f_args)...}) { + for (const auto &argument : parent_parser.m_positional_arguments) { + auto it = m_positional_arguments.insert( + std::cend(m_positional_arguments), argument); + index_argument(it); + } + for (const auto &argument : parent_parser.m_optional_arguments) { + auto it = m_optional_arguments.insert(std::cend(m_optional_arguments), + argument); + index_argument(it); + } + } + return *this; + } + + // Ask for the next optional arguments to be displayed on a separate + // line in usage() output. Only effective if set_usage_max_line_width() is + // also used. + ArgumentParser &add_usage_newline() { + ++m_usage_newline_counter; + return *this; + } + + // Ask for the next optional arguments to be displayed in a separate section + // in usage() and help (<< *this) output. + // For usage(), this is only effective if set_usage_max_line_width() is + // also used. + ArgumentParser &add_group(std::string group_name) { + m_group_names.emplace_back(std::move(group_name)); + return *this; + } + + ArgumentParser &add_description(std::string description) { + m_description = std::move(description); + return *this; + } + + ArgumentParser &add_epilog(std::string epilog) { + m_epilog = std::move(epilog); + return *this; + } + + // Add a un-documented/hidden alias for an argument. + // Ideally we'd want this to be a method of Argument, but Argument + // does not own its owing ArgumentParser. + ArgumentParser &add_hidden_alias_for(Argument &arg, std::string_view alias) { + for (auto it = m_optional_arguments.begin(); + it != m_optional_arguments.end(); ++it) { + if (&(*it) == &arg) { + m_argument_map.insert_or_assign(std::string(alias), it); + return *this; + } + } + throw std::logic_error( + "Argument is not an optional argument of this parser"); + } + + /* Getter for arguments and subparsers. + * @throws std::logic_error in case of an invalid argument or subparser name + */ + template T &at(std::string_view name) { + if constexpr (std::is_same_v) { + return (*this)[name]; + } else { + std::string str_name(name); + auto subparser_it = m_subparser_map.find(str_name); + if (subparser_it != m_subparser_map.end()) { + return subparser_it->second->get(); + } + throw std::logic_error("No such subparser: " + str_name); + } + } + + ArgumentParser &set_prefix_chars(std::string prefix_chars) { + m_prefix_chars = std::move(prefix_chars); + return *this; + } + + ArgumentParser &set_assign_chars(std::string assign_chars) { + m_assign_chars = std::move(assign_chars); + return *this; + } + + /* Call parse_args_internal - which does all the work + * Then, validate the parsed arguments + * This variant is used mainly for testing + * @throws std::runtime_error in case of any invalid argument + */ + void parse_args(const std::vector &arguments) { + parse_args_internal(arguments); + // Check if all arguments are parsed + for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) { + argument->validate(); + } + + // Check each mutually exclusive group and make sure + // there are no constraint violations + for (const auto &group : m_mutually_exclusive_groups) { + auto mutex_argument_used{false}; + Argument *mutex_argument_it{nullptr}; + for (Argument *arg : group.m_elements) { + if (!mutex_argument_used && arg->m_is_used) { + mutex_argument_used = true; + mutex_argument_it = arg; + } else if (mutex_argument_used && arg->m_is_used) { + // Violation + throw std::runtime_error("Argument '" + arg->get_usage_full() + + "' not allowed with '" + + mutex_argument_it->get_usage_full() + "'"); + } + } + + if (!mutex_argument_used && group.m_required) { + // at least one argument from the group is + // required + std::string argument_names{}; + std::size_t i = 0; + std::size_t size = group.m_elements.size(); + for (Argument *arg : group.m_elements) { + if (i + 1 == size) { + // last + argument_names += std::string("'") + arg->get_usage_full() + std::string("' "); + } else { + argument_names += std::string("'") + arg->get_usage_full() + std::string("' or "); + } + i += 1; + } + throw std::runtime_error("One of the arguments " + argument_names + + "is required"); + } + } + } + + /* Call parse_known_args_internal - which does all the work + * Then, validate the parsed arguments + * This variant is used mainly for testing + * @throws std::runtime_error in case of any invalid argument + */ + std::vector + parse_known_args(const std::vector &arguments) { + auto unknown_arguments = parse_known_args_internal(arguments); + // Check if all arguments are parsed + for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) { + argument->validate(); + } + return unknown_arguments; + } + + /* Main entry point for parsing command-line arguments using this + * ArgumentParser + * @throws std::runtime_error in case of any invalid argument + */ + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) + void parse_args(int argc, const char *const argv[]) { + parse_args({argv, argv + argc}); + } + + /* Main entry point for parsing command-line arguments using this + * ArgumentParser + * @throws std::runtime_error in case of any invalid argument + */ + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) + auto parse_known_args(int argc, const char *const argv[]) { + return parse_known_args({argv, argv + argc}); + } + + /* Getter for options with default values. + * @throws std::logic_error if parse_args() has not been previously called + * @throws std::logic_error if there is no such option + * @throws std::logic_error if the option has no value + * @throws std::bad_any_cast if the option is not of type T + */ + template T get(std::string_view arg_name) const { + if (!m_is_parsed) { + throw std::logic_error("Nothing parsed, no arguments are available."); + } + return (*this)[arg_name].get(); + } + + /* Getter for options without default values. + * @pre The option has no default value. + * @throws std::logic_error if there is no such option + * @throws std::bad_any_cast if the option is not of type T + */ + template + auto present(std::string_view arg_name) const -> std::optional { + return (*this)[arg_name].present(); + } + + /* Getter that returns true for user-supplied options. Returns false if not + * user-supplied, even with a default value. + */ + auto is_used(std::string_view arg_name) const { + return (*this)[arg_name].m_is_used; + } + + /* Getter that returns true if a subcommand is used. + */ + auto is_subcommand_used(std::string_view subcommand_name) const { + return m_subparser_used.at(std::string(subcommand_name)); + } + + /* Getter that returns true if a subcommand is used. + */ + auto is_subcommand_used(const ArgumentParser &subparser) const { + return is_subcommand_used(subparser.m_program_name); + } + + /* Indexing operator. Return a reference to an Argument object + * Used in conjunction with Argument.operator== e.g., parser["foo"] == true + * @throws std::logic_error in case of an invalid argument name + */ + Argument &operator[](std::string_view arg_name) const { + std::string name(arg_name); + auto it = m_argument_map.find(name); + if (it != m_argument_map.end()) { + return *(it->second); + } + if (!is_valid_prefix_char(arg_name.front())) { + const auto legal_prefix_char = get_any_valid_prefix_char(); + const auto prefix = std::string(1, legal_prefix_char); + + // "-" + arg_name + name = prefix + name; + it = m_argument_map.find(name); + if (it != m_argument_map.end()) { + return *(it->second); + } + // "--" + arg_name + name = prefix + name; + it = m_argument_map.find(name); + if (it != m_argument_map.end()) { + return *(it->second); + } + } + throw std::logic_error("No such argument: " + std::string(arg_name)); + } + + // Print help message + friend auto operator<<(std::ostream &stream, const ArgumentParser &parser) + -> std::ostream & { + stream.setf(std::ios_base::left); + + auto longest_arg_length = parser.get_length_of_longest_argument(); + + stream << parser.usage() << "\n\n"; + + if (!parser.m_description.empty()) { + stream << parser.m_description << "\n\n"; + } + + const bool has_visible_positional_args = std::find_if( + parser.m_positional_arguments.begin(), + parser.m_positional_arguments.end(), + [](const auto &argument) { + return !argument.m_is_hidden; }) != + parser.m_positional_arguments.end(); + if (has_visible_positional_args) { + stream << "Positional arguments:\n"; + } + + for (const auto &argument : parser.m_positional_arguments) { + if (!argument.m_is_hidden) { + stream.width(static_cast(longest_arg_length)); + stream << argument; + } + } + + if (!parser.m_optional_arguments.empty()) { + stream << (!has_visible_positional_args ? "" : "\n") + << "Optional arguments:\n"; + } + + for (const auto &argument : parser.m_optional_arguments) { + if (argument.m_group_idx == 0 && !argument.m_is_hidden) { + stream.width(static_cast(longest_arg_length)); + stream << argument; + } + } + + for (size_t i_group = 0; i_group < parser.m_group_names.size(); ++i_group) { + stream << "\n" << parser.m_group_names[i_group] << " (detailed usage):\n"; + for (const auto &argument : parser.m_optional_arguments) { + if (argument.m_group_idx == i_group + 1 && !argument.m_is_hidden) { + stream.width(static_cast(longest_arg_length)); + stream << argument; + } + } + } + + bool has_visible_subcommands = std::any_of( + parser.m_subparser_map.begin(), parser.m_subparser_map.end(), + [](auto &p) { return !p.second->get().m_suppress; }); + + if (has_visible_subcommands) { + stream << (parser.m_positional_arguments.empty() + ? (parser.m_optional_arguments.empty() ? "" : "\n") + : "\n") + << "Subcommands:\n"; + for (const auto &[command, subparser] : parser.m_subparser_map) { + if (subparser->get().m_suppress) { + continue; + } + + stream << std::setw(2) << " "; + stream << std::setw(static_cast(longest_arg_length - 2)) + << command; + stream << " " << subparser->get().m_description << "\n"; + } + } + + if (!parser.m_epilog.empty()) { + stream << '\n'; + stream << parser.m_epilog << "\n\n"; + } + + return stream; + } + + // Format help message + auto help() const -> std::stringstream { + std::stringstream out; + out << *this; + return out; + } + + // Sets the maximum width for a line of the Usage message + ArgumentParser &set_usage_max_line_width(size_t w) { + this->m_usage_max_line_width = w; + return *this; + } + + // Asks to display arguments of mutually exclusive group on separate lines in + // the Usage message + ArgumentParser &set_usage_break_on_mutex() { + this->m_usage_break_on_mutex = true; + return *this; + } + + // Format usage part of help only + auto usage() const -> std::string { + std::stringstream stream; + + std::string curline("Usage: "); + curline += this->m_program_name; + const bool multiline_usage = + this->m_usage_max_line_width < (std::numeric_limits::max)(); + const size_t indent_size = curline.size(); + + const auto deal_with_options_of_group = [&](std::size_t group_idx) { + bool found_options = false; + // Add any options inline here + const MutuallyExclusiveGroup *cur_mutex = nullptr; + int usage_newline_counter = -1; + for (const auto &argument : this->m_optional_arguments) { + if (argument.m_is_hidden) { + continue; + } + if (multiline_usage) { + if (argument.m_group_idx != group_idx) { + continue; + } + if (usage_newline_counter != argument.m_usage_newline_counter) { + if (usage_newline_counter >= 0) { + if (curline.size() > indent_size) { + stream << curline << std::endl; + curline = std::string(indent_size, ' '); + } + } + usage_newline_counter = argument.m_usage_newline_counter; + } + } + found_options = true; + const std::string arg_inline_usage = argument.get_inline_usage(); + const MutuallyExclusiveGroup *arg_mutex = + get_belonging_mutex(&argument); + if ((cur_mutex != nullptr) && (arg_mutex == nullptr)) { + curline += ']'; + if (this->m_usage_break_on_mutex) { + stream << curline << std::endl; + curline = std::string(indent_size, ' '); + } + } else if ((cur_mutex == nullptr) && (arg_mutex != nullptr)) { + if ((this->m_usage_break_on_mutex && curline.size() > indent_size) || + curline.size() + 3 + arg_inline_usage.size() > + this->m_usage_max_line_width) { + stream << curline << std::endl; + curline = std::string(indent_size, ' '); + } + curline += " ["; + } else if ((cur_mutex != nullptr) && (arg_mutex != nullptr)) { + if (cur_mutex != arg_mutex) { + curline += ']'; + if (this->m_usage_break_on_mutex || + curline.size() + 3 + arg_inline_usage.size() > + this->m_usage_max_line_width) { + stream << curline << std::endl; + curline = std::string(indent_size, ' '); + } + curline += " ["; + } else { + curline += '|'; + } + } + cur_mutex = arg_mutex; + if (curline.size() + 1 + arg_inline_usage.size() > + this->m_usage_max_line_width) { + stream << curline << std::endl; + curline = std::string(indent_size, ' '); + curline += " "; + } else if (cur_mutex == nullptr) { + curline += " "; + } + curline += arg_inline_usage; + } + if (cur_mutex != nullptr) { + curline += ']'; + } + return found_options; + }; + + const bool found_options = deal_with_options_of_group(0); + + if (found_options && multiline_usage && + !this->m_positional_arguments.empty()) { + stream << curline << std::endl; + curline = std::string(indent_size, ' '); + } + // Put positional arguments after the optionals + for (const auto &argument : this->m_positional_arguments) { + if (argument.m_is_hidden) { + continue; + } + const std::string pos_arg = !argument.m_metavar.empty() + ? argument.m_metavar + : argument.m_names.front(); + if (curline.size() + 1 + pos_arg.size() > this->m_usage_max_line_width) { + stream << curline << std::endl; + curline = std::string(indent_size, ' '); + } + curline += " "; + if (argument.m_num_args_range.get_min() == 0 && + !argument.m_num_args_range.is_right_bounded()) { + curline += "["; + curline += pos_arg; + curline += "]..."; + } else if (argument.m_num_args_range.get_min() == 1 && + !argument.m_num_args_range.is_right_bounded()) { + curline += pos_arg; + curline += "..."; + } else { + curline += pos_arg; + } + } + + if (multiline_usage) { + // Display options of other groups + for (std::size_t i = 0; i < m_group_names.size(); ++i) { + stream << curline << std::endl << std::endl; + stream << m_group_names[i] << ":" << std::endl; + curline = std::string(indent_size, ' '); + deal_with_options_of_group(i + 1); + } + } + + stream << curline; + + // Put subcommands after positional arguments + if (!m_subparser_map.empty()) { + stream << " {"; + std::size_t i{0}; + for (const auto &[command, subparser] : m_subparser_map) { + if (subparser->get().m_suppress) { + continue; + } + + if (i == 0) { + stream << command; + } else { + stream << "," << command; + } + ++i; + } + stream << "}"; + } + + return stream.str(); + } + + // Printing the one and only help message + // I've stuck with a simple message format, nothing fancy. + [[deprecated("Use cout << program; instead. See also help().")]] std::string + print_help() const { + auto out = help(); + std::cout << out.rdbuf(); + return out.str(); + } + + void add_subparser(ArgumentParser &parser) { + parser.m_parser_path = m_program_name + " " + parser.m_program_name; + auto it = m_subparsers.emplace(std::cend(m_subparsers), parser); + m_subparser_map.insert_or_assign(parser.m_program_name, it); + m_subparser_used.insert_or_assign(parser.m_program_name, false); + } + + void set_suppress(bool suppress) { m_suppress = suppress; } + +protected: + const MutuallyExclusiveGroup *get_belonging_mutex(const Argument *arg) const { + for (const auto &mutex : m_mutually_exclusive_groups) { + if (std::find(mutex.m_elements.begin(), mutex.m_elements.end(), arg) != + mutex.m_elements.end()) { + return &mutex; + } + } + return nullptr; + } + + bool is_valid_prefix_char(char c) const { + return m_prefix_chars.find(c) != std::string::npos; + } + + char get_any_valid_prefix_char() const { return m_prefix_chars[0]; } + + /* + * Pre-process this argument list. Anything starting with "--", that + * contains an =, where the prefix before the = has an entry in the + * options table, should be split. + */ + std::vector + preprocess_arguments(const std::vector &raw_arguments) const { + std::vector arguments{}; + for (const auto &arg : raw_arguments) { + + const auto argument_starts_with_prefix_chars = + [this](const std::string &a) -> bool { + if (!a.empty()) { + + const auto legal_prefix = [this](char c) -> bool { + return m_prefix_chars.find(c) != std::string::npos; + }; + + // Windows-style + // if '/' is a legal prefix char + // then allow single '/' followed by argument name, followed by an + // assign char, e.g., ':' e.g., 'test.exe /A:Foo' + const auto windows_style = legal_prefix('/'); + + if (windows_style) { + if (legal_prefix(a[0])) { + return true; + } + } else { + // Slash '/' is not a legal prefix char + // For all other characters, only support long arguments + // i.e., the argument must start with 2 prefix chars, e.g, + // '--foo' e,g, './test --foo=Bar -DARG=yes' + if (a.size() > 1) { + return (legal_prefix(a[0]) && legal_prefix(a[1])); + } + } + } + return false; + }; + + // Check that: + // - We don't have an argument named exactly this + // - The argument starts with a prefix char, e.g., "--" + // - The argument contains an assign char, e.g., "=" + auto assign_char_pos = arg.find_first_of(m_assign_chars); + + if (m_argument_map.find(arg) == m_argument_map.end() && + argument_starts_with_prefix_chars(arg) && + assign_char_pos != std::string::npos) { + // Get the name of the potential option, and check it exists + std::string opt_name = arg.substr(0, assign_char_pos); + if (m_argument_map.find(opt_name) != m_argument_map.end()) { + // This is the name of an option! Split it into two parts + arguments.push_back(std::move(opt_name)); + arguments.push_back(arg.substr(assign_char_pos + 1)); + continue; + } + } + // If we've fallen through to here, then it's a standard argument + arguments.push_back(arg); + } + return arguments; + } + + /* + * @throws std::runtime_error in case of any invalid argument + */ + void parse_args_internal(const std::vector &raw_arguments) { + auto arguments = preprocess_arguments(raw_arguments); + if (m_program_name.empty() && !arguments.empty()) { + m_program_name = arguments.front(); + } + auto end = std::end(arguments); + auto positional_argument_it = std::begin(m_positional_arguments); + for (auto it = std::next(std::begin(arguments)); it != end;) { + const auto ¤t_argument = *it; + if (Argument::is_positional(current_argument, m_prefix_chars)) { + if (positional_argument_it == std::end(m_positional_arguments)) { + + // Check sub-parsers + auto subparser_it = m_subparser_map.find(current_argument); + if (subparser_it != m_subparser_map.end()) { + + // build list of remaining args + const auto unprocessed_arguments = + std::vector(it, end); + + // invoke subparser + m_is_parsed = true; + m_subparser_used[current_argument] = true; + return subparser_it->second->get().parse_args( + unprocessed_arguments); + } + + if (m_positional_arguments.empty()) { + + // Ask the user if they argument they provided was a typo + // for some sub-parser, + // e.g., user provided `git totes` instead of `git notes` + if (!m_subparser_map.empty()) { + throw std::runtime_error( + "Failed to parse '" + current_argument + "', did you mean '" + + std::string{details::get_most_similar_string( + m_subparser_map, current_argument)} + + "'"); + } + + // Ask the user if they meant to use a specific optional argument + if (!m_optional_arguments.empty()) { + for (const auto &opt : m_optional_arguments) { + if (!opt.m_implicit_value.has_value()) { + // not a flag, requires a value + if (!opt.m_is_used) { + throw std::runtime_error( + "Zero positional arguments expected, did you mean " + + opt.get_usage_full()); + } + } + } + + throw std::runtime_error("Zero positional arguments expected"); + } else { + throw std::runtime_error("Zero positional arguments expected"); + } + } else { + throw std::runtime_error("Maximum number of positional arguments " + "exceeded, failed to parse '" + + current_argument + "'"); + } + } + auto argument = positional_argument_it++; + + // Deal with the situation of ... + if (argument->m_num_args_range.get_min() == 1 && + argument->m_num_args_range.get_max() == (std::numeric_limits::max)() && + positional_argument_it != std::end(m_positional_arguments) && + std::next(positional_argument_it) == std::end(m_positional_arguments) && + positional_argument_it->m_num_args_range.get_min() == 1 && + positional_argument_it->m_num_args_range.get_max() == 1 ) { + if (std::next(it) != end) { + positional_argument_it->consume(std::prev(end), end); + end = std::prev(end); + } else { + throw std::runtime_error("Missing " + positional_argument_it->m_names.front()); + } + } + + it = argument->consume(it, end); + continue; + } + + auto arg_map_it = m_argument_map.find(current_argument); + if (arg_map_it != m_argument_map.end()) { + auto argument = arg_map_it->second; + it = argument->consume(std::next(it), end, arg_map_it->first); + } else if (const auto &compound_arg = current_argument; + compound_arg.size() > 1 && + is_valid_prefix_char(compound_arg[0]) && + !is_valid_prefix_char(compound_arg[1])) { + ++it; + for (std::size_t j = 1; j < compound_arg.size(); j++) { + auto hypothetical_arg = std::string{'-', compound_arg[j]}; + auto arg_map_it2 = m_argument_map.find(hypothetical_arg); + if (arg_map_it2 != m_argument_map.end()) { + auto argument = arg_map_it2->second; + it = argument->consume(it, end, arg_map_it2->first); + } else { + throw std::runtime_error("Unknown argument: " + current_argument); + } + } + } else { + throw std::runtime_error("Unknown argument: " + current_argument); + } + } + m_is_parsed = true; + } + + /* + * Like parse_args_internal but collects unused args into a vector + */ + std::vector + parse_known_args_internal(const std::vector &raw_arguments) { + auto arguments = preprocess_arguments(raw_arguments); + + std::vector unknown_arguments{}; + + if (m_program_name.empty() && !arguments.empty()) { + m_program_name = arguments.front(); + } + auto end = std::end(arguments); + auto positional_argument_it = std::begin(m_positional_arguments); + for (auto it = std::next(std::begin(arguments)); it != end;) { + const auto ¤t_argument = *it; + if (Argument::is_positional(current_argument, m_prefix_chars)) { + if (positional_argument_it == std::end(m_positional_arguments)) { + + // Check sub-parsers + auto subparser_it = m_subparser_map.find(current_argument); + if (subparser_it != m_subparser_map.end()) { + + // build list of remaining args + const auto unprocessed_arguments = + std::vector(it, end); + + // invoke subparser + m_is_parsed = true; + m_subparser_used[current_argument] = true; + return subparser_it->second->get().parse_known_args_internal( + unprocessed_arguments); + } + + // save current argument as unknown and go to next argument + unknown_arguments.push_back(current_argument); + ++it; + } else { + // current argument is the value of a positional argument + // consume it + auto argument = positional_argument_it++; + it = argument->consume(it, end); + } + continue; + } + + auto arg_map_it = m_argument_map.find(current_argument); + if (arg_map_it != m_argument_map.end()) { + auto argument = arg_map_it->second; + it = argument->consume(std::next(it), end, arg_map_it->first); + } else if (const auto &compound_arg = current_argument; + compound_arg.size() > 1 && + is_valid_prefix_char(compound_arg[0]) && + !is_valid_prefix_char(compound_arg[1])) { + ++it; + for (std::size_t j = 1; j < compound_arg.size(); j++) { + auto hypothetical_arg = std::string{'-', compound_arg[j]}; + auto arg_map_it2 = m_argument_map.find(hypothetical_arg); + if (arg_map_it2 != m_argument_map.end()) { + auto argument = arg_map_it2->second; + it = argument->consume(it, end, arg_map_it2->first); + } else { + unknown_arguments.push_back(current_argument); + break; + } + } + } else { + // current argument is an optional-like argument that is unknown + // save it and move to next argument + unknown_arguments.push_back(current_argument); + ++it; + } + } + m_is_parsed = true; + return unknown_arguments; + } + + // Used by print_help. + std::size_t get_length_of_longest_argument() const { + if (m_argument_map.empty()) { + return 0; + } + std::size_t max_size = 0; + for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) { + max_size = + std::max(max_size, argument->get_arguments_length()); + } + for ([[maybe_unused]] const auto &[command, unused] : m_subparser_map) { + max_size = std::max(max_size, command.size()); + } + return max_size; + } + + using argument_it = std::list::iterator; + using mutex_group_it = std::vector::iterator; + using argument_parser_it = + std::list>::iterator; + + void index_argument(argument_it it) { + for (const auto &name : std::as_const(it->m_names)) { + m_argument_map.insert_or_assign(name, it); + } + } + + std::string m_program_name; + std::string m_version; + std::string m_description; + std::string m_epilog; + bool m_exit_on_default_arguments = true; + std::string m_prefix_chars{"-"}; + std::string m_assign_chars{"="}; + bool m_is_parsed = false; + std::list m_positional_arguments; + std::list m_optional_arguments; + std::map m_argument_map; + std::string m_parser_path; + std::list> m_subparsers; + std::map m_subparser_map; + std::map m_subparser_used; + std::vector m_mutually_exclusive_groups; + bool m_suppress = false; + std::size_t m_usage_max_line_width = (std::numeric_limits::max)(); + bool m_usage_break_on_mutex = false; + int m_usage_newline_counter = 0; + std::vector m_group_names; +}; + +} // namespace argparse \ No newline at end of file From 9b79798648ade16db849d4bddaa70a93cbe9cfab Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 10 Sep 2024 23:38:26 -0400 Subject: [PATCH 03/35] turn stuff back on --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 89dfa37c566..22569fbaec1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,9 +85,9 @@ cmake_dependent_option( WITH_JAVA OFF ) -option(WITH_WPILIB "Build hal, wpilibc/j, and developerRobot (needs OpenCV)" ON) +option(WITH_WPILIB "Build hal, wpilibc/j, and developerRobot (needs OpenCV)" OFF) option(WITH_EXAMPLES "Build examples" OFF) -option(WITH_TESTS "Build unit tests (requires internet connection)" ON) +option(WITH_TESTS "Build unit tests (requires internet connection)" OFF) option(WITH_GUI "Build GUI items" ON) option(WITH_SIMULATION_MODULES "Build simulation modules" ON) option(WITH_PROTOBUF "Build protobuf support" ON) From 1b172439c34d873baabbeb4ac8b3d083e30ed7c7 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 10 Sep 2024 23:38:38 -0400 Subject: [PATCH 04/35] add cli commands --- wpilogcli/src/main/native/cpp/main.cpp | 90 ++++++++++++++++++++------ 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/wpilogcli/src/main/native/cpp/main.cpp b/wpilogcli/src/main/native/cpp/main.cpp index ccddaa68a9b..a08bf5882a1 100644 --- a/wpilogcli/src/main/native/cpp/main.cpp +++ b/wpilogcli/src/main/native/cpp/main.cpp @@ -1,29 +1,83 @@ #include +#include #include -#include +#include "argparse/argparse.hpp" -void show(std::string_view message_type, bool raw) { +void export_json(std::string_view output_path) { } -void list() { +void export_csv(std::string_view output_path) { } -int main(int argc, char *argv[]) { - if (argc > 1) { // - std::string_view command{argv[1]}; - std::string_view message_type{}; - - if (command.compare("json")) { // - - } else if (command.compare("csv")) { - - } else if (command.compare("extract")) { - - } - } else { - // no args given - // report error +void extract_field(std::string_view field_name, std::string_view output_path, bool use_json) { + +} + +void open_log(std::string_view log_path) { + // TODO: Add Log reader (base it on SysId?) +} + +int main(int argc, char* argv[]) { + argparse::ArgumentParser cli{"wpilog-cli"}; + + argparse::ArgumentParser export_json_command{"json"}; + export_json_command.add_description( + "Export a JSON representation of a WPILOG file"); + export_json_command.add_argument("log_file") + .help("Path to the WPILOG file to export"); + export_json_command.add_argument("json_file") + .help( + "Path of the JSON file to create with the exported data. If it " + "exists, it will be overwritten."); + + argparse::ArgumentParser export_csv_command{"csv"}; + export_csv_command.add_description( + "Export a CSV representation of a WPILOG file"); + export_csv_command.add_argument("-l", "--log-file").help("The WPILOG file to export"); + export_csv_command.add_argument("-o", "--output-file") + .required() + .help( + "The CSV file to create with the exported data. If it " + "exists, it will be overwritten."); + + argparse::ArgumentParser extract_field_command{"extract"}; + extract_field_command.add_description( + "Extract the histoy of one field from a WPILOG file and store it in a " + "JSON or CSV file"); + extract_field_command.add_argument("-f", "--field") + .required() + .help("The field to extract from the WPILOG"); + extract_field_command.add_argument("-l", "--log") + .required() + .help("The WPILOG file to extract from"); + extract_field_command.add_argument("--time-start") + .help("The timestamp to start extracting at"); + extract_field_command.add_argument("-o", "--output") + .required() + .help( + "The file to export the field and data to. It will be created or " + "overwritten if it already exists"); + + cli.add_subparser(export_json_command); + cli.add_subparser(export_csv_command); + cli.add_subparser(extract_field_command); + + try { + cli.parse_args(argc, argv); + } catch (const std::exception& err) { + std::cerr << err.what() << std::endl; + std::cerr << cli; + return 1; + } + + // see which one was called + if (export_json_command) { + //export_json(); + } else if (export_csv_command) { + export_csv(export_csv_command.get("--log-file")); + } else if (extract_field_command) { + //extract_field(); } } \ No newline at end of file From dc46ca8adde00055217cfcc9eb7411491ed67bac Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Thu, 12 Sep 2024 11:51:55 -0400 Subject: [PATCH 05/35] notional log reader impl --- wpilogcli/CMakeLists.txt | 2 + wpilogcli/src/main/native/cpp/LogLoader.cpp | 214 ++++++++++++++++++++ wpilogcli/src/main/native/cpp/LogLoader.h | 74 +++++++ 3 files changed, 290 insertions(+) create mode 100644 wpilogcli/src/main/native/cpp/LogLoader.cpp create mode 100644 wpilogcli/src/main/native/cpp/LogLoader.h diff --git a/wpilogcli/CMakeLists.txt b/wpilogcli/CMakeLists.txt index b4c7ee8a746..ceca9f158e4 100644 --- a/wpilogcli/CMakeLists.txt +++ b/wpilogcli/CMakeLists.txt @@ -23,6 +23,8 @@ add_executable( ${APP_ICON_MACOSX} ) +target_include_directories(wpilogcli PUBLIC src/main/native/cpp) + target_link_libraries(wpilogcli PRIVATE wpiutil) if(WIN32) diff --git a/wpilogcli/src/main/native/cpp/LogLoader.cpp b/wpilogcli/src/main/native/cpp/LogLoader.cpp new file mode 100644 index 00000000000..e6fdf8524b9 --- /dev/null +++ b/wpilogcli/src/main/native/cpp/LogLoader.cpp @@ -0,0 +1,214 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "LogLoader.h" + +#include +#include +#include +#include +#include +#include "fmt/base.h" +#include "wpi/DataLogReader.h" + +#include +#include +#include +#include + +using namespace wpilogcli; + +LogLoader::LogLoader(glass::Storage& storage, wpi::Logger& logger) {} + +LogLoader::~LogLoader() = default; + +void LogLoader::Load(std::string_view log_path) { + + // Handle opening the file + std::error_code ec; + auto buf = wpi::MemoryBuffer::GetFile(log_path, ec); + if (ec) { + m_error = fmt::format("Could not open file: {}", ec.message()); + return; + } + + wpi::log::DataLogReader reader{std::move(buf)}; + if (!reader.IsValid()) { + m_error = "Not a valid datalog file"; + return; + } + unload(); // release the actual file, we have the data in the reader now + m_reader = std::make_unique(std::move(reader)); + m_entryTree.clear(); + + // Handle Errors + fmt::println("{}", m_error); + + if (!m_reader) { + return; + } + + // Summary info + fmt::println("{}", fs::path{m_filename}.stem().string().c_str()); + fmt::println("%u records, %u entries%s", m_reader->GetNumRecords(), + m_reader->GetNumEntries(), + m_reader->IsDone() ? "" : " (working)"); + + if (!m_reader->IsDone()) { + return; + } + + /*ImGui::BeginTable( + "Entries", 2, + ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp); + ImGui::TableSetupColumn("Name"); + ImGui::TableSetupColumn("Type"); + // ImGui::TableSetupColumn("Metadata"); + ImGui::TableHeadersRow(); + DisplayEntryTree(m_entryTree); + ImGui::EndTable();*/ +} + +std::vector LogLoader::GetRecords(std::string_view field_name) { + int entry_id{GetTargetEntryId(field_name)}; + std::vector record_list{}; + + if (entry_id == -1) { + // didnt find a record with the name we want + return record_list; + } + + auto iter = m_reader->GetReader().begin(); + + while(!m_reader->IsDone()) { + if (iter->GetEntry() == entry_id) { + // this is the one we want + record_list.push_back(*iter); + } + iter++; + } + + return record_list; +} + +int LogLoader::GetTargetEntryId(std::string_view name) { + // get first entry + auto iter = m_reader->GetReader().begin(); + // is it what we want? + if (iter->IsStart()) { // looking for a start record to get the entry ID + wpi::log::StartRecordData* entryData{}; + iter->GetStartData(entryData); + if (entryData->name.compare(name) != 0) { + // this is it! + return entryData->entry; + } + } + return -1; +} + +/*void LogLoader::RebuildEntryTree() { + m_entryTree.clear(); + wpi::SmallVector parts; + m_reader->ForEachEntryName([&](const glass::DataLogReaderEntry& entry) { + // only show double/float/string entries (TODO: support struct/protobuf) + if (entry.type != "double" && entry.type != "float" && + entry.type != "string") { + return; + } + + // filter on name + if (!m_filter.empty() && !wpi::contains_lower(entry.name, m_filter)) { + return; + } + + parts.clear(); + // split on first : if one is present + auto [prefix, mainpart] = wpi::split(entry.name, ':'); + if (mainpart.empty() || wpi::contains(prefix, '/')) { + mainpart = entry.name; + } else { + parts.emplace_back(prefix); + } + wpi::split(mainpart, parts, '/', -1, false); + + // ignore a raw "/" key + if (parts.empty()) { + return; + } + + // get to leaf + auto nodes = &m_entryTree; + for (auto part : wpi::drop_back(std::span{parts.begin(), parts.end()})) { + auto it = + std::find_if(nodes->begin(), nodes->end(), + [&](const auto& node) { return node.name == part; }); + if (it == nodes->end()) { + nodes->emplace_back(part); + // path is from the beginning of the string to the end of the current + // part; this works because part is a reference to the internals of + // entry.name + nodes->back().path.assign( + entry.name.data(), part.data() + part.size() - entry.name.data()); + it = nodes->end() - 1; + } + nodes = &it->children; + } + + auto it = std::find_if(nodes->begin(), nodes->end(), [&](const auto& node) { + return node.name == parts.back(); + }); + if (it == nodes->end()) { + nodes->emplace_back(parts.back()); + // no need to set path, as it's identical to entry.name + it = nodes->end() - 1; + } + it->entry = &entry; + }); +} + +static void EmitEntry(const std::string& name, + const glass::DataLogReaderEntry& entry) { + ImGui::TableNextColumn(); + ImGui::Selectable(name.c_str()); + if (ImGui::BeginDragDropSource()) { + auto entryPtr = &entry; + ImGui::SetDragDropPayload( + entry.type == "string" ? "DataLogEntryString" : "DataLogEntry", + &entryPtr, + sizeof(entryPtr)); // NOLINT + ImGui::TextUnformatted(entry.name.data(), + entry.name.data() + entry.name.size()); + ImGui::EndDragDropSource(); + } + ImGui::TableNextColumn(); + ImGui::TextUnformatted(entry.type.data(), + entry.type.data() + entry.type.size()); +#if 0 + ImGui::TableNextColumn(); + ImGui::TextUnformatted(entry.metadata.data(), + entry.metadata.data() + entry.metadata.size()); +#endif +} + +void LogLoader::DisplayEntryTree(const std::vector& tree) { + for (auto&& node : tree) { + if (node.entry) { + EmitEntry(node.name, *node.entry); + } + + if (!node.children.empty()) { + ImGui::TableNextColumn(); + bool open = ImGui::TreeNodeEx(node.name.c_str(), + ImGuiTreeNodeFlags_SpanFullWidth); + ImGui::TableNextColumn(); +#if 0 + ImGui::TableNextColumn(); +#endif + if (open) { + DisplayEntryTree(node.children); + ImGui::TreePop(); + } + } + } +}*/ diff --git a/wpilogcli/src/main/native/cpp/LogLoader.h b/wpilogcli/src/main/native/cpp/LogLoader.h new file mode 100644 index 00000000000..0fd56b58657 --- /dev/null +++ b/wpilogcli/src/main/native/cpp/LogLoader.h @@ -0,0 +1,74 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include +#include +#include "wpi/DataLogReader.h" + +#include + +namespace glass { +class DataLogReaderEntry; +class DataLogReaderThread; +class Storage; +} // namespace glass + +namespace wpi { +class Logger; +} // namespace wpi + +namespace wpilogcli { +/** + * Helps with loading datalog files. + */ +class LogLoader { + public: + /** + * Creates a log loader widget + * + * @param logger The program logger + */ + explicit LogLoader(glass::Storage& storage, wpi::Logger& logger); + + ~LogLoader(); + + /** + * Signal called when the current file is unloaded (invalidates any + * LogEntry*). + */ + wpi::sig::Signal<> unload; + + void Load(std::string_view log_path); + + std::vector GetRecords(std::string_view field_name); + + private: + // wpi::Logger& m_logger; + + std::string m_filename; + std::unique_ptr m_reader; + + std::string m_error; + + std::string m_filter; + + wpi::log::StartRecordData* entryData; + + struct EntryTreeNode { + explicit EntryTreeNode(std::string_view name) : name{name} {} + std::string name; // name of just this node + std::string path; // full path if entry is nullptr + const glass::DataLogReaderEntry* entry = nullptr; + std::vector children; // children, sorted by name + }; + std::vector m_entryTree; + + void RebuildEntryTree(); + int GetTargetEntryId(std::string_view name); +}; +} // namespace wpilogcli From 7871d70fd7b1972cc81e58840d0c94df1f5c9c65 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 11:04:21 -0600 Subject: [PATCH 06/35] rename to datalogcli --- CMakeLists.txt | 2 +- datalogcli/CMakeLists.txt | 34 +++++++++++++++++++ .../src/main/generate/WPILibVersion.cpp.in | 0 .../src/main/native/cpp/LogLoader.cpp | 2 +- .../src/main/native/cpp/LogLoader.h | 4 +-- .../src/main/native/cpp/main.cpp | 0 wpilogcli/CMakeLists.txt | 34 ------------------- 7 files changed, 38 insertions(+), 38 deletions(-) create mode 100644 datalogcli/CMakeLists.txt rename {wpilogcli => datalogcli}/src/main/generate/WPILibVersion.cpp.in (100%) rename {wpilogcli => datalogcli}/src/main/native/cpp/LogLoader.cpp (99%) rename {wpilogcli => datalogcli}/src/main/native/cpp/LogLoader.h (97%) rename {wpilogcli => datalogcli}/src/main/native/cpp/main.cpp (100%) delete mode 100644 wpilogcli/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 22569fbaec1..64b2d0930da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -300,7 +300,7 @@ if(WITH_NTCORE) add_subdirectory(ntcore) endif() -add_subdirectory(wpilogcli) +add_subdirectory(datalogcli) add_subdirectory(protoplugin) diff --git a/datalogcli/CMakeLists.txt b/datalogcli/CMakeLists.txt new file mode 100644 index 00000000000..bcb911bbc39 --- /dev/null +++ b/datalogcli/CMakeLists.txt @@ -0,0 +1,34 @@ +project(datalogcli) + +include(CompileWarnings) +include(GenResources) + +configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) +generate_resources(src/main/native/resources generated/main/cpp DLT dlt datalogcli_resources_src) + +file(GLOB datalogcli_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) + +if(WIN32) + set(datalogcli_rc src/main/native/win/datalogcli.rc) +elseif(APPLE) + set(MACOSX_BUNDLE_ICON_FILE datalogcli.icns) + set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") +endif() + +add_executable( + datalogcli + ${datalogcli_src} + ${datalogcli_resources_src} + ${datalogcli_rc} + ${APP_ICON_MACOSX} +) + +target_include_directories(datalogcli PUBLIC src/main/native/cpp) + +target_link_libraries(datalogcli PRIVATE wpiutil) + +if(WIN32) + set_target_properties(datalogcli PROPERTIES WIN32_EXECUTABLE YES) +elseif(APPLE) + set_target_properties(datalogcli PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "datalogcli") +endif() \ No newline at end of file diff --git a/wpilogcli/src/main/generate/WPILibVersion.cpp.in b/datalogcli/src/main/generate/WPILibVersion.cpp.in similarity index 100% rename from wpilogcli/src/main/generate/WPILibVersion.cpp.in rename to datalogcli/src/main/generate/WPILibVersion.cpp.in diff --git a/wpilogcli/src/main/native/cpp/LogLoader.cpp b/datalogcli/src/main/native/cpp/LogLoader.cpp similarity index 99% rename from wpilogcli/src/main/native/cpp/LogLoader.cpp rename to datalogcli/src/main/native/cpp/LogLoader.cpp index e6fdf8524b9..4a0cdb4d69e 100644 --- a/wpilogcli/src/main/native/cpp/LogLoader.cpp +++ b/datalogcli/src/main/native/cpp/LogLoader.cpp @@ -17,7 +17,7 @@ #include #include -using namespace wpilogcli; +using namespace datalogcli; LogLoader::LogLoader(glass::Storage& storage, wpi::Logger& logger) {} diff --git a/wpilogcli/src/main/native/cpp/LogLoader.h b/datalogcli/src/main/native/cpp/LogLoader.h similarity index 97% rename from wpilogcli/src/main/native/cpp/LogLoader.h rename to datalogcli/src/main/native/cpp/LogLoader.h index 0fd56b58657..fa21662fd75 100644 --- a/wpilogcli/src/main/native/cpp/LogLoader.h +++ b/datalogcli/src/main/native/cpp/LogLoader.h @@ -22,7 +22,7 @@ namespace wpi { class Logger; } // namespace wpi -namespace wpilogcli { +namespace datalogcli { /** * Helps with loading datalog files. */ @@ -71,4 +71,4 @@ class LogLoader { void RebuildEntryTree(); int GetTargetEntryId(std::string_view name); }; -} // namespace wpilogcli +} // namespace datalogcli diff --git a/wpilogcli/src/main/native/cpp/main.cpp b/datalogcli/src/main/native/cpp/main.cpp similarity index 100% rename from wpilogcli/src/main/native/cpp/main.cpp rename to datalogcli/src/main/native/cpp/main.cpp diff --git a/wpilogcli/CMakeLists.txt b/wpilogcli/CMakeLists.txt deleted file mode 100644 index ceca9f158e4..00000000000 --- a/wpilogcli/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -project(wpilogcli) - -include(CompileWarnings) -include(GenResources) - -configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) -generate_resources(src/main/native/resources generated/main/cpp DLT dlt wpilogcli_resources_src) - -file(GLOB wpilogcli_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) - -if(WIN32) - set(wpilogcli_rc src/main/native/win/wpilogcli.rc) -elseif(APPLE) - set(MACOSX_BUNDLE_ICON_FILE wpilogcli.icns) - set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") -endif() - -add_executable( - wpilogcli - ${wpilogcli_src} - ${wpilogcli_resources_src} - ${wpilogcli_rc} - ${APP_ICON_MACOSX} -) - -target_include_directories(wpilogcli PUBLIC src/main/native/cpp) - -target_link_libraries(wpilogcli PRIVATE wpiutil) - -if(WIN32) - set_target_properties(wpilogcli PROPERTIES WIN32_EXECUTABLE YES) -elseif(APPLE) - set_target_properties(wpilogcli PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "wpilogcli") -endif() \ No newline at end of file From c1780edd146cf7c146b248c1fd74c775d85b8cb4 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 11:04:28 -0600 Subject: [PATCH 07/35] remove local argparse --- .../src/main/native/cpp/argparse/argparse.hpp | 2543 ----------------- 1 file changed, 2543 deletions(-) delete mode 100644 wpilogcli/src/main/native/cpp/argparse/argparse.hpp diff --git a/wpilogcli/src/main/native/cpp/argparse/argparse.hpp b/wpilogcli/src/main/native/cpp/argparse/argparse.hpp deleted file mode 100644 index 0843613bab8..00000000000 --- a/wpilogcli/src/main/native/cpp/argparse/argparse.hpp +++ /dev/null @@ -1,2543 +0,0 @@ -/* - __ _ _ __ __ _ _ __ __ _ _ __ ___ ___ - / _` | '__/ _` | '_ \ / _` | '__/ __|/ _ \ Argument Parser for Modern C++ -| (_| | | | (_| | |_) | (_| | | \__ \ __/ http://github.com/p-ranav/argparse - \__,_|_| \__, | .__/ \__,_|_| |___/\___| - |___/|_| - -Licensed under the MIT License . -SPDX-License-Identifier: MIT -Copyright (c) 2019-2022 Pranav Srinivas Kumar -and other contributors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ -#pragma once - -#include - -#ifndef ARGPARSE_MODULE_USE_STD_MODULE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#endif - -#ifndef ARGPARSE_CUSTOM_STRTOF -#define ARGPARSE_CUSTOM_STRTOF strtof -#endif - -#ifndef ARGPARSE_CUSTOM_STRTOD -#define ARGPARSE_CUSTOM_STRTOD strtod -#endif - -#ifndef ARGPARSE_CUSTOM_STRTOLD -#define ARGPARSE_CUSTOM_STRTOLD strtold -#endif - -namespace argparse { - -namespace details { // namespace for helper methods - -template -struct HasContainerTraits : std::false_type {}; - -template <> struct HasContainerTraits : std::false_type {}; - -template <> struct HasContainerTraits : std::false_type {}; - -template -struct HasContainerTraits< - T, std::void_t().begin()), - decltype(std::declval().end()), - decltype(std::declval().size())>> : std::true_type {}; - -template -inline constexpr bool IsContainer = HasContainerTraits::value; - -template -struct HasStreamableTraits : std::false_type {}; - -template -struct HasStreamableTraits< - T, - std::void_t() << std::declval())>> - : std::true_type {}; - -template -inline constexpr bool IsStreamable = HasStreamableTraits::value; - -constexpr std::size_t repr_max_container_size = 5; - -template std::string repr(T const &val) { - if constexpr (std::is_same_v) { - return val ? "true" : "false"; - } else if constexpr (std::is_convertible_v) { - return '"' + std::string{std::string_view{val}} + '"'; - } else if constexpr (IsContainer) { - std::stringstream out; - out << "{"; - const auto size = val.size(); - if (size > 1) { - out << repr(*val.begin()); - std::for_each( - std::next(val.begin()), - std::next( - val.begin(), - static_cast( - std::min(size, repr_max_container_size) - 1)), - [&out](const auto &v) { out << " " << repr(v); }); - if (size <= repr_max_container_size) { - out << " "; - } else { - out << "..."; - } - } - if (size > 0) { - out << repr(*std::prev(val.end())); - } - out << "}"; - return out.str(); - } else if constexpr (IsStreamable) { - std::stringstream out; - out << val; - return out.str(); - } else { - return ""; - } -} - -namespace { - -template constexpr bool standard_signed_integer = false; -template <> constexpr bool standard_signed_integer = true; -template <> constexpr bool standard_signed_integer = true; -template <> constexpr bool standard_signed_integer = true; -template <> constexpr bool standard_signed_integer = true; -template <> constexpr bool standard_signed_integer = true; - -template constexpr bool standard_unsigned_integer = false; -template <> constexpr bool standard_unsigned_integer = true; -template <> constexpr bool standard_unsigned_integer = true; -template <> constexpr bool standard_unsigned_integer = true; -template <> constexpr bool standard_unsigned_integer = true; -template <> -constexpr bool standard_unsigned_integer = true; - -} // namespace - -constexpr int radix_2 = 2; -constexpr int radix_8 = 8; -constexpr int radix_10 = 10; -constexpr int radix_16 = 16; - -template -constexpr bool standard_integer = - standard_signed_integer || standard_unsigned_integer; - -template -constexpr decltype(auto) -apply_plus_one_impl(F &&f, Tuple &&t, Extra &&x, - std::index_sequence /*unused*/) { - return std::invoke(std::forward(f), std::get(std::forward(t))..., - std::forward(x)); -} - -template -constexpr decltype(auto) apply_plus_one(F &&f, Tuple &&t, Extra &&x) { - return details::apply_plus_one_impl( - std::forward(f), std::forward(t), std::forward(x), - std::make_index_sequence< - std::tuple_size_v>>{}); -} - -constexpr auto pointer_range(std::string_view s) noexcept { - return std::tuple(s.data(), s.data() + s.size()); -} - -template -constexpr bool starts_with(std::basic_string_view prefix, - std::basic_string_view s) noexcept { - return s.substr(0, prefix.size()) == prefix; -} - -enum class chars_format { - scientific = 0xf1, - fixed = 0xf2, - hex = 0xf4, - binary = 0xf8, - general = fixed | scientific -}; - -struct ConsumeBinaryPrefixResult { - bool is_binary; - std::string_view rest; -}; - -constexpr auto consume_binary_prefix(std::string_view s) - -> ConsumeBinaryPrefixResult { - if (starts_with(std::string_view{"0b"}, s) || - starts_with(std::string_view{"0B"}, s)) { - s.remove_prefix(2); - return {true, s}; - } - return {false, s}; -} - -struct ConsumeHexPrefixResult { - bool is_hexadecimal; - std::string_view rest; -}; - -using namespace std::literals; - -constexpr auto consume_hex_prefix(std::string_view s) - -> ConsumeHexPrefixResult { - if (starts_with("0x"sv, s) || starts_with("0X"sv, s)) { - s.remove_prefix(2); - return {true, s}; - } - return {false, s}; -} - -template -inline auto do_from_chars(std::string_view s) -> T { - T x{0}; - auto [first, last] = pointer_range(s); - auto [ptr, ec] = std::from_chars(first, last, x, Param); - if (ec == std::errc()) { - if (ptr == last) { - return x; - } - throw std::invalid_argument{"pattern '" + std::string(s) + - "' does not match to the end"}; - } - if (ec == std::errc::invalid_argument) { - throw std::invalid_argument{"pattern '" + std::string(s) + "' not found"}; - } - if (ec == std::errc::result_out_of_range) { - throw std::range_error{"'" + std::string(s) + "' not representable"}; - } - return x; // unreachable -} - -template struct parse_number { - auto operator()(std::string_view s) -> T { - return do_from_chars(s); - } -}; - -template struct parse_number { - auto operator()(std::string_view s) -> T { - if (auto [ok, rest] = consume_binary_prefix(s); ok) { - return do_from_chars(rest); - } - throw std::invalid_argument{"pattern not found"}; - } -}; - -template struct parse_number { - auto operator()(std::string_view s) -> T { - if (starts_with("0x"sv, s) || starts_with("0X"sv, s)) { - if (auto [ok, rest] = consume_hex_prefix(s); ok) { - try { - return do_from_chars(rest); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + std::string(s) + - "' as hexadecimal: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + std::string(s) + - "' as hexadecimal: " + err.what()); - } - } - } else { - // Allow passing hex numbers without prefix - // Shape 'x' already has to be specified - try { - return do_from_chars(s); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + std::string(s) + - "' as hexadecimal: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + std::string(s) + - "' as hexadecimal: " + err.what()); - } - } - - throw std::invalid_argument{"pattern '" + std::string(s) + - "' not identified as hexadecimal"}; - } -}; - -template struct parse_number { - auto operator()(std::string_view s) -> T { - auto [ok, rest] = consume_hex_prefix(s); - if (ok) { - try { - return do_from_chars(rest); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + std::string(s) + - "' as hexadecimal: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + std::string(s) + - "' as hexadecimal: " + err.what()); - } - } - - auto [ok_binary, rest_binary] = consume_binary_prefix(s); - if (ok_binary) { - try { - return do_from_chars(rest_binary); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + std::string(s) + - "' as binary: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + std::string(s) + - "' as binary: " + err.what()); - } - } - - if (starts_with("0"sv, s)) { - try { - return do_from_chars(rest); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + std::string(s) + - "' as octal: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + std::string(s) + - "' as octal: " + err.what()); - } - } - - try { - return do_from_chars(rest); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + std::string(s) + - "' as decimal integer: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + std::string(s) + - "' as decimal integer: " + err.what()); - } - } -}; - -namespace { - -template inline const auto generic_strtod = nullptr; -template <> inline const auto generic_strtod = ARGPARSE_CUSTOM_STRTOF; -template <> inline const auto generic_strtod = ARGPARSE_CUSTOM_STRTOD; -template <> -inline const auto generic_strtod = ARGPARSE_CUSTOM_STRTOLD; - -} // namespace - -template inline auto do_strtod(std::string const &s) -> T { - if (isspace(static_cast(s[0])) || s[0] == '+') { - throw std::invalid_argument{"pattern '" + s + "' not found"}; - } - - auto [first, last] = pointer_range(s); - char *ptr; - - errno = 0; - auto x = generic_strtod(first, &ptr); - if (errno == 0) { - if (ptr == last) { - return x; - } - throw std::invalid_argument{"pattern '" + s + - "' does not match to the end"}; - } - if (errno == ERANGE) { - throw std::range_error{"'" + s + "' not representable"}; - } - return x; // unreachable -} - -template struct parse_number { - auto operator()(std::string const &s) -> T { - if (auto r = consume_hex_prefix(s); r.is_hexadecimal) { - throw std::invalid_argument{ - "chars_format::general does not parse hexfloat"}; - } - if (auto r = consume_binary_prefix(s); r.is_binary) { - throw std::invalid_argument{ - "chars_format::general does not parse binfloat"}; - } - - try { - return do_strtod(s); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + s + - "' as number: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + s + - "' as number: " + err.what()); - } - } -}; - -template struct parse_number { - auto operator()(std::string const &s) -> T { - if (auto r = consume_hex_prefix(s); !r.is_hexadecimal) { - throw std::invalid_argument{"chars_format::hex parses hexfloat"}; - } - if (auto r = consume_binary_prefix(s); r.is_binary) { - throw std::invalid_argument{"chars_format::hex does not parse binfloat"}; - } - - try { - return do_strtod(s); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + s + - "' as hexadecimal: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + s + - "' as hexadecimal: " + err.what()); - } - } -}; - -template struct parse_number { - auto operator()(std::string const &s) -> T { - if (auto r = consume_hex_prefix(s); r.is_hexadecimal) { - throw std::invalid_argument{ - "chars_format::binary does not parse hexfloat"}; - } - if (auto r = consume_binary_prefix(s); !r.is_binary) { - throw std::invalid_argument{"chars_format::binary parses binfloat"}; - } - - return do_strtod(s); - } -}; - -template struct parse_number { - auto operator()(std::string const &s) -> T { - if (auto r = consume_hex_prefix(s); r.is_hexadecimal) { - throw std::invalid_argument{ - "chars_format::scientific does not parse hexfloat"}; - } - if (auto r = consume_binary_prefix(s); r.is_binary) { - throw std::invalid_argument{ - "chars_format::scientific does not parse binfloat"}; - } - if (s.find_first_of("eE") == std::string::npos) { - throw std::invalid_argument{ - "chars_format::scientific requires exponent part"}; - } - - try { - return do_strtod(s); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + s + - "' as scientific notation: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + s + - "' as scientific notation: " + err.what()); - } - } -}; - -template struct parse_number { - auto operator()(std::string const &s) -> T { - if (auto r = consume_hex_prefix(s); r.is_hexadecimal) { - throw std::invalid_argument{ - "chars_format::fixed does not parse hexfloat"}; - } - if (auto r = consume_binary_prefix(s); r.is_binary) { - throw std::invalid_argument{ - "chars_format::fixed does not parse binfloat"}; - } - if (s.find_first_of("eE") != std::string::npos) { - throw std::invalid_argument{ - "chars_format::fixed does not parse exponent part"}; - } - - try { - return do_strtod(s); - } catch (const std::invalid_argument &err) { - throw std::invalid_argument("Failed to parse '" + s + - "' as fixed notation: " + err.what()); - } catch (const std::range_error &err) { - throw std::range_error("Failed to parse '" + s + - "' as fixed notation: " + err.what()); - } - } -}; - -template -std::string join(StrIt first, StrIt last, const std::string &separator) { - if (first == last) { - return ""; - } - std::stringstream value; - value << *first; - ++first; - while (first != last) { - value << separator << *first; - ++first; - } - return value.str(); -} - -template struct can_invoke_to_string { - template - static auto test(int) - -> decltype(std::to_string(std::declval()), std::true_type{}); - - template static auto test(...) -> std::false_type; - - static constexpr bool value = decltype(test(0))::value; -}; - -template struct IsChoiceTypeSupported { - using CleanType = typename std::decay::type; - static const bool value = std::is_integral::value || - std::is_same::value || - std::is_same::value || - std::is_same::value; -}; - -template -std::size_t get_levenshtein_distance(const StringType &s1, - const StringType &s2) { - std::vector> dp( - s1.size() + 1, std::vector(s2.size() + 1, 0)); - - for (std::size_t i = 0; i <= s1.size(); ++i) { - for (std::size_t j = 0; j <= s2.size(); ++j) { - if (i == 0) { - dp[i][j] = j; - } else if (j == 0) { - dp[i][j] = i; - } else if (s1[i - 1] == s2[j - 1]) { - dp[i][j] = dp[i - 1][j - 1]; - } else { - dp[i][j] = 1 + std::min({dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]}); - } - } - } - - return dp[s1.size()][s2.size()]; -} - -template -std::string get_most_similar_string(const std::map &map, - const std::string &input) { - std::string most_similar{}; - std::size_t min_distance = (std::numeric_limits::max)(); - - for (const auto &entry : map) { - std::size_t distance = get_levenshtein_distance(entry.first, input); - if (distance < min_distance) { - min_distance = distance; - most_similar = entry.first; - } - } - - return most_similar; -} - -} // namespace details - -enum class nargs_pattern { optional, any, at_least_one }; - -enum class default_arguments : unsigned int { - none = 0, - help = 1, - version = 2, - all = help | version, -}; - -inline default_arguments operator&(const default_arguments &a, - const default_arguments &b) { - return static_cast( - static_cast::type>(a) & - static_cast::type>(b)); -} - -class ArgumentParser; - -class Argument { - friend class ArgumentParser; - friend auto operator<<(std::ostream &stream, const ArgumentParser &parser) - -> std::ostream &; - - template - explicit Argument(std::string_view prefix_chars, - std::array &&a, - std::index_sequence /*unused*/) - : m_accepts_optional_like_value(false), - m_is_optional((is_optional(a[I], prefix_chars) || ...)), - m_is_required(false), m_is_repeatable(false), m_is_used(false), - m_is_hidden(false), m_prefix_chars(prefix_chars) { - ((void)m_names.emplace_back(a[I]), ...); - std::sort( - m_names.begin(), m_names.end(), [](const auto &lhs, const auto &rhs) { - return lhs.size() == rhs.size() ? lhs < rhs : lhs.size() < rhs.size(); - }); - } - -public: - template - explicit Argument(std::string_view prefix_chars, - std::array &&a) - : Argument(prefix_chars, std::move(a), std::make_index_sequence{}) {} - - Argument &help(std::string help_text) { - m_help = std::move(help_text); - return *this; - } - - Argument &metavar(std::string metavar) { - m_metavar = std::move(metavar); - return *this; - } - - template Argument &default_value(T &&value) { - m_num_args_range = NArgsRange{0, m_num_args_range.get_max()}; - m_default_value_repr = details::repr(value); - - if constexpr (std::is_convertible_v) { - m_default_value_str = std::string{std::string_view{value}}; - } else if constexpr (details::can_invoke_to_string::value) { - m_default_value_str = std::to_string(value); - } - - m_default_value = std::forward(value); - return *this; - } - - Argument &default_value(const char *value) { - return default_value(std::string(value)); - } - - Argument &required() { - m_is_required = true; - return *this; - } - - Argument &implicit_value(std::any value) { - m_implicit_value = std::move(value); - m_num_args_range = NArgsRange{0, 0}; - return *this; - } - - // This is shorthand for: - // program.add_argument("foo") - // .default_value(false) - // .implicit_value(true) - Argument &flag() { - default_value(false); - implicit_value(true); - return *this; - } - - template - auto action(F &&callable, Args &&... bound_args) - -> std::enable_if_t, - Argument &> { - using action_type = std::conditional_t< - std::is_void_v>, - void_action, valued_action>; - if constexpr (sizeof...(Args) == 0) { - m_action.emplace(std::forward(callable)); - } else { - m_action.emplace( - [f = std::forward(callable), - tup = std::make_tuple(std::forward(bound_args)...)]( - std::string const &opt) mutable { - return details::apply_plus_one(f, tup, opt); - }); - } - return *this; - } - - auto &store_into(bool &var) { - flag(); - if (m_default_value.has_value()) { - var = std::any_cast(m_default_value); - } - action([&var](const auto & /*unused*/) { var = true; }); - return *this; - } - - template ::value>::type * = nullptr> - auto &store_into(T &var) { - if (m_default_value.has_value()) { - var = std::any_cast(m_default_value); - } - action([&var](const auto &s) { - var = details::parse_number()(s); - }); - return *this; - } - - auto &store_into(double &var) { - if (m_default_value.has_value()) { - var = std::any_cast(m_default_value); - } - action([&var](const auto &s) { - var = details::parse_number()(s); - }); - return *this; - } - - auto &store_into(std::string &var) { - if (m_default_value.has_value()) { - var = std::any_cast(m_default_value); - } - action([&var](const std::string &s) { var = s; }); - return *this; - } - - auto &store_into(std::vector &var) { - if (m_default_value.has_value()) { - var = std::any_cast>(m_default_value); - } - action([this, &var](const std::string &s) { - if (!m_is_used) { - var.clear(); - } - m_is_used = true; - var.push_back(s); - }); - return *this; - } - - auto &store_into(std::vector &var) { - if (m_default_value.has_value()) { - var = std::any_cast>(m_default_value); - } - action([this, &var](const std::string &s) { - if (!m_is_used) { - var.clear(); - } - m_is_used = true; - var.push_back(details::parse_number()(s)); - }); - return *this; - } - - auto &store_into(std::set &var) { - if (m_default_value.has_value()) { - var = std::any_cast>(m_default_value); - } - action([this, &var](const std::string &s) { - if (!m_is_used) { - var.clear(); - } - m_is_used = true; - var.insert(s); - }); - return *this; - } - - auto &store_into(std::set &var) { - if (m_default_value.has_value()) { - var = std::any_cast>(m_default_value); - } - action([this, &var](const std::string &s) { - if (!m_is_used) { - var.clear(); - } - m_is_used = true; - var.insert(details::parse_number()(s)); - }); - return *this; - } - - auto &append() { - m_is_repeatable = true; - return *this; - } - - // Cause the argument to be invisible in usage and help - auto &hidden() { - m_is_hidden = true; - return *this; - } - - template - auto scan() -> std::enable_if_t, Argument &> { - static_assert(!(std::is_const_v || std::is_volatile_v), - "T should not be cv-qualified"); - auto is_one_of = [](char c, auto... x) constexpr { - return ((c == x) || ...); - }; - - if constexpr (is_one_of(Shape, 'd') && details::standard_integer) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'i') && - details::standard_integer) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'u') && - details::standard_unsigned_integer) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'b') && - details::standard_unsigned_integer) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'o') && - details::standard_unsigned_integer) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'x', 'X') && - details::standard_unsigned_integer) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'a', 'A') && - std::is_floating_point_v) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'e', 'E') && - std::is_floating_point_v) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'f', 'F') && - std::is_floating_point_v) { - action(details::parse_number()); - } else if constexpr (is_one_of(Shape, 'g', 'G') && - std::is_floating_point_v) { - action(details::parse_number()); - } else { - static_assert(alignof(T) == 0, "No scan specification for T"); - } - - return *this; - } - - Argument &nargs(std::size_t num_args) { - m_num_args_range = NArgsRange{num_args, num_args}; - return *this; - } - - Argument &nargs(std::size_t num_args_min, std::size_t num_args_max) { - m_num_args_range = NArgsRange{num_args_min, num_args_max}; - return *this; - } - - Argument &nargs(nargs_pattern pattern) { - switch (pattern) { - case nargs_pattern::optional: - m_num_args_range = NArgsRange{0, 1}; - break; - case nargs_pattern::any: - m_num_args_range = - NArgsRange{0, (std::numeric_limits::max)()}; - break; - case nargs_pattern::at_least_one: - m_num_args_range = - NArgsRange{1, (std::numeric_limits::max)()}; - break; - } - return *this; - } - - Argument &remaining() { - m_accepts_optional_like_value = true; - return nargs(nargs_pattern::any); - } - - template void add_choice(T &&choice) { - static_assert(details::IsChoiceTypeSupported::value, - "Only string or integer type supported for choice"); - static_assert(std::is_convertible_v || - details::can_invoke_to_string::value, - "Choice is not convertible to string_type"); - if (!m_choices.has_value()) { - m_choices = std::vector{}; - } - - if constexpr (std::is_convertible_v) { - m_choices.value().push_back( - std::string{std::string_view{std::forward(choice)}}); - } else if constexpr (details::can_invoke_to_string::value) { - m_choices.value().push_back(std::to_string(std::forward(choice))); - } - } - - Argument &choices() { - if (!m_choices.has_value()) { - throw std::runtime_error("Zero choices provided"); - } - return *this; - } - - template - Argument &choices(T &&first, U &&... rest) { - add_choice(std::forward(first)); - choices(std::forward(rest)...); - return *this; - } - - void find_default_value_in_choices_or_throw() const { - - const auto &choices = m_choices.value(); - - if (m_default_value.has_value()) { - if (std::find(choices.begin(), choices.end(), m_default_value_str) == - choices.end()) { - // provided arg not in list of allowed choices - // report error - - std::string choices_as_csv = - std::accumulate(choices.begin(), choices.end(), std::string(), - [](const std::string &a, const std::string &b) { - return a + (a.empty() ? "" : ", ") + b; - }); - - throw std::runtime_error( - std::string{"Invalid default value "} + m_default_value_repr + - " - allowed options: {" + choices_as_csv + "}"); - } - } - } - - template - void find_value_in_choices_or_throw(Iterator it) const { - - const auto &choices = m_choices.value(); - - if (std::find(choices.begin(), choices.end(), *it) == choices.end()) { - // provided arg not in list of allowed choices - // report error - - std::string choices_as_csv = - std::accumulate(choices.begin(), choices.end(), std::string(), - [](const std::string &a, const std::string &b) { - return a + (a.empty() ? "" : ", ") + b; - }); - - throw std::runtime_error(std::string{"Invalid argument "} + - details::repr(*it) + " - allowed options: {" + - choices_as_csv + "}"); - } - } - - /* The dry_run parameter can be set to true to avoid running the actions, - * and setting m_is_used. This may be used by a pre-processing step to do - * a first iteration over arguments. - */ - template - Iterator consume(Iterator start, Iterator end, - std::string_view used_name = {}, bool dry_run = false) { - if (!m_is_repeatable && m_is_used) { - throw std::runtime_error( - std::string("Duplicate argument ").append(used_name)); - } - m_used_name = used_name; - - if (m_choices.has_value()) { - // Check each value in (start, end) and make sure - // it is in the list of allowed choices/options - std::size_t i = 0; - auto max_number_of_args = m_num_args_range.get_max(); - for (auto it = start; it != end; ++it) { - if (i == max_number_of_args) { - break; - } - find_value_in_choices_or_throw(it); - i += 1; - } - } - - const auto num_args_max = m_num_args_range.get_max(); - const auto num_args_min = m_num_args_range.get_min(); - std::size_t dist = 0; - if (num_args_max == 0) { - if (!dry_run) { - m_values.emplace_back(m_implicit_value); - std::visit([](const auto &f) { f({}); }, m_action); - m_is_used = true; - } - return start; - } - if ((dist = static_cast(std::distance(start, end))) >= - num_args_min) { - if (num_args_max < dist) { - end = std::next(start, static_cast( - num_args_max)); - } - if (!m_accepts_optional_like_value) { - end = std::find_if( - start, end, - std::bind(is_optional, std::placeholders::_1, m_prefix_chars)); - dist = static_cast(std::distance(start, end)); - if (dist < num_args_min) { - throw std::runtime_error("Too few arguments for '" + - std::string(m_used_name) + "'."); - } - } - - struct ActionApply { - void operator()(valued_action &f) { - std::transform(first, last, std::back_inserter(self.m_values), f); - } - - void operator()(void_action &f) { - std::for_each(first, last, f); - if (!self.m_default_value.has_value()) { - if (!self.m_accepts_optional_like_value) { - self.m_values.resize( - static_cast(std::distance(first, last))); - } - } - } - - Iterator first, last; - Argument &self; - }; - if (!dry_run) { - std::visit(ActionApply{start, end, *this}, m_action); - m_is_used = true; - } - return end; - } - if (m_default_value.has_value()) { - if (!dry_run) { - m_is_used = true; - } - return start; - } - throw std::runtime_error("Too few arguments for '" + - std::string(m_used_name) + "'."); - } - - /* - * @throws std::runtime_error if argument values are not valid - */ - void validate() const { - if (m_is_optional) { - // TODO: check if an implicit value was programmed for this argument - if (!m_is_used && !m_default_value.has_value() && m_is_required) { - throw_required_arg_not_used_error(); - } - if (m_is_used && m_is_required && m_values.empty()) { - throw_required_arg_no_value_provided_error(); - } - } else { - if (!m_num_args_range.contains(m_values.size()) && - !m_default_value.has_value()) { - throw_nargs_range_validation_error(); - } - } - - if (m_choices.has_value()) { - // Make sure the default value (if provided) - // is in the list of choices - find_default_value_in_choices_or_throw(); - } - } - - std::string get_names_csv(char separator = ',') const { - return std::accumulate( - m_names.begin(), m_names.end(), std::string{""}, - [&](const std::string &result, const std::string &name) { - return result.empty() ? name : result + separator + name; - }); - } - - std::string get_usage_full() const { - std::stringstream usage; - - usage << get_names_csv('/'); - const std::string metavar = !m_metavar.empty() ? m_metavar : "VAR"; - if (m_num_args_range.get_max() > 0) { - usage << " " << metavar; - if (m_num_args_range.get_max() > 1) { - usage << "..."; - } - } - return usage.str(); - } - - std::string get_inline_usage() const { - std::stringstream usage; - // Find the longest variant to show in the usage string - std::string longest_name = m_names.front(); - for (const auto &s : m_names) { - if (s.size() > longest_name.size()) { - longest_name = s; - } - } - if (!m_is_required) { - usage << "["; - } - usage << longest_name; - const std::string metavar = !m_metavar.empty() ? m_metavar : "VAR"; - if (m_num_args_range.get_max() > 0) { - usage << " " << metavar; - if (m_num_args_range.get_max() > 1 && - m_metavar.find("> <") == std::string::npos) { - usage << "..."; - } - } - if (!m_is_required) { - usage << "]"; - } - if (m_is_repeatable) { - usage << "..."; - } - return usage.str(); - } - - std::size_t get_arguments_length() const { - - std::size_t names_size = std::accumulate( - std::begin(m_names), std::end(m_names), std::size_t(0), - [](const auto &sum, const auto &s) { return sum + s.size(); }); - - if (is_positional(m_names.front(), m_prefix_chars)) { - // A set metavar means this replaces the names - if (!m_metavar.empty()) { - // Indent and metavar - return 2 + m_metavar.size(); - } - - // Indent and space-separated - return 2 + names_size + (m_names.size() - 1); - } - // Is an option - include both names _and_ metavar - // size = text + (", " between names) - std::size_t size = names_size + 2 * (m_names.size() - 1); - if (!m_metavar.empty() && m_num_args_range == NArgsRange{1, 1}) { - size += m_metavar.size() + 1; - } - return size + 2; // indent - } - - friend std::ostream &operator<<(std::ostream &stream, - const Argument &argument) { - std::stringstream name_stream; - name_stream << " "; // indent - if (argument.is_positional(argument.m_names.front(), - argument.m_prefix_chars)) { - if (!argument.m_metavar.empty()) { - name_stream << argument.m_metavar; - } else { - name_stream << details::join(argument.m_names.begin(), - argument.m_names.end(), " "); - } - } else { - name_stream << details::join(argument.m_names.begin(), - argument.m_names.end(), ", "); - // If we have a metavar, and one narg - print the metavar - if (!argument.m_metavar.empty() && - argument.m_num_args_range == NArgsRange{1, 1}) { - name_stream << " " << argument.m_metavar; - } - else if (!argument.m_metavar.empty() && - argument.m_num_args_range.get_min() == argument.m_num_args_range.get_max() && - argument.m_metavar.find("> <") != std::string::npos) { - name_stream << " " << argument.m_metavar; - } - } - - // align multiline help message - auto stream_width = stream.width(); - auto name_padding = std::string(name_stream.str().size(), ' '); - auto pos = std::string::size_type{}; - auto prev = std::string::size_type{}; - auto first_line = true; - auto hspace = " "; // minimal space between name and help message - stream << name_stream.str(); - std::string_view help_view(argument.m_help); - while ((pos = argument.m_help.find('\n', prev)) != std::string::npos) { - auto line = help_view.substr(prev, pos - prev + 1); - if (first_line) { - stream << hspace << line; - first_line = false; - } else { - stream.width(stream_width); - stream << name_padding << hspace << line; - } - prev += pos - prev + 1; - } - if (first_line) { - stream << hspace << argument.m_help; - } else { - auto leftover = help_view.substr(prev, argument.m_help.size() - prev); - if (!leftover.empty()) { - stream.width(stream_width); - stream << name_padding << hspace << leftover; - } - } - - // print nargs spec - if (!argument.m_help.empty()) { - stream << " "; - } - stream << argument.m_num_args_range; - - bool add_space = false; - if (argument.m_default_value.has_value() && - argument.m_num_args_range != NArgsRange{0, 0}) { - stream << "[default: " << argument.m_default_value_repr << "]"; - add_space = true; - } else if (argument.m_is_required) { - stream << "[required]"; - add_space = true; - } - if (argument.m_is_repeatable) { - if (add_space) { - stream << " "; - } - stream << "[may be repeated]"; - } - stream << "\n"; - return stream; - } - - template bool operator!=(const T &rhs) const { - return !(*this == rhs); - } - - /* - * Compare to an argument value of known type - * @throws std::logic_error in case of incompatible types - */ - template bool operator==(const T &rhs) const { - if constexpr (!details::IsContainer) { - return get() == rhs; - } else { - using ValueType = typename T::value_type; - auto lhs = get(); - return std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs), - std::end(rhs), [](const auto &a, const auto &b) { - return std::any_cast(a) == b; - }); - } - } - - /* - * positional: - * _empty_ - * '-' - * '-' decimal-literal - * !'-' anything - */ - static bool is_positional(std::string_view name, - std::string_view prefix_chars) { - auto first = lookahead(name); - - if (first == eof) { - return true; - } - if (prefix_chars.find(static_cast(first)) != - std::string_view::npos) { - name.remove_prefix(1); - if (name.empty()) { - return true; - } - return is_decimal_literal(name); - } - return true; - } - -private: - class NArgsRange { - std::size_t m_min; - std::size_t m_max; - - public: - NArgsRange(std::size_t minimum, std::size_t maximum) - : m_min(minimum), m_max(maximum) { - if (minimum > maximum) { - throw std::logic_error("Range of number of arguments is invalid"); - } - } - - bool contains(std::size_t value) const { - return value >= m_min && value <= m_max; - } - - bool is_exact() const { return m_min == m_max; } - - bool is_right_bounded() const { - return m_max < (std::numeric_limits::max)(); - } - - std::size_t get_min() const { return m_min; } - - std::size_t get_max() const { return m_max; } - - // Print help message - friend auto operator<<(std::ostream &stream, const NArgsRange &range) - -> std::ostream & { - if (range.m_min == range.m_max) { - if (range.m_min != 0 && range.m_min != 1) { - stream << "[nargs: " << range.m_min << "] "; - } - } else { - if (range.m_max == (std::numeric_limits::max)()) { - stream << "[nargs: " << range.m_min << " or more] "; - } else { - stream << "[nargs=" << range.m_min << ".." << range.m_max << "] "; - } - } - return stream; - } - - bool operator==(const NArgsRange &rhs) const { - return rhs.m_min == m_min && rhs.m_max == m_max; - } - - bool operator!=(const NArgsRange &rhs) const { return !(*this == rhs); } - }; - - void throw_nargs_range_validation_error() const { - std::stringstream stream; - if (!m_used_name.empty()) { - stream << m_used_name << ": "; - } else { - stream << m_names.front() << ": "; - } - if (m_num_args_range.is_exact()) { - stream << m_num_args_range.get_min(); - } else if (m_num_args_range.is_right_bounded()) { - stream << m_num_args_range.get_min() << " to " - << m_num_args_range.get_max(); - } else { - stream << m_num_args_range.get_min() << " or more"; - } - stream << " argument(s) expected. " << m_values.size() << " provided."; - throw std::runtime_error(stream.str()); - } - - void throw_required_arg_not_used_error() const { - std::stringstream stream; - stream << m_names.front() << ": required."; - throw std::runtime_error(stream.str()); - } - - void throw_required_arg_no_value_provided_error() const { - std::stringstream stream; - stream << m_used_name << ": no value provided."; - throw std::runtime_error(stream.str()); - } - - static constexpr int eof = std::char_traits::eof(); - - static auto lookahead(std::string_view s) -> int { - if (s.empty()) { - return eof; - } - return static_cast(static_cast(s[0])); - } - - /* - * decimal-literal: - * '0' - * nonzero-digit digit-sequence_opt - * integer-part fractional-part - * fractional-part - * integer-part '.' exponent-part_opt - * integer-part exponent-part - * - * integer-part: - * digit-sequence - * - * fractional-part: - * '.' post-decimal-point - * - * post-decimal-point: - * digit-sequence exponent-part_opt - * - * exponent-part: - * 'e' post-e - * 'E' post-e - * - * post-e: - * sign_opt digit-sequence - * - * sign: one of - * '+' '-' - */ - static bool is_decimal_literal(std::string_view s) { - auto is_digit = [](auto c) constexpr { - switch (c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return true; - default: - return false; - } - }; - - // precondition: we have consumed or will consume at least one digit - auto consume_digits = [=](std::string_view sd) { - // NOLINTNEXTLINE(readability-qualified-auto) - auto it = std::find_if_not(std::begin(sd), std::end(sd), is_digit); - return sd.substr(static_cast(it - std::begin(sd))); - }; - - switch (lookahead(s)) { - case '0': { - s.remove_prefix(1); - if (s.empty()) { - return true; - } - goto integer_part; - } - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - s = consume_digits(s); - if (s.empty()) { - return true; - } - goto integer_part_consumed; - } - case '.': { - s.remove_prefix(1); - goto post_decimal_point; - } - default: - return false; - } - - integer_part: - s = consume_digits(s); - integer_part_consumed: - switch (lookahead(s)) { - case '.': { - s.remove_prefix(1); - if (is_digit(lookahead(s))) { - goto post_decimal_point; - } else { - goto exponent_part_opt; - } - } - case 'e': - case 'E': { - s.remove_prefix(1); - goto post_e; - } - default: - return false; - } - - post_decimal_point: - if (is_digit(lookahead(s))) { - s = consume_digits(s); - goto exponent_part_opt; - } - return false; - - exponent_part_opt: - switch (lookahead(s)) { - case eof: - return true; - case 'e': - case 'E': { - s.remove_prefix(1); - goto post_e; - } - default: - return false; - } - - post_e: - switch (lookahead(s)) { - case '-': - case '+': - s.remove_prefix(1); - } - if (is_digit(lookahead(s))) { - s = consume_digits(s); - return s.empty(); - } - return false; - } - - static bool is_optional(std::string_view name, - std::string_view prefix_chars) { - return !is_positional(name, prefix_chars); - } - - /* - * Get argument value given a type - * @throws std::logic_error in case of incompatible types - */ - template T get() const { - if (!m_values.empty()) { - if constexpr (details::IsContainer) { - return any_cast_container(m_values); - } else { - return std::any_cast(m_values.front()); - } - } - if (m_default_value.has_value()) { - return std::any_cast(m_default_value); - } - if constexpr (details::IsContainer) { - if (!m_accepts_optional_like_value) { - return any_cast_container(m_values); - } - } - - throw std::logic_error("No value provided for '" + m_names.back() + "'."); - } - - /* - * Get argument value given a type. - * @pre The object has no default value. - * @returns The stored value if any, std::nullopt otherwise. - */ - template auto present() const -> std::optional { - if (m_default_value.has_value()) { - throw std::logic_error("Argument with default value always presents"); - } - if (m_values.empty()) { - return std::nullopt; - } - if constexpr (details::IsContainer) { - return any_cast_container(m_values); - } - return std::any_cast(m_values.front()); - } - - template - static auto any_cast_container(const std::vector &operand) -> T { - using ValueType = typename T::value_type; - - T result; - std::transform( - std::begin(operand), std::end(operand), std::back_inserter(result), - [](const auto &value) { return std::any_cast(value); }); - return result; - } - - void set_usage_newline_counter(int i) { m_usage_newline_counter = i; } - - void set_group_idx(std::size_t i) { m_group_idx = i; } - - std::vector m_names; - std::string_view m_used_name; - std::string m_help; - std::string m_metavar; - std::any m_default_value; - std::string m_default_value_repr; - std::optional - m_default_value_str; // used for checking default_value against choices - std::any m_implicit_value; - std::optional> m_choices{std::nullopt}; - using valued_action = std::function; - using void_action = std::function; - std::variant m_action{ - std::in_place_type, - [](const std::string &value) { return value; }}; - std::vector m_values; - NArgsRange m_num_args_range{1, 1}; - // Bit field of bool values. Set default value in ctor. - bool m_accepts_optional_like_value : 1; - bool m_is_optional : 1; - bool m_is_required : 1; - bool m_is_repeatable : 1; - bool m_is_used : 1; - bool m_is_hidden : 1; // if set, does not appear in usage or help - std::string_view m_prefix_chars; // ArgumentParser has the prefix_chars - int m_usage_newline_counter = 0; - std::size_t m_group_idx = 0; -}; - -class ArgumentParser { -public: - explicit ArgumentParser(std::string program_name = {}, - std::string version = "1.0", - default_arguments add_args = default_arguments::all, - bool exit_on_default_arguments = true, - std::ostream &os = std::cout) - : m_program_name(std::move(program_name)), m_version(std::move(version)), - m_exit_on_default_arguments(exit_on_default_arguments), - m_parser_path(m_program_name) { - if ((add_args & default_arguments::help) == default_arguments::help) { - add_argument("-h", "--help") - .action([&](const auto & /*unused*/) { - os << help().str(); - if (m_exit_on_default_arguments) { - std::exit(0); - } - }) - .default_value(false) - .help("shows help message and exits") - .implicit_value(true) - .nargs(0); - } - if ((add_args & default_arguments::version) == default_arguments::version) { - add_argument("-v", "--version") - .action([&](const auto & /*unused*/) { - os << m_version << std::endl; - if (m_exit_on_default_arguments) { - std::exit(0); - } - }) - .default_value(false) - .help("prints version information and exits") - .implicit_value(true) - .nargs(0); - } - } - - ~ArgumentParser() = default; - - // ArgumentParser is meant to be used in a single function. - // Setup everything and parse arguments in one place. - // - // ArgumentParser internally uses std::string_views, - // references, iterators, etc. - // Many of these elements become invalidated after a copy or move. - ArgumentParser(const ArgumentParser &other) = delete; - ArgumentParser &operator=(const ArgumentParser &other) = delete; - ArgumentParser(ArgumentParser &&) noexcept = delete; - ArgumentParser &operator=(ArgumentParser &&) = delete; - - explicit operator bool() const { - auto arg_used = std::any_of(m_argument_map.cbegin(), m_argument_map.cend(), - [](auto &it) { return it.second->m_is_used; }); - auto subparser_used = - std::any_of(m_subparser_used.cbegin(), m_subparser_used.cend(), - [](auto &it) { return it.second; }); - - return m_is_parsed && (arg_used || subparser_used); - } - - // Parameter packing - // Call add_argument with variadic number of string arguments - template Argument &add_argument(Targs... f_args) { - using array_of_sv = std::array; - auto argument = - m_optional_arguments.emplace(std::cend(m_optional_arguments), - m_prefix_chars, array_of_sv{f_args...}); - - if (!argument->m_is_optional) { - m_positional_arguments.splice(std::cend(m_positional_arguments), - m_optional_arguments, argument); - } - argument->set_usage_newline_counter(m_usage_newline_counter); - argument->set_group_idx(m_group_names.size()); - - index_argument(argument); - return *argument; - } - - class MutuallyExclusiveGroup { - friend class ArgumentParser; - - public: - MutuallyExclusiveGroup() = delete; - - explicit MutuallyExclusiveGroup(ArgumentParser &parent, - bool required = false) - : m_parent(parent), m_required(required), m_elements({}) {} - - MutuallyExclusiveGroup(const MutuallyExclusiveGroup &other) = delete; - MutuallyExclusiveGroup & - operator=(const MutuallyExclusiveGroup &other) = delete; - - MutuallyExclusiveGroup(MutuallyExclusiveGroup &&other) noexcept - : m_parent(other.m_parent), m_required(other.m_required), - m_elements(std::move(other.m_elements)) { - other.m_elements.clear(); - } - - template Argument &add_argument(Targs... f_args) { - auto &argument = m_parent.add_argument(std::forward(f_args)...); - m_elements.push_back(&argument); - argument.set_usage_newline_counter(m_parent.m_usage_newline_counter); - argument.set_group_idx(m_parent.m_group_names.size()); - return argument; - } - - private: - ArgumentParser &m_parent; - bool m_required{false}; - std::vector m_elements{}; - }; - - MutuallyExclusiveGroup &add_mutually_exclusive_group(bool required = false) { - m_mutually_exclusive_groups.emplace_back(*this, required); - return m_mutually_exclusive_groups.back(); - } - - // Parameter packed add_parents method - // Accepts a variadic number of ArgumentParser objects - template - ArgumentParser &add_parents(const Targs &... f_args) { - for (const ArgumentParser &parent_parser : {std::ref(f_args)...}) { - for (const auto &argument : parent_parser.m_positional_arguments) { - auto it = m_positional_arguments.insert( - std::cend(m_positional_arguments), argument); - index_argument(it); - } - for (const auto &argument : parent_parser.m_optional_arguments) { - auto it = m_optional_arguments.insert(std::cend(m_optional_arguments), - argument); - index_argument(it); - } - } - return *this; - } - - // Ask for the next optional arguments to be displayed on a separate - // line in usage() output. Only effective if set_usage_max_line_width() is - // also used. - ArgumentParser &add_usage_newline() { - ++m_usage_newline_counter; - return *this; - } - - // Ask for the next optional arguments to be displayed in a separate section - // in usage() and help (<< *this) output. - // For usage(), this is only effective if set_usage_max_line_width() is - // also used. - ArgumentParser &add_group(std::string group_name) { - m_group_names.emplace_back(std::move(group_name)); - return *this; - } - - ArgumentParser &add_description(std::string description) { - m_description = std::move(description); - return *this; - } - - ArgumentParser &add_epilog(std::string epilog) { - m_epilog = std::move(epilog); - return *this; - } - - // Add a un-documented/hidden alias for an argument. - // Ideally we'd want this to be a method of Argument, but Argument - // does not own its owing ArgumentParser. - ArgumentParser &add_hidden_alias_for(Argument &arg, std::string_view alias) { - for (auto it = m_optional_arguments.begin(); - it != m_optional_arguments.end(); ++it) { - if (&(*it) == &arg) { - m_argument_map.insert_or_assign(std::string(alias), it); - return *this; - } - } - throw std::logic_error( - "Argument is not an optional argument of this parser"); - } - - /* Getter for arguments and subparsers. - * @throws std::logic_error in case of an invalid argument or subparser name - */ - template T &at(std::string_view name) { - if constexpr (std::is_same_v) { - return (*this)[name]; - } else { - std::string str_name(name); - auto subparser_it = m_subparser_map.find(str_name); - if (subparser_it != m_subparser_map.end()) { - return subparser_it->second->get(); - } - throw std::logic_error("No such subparser: " + str_name); - } - } - - ArgumentParser &set_prefix_chars(std::string prefix_chars) { - m_prefix_chars = std::move(prefix_chars); - return *this; - } - - ArgumentParser &set_assign_chars(std::string assign_chars) { - m_assign_chars = std::move(assign_chars); - return *this; - } - - /* Call parse_args_internal - which does all the work - * Then, validate the parsed arguments - * This variant is used mainly for testing - * @throws std::runtime_error in case of any invalid argument - */ - void parse_args(const std::vector &arguments) { - parse_args_internal(arguments); - // Check if all arguments are parsed - for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) { - argument->validate(); - } - - // Check each mutually exclusive group and make sure - // there are no constraint violations - for (const auto &group : m_mutually_exclusive_groups) { - auto mutex_argument_used{false}; - Argument *mutex_argument_it{nullptr}; - for (Argument *arg : group.m_elements) { - if (!mutex_argument_used && arg->m_is_used) { - mutex_argument_used = true; - mutex_argument_it = arg; - } else if (mutex_argument_used && arg->m_is_used) { - // Violation - throw std::runtime_error("Argument '" + arg->get_usage_full() + - "' not allowed with '" + - mutex_argument_it->get_usage_full() + "'"); - } - } - - if (!mutex_argument_used && group.m_required) { - // at least one argument from the group is - // required - std::string argument_names{}; - std::size_t i = 0; - std::size_t size = group.m_elements.size(); - for (Argument *arg : group.m_elements) { - if (i + 1 == size) { - // last - argument_names += std::string("'") + arg->get_usage_full() + std::string("' "); - } else { - argument_names += std::string("'") + arg->get_usage_full() + std::string("' or "); - } - i += 1; - } - throw std::runtime_error("One of the arguments " + argument_names + - "is required"); - } - } - } - - /* Call parse_known_args_internal - which does all the work - * Then, validate the parsed arguments - * This variant is used mainly for testing - * @throws std::runtime_error in case of any invalid argument - */ - std::vector - parse_known_args(const std::vector &arguments) { - auto unknown_arguments = parse_known_args_internal(arguments); - // Check if all arguments are parsed - for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) { - argument->validate(); - } - return unknown_arguments; - } - - /* Main entry point for parsing command-line arguments using this - * ArgumentParser - * @throws std::runtime_error in case of any invalid argument - */ - // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) - void parse_args(int argc, const char *const argv[]) { - parse_args({argv, argv + argc}); - } - - /* Main entry point for parsing command-line arguments using this - * ArgumentParser - * @throws std::runtime_error in case of any invalid argument - */ - // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays) - auto parse_known_args(int argc, const char *const argv[]) { - return parse_known_args({argv, argv + argc}); - } - - /* Getter for options with default values. - * @throws std::logic_error if parse_args() has not been previously called - * @throws std::logic_error if there is no such option - * @throws std::logic_error if the option has no value - * @throws std::bad_any_cast if the option is not of type T - */ - template T get(std::string_view arg_name) const { - if (!m_is_parsed) { - throw std::logic_error("Nothing parsed, no arguments are available."); - } - return (*this)[arg_name].get(); - } - - /* Getter for options without default values. - * @pre The option has no default value. - * @throws std::logic_error if there is no such option - * @throws std::bad_any_cast if the option is not of type T - */ - template - auto present(std::string_view arg_name) const -> std::optional { - return (*this)[arg_name].present(); - } - - /* Getter that returns true for user-supplied options. Returns false if not - * user-supplied, even with a default value. - */ - auto is_used(std::string_view arg_name) const { - return (*this)[arg_name].m_is_used; - } - - /* Getter that returns true if a subcommand is used. - */ - auto is_subcommand_used(std::string_view subcommand_name) const { - return m_subparser_used.at(std::string(subcommand_name)); - } - - /* Getter that returns true if a subcommand is used. - */ - auto is_subcommand_used(const ArgumentParser &subparser) const { - return is_subcommand_used(subparser.m_program_name); - } - - /* Indexing operator. Return a reference to an Argument object - * Used in conjunction with Argument.operator== e.g., parser["foo"] == true - * @throws std::logic_error in case of an invalid argument name - */ - Argument &operator[](std::string_view arg_name) const { - std::string name(arg_name); - auto it = m_argument_map.find(name); - if (it != m_argument_map.end()) { - return *(it->second); - } - if (!is_valid_prefix_char(arg_name.front())) { - const auto legal_prefix_char = get_any_valid_prefix_char(); - const auto prefix = std::string(1, legal_prefix_char); - - // "-" + arg_name - name = prefix + name; - it = m_argument_map.find(name); - if (it != m_argument_map.end()) { - return *(it->second); - } - // "--" + arg_name - name = prefix + name; - it = m_argument_map.find(name); - if (it != m_argument_map.end()) { - return *(it->second); - } - } - throw std::logic_error("No such argument: " + std::string(arg_name)); - } - - // Print help message - friend auto operator<<(std::ostream &stream, const ArgumentParser &parser) - -> std::ostream & { - stream.setf(std::ios_base::left); - - auto longest_arg_length = parser.get_length_of_longest_argument(); - - stream << parser.usage() << "\n\n"; - - if (!parser.m_description.empty()) { - stream << parser.m_description << "\n\n"; - } - - const bool has_visible_positional_args = std::find_if( - parser.m_positional_arguments.begin(), - parser.m_positional_arguments.end(), - [](const auto &argument) { - return !argument.m_is_hidden; }) != - parser.m_positional_arguments.end(); - if (has_visible_positional_args) { - stream << "Positional arguments:\n"; - } - - for (const auto &argument : parser.m_positional_arguments) { - if (!argument.m_is_hidden) { - stream.width(static_cast(longest_arg_length)); - stream << argument; - } - } - - if (!parser.m_optional_arguments.empty()) { - stream << (!has_visible_positional_args ? "" : "\n") - << "Optional arguments:\n"; - } - - for (const auto &argument : parser.m_optional_arguments) { - if (argument.m_group_idx == 0 && !argument.m_is_hidden) { - stream.width(static_cast(longest_arg_length)); - stream << argument; - } - } - - for (size_t i_group = 0; i_group < parser.m_group_names.size(); ++i_group) { - stream << "\n" << parser.m_group_names[i_group] << " (detailed usage):\n"; - for (const auto &argument : parser.m_optional_arguments) { - if (argument.m_group_idx == i_group + 1 && !argument.m_is_hidden) { - stream.width(static_cast(longest_arg_length)); - stream << argument; - } - } - } - - bool has_visible_subcommands = std::any_of( - parser.m_subparser_map.begin(), parser.m_subparser_map.end(), - [](auto &p) { return !p.second->get().m_suppress; }); - - if (has_visible_subcommands) { - stream << (parser.m_positional_arguments.empty() - ? (parser.m_optional_arguments.empty() ? "" : "\n") - : "\n") - << "Subcommands:\n"; - for (const auto &[command, subparser] : parser.m_subparser_map) { - if (subparser->get().m_suppress) { - continue; - } - - stream << std::setw(2) << " "; - stream << std::setw(static_cast(longest_arg_length - 2)) - << command; - stream << " " << subparser->get().m_description << "\n"; - } - } - - if (!parser.m_epilog.empty()) { - stream << '\n'; - stream << parser.m_epilog << "\n\n"; - } - - return stream; - } - - // Format help message - auto help() const -> std::stringstream { - std::stringstream out; - out << *this; - return out; - } - - // Sets the maximum width for a line of the Usage message - ArgumentParser &set_usage_max_line_width(size_t w) { - this->m_usage_max_line_width = w; - return *this; - } - - // Asks to display arguments of mutually exclusive group on separate lines in - // the Usage message - ArgumentParser &set_usage_break_on_mutex() { - this->m_usage_break_on_mutex = true; - return *this; - } - - // Format usage part of help only - auto usage() const -> std::string { - std::stringstream stream; - - std::string curline("Usage: "); - curline += this->m_program_name; - const bool multiline_usage = - this->m_usage_max_line_width < (std::numeric_limits::max)(); - const size_t indent_size = curline.size(); - - const auto deal_with_options_of_group = [&](std::size_t group_idx) { - bool found_options = false; - // Add any options inline here - const MutuallyExclusiveGroup *cur_mutex = nullptr; - int usage_newline_counter = -1; - for (const auto &argument : this->m_optional_arguments) { - if (argument.m_is_hidden) { - continue; - } - if (multiline_usage) { - if (argument.m_group_idx != group_idx) { - continue; - } - if (usage_newline_counter != argument.m_usage_newline_counter) { - if (usage_newline_counter >= 0) { - if (curline.size() > indent_size) { - stream << curline << std::endl; - curline = std::string(indent_size, ' '); - } - } - usage_newline_counter = argument.m_usage_newline_counter; - } - } - found_options = true; - const std::string arg_inline_usage = argument.get_inline_usage(); - const MutuallyExclusiveGroup *arg_mutex = - get_belonging_mutex(&argument); - if ((cur_mutex != nullptr) && (arg_mutex == nullptr)) { - curline += ']'; - if (this->m_usage_break_on_mutex) { - stream << curline << std::endl; - curline = std::string(indent_size, ' '); - } - } else if ((cur_mutex == nullptr) && (arg_mutex != nullptr)) { - if ((this->m_usage_break_on_mutex && curline.size() > indent_size) || - curline.size() + 3 + arg_inline_usage.size() > - this->m_usage_max_line_width) { - stream << curline << std::endl; - curline = std::string(indent_size, ' '); - } - curline += " ["; - } else if ((cur_mutex != nullptr) && (arg_mutex != nullptr)) { - if (cur_mutex != arg_mutex) { - curline += ']'; - if (this->m_usage_break_on_mutex || - curline.size() + 3 + arg_inline_usage.size() > - this->m_usage_max_line_width) { - stream << curline << std::endl; - curline = std::string(indent_size, ' '); - } - curline += " ["; - } else { - curline += '|'; - } - } - cur_mutex = arg_mutex; - if (curline.size() + 1 + arg_inline_usage.size() > - this->m_usage_max_line_width) { - stream << curline << std::endl; - curline = std::string(indent_size, ' '); - curline += " "; - } else if (cur_mutex == nullptr) { - curline += " "; - } - curline += arg_inline_usage; - } - if (cur_mutex != nullptr) { - curline += ']'; - } - return found_options; - }; - - const bool found_options = deal_with_options_of_group(0); - - if (found_options && multiline_usage && - !this->m_positional_arguments.empty()) { - stream << curline << std::endl; - curline = std::string(indent_size, ' '); - } - // Put positional arguments after the optionals - for (const auto &argument : this->m_positional_arguments) { - if (argument.m_is_hidden) { - continue; - } - const std::string pos_arg = !argument.m_metavar.empty() - ? argument.m_metavar - : argument.m_names.front(); - if (curline.size() + 1 + pos_arg.size() > this->m_usage_max_line_width) { - stream << curline << std::endl; - curline = std::string(indent_size, ' '); - } - curline += " "; - if (argument.m_num_args_range.get_min() == 0 && - !argument.m_num_args_range.is_right_bounded()) { - curline += "["; - curline += pos_arg; - curline += "]..."; - } else if (argument.m_num_args_range.get_min() == 1 && - !argument.m_num_args_range.is_right_bounded()) { - curline += pos_arg; - curline += "..."; - } else { - curline += pos_arg; - } - } - - if (multiline_usage) { - // Display options of other groups - for (std::size_t i = 0; i < m_group_names.size(); ++i) { - stream << curline << std::endl << std::endl; - stream << m_group_names[i] << ":" << std::endl; - curline = std::string(indent_size, ' '); - deal_with_options_of_group(i + 1); - } - } - - stream << curline; - - // Put subcommands after positional arguments - if (!m_subparser_map.empty()) { - stream << " {"; - std::size_t i{0}; - for (const auto &[command, subparser] : m_subparser_map) { - if (subparser->get().m_suppress) { - continue; - } - - if (i == 0) { - stream << command; - } else { - stream << "," << command; - } - ++i; - } - stream << "}"; - } - - return stream.str(); - } - - // Printing the one and only help message - // I've stuck with a simple message format, nothing fancy. - [[deprecated("Use cout << program; instead. See also help().")]] std::string - print_help() const { - auto out = help(); - std::cout << out.rdbuf(); - return out.str(); - } - - void add_subparser(ArgumentParser &parser) { - parser.m_parser_path = m_program_name + " " + parser.m_program_name; - auto it = m_subparsers.emplace(std::cend(m_subparsers), parser); - m_subparser_map.insert_or_assign(parser.m_program_name, it); - m_subparser_used.insert_or_assign(parser.m_program_name, false); - } - - void set_suppress(bool suppress) { m_suppress = suppress; } - -protected: - const MutuallyExclusiveGroup *get_belonging_mutex(const Argument *arg) const { - for (const auto &mutex : m_mutually_exclusive_groups) { - if (std::find(mutex.m_elements.begin(), mutex.m_elements.end(), arg) != - mutex.m_elements.end()) { - return &mutex; - } - } - return nullptr; - } - - bool is_valid_prefix_char(char c) const { - return m_prefix_chars.find(c) != std::string::npos; - } - - char get_any_valid_prefix_char() const { return m_prefix_chars[0]; } - - /* - * Pre-process this argument list. Anything starting with "--", that - * contains an =, where the prefix before the = has an entry in the - * options table, should be split. - */ - std::vector - preprocess_arguments(const std::vector &raw_arguments) const { - std::vector arguments{}; - for (const auto &arg : raw_arguments) { - - const auto argument_starts_with_prefix_chars = - [this](const std::string &a) -> bool { - if (!a.empty()) { - - const auto legal_prefix = [this](char c) -> bool { - return m_prefix_chars.find(c) != std::string::npos; - }; - - // Windows-style - // if '/' is a legal prefix char - // then allow single '/' followed by argument name, followed by an - // assign char, e.g., ':' e.g., 'test.exe /A:Foo' - const auto windows_style = legal_prefix('/'); - - if (windows_style) { - if (legal_prefix(a[0])) { - return true; - } - } else { - // Slash '/' is not a legal prefix char - // For all other characters, only support long arguments - // i.e., the argument must start with 2 prefix chars, e.g, - // '--foo' e,g, './test --foo=Bar -DARG=yes' - if (a.size() > 1) { - return (legal_prefix(a[0]) && legal_prefix(a[1])); - } - } - } - return false; - }; - - // Check that: - // - We don't have an argument named exactly this - // - The argument starts with a prefix char, e.g., "--" - // - The argument contains an assign char, e.g., "=" - auto assign_char_pos = arg.find_first_of(m_assign_chars); - - if (m_argument_map.find(arg) == m_argument_map.end() && - argument_starts_with_prefix_chars(arg) && - assign_char_pos != std::string::npos) { - // Get the name of the potential option, and check it exists - std::string opt_name = arg.substr(0, assign_char_pos); - if (m_argument_map.find(opt_name) != m_argument_map.end()) { - // This is the name of an option! Split it into two parts - arguments.push_back(std::move(opt_name)); - arguments.push_back(arg.substr(assign_char_pos + 1)); - continue; - } - } - // If we've fallen through to here, then it's a standard argument - arguments.push_back(arg); - } - return arguments; - } - - /* - * @throws std::runtime_error in case of any invalid argument - */ - void parse_args_internal(const std::vector &raw_arguments) { - auto arguments = preprocess_arguments(raw_arguments); - if (m_program_name.empty() && !arguments.empty()) { - m_program_name = arguments.front(); - } - auto end = std::end(arguments); - auto positional_argument_it = std::begin(m_positional_arguments); - for (auto it = std::next(std::begin(arguments)); it != end;) { - const auto ¤t_argument = *it; - if (Argument::is_positional(current_argument, m_prefix_chars)) { - if (positional_argument_it == std::end(m_positional_arguments)) { - - // Check sub-parsers - auto subparser_it = m_subparser_map.find(current_argument); - if (subparser_it != m_subparser_map.end()) { - - // build list of remaining args - const auto unprocessed_arguments = - std::vector(it, end); - - // invoke subparser - m_is_parsed = true; - m_subparser_used[current_argument] = true; - return subparser_it->second->get().parse_args( - unprocessed_arguments); - } - - if (m_positional_arguments.empty()) { - - // Ask the user if they argument they provided was a typo - // for some sub-parser, - // e.g., user provided `git totes` instead of `git notes` - if (!m_subparser_map.empty()) { - throw std::runtime_error( - "Failed to parse '" + current_argument + "', did you mean '" + - std::string{details::get_most_similar_string( - m_subparser_map, current_argument)} + - "'"); - } - - // Ask the user if they meant to use a specific optional argument - if (!m_optional_arguments.empty()) { - for (const auto &opt : m_optional_arguments) { - if (!opt.m_implicit_value.has_value()) { - // not a flag, requires a value - if (!opt.m_is_used) { - throw std::runtime_error( - "Zero positional arguments expected, did you mean " + - opt.get_usage_full()); - } - } - } - - throw std::runtime_error("Zero positional arguments expected"); - } else { - throw std::runtime_error("Zero positional arguments expected"); - } - } else { - throw std::runtime_error("Maximum number of positional arguments " - "exceeded, failed to parse '" + - current_argument + "'"); - } - } - auto argument = positional_argument_it++; - - // Deal with the situation of ... - if (argument->m_num_args_range.get_min() == 1 && - argument->m_num_args_range.get_max() == (std::numeric_limits::max)() && - positional_argument_it != std::end(m_positional_arguments) && - std::next(positional_argument_it) == std::end(m_positional_arguments) && - positional_argument_it->m_num_args_range.get_min() == 1 && - positional_argument_it->m_num_args_range.get_max() == 1 ) { - if (std::next(it) != end) { - positional_argument_it->consume(std::prev(end), end); - end = std::prev(end); - } else { - throw std::runtime_error("Missing " + positional_argument_it->m_names.front()); - } - } - - it = argument->consume(it, end); - continue; - } - - auto arg_map_it = m_argument_map.find(current_argument); - if (arg_map_it != m_argument_map.end()) { - auto argument = arg_map_it->second; - it = argument->consume(std::next(it), end, arg_map_it->first); - } else if (const auto &compound_arg = current_argument; - compound_arg.size() > 1 && - is_valid_prefix_char(compound_arg[0]) && - !is_valid_prefix_char(compound_arg[1])) { - ++it; - for (std::size_t j = 1; j < compound_arg.size(); j++) { - auto hypothetical_arg = std::string{'-', compound_arg[j]}; - auto arg_map_it2 = m_argument_map.find(hypothetical_arg); - if (arg_map_it2 != m_argument_map.end()) { - auto argument = arg_map_it2->second; - it = argument->consume(it, end, arg_map_it2->first); - } else { - throw std::runtime_error("Unknown argument: " + current_argument); - } - } - } else { - throw std::runtime_error("Unknown argument: " + current_argument); - } - } - m_is_parsed = true; - } - - /* - * Like parse_args_internal but collects unused args into a vector - */ - std::vector - parse_known_args_internal(const std::vector &raw_arguments) { - auto arguments = preprocess_arguments(raw_arguments); - - std::vector unknown_arguments{}; - - if (m_program_name.empty() && !arguments.empty()) { - m_program_name = arguments.front(); - } - auto end = std::end(arguments); - auto positional_argument_it = std::begin(m_positional_arguments); - for (auto it = std::next(std::begin(arguments)); it != end;) { - const auto ¤t_argument = *it; - if (Argument::is_positional(current_argument, m_prefix_chars)) { - if (positional_argument_it == std::end(m_positional_arguments)) { - - // Check sub-parsers - auto subparser_it = m_subparser_map.find(current_argument); - if (subparser_it != m_subparser_map.end()) { - - // build list of remaining args - const auto unprocessed_arguments = - std::vector(it, end); - - // invoke subparser - m_is_parsed = true; - m_subparser_used[current_argument] = true; - return subparser_it->second->get().parse_known_args_internal( - unprocessed_arguments); - } - - // save current argument as unknown and go to next argument - unknown_arguments.push_back(current_argument); - ++it; - } else { - // current argument is the value of a positional argument - // consume it - auto argument = positional_argument_it++; - it = argument->consume(it, end); - } - continue; - } - - auto arg_map_it = m_argument_map.find(current_argument); - if (arg_map_it != m_argument_map.end()) { - auto argument = arg_map_it->second; - it = argument->consume(std::next(it), end, arg_map_it->first); - } else if (const auto &compound_arg = current_argument; - compound_arg.size() > 1 && - is_valid_prefix_char(compound_arg[0]) && - !is_valid_prefix_char(compound_arg[1])) { - ++it; - for (std::size_t j = 1; j < compound_arg.size(); j++) { - auto hypothetical_arg = std::string{'-', compound_arg[j]}; - auto arg_map_it2 = m_argument_map.find(hypothetical_arg); - if (arg_map_it2 != m_argument_map.end()) { - auto argument = arg_map_it2->second; - it = argument->consume(it, end, arg_map_it2->first); - } else { - unknown_arguments.push_back(current_argument); - break; - } - } - } else { - // current argument is an optional-like argument that is unknown - // save it and move to next argument - unknown_arguments.push_back(current_argument); - ++it; - } - } - m_is_parsed = true; - return unknown_arguments; - } - - // Used by print_help. - std::size_t get_length_of_longest_argument() const { - if (m_argument_map.empty()) { - return 0; - } - std::size_t max_size = 0; - for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) { - max_size = - std::max(max_size, argument->get_arguments_length()); - } - for ([[maybe_unused]] const auto &[command, unused] : m_subparser_map) { - max_size = std::max(max_size, command.size()); - } - return max_size; - } - - using argument_it = std::list::iterator; - using mutex_group_it = std::vector::iterator; - using argument_parser_it = - std::list>::iterator; - - void index_argument(argument_it it) { - for (const auto &name : std::as_const(it->m_names)) { - m_argument_map.insert_or_assign(name, it); - } - } - - std::string m_program_name; - std::string m_version; - std::string m_description; - std::string m_epilog; - bool m_exit_on_default_arguments = true; - std::string m_prefix_chars{"-"}; - std::string m_assign_chars{"="}; - bool m_is_parsed = false; - std::list m_positional_arguments; - std::list m_optional_arguments; - std::map m_argument_map; - std::string m_parser_path; - std::list> m_subparsers; - std::map m_subparser_map; - std::map m_subparser_used; - std::vector m_mutually_exclusive_groups; - bool m_suppress = false; - std::size_t m_usage_max_line_width = (std::numeric_limits::max)(); - bool m_usage_break_on_mutex = false; - int m_usage_newline_counter = 0; - std::vector m_group_names; -}; - -} // namespace argparse \ No newline at end of file From 6cb318e7b7be68dfde052cd8327e2bf89293bf38 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 11:13:47 -0600 Subject: [PATCH 08/35] move ehader to indclude directory --- CMakeLists.txt | 2 -- datalogcli/CMakeLists.txt | 2 +- datalogcli/src/main/native/{cpp => include}/LogLoader.h | 0 3 files changed, 1 insertion(+), 3 deletions(-) rename datalogcli/src/main/native/{cpp => include}/LogLoader.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64b2d0930da..b0cedd9b8ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -302,8 +302,6 @@ endif() add_subdirectory(datalogcli) -add_subdirectory(protoplugin) - if(WITH_WPIMATH) if(WITH_JAVA) set(WPIUNITS_DEP_REPLACE ${WPIUNITS_DEP_REPLACE_IMPL}) diff --git a/datalogcli/CMakeLists.txt b/datalogcli/CMakeLists.txt index bcb911bbc39..44487eead73 100644 --- a/datalogcli/CMakeLists.txt +++ b/datalogcli/CMakeLists.txt @@ -23,7 +23,7 @@ add_executable( ${APP_ICON_MACOSX} ) -target_include_directories(datalogcli PUBLIC src/main/native/cpp) +target_include_directories(datalogcli PUBLIC src/main/native/include) target_link_libraries(datalogcli PRIVATE wpiutil) diff --git a/datalogcli/src/main/native/cpp/LogLoader.h b/datalogcli/src/main/native/include/LogLoader.h similarity index 100% rename from datalogcli/src/main/native/cpp/LogLoader.h rename to datalogcli/src/main/native/include/LogLoader.h From f328cb359137a1217671c63d0725c0da29a5b0df Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 12:42:00 -0600 Subject: [PATCH 09/35] use wpilib argparse --- datalogcli/src/main/native/cpp/main.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datalogcli/src/main/native/cpp/main.cpp b/datalogcli/src/main/native/cpp/main.cpp index a08bf5882a1..b5b9f2bed93 100644 --- a/datalogcli/src/main/native/cpp/main.cpp +++ b/datalogcli/src/main/native/cpp/main.cpp @@ -1,7 +1,7 @@ #include #include #include -#include "argparse/argparse.hpp" +#include "wpi/argparse.h" void export_json(std::string_view output_path) { @@ -20,9 +20,9 @@ void open_log(std::string_view log_path) { } int main(int argc, char* argv[]) { - argparse::ArgumentParser cli{"wpilog-cli"}; + wpi::ArgumentParser cli{"wpilog-cli"}; - argparse::ArgumentParser export_json_command{"json"}; + wpi::ArgumentParser export_json_command{"json"}; export_json_command.add_description( "Export a JSON representation of a WPILOG file"); export_json_command.add_argument("log_file") @@ -32,7 +32,7 @@ int main(int argc, char* argv[]) { "Path of the JSON file to create with the exported data. If it " "exists, it will be overwritten."); - argparse::ArgumentParser export_csv_command{"csv"}; + wpi::ArgumentParser export_csv_command{"csv"}; export_csv_command.add_description( "Export a CSV representation of a WPILOG file"); export_csv_command.add_argument("-l", "--log-file").help("The WPILOG file to export"); @@ -42,7 +42,7 @@ int main(int argc, char* argv[]) { "The CSV file to create with the exported data. If it " "exists, it will be overwritten."); - argparse::ArgumentParser extract_field_command{"extract"}; + wpi::ArgumentParser extract_field_command{"extract"}; extract_field_command.add_description( "Extract the histoy of one field from a WPILOG file and store it in a " "JSON or CSV file"); From f62505e616a41260ae6901f658c5c290b31069ed Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 12:42:39 -0600 Subject: [PATCH 10/35] apply datalog move to wpiutil --- .../include/wpi/datalog/DataLogReaderThread.h | 6 + datalogcli/src/main/native/cpp/LogLoader.cpp | 163 +----------------- .../src/main/native/include/LogLoader.h | 13 +- datalogtool/src/main/native/cpp/Exporter.cpp | 1 + sysid/src/main/native/cpp/view/LogLoader.cpp | 1 + .../main/native/cpp/DataLogReaderThread.cpp | 136 +++++++++++++++ .../native/include/wpi/DataLogReaderThread.h | 115 ++++++++++++ 7 files changed, 269 insertions(+), 166 deletions(-) create mode 100644 wpiutil/src/main/native/cpp/DataLogReaderThread.cpp create mode 100644 wpiutil/src/main/native/include/wpi/DataLogReaderThread.h diff --git a/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h b/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h index 349b7001aec..b9a269f44f2 100644 --- a/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h +++ b/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h @@ -4,6 +4,12 @@ #pragma once +#include +#include +#include +#include +#include + #include #include #include diff --git a/datalogcli/src/main/native/cpp/LogLoader.cpp b/datalogcli/src/main/native/cpp/LogLoader.cpp index 4a0cdb4d69e..5ceb2c8a5be 100644 --- a/datalogcli/src/main/native/cpp/LogLoader.cpp +++ b/datalogcli/src/main/native/cpp/LogLoader.cpp @@ -12,14 +12,14 @@ #include "fmt/base.h" #include "wpi/DataLogReader.h" -#include +#include #include #include #include using namespace datalogcli; -LogLoader::LogLoader(glass::Storage& storage, wpi::Logger& logger) {} +LogLoader::LogLoader(wpi::Logger& logger) {} LogLoader::~LogLoader() = default; @@ -27,19 +27,19 @@ void LogLoader::Load(std::string_view log_path) { // Handle opening the file std::error_code ec; - auto buf = wpi::MemoryBuffer::GetFile(log_path, ec); + auto buf = wpi::MemoryBuffer::GetFile(log_path); if (ec) { m_error = fmt::format("Could not open file: {}", ec.message()); return; } - wpi::log::DataLogReader reader{std::move(buf)}; + wpi::log::DataLogReader reader{*std::move(buf)}; if (!reader.IsValid()) { m_error = "Not a valid datalog file"; return; } unload(); // release the actual file, we have the data in the reader now - m_reader = std::make_unique(std::move(reader)); + m_reader = std::make_unique(std::move(reader)); m_entryTree.clear(); // Handle Errors @@ -58,157 +58,4 @@ void LogLoader::Load(std::string_view log_path) { if (!m_reader->IsDone()) { return; } - - /*ImGui::BeginTable( - "Entries", 2, - ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp); - ImGui::TableSetupColumn("Name"); - ImGui::TableSetupColumn("Type"); - // ImGui::TableSetupColumn("Metadata"); - ImGui::TableHeadersRow(); - DisplayEntryTree(m_entryTree); - ImGui::EndTable();*/ -} - -std::vector LogLoader::GetRecords(std::string_view field_name) { - int entry_id{GetTargetEntryId(field_name)}; - std::vector record_list{}; - - if (entry_id == -1) { - // didnt find a record with the name we want - return record_list; - } - - auto iter = m_reader->GetReader().begin(); - - while(!m_reader->IsDone()) { - if (iter->GetEntry() == entry_id) { - // this is the one we want - record_list.push_back(*iter); - } - iter++; - } - - return record_list; -} - -int LogLoader::GetTargetEntryId(std::string_view name) { - // get first entry - auto iter = m_reader->GetReader().begin(); - // is it what we want? - if (iter->IsStart()) { // looking for a start record to get the entry ID - wpi::log::StartRecordData* entryData{}; - iter->GetStartData(entryData); - if (entryData->name.compare(name) != 0) { - // this is it! - return entryData->entry; - } - } - return -1; -} - -/*void LogLoader::RebuildEntryTree() { - m_entryTree.clear(); - wpi::SmallVector parts; - m_reader->ForEachEntryName([&](const glass::DataLogReaderEntry& entry) { - // only show double/float/string entries (TODO: support struct/protobuf) - if (entry.type != "double" && entry.type != "float" && - entry.type != "string") { - return; - } - - // filter on name - if (!m_filter.empty() && !wpi::contains_lower(entry.name, m_filter)) { - return; - } - - parts.clear(); - // split on first : if one is present - auto [prefix, mainpart] = wpi::split(entry.name, ':'); - if (mainpart.empty() || wpi::contains(prefix, '/')) { - mainpart = entry.name; - } else { - parts.emplace_back(prefix); - } - wpi::split(mainpart, parts, '/', -1, false); - - // ignore a raw "/" key - if (parts.empty()) { - return; - } - - // get to leaf - auto nodes = &m_entryTree; - for (auto part : wpi::drop_back(std::span{parts.begin(), parts.end()})) { - auto it = - std::find_if(nodes->begin(), nodes->end(), - [&](const auto& node) { return node.name == part; }); - if (it == nodes->end()) { - nodes->emplace_back(part); - // path is from the beginning of the string to the end of the current - // part; this works because part is a reference to the internals of - // entry.name - nodes->back().path.assign( - entry.name.data(), part.data() + part.size() - entry.name.data()); - it = nodes->end() - 1; - } - nodes = &it->children; - } - - auto it = std::find_if(nodes->begin(), nodes->end(), [&](const auto& node) { - return node.name == parts.back(); - }); - if (it == nodes->end()) { - nodes->emplace_back(parts.back()); - // no need to set path, as it's identical to entry.name - it = nodes->end() - 1; - } - it->entry = &entry; - }); } - -static void EmitEntry(const std::string& name, - const glass::DataLogReaderEntry& entry) { - ImGui::TableNextColumn(); - ImGui::Selectable(name.c_str()); - if (ImGui::BeginDragDropSource()) { - auto entryPtr = &entry; - ImGui::SetDragDropPayload( - entry.type == "string" ? "DataLogEntryString" : "DataLogEntry", - &entryPtr, - sizeof(entryPtr)); // NOLINT - ImGui::TextUnformatted(entry.name.data(), - entry.name.data() + entry.name.size()); - ImGui::EndDragDropSource(); - } - ImGui::TableNextColumn(); - ImGui::TextUnformatted(entry.type.data(), - entry.type.data() + entry.type.size()); -#if 0 - ImGui::TableNextColumn(); - ImGui::TextUnformatted(entry.metadata.data(), - entry.metadata.data() + entry.metadata.size()); -#endif -} - -void LogLoader::DisplayEntryTree(const std::vector& tree) { - for (auto&& node : tree) { - if (node.entry) { - EmitEntry(node.name, *node.entry); - } - - if (!node.children.empty()) { - ImGui::TableNextColumn(); - bool open = ImGui::TreeNodeEx(node.name.c_str(), - ImGuiTreeNodeFlags_SpanFullWidth); - ImGui::TableNextColumn(); -#if 0 - ImGui::TableNextColumn(); -#endif - if (open) { - DisplayEntryTree(node.children); - ImGui::TreePop(); - } - } - } -}*/ diff --git a/datalogcli/src/main/native/include/LogLoader.h b/datalogcli/src/main/native/include/LogLoader.h index fa21662fd75..e9b645adabe 100644 --- a/datalogcli/src/main/native/include/LogLoader.h +++ b/datalogcli/src/main/native/include/LogLoader.h @@ -13,12 +13,12 @@ #include namespace glass { -class DataLogReaderEntry; -class DataLogReaderThread; class Storage; } // namespace glass namespace wpi { +class DataLogReaderEntry; +class DataLogReaderThread; class Logger; } // namespace wpi @@ -33,7 +33,7 @@ class LogLoader { * * @param logger The program logger */ - explicit LogLoader(glass::Storage& storage, wpi::Logger& logger); + explicit LogLoader(wpi::Logger& logger); ~LogLoader(); @@ -51,7 +51,7 @@ class LogLoader { // wpi::Logger& m_logger; std::string m_filename; - std::unique_ptr m_reader; + std::unique_ptr m_reader; std::string m_error; @@ -63,12 +63,9 @@ class LogLoader { explicit EntryTreeNode(std::string_view name) : name{name} {} std::string name; // name of just this node std::string path; // full path if entry is nullptr - const glass::DataLogReaderEntry* entry = nullptr; + const wpi::DataLogReaderEntry* entry = nullptr; std::vector children; // children, sorted by name }; std::vector m_entryTree; - - void RebuildEntryTree(); - int GetTargetEntryId(std::string_view name); }; } // namespace datalogcli diff --git a/datalogtool/src/main/native/cpp/Exporter.cpp b/datalogtool/src/main/native/cpp/Exporter.cpp index 2cf399b71c2..7432c444b8d 100644 --- a/datalogtool/src/main/native/cpp/Exporter.cpp +++ b/datalogtool/src/main/native/cpp/Exporter.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/sysid/src/main/native/cpp/view/LogLoader.cpp b/sysid/src/main/native/cpp/view/LogLoader.cpp index b21bea46158..e5c56a9839f 100644 --- a/sysid/src/main/native/cpp/view/LogLoader.cpp +++ b/sysid/src/main/native/cpp/view/LogLoader.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/wpiutil/src/main/native/cpp/DataLogReaderThread.cpp b/wpiutil/src/main/native/cpp/DataLogReaderThread.cpp new file mode 100644 index 00000000000..bd5452a9b32 --- /dev/null +++ b/wpiutil/src/main/native/cpp/DataLogReaderThread.cpp @@ -0,0 +1,136 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +<<<<<<<< HEAD:datalog/src/main/native/cpp/DataLogReaderThread.cpp +#include +#include +======== +#include "wpi/DataLogReaderThread.h" +>>>>>>>> ae195463f (apply datalog move to wpiutil):wpiutil/src/main/native/cpp/DataLogReaderThread.cpp + +#include +#include + +<<<<<<<< HEAD:datalog/src/main/native/cpp/DataLogReaderThread.cpp +#include "wpi/datalog/DataLogReaderThread.h" + +using namespace wpi::log; +======== +#include +#include + +using namespace wpi; +>>>>>>>> ae195463f (apply datalog move to wpiutil):wpiutil/src/main/native/cpp/DataLogReaderThread.cpp + +DataLogReaderThread::~DataLogReaderThread() { + if (m_thread.joinable()) { + m_active = false; + m_thread.join(); + } +} + +void DataLogReaderThread::ReadMain() { + wpi::SmallDenseMap< + int, std::pair>, 8> + schemaEntries; + + for (auto recordIt = m_reader.begin(), recordEnd = m_reader.end(); + recordIt != recordEnd; ++recordIt) { + auto& record = *recordIt; + if (!m_active) { + break; + } + ++m_numRecords; + if (record.IsStart()) { + DataLogReaderEntry data; + if (record.GetStartData(&data)) { + std::scoped_lock lock{m_mutex}; + auto& entryPtr = m_entriesById[data.entry]; + if (entryPtr) { + wpi::print("...DUPLICATE entry ID, overriding\n"); + } + auto [it, isNew] = m_entriesByName.emplace(data.name, data); + if (isNew) { + it->second.ranges.emplace_back(recordIt, recordEnd); + } + entryPtr = &it->second; + if (data.type == "structschema" || + data.type == "proto:FileDescriptorProto") { + schemaEntries.try_emplace(data.entry, entryPtr, + std::span{}); + } + sigEntryAdded(data); + } else { + wpi::print("Start(INVALID)\n"); + } + } else if (record.IsFinish()) { + int entry; + if (record.GetFinishEntry(&entry)) { + std::scoped_lock lock{m_mutex}; + auto it = m_entriesById.find(entry); + if (it == m_entriesById.end()) { + wpi::print("...ID not found\n"); + } else { + it->second->ranges.back().m_end = recordIt; + m_entriesById.erase(it); + } + } else { + wpi::print("Finish(INVALID)\n"); + } + } else if (record.IsSetMetadata()) { + wpi::log::MetadataRecordData data; + if (record.GetSetMetadataData(&data)) { + std::scoped_lock lock{m_mutex}; + auto it = m_entriesById.find(data.entry); + if (it == m_entriesById.end()) { + wpi::print("...ID not found\n"); + } else { + it->second->metadata = data.metadata; + } + } else { + wpi::print("SetMetadata(INVALID)\n"); + } + } else if (record.IsControl()) { + wpi::print("Unrecognized control record\n"); + } else { + auto it = schemaEntries.find(record.GetEntry()); + if (it != schemaEntries.end()) { + it->second.second = record.GetRaw(); + } + } + } + + // build schema databases + for (auto&& schemaPair : schemaEntries) { + auto name = schemaPair.second.first->name; + auto data = schemaPair.second.second; + if (data.empty()) { + continue; + } + if (auto strippedName = wpi::remove_prefix(name, "NT:")) { + name = *strippedName; + } + if (auto typeStr = wpi::remove_prefix(name, "/.schema/struct:")) { + std::string_view schema{reinterpret_cast(data.data()), + data.size()}; + std::string err; + auto desc = m_structDb.Add(*typeStr, schema, &err); + if (!desc) { + wpi::print("could not decode struct '{}' schema '{}': {}\n", name, + schema, err); + } + } else if (auto filename = wpi::remove_prefix(name, "/.schema/proto:")) { +#ifndef NO_PROTOBUF + // protobuf descriptor handling + if (!m_protoDb.Add(*filename, data)) { + wpi::print("could not decode protobuf '{}' filename '{}'\n", name, + *filename); + } +#endif + } + } + + sigDone(); + m_done = true; +} diff --git a/wpiutil/src/main/native/include/wpi/DataLogReaderThread.h b/wpiutil/src/main/native/include/wpi/DataLogReaderThread.h new file mode 100644 index 00000000000..b9a269f44f2 --- /dev/null +++ b/wpiutil/src/main/native/include/wpi/DataLogReaderThread.h @@ -0,0 +1,115 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "wpi/datalog/DataLogReader.h" + +#ifndef NO_PROTOBUF +#include +#endif + +namespace wpi::log { + +class DataLogReaderRange { + public: + DataLogReaderRange(wpi::log::DataLogReader::iterator begin, + wpi::log::DataLogReader::iterator end) + : m_begin{begin}, m_end{end} {} + + wpi::log::DataLogReader::iterator begin() const { return m_begin; } + wpi::log::DataLogReader::iterator end() const { return m_end; } + + wpi::log::DataLogReader::iterator m_begin; + wpi::log::DataLogReader::iterator m_end; +}; + +class DataLogReaderEntry : public wpi::log::StartRecordData { + public: + std::vector ranges; // ranges where this entry is valid +}; + +class DataLogReaderThread { + public: + explicit DataLogReaderThread(wpi::log::DataLogReader reader) + : m_reader{std::move(reader)}, m_thread{[this] { ReadMain(); }} {} + ~DataLogReaderThread(); + + bool IsDone() const { return m_done; } + std::string_view GetBufferIdentifier() const { + return m_reader.GetBufferIdentifier(); + } + unsigned int GetNumRecords() const { return m_numRecords; } + unsigned int GetNumEntries() const { + std::scoped_lock lock{m_mutex}; + return m_entriesByName.size(); + } + + // Passes Entry& to func + template + void ForEachEntryName(T&& func) { + std::scoped_lock lock{m_mutex}; + for (auto&& kv : m_entriesByName) { + func(kv.second); + } + } + + const DataLogReaderEntry* GetEntry(std::string_view name) const { + std::scoped_lock lock{m_mutex}; + auto it = m_entriesByName.find(name); + if (it == m_entriesByName.end()) { + return nullptr; + } + return &it->second; + } + + wpi::StructDescriptorDatabase& GetStructDatabase() { return m_structDb; } +#ifndef NO_PROTOBUF + wpi::ProtobufMessageDatabase& GetProtobufDatabase() { return m_protoDb; } +#endif + + const wpi::log::DataLogReader& GetReader() const { return m_reader; } + + // note: these are called on separate thread + wpi::sig::Signal_mt sigEntryAdded; + wpi::sig::Signal_mt<> sigDone; + + private: + void ReadMain(); + + wpi::log::DataLogReader m_reader; + mutable wpi::mutex m_mutex; + std::atomic_bool m_active{true}; + std::atomic_bool m_done{false}; + std::atomic m_numRecords{0}; + std::map> m_entriesByName; + wpi::DenseMap m_entriesById; + wpi::StructDescriptorDatabase m_structDb; +#ifndef NO_PROTOBUF + wpi::ProtobufMessageDatabase m_protoDb; +#endif + std::thread m_thread; +}; + +} // namespace wpi::log From d7728f65d2858a961688537531920345fe709d2d Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 13:58:44 -0600 Subject: [PATCH 11/35] extract regards and stub writer methods --- datalogcli/src/main/native/cpp/LogLoader.cpp | 24 +++- datalogcli/src/main/native/cpp/main.cpp | 109 ++++++++++++++---- .../src/main/native/include/LogLoader.h | 3 +- 3 files changed, 110 insertions(+), 26 deletions(-) diff --git a/datalogcli/src/main/native/cpp/LogLoader.cpp b/datalogcli/src/main/native/cpp/LogLoader.cpp index 5ceb2c8a5be..532346db2ef 100644 --- a/datalogcli/src/main/native/cpp/LogLoader.cpp +++ b/datalogcli/src/main/native/cpp/LogLoader.cpp @@ -19,7 +19,7 @@ using namespace datalogcli; -LogLoader::LogLoader(wpi::Logger& logger) {} +LogLoader::LogLoader() {} LogLoader::~LogLoader() = default; @@ -59,3 +59,25 @@ void LogLoader::Load(std::string_view log_path) { return; } } + +std::vector LogLoader::GetRecords(std::string_view field_name) { + std::vector record_list{}; + + const wpi::DataLogReaderEntry* entry = m_reader->GetEntry(field_name); + for (wpi::DataLogReaderRange range : entry->ranges) + { + wpi::log::DataLogReader::iterator rangeReader = range.begin(); + while (!rangeReader->IsFinish()) + { + record_list.push_back(*rangeReader); + } + } + + + + + return record_list; +} + + + diff --git a/datalogcli/src/main/native/cpp/main.cpp b/datalogcli/src/main/native/cpp/main.cpp index b5b9f2bed93..ccf03a5687c 100644 --- a/datalogcli/src/main/native/cpp/main.cpp +++ b/datalogcli/src/main/native/cpp/main.cpp @@ -1,22 +1,38 @@ #include #include #include +#include #include "wpi/argparse.h" +#include "LogLoader.h" -void export_json(std::string_view output_path) { +namespace fs = std::filesystem; + +void export_json(fs::path log_path, fs::path output_path) { + +} + +void export_csv(fs::path log_path, fs::path output_path) { } -void export_csv(std::string_view output_path) { +void write_json(std::vector records, fs::path output_path) { } -void extract_field(std::string_view field_name, std::string_view output_path, bool use_json) { +void write_csv(std::vector records, fs::path output_path) { + std::string header{"timestamp,value"}; } -void open_log(std::string_view log_path) { - // TODO: Add Log reader (base it on SysId?) +void extract_entry(std::string_view entry_name, fs::path log_path, fs::path output_path, bool use_json) { + // represent entry as a list of records + datalogcli::LogLoader loader = open_log(log_path); + std::vector records = loader.GetRecords(entry_name); +} + +datalogcli::LogLoader open_log(fs::path log_path) { + datalogcli::LogLoader loader{}; + loader.Load(log_path.string()); } int main(int argc, char* argv[]) { @@ -24,9 +40,9 @@ int main(int argc, char* argv[]) { wpi::ArgumentParser export_json_command{"json"}; export_json_command.add_description( - "Export a JSON representation of a WPILOG file"); + "Export a JSON representation of a DataLog file"); export_json_command.add_argument("log_file") - .help("Path to the WPILOG file to export"); + .help("Path to the DataLog file to export"); export_json_command.add_argument("json_file") .help( "Path of the JSON file to create with the exported data. If it " @@ -34,27 +50,27 @@ int main(int argc, char* argv[]) { wpi::ArgumentParser export_csv_command{"csv"}; export_csv_command.add_description( - "Export a CSV representation of a WPILOG file"); - export_csv_command.add_argument("-l", "--log-file").help("The WPILOG file to export"); + "Export a CSV representation of a DataLog file"); + export_csv_command.add_argument("-l", "--log-file").help("The DataLog file to export"); export_csv_command.add_argument("-o", "--output-file") .required() .help( "The CSV file to create with the exported data. If it " "exists, it will be overwritten."); - wpi::ArgumentParser extract_field_command{"extract"}; - extract_field_command.add_description( - "Extract the histoy of one field from a WPILOG file and store it in a " + wpi::ArgumentParser extract_entry_command{"extract"}; + extract_entry_command.add_description( + "Extract the history of one entry from a DataLog file and store it in a " "JSON or CSV file"); - extract_field_command.add_argument("-f", "--field") + extract_entry_command.add_argument("-e", "--entry") .required() - .help("The field to extract from the WPILOG"); - extract_field_command.add_argument("-l", "--log") + .help("The entry to extract from the Log"); + extract_entry_command.add_argument("-l", "--log-file") .required() - .help("The WPILOG file to extract from"); - extract_field_command.add_argument("--time-start") + .help("The DataLog file to extract from"); + extract_entry_command.add_argument("--time-start") .help("The timestamp to start extracting at"); - extract_field_command.add_argument("-o", "--output") + extract_entry_command.add_argument("-o", "--output") .required() .help( "The file to export the field and data to. It will be created or " @@ -62,7 +78,7 @@ int main(int argc, char* argv[]) { cli.add_subparser(export_json_command); cli.add_subparser(export_csv_command); - cli.add_subparser(extract_field_command); + cli.add_subparser(extract_entry_command); try { cli.parse_args(argc, argv); @@ -74,10 +90,57 @@ int main(int argc, char* argv[]) { // see which one was called if (export_json_command) { - //export_json(); + // validate paths + fs::path logPath{export_json_command.get("--log-file")}; + if (logPath.extension() != ".wpilog") + { + std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; + return 1; + } + + fs::path outputPath{export_json_command.get("--output")}; + if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") + { + std::cerr << "Only JSON and CSV are currently supported as output formats." << std::endl; + return 1; + } + + export_json(logPath, outputPath); } else if (export_csv_command) { - export_csv(export_csv_command.get("--log-file")); - } else if (extract_field_command) { - //extract_field(); + // validate paths + fs::path logPath{export_csv_command.get("--log-file")}; + if (logPath.extension() != ".wpilog") + { + std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; + return 1; + } + + fs::path outputPath{export_csv_command.get("--output")}; + if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") + { + std::cerr << "Only JSON and CSV are currently supported as output formats." << std::endl; + return 1; + } + + export_csv(logPath, outputPath); + } else if (extract_entry_command) { + // validate paths + fs::path logPath{extract_entry_command.get("--log-file")}; + if (logPath.extension() != ".wpilog") + { + std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; + return 1; + } + + fs::path outputPath{extract_entry_command.get("--output")}; + if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") + { + std::cerr << "Only JSON and CSV are currently supported as output formats." << std::endl; + return 1; + } + + bool json{outputPath.extension() == ".json" ? true : false}; + + extract_entry(extract_entry_command.get("--entry"), logPath, outputPath, json); } } \ No newline at end of file diff --git a/datalogcli/src/main/native/include/LogLoader.h b/datalogcli/src/main/native/include/LogLoader.h index e9b645adabe..85a27002596 100644 --- a/datalogcli/src/main/native/include/LogLoader.h +++ b/datalogcli/src/main/native/include/LogLoader.h @@ -33,7 +33,7 @@ class LogLoader { * * @param logger The program logger */ - explicit LogLoader(wpi::Logger& logger); + explicit LogLoader(); ~LogLoader(); @@ -48,7 +48,6 @@ class LogLoader { std::vector GetRecords(std::string_view field_name); private: - // wpi::Logger& m_logger; std::string m_filename; std::unique_ptr m_reader; From 297ef33db7719905e83a0bc3106793c2511ae0a7 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 14:25:27 -0600 Subject: [PATCH 12/35] need copy constructor --- datalogcli/CMakeLists.txt | 26 +++++++++---------- datalogcli/src/main/native/cpp/LogLoader.cpp | 11 +++++--- datalogcli/src/main/native/cpp/main.cpp | 13 +++++----- .../src/main/native/include/LogLoader.h | 4 ++- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/datalogcli/CMakeLists.txt b/datalogcli/CMakeLists.txt index 44487eead73..b726259b880 100644 --- a/datalogcli/CMakeLists.txt +++ b/datalogcli/CMakeLists.txt @@ -1,34 +1,34 @@ -project(datalogcli) +project(datalog-export) include(CompileWarnings) include(GenResources) configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) -generate_resources(src/main/native/resources generated/main/cpp DLT dlt datalogcli_resources_src) +generate_resources(src/main/native/resources generated/main/cpp DLT dlt datalog-export_resources_src) -file(GLOB datalogcli_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) +file(GLOB datalog-export_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) if(WIN32) - set(datalogcli_rc src/main/native/win/datalogcli.rc) + set(datalog-export_rc src/main/native/win/datalog-export.rc) elseif(APPLE) - set(MACOSX_BUNDLE_ICON_FILE datalogcli.icns) + set(MACOSX_BUNDLE_ICON_FILE datalog-export.icns) set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") endif() add_executable( - datalogcli - ${datalogcli_src} - ${datalogcli_resources_src} - ${datalogcli_rc} + datalog-export + ${datalog-export_src} + ${datalog-export_resources_src} + ${datalog-export_rc} ${APP_ICON_MACOSX} ) -target_include_directories(datalogcli PUBLIC src/main/native/include) +target_include_directories(datalog-export PUBLIC src/main/native/include) -target_link_libraries(datalogcli PRIVATE wpiutil) +target_link_libraries(datalog-export PRIVATE wpiutil) if(WIN32) - set_target_properties(datalogcli PROPERTIES WIN32_EXECUTABLE YES) + set_target_properties(datalog-export PROPERTIES WIN32_EXECUTABLE YES) elseif(APPLE) - set_target_properties(datalogcli PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "datalogcli") + set_target_properties(datalog-export PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "datalog-export") endif() \ No newline at end of file diff --git a/datalogcli/src/main/native/cpp/LogLoader.cpp b/datalogcli/src/main/native/cpp/LogLoader.cpp index 532346db2ef..6dca7d44ee4 100644 --- a/datalogcli/src/main/native/cpp/LogLoader.cpp +++ b/datalogcli/src/main/native/cpp/LogLoader.cpp @@ -21,6 +21,14 @@ using namespace datalogcli; LogLoader::LogLoader() {} +datalogcli::LogLoader::LogLoader(const LogLoader& original) { + m_reader = std::make_unique(original.m_reader.get()); + m_filename = original.m_filename; + m_error = original.m_error; + m_entryTree = original.m_entryTree; + m_filter = original.m_filter; +} + LogLoader::~LogLoader() = default; void LogLoader::Load(std::string_view log_path) { @@ -72,9 +80,6 @@ std::vector LogLoader::GetRecords(std::string_view fiel record_list.push_back(*rangeReader); } } - - - return record_list; } diff --git a/datalogcli/src/main/native/cpp/main.cpp b/datalogcli/src/main/native/cpp/main.cpp index ccf03a5687c..261b80fd029 100644 --- a/datalogcli/src/main/native/cpp/main.cpp +++ b/datalogcli/src/main/native/cpp/main.cpp @@ -5,7 +5,11 @@ #include "wpi/argparse.h" #include "LogLoader.h" -namespace fs = std::filesystem; +datalogcli::LogLoader open_log(fs::path log_path) { + datalogcli::LogLoader loader{}; + loader.Load(log_path.string()); + return loader; +} void export_json(fs::path log_path, fs::path output_path) { @@ -30,13 +34,8 @@ void extract_entry(std::string_view entry_name, fs::path log_path, fs::path outp std::vector records = loader.GetRecords(entry_name); } -datalogcli::LogLoader open_log(fs::path log_path) { - datalogcli::LogLoader loader{}; - loader.Load(log_path.string()); -} - int main(int argc, char* argv[]) { - wpi::ArgumentParser cli{"wpilog-cli"}; + wpi::ArgumentParser cli{"datalogcli"}; wpi::ArgumentParser export_json_command{"json"}; export_json_command.add_description( diff --git a/datalogcli/src/main/native/include/LogLoader.h b/datalogcli/src/main/native/include/LogLoader.h index 85a27002596..1bd1068bef3 100644 --- a/datalogcli/src/main/native/include/LogLoader.h +++ b/datalogcli/src/main/native/include/LogLoader.h @@ -29,12 +29,14 @@ namespace datalogcli { class LogLoader { public: /** - * Creates a log loader widget + * Creates a log loader * * @param logger The program logger */ explicit LogLoader(); + explicit LogLoader(const LogLoader&); + ~LogLoader(); /** From 9ca69f74ba40e117676f781a87a14088466b8743 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 14:38:45 -0600 Subject: [PATCH 13/35] just make the loader in situ --- datalogcli/src/main/native/cpp/LogLoader.cpp | 8 -------- datalogcli/src/main/native/cpp/main.cpp | 9 ++------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/datalogcli/src/main/native/cpp/LogLoader.cpp b/datalogcli/src/main/native/cpp/LogLoader.cpp index 6dca7d44ee4..9eb97de2f1a 100644 --- a/datalogcli/src/main/native/cpp/LogLoader.cpp +++ b/datalogcli/src/main/native/cpp/LogLoader.cpp @@ -21,14 +21,6 @@ using namespace datalogcli; LogLoader::LogLoader() {} -datalogcli::LogLoader::LogLoader(const LogLoader& original) { - m_reader = std::make_unique(original.m_reader.get()); - m_filename = original.m_filename; - m_error = original.m_error; - m_entryTree = original.m_entryTree; - m_filter = original.m_filter; -} - LogLoader::~LogLoader() = default; void LogLoader::Load(std::string_view log_path) { diff --git a/datalogcli/src/main/native/cpp/main.cpp b/datalogcli/src/main/native/cpp/main.cpp index 261b80fd029..8db71452799 100644 --- a/datalogcli/src/main/native/cpp/main.cpp +++ b/datalogcli/src/main/native/cpp/main.cpp @@ -5,12 +5,6 @@ #include "wpi/argparse.h" #include "LogLoader.h" -datalogcli::LogLoader open_log(fs::path log_path) { - datalogcli::LogLoader loader{}; - loader.Load(log_path.string()); - return loader; -} - void export_json(fs::path log_path, fs::path output_path) { } @@ -30,7 +24,8 @@ void write_csv(std::vector records, fs::path output_pat void extract_entry(std::string_view entry_name, fs::path log_path, fs::path output_path, bool use_json) { // represent entry as a list of records - datalogcli::LogLoader loader = open_log(log_path); + datalogcli::LogLoader loader{}; + loader.Load(log_path.string()); std::vector records = loader.GetRecords(entry_name); } From 01a65a11b203e0bcacf4428adad41626baf19c4d Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 15:17:30 -0600 Subject: [PATCH 14/35] rename and add csv header --- CMakeLists.txt | 2 +- {datalogcli => datalog-export}/CMakeLists.txt | 0 .../src/main/generate/WPILibVersion.cpp.in | 0 .../src/main/native/cpp/DataLogCSVWriter.cpp | 293 ++++++++++++++++++ .../src/main/native/cpp/LogLoader.cpp | 0 .../src/main/native/cpp/main.cpp | 4 +- .../main/native/include/DataLogCSVWriter.h | 9 + .../src/main/native/include/LogLoader.h | 0 8 files changed, 305 insertions(+), 3 deletions(-) rename {datalogcli => datalog-export}/CMakeLists.txt (100%) rename {datalogcli => datalog-export}/src/main/generate/WPILibVersion.cpp.in (100%) create mode 100644 datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp rename {datalogcli => datalog-export}/src/main/native/cpp/LogLoader.cpp (100%) rename {datalogcli => datalog-export}/src/main/native/cpp/main.cpp (98%) create mode 100644 datalog-export/src/main/native/include/DataLogCSVWriter.h rename {datalogcli => datalog-export}/src/main/native/include/LogLoader.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index b0cedd9b8ba..1eec148b21b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -300,7 +300,7 @@ if(WITH_NTCORE) add_subdirectory(ntcore) endif() -add_subdirectory(datalogcli) +add_subdirectory(datalog-export) if(WITH_WPIMATH) if(WITH_JAVA) diff --git a/datalogcli/CMakeLists.txt b/datalog-export/CMakeLists.txt similarity index 100% rename from datalogcli/CMakeLists.txt rename to datalog-export/CMakeLists.txt diff --git a/datalogcli/src/main/generate/WPILibVersion.cpp.in b/datalog-export/src/main/generate/WPILibVersion.cpp.in similarity index 100% rename from datalogcli/src/main/generate/WPILibVersion.cpp.in rename to datalog-export/src/main/generate/WPILibVersion.cpp.in diff --git a/datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp b/datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp new file mode 100644 index 00000000000..4c40fc2482c --- /dev/null +++ b/datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp @@ -0,0 +1,293 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "DataLogCSVWriter.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +struct InputFile { + explicit InputFile(std::unique_ptr datalog); + + InputFile(std::string_view filename, std::string_view status) + : filename{filename}, + stem{fs::path{filename}.stem().string()}, + status{status} {} + + ~InputFile(); + + std::string filename; + std::string stem; + std::unique_ptr datalog; + std::string status; + bool highlight = false; +}; + +struct Entry { + explicit Entry(const wpi::log::StartRecordData& srd) + : name{srd.name}, type{srd.type}, metadata{srd.metadata} {} + + std::string name; + std::string type; + std::string metadata; + std::set inputFiles; + bool typeConflict = false; + bool metadataConflict = false; + bool selected = true; + + // used only during export + int column = -1; +}; +} // namespace + +static std::map, std::less<>> + gInputFiles; +static wpi::mutex gEntriesMutex; +static std::map, std::less<>> gEntries; +std::atomic_int gExportCount{0}; + +InputFile::InputFile(std::unique_ptr datalog_) + : filename{datalog_->GetBufferIdentifier()}, + stem{fs::path{filename}.stem().string()}, + datalog{std::move(datalog_)} { + datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) { + std::scoped_lock lock{gEntriesMutex}; + auto it = gEntries.find(srd.name); + if (it == gEntries.end()) { + it = gEntries.emplace(srd.name, std::make_unique(srd)).first; + } else { + if (it->second->type != srd.type) { + it->second->typeConflict = true; + } + if (it->second->metadata != srd.metadata) { + it->second->metadataConflict = true; + } + } + it->second->inputFiles.emplace(this); + }); +} + +InputFile::~InputFile() { + if (!datalog) { + return; + } + std::scoped_lock lock{gEntriesMutex}; + bool changed = false; + for (auto it = gEntries.begin(); it != gEntries.end();) { + it->second->inputFiles.erase(this); + if (it->second->inputFiles.empty()) { + it = gEntries.erase(it); + changed = true; + } else { + ++it; + } + } +} + +static wpi::mutex gExportMutex; +static std::vector gExportErrors; + +static void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str) { + auto s = str; + while (!s.empty()) { + std::string_view fragment; + std::tie(fragment, s) = wpi::split(s, '"'); + os << fragment; + if (!s.empty()) { + os << '"' << '"'; + } + } + if (wpi::ends_with(str, '"')) { + os << '"' << '"'; + } +} + +static void ValueToCsv(wpi::raw_ostream& os, const Entry& entry, + const wpi::log::DataLogRecord& record) { + // handle systemTime specially + if (entry.name == "systemTime" && entry.type == "int64") { + int64_t val; + if (record.GetInteger(&val)) { + std::time_t timeval = val / 1000000; + wpi::print(os, "{:%Y-%m-%d %H:%M:%S}.{:06}", *std::localtime(&timeval), + val % 1000000); + return; + } + } else if (entry.type == "double") { + double val; + if (record.GetDouble(&val)) { + wpi::print(os, "{}", val); + return; + } + } else if (entry.type == "int64" || entry.type == "int") { + // support "int" for compatibility with old NT4 datalogs + int64_t val; + if (record.GetInteger(&val)) { + wpi::print(os, "{}", val); + return; + } + } else if (entry.type == "string" || entry.type == "json") { + std::string_view val; + record.GetString(&val); + os << '"'; + PrintEscapedCsvString(os, val); + os << '"'; + return; + } else if (entry.type == "boolean") { + bool val; + if (record.GetBoolean(&val)) { + wpi::print(os, "{}", val); + return; + } + } else if (entry.type == "boolean[]") { + std::vector val; + if (record.GetBooleanArray(&val)) { + wpi::print(os, "{}", fmt::join(val, ";")); + return; + } + } else if (entry.type == "double[]") { + std::vector val; + if (record.GetDoubleArray(&val)) { + wpi::print(os, "{}", fmt::join(val, ";")); + return; + } + } else if (entry.type == "float[]") { + std::vector val; + if (record.GetFloatArray(&val)) { + wpi::print(os, "{}", fmt::join(val, ";")); + return; + } + } else if (entry.type == "int64[]") { + std::vector val; + if (record.GetIntegerArray(&val)) { + wpi::print(os, "{}", fmt::join(val, ";")); + return; + } + } else if (entry.type == "string[]") { + std::vector val; + if (record.GetStringArray(&val)) { + os << '"'; + bool first = true; + for (auto&& v : val) { + if (!first) { + os << ';'; + } + first = false; + PrintEscapedCsvString(os, v); + } + os << '"'; + return; + } + } + wpi::print(os, ""); +} + +static void ExportCsvFile(InputFile& f, wpi::raw_ostream& os, int style) { + // header + if (style == 0) { + os << "Timestamp,Name,Value\n"; + } else if (style == 1) { + // scan for exported fields for this file to print header and assign columns + os << "Timestamp"; + int columnNum = 0; + for (auto&& entry : gEntries) { + if (entry.second->selected && + entry.second->inputFiles.find(&f) != entry.second->inputFiles.end()) { + os << ',' << '"'; + PrintEscapedCsvString(os, entry.first); + os << '"'; + entry.second->column = columnNum++; + } else { + entry.second->column = -1; + } + } + os << '\n'; + } + + wpi::DenseMap nameMap; + for (wpi::log::DataLogRecord record : f.datalog->GetReader()) { + if (record.IsStart()) { + wpi::log::StartRecordData data; + if (record.GetStartData(&data)) { + auto it = gEntries.find(data.name); + if (it != gEntries.end() && it->second->selected) { + nameMap[data.entry] = it->second.get(); + } + } + } else if (record.IsFinish()) { + int entry; + if (record.GetFinishEntry(&entry)) { + nameMap.erase(entry); + } + } else if (!record.IsControl()) { + auto entryIt = nameMap.find(record.GetEntry()); + if (entryIt == nameMap.end()) { + continue; + } + Entry* entry = entryIt->second; + + if (style == 0) { + wpi::print(os, "{},\"", record.GetTimestamp() / 1000000.0); + PrintEscapedCsvString(os, entry->name); + os << '"' << ','; + ValueToCsv(os, *entry, record); + os << '\n'; + } else if (style == 1 && entry->column != -1) { + wpi::print(os, "{},", record.GetTimestamp() / 1000000.0); + for (int i = 0; i < entry->column; ++i) { + os << ','; + } + ValueToCsv(os, *entry, record); + os << '\n'; + } + } + } +} + +static void ExportCsv(std::string_view outputFolder, int style) { + fs::path outPath{outputFolder}; + for (auto&& f : gInputFiles) { + if (f.second->datalog) { + std::error_code ec; + auto of = fs::OpenFileForWrite( + outPath / fs::path{f.first}.replace_extension("csv"), ec, + fs::CD_CreateNew, fs::OF_Text); + if (ec) { + std::scoped_lock lock{gExportMutex}; + gExportErrors.emplace_back( + fmt::format("{}: {}", f.first, ec.message())); + ++gExportCount; + continue; + } + wpi::raw_fd_ostream os{fs::FileToFd(of, ec, fs::OF_Text), true}; + ExportCsvFile(*f.second, os, style); + } + ++gExportCount; + } +} diff --git a/datalogcli/src/main/native/cpp/LogLoader.cpp b/datalog-export/src/main/native/cpp/LogLoader.cpp similarity index 100% rename from datalogcli/src/main/native/cpp/LogLoader.cpp rename to datalog-export/src/main/native/cpp/LogLoader.cpp diff --git a/datalogcli/src/main/native/cpp/main.cpp b/datalog-export/src/main/native/cpp/main.cpp similarity index 98% rename from datalogcli/src/main/native/cpp/main.cpp rename to datalog-export/src/main/native/cpp/main.cpp index 8db71452799..204db14b2f6 100644 --- a/datalogcli/src/main/native/cpp/main.cpp +++ b/datalog-export/src/main/native/cpp/main.cpp @@ -4,6 +4,7 @@ #include #include "wpi/argparse.h" #include "LogLoader.h" +#include "DataLogCSVWriter.h" void export_json(fs::path log_path, fs::path output_path) { @@ -18,8 +19,7 @@ void write_json(std::vector records, fs::path output_pa } void write_csv(std::vector records, fs::path output_path) { - std::string header{"timestamp,value"}; - + ExportCsv(output_path.string(), 0); } void extract_entry(std::string_view entry_name, fs::path log_path, fs::path output_path, bool use_json) { diff --git a/datalog-export/src/main/native/include/DataLogCSVWriter.h b/datalog-export/src/main/native/include/DataLogCSVWriter.h new file mode 100644 index 00000000000..0ae7b6a4d67 --- /dev/null +++ b/datalog-export/src/main/native/include/DataLogCSVWriter.h @@ -0,0 +1,9 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +static void ExportCsv(std::string_view outputFolder, int style); \ No newline at end of file diff --git a/datalogcli/src/main/native/include/LogLoader.h b/datalog-export/src/main/native/include/LogLoader.h similarity index 100% rename from datalogcli/src/main/native/include/LogLoader.h rename to datalog-export/src/main/native/include/LogLoader.h From e718bc367a3c03be1868538290eb5664280ecc77 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Sun, 5 Jan 2025 16:30:57 -0600 Subject: [PATCH 15/35] make writer a class --- .../src/main/native/cpp/DataLogCSVWriter.cpp | 2 ++ datalog-export/src/main/native/cpp/main.cpp | 3 ++- .../src/main/native/include/DataLogCSVWriter.h | 10 +++++++++- datalog-export/src/main/native/include/LogLoader.h | 4 ---- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp b/datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp index 4c40fc2482c..70396f235d1 100644 --- a/datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp +++ b/datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp @@ -110,6 +110,8 @@ InputFile::~InputFile() { } } +using namespace datalogcli; + static wpi::mutex gExportMutex; static std::vector gExportErrors; diff --git a/datalog-export/src/main/native/cpp/main.cpp b/datalog-export/src/main/native/cpp/main.cpp index 204db14b2f6..869a550391b 100644 --- a/datalog-export/src/main/native/cpp/main.cpp +++ b/datalog-export/src/main/native/cpp/main.cpp @@ -19,7 +19,8 @@ void write_json(std::vector records, fs::path output_pa } void write_csv(std::vector records, fs::path output_path) { - ExportCsv(output_path.string(), 0); + datalogcli::DataLogCSVWriter writer{}; + writer.ExportCsv(output_path.string(), 0); } void extract_entry(std::string_view entry_name, fs::path log_path, fs::path output_path, bool use_json) { diff --git a/datalog-export/src/main/native/include/DataLogCSVWriter.h b/datalog-export/src/main/native/include/DataLogCSVWriter.h index 0ae7b6a4d67..1f8dca8b37b 100644 --- a/datalog-export/src/main/native/include/DataLogCSVWriter.h +++ b/datalog-export/src/main/native/include/DataLogCSVWriter.h @@ -6,4 +6,12 @@ #include -static void ExportCsv(std::string_view outputFolder, int style); \ No newline at end of file +namespace datalogcli { + class DataLogCSVWriter { + public: + explicit DataLogCSVWriter(); + ~DataLogCSVWriter(); + + void ExportCsv(std::string_view outputFolder, int style); + }; +} diff --git a/datalog-export/src/main/native/include/LogLoader.h b/datalog-export/src/main/native/include/LogLoader.h index 1bd1068bef3..e3d684a4b72 100644 --- a/datalog-export/src/main/native/include/LogLoader.h +++ b/datalog-export/src/main/native/include/LogLoader.h @@ -30,13 +30,9 @@ class LogLoader { public: /** * Creates a log loader - * - * @param logger The program logger */ explicit LogLoader(); - explicit LogLoader(const LogLoader&); - ~LogLoader(); /** From c27a70f50e2fb768715ecaaf0454d069196ceb1d Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 17:19:19 -0600 Subject: [PATCH 16/35] format --- datalog-export/CMakeLists.txt | 16 +++- .../src/main/generate/WPILibVersion.cpp.in | 2 +- .../src/main/native/cpp/LogLoader.cpp | 27 +++--- datalog-export/src/main/native/cpp/main.cpp | 90 ++++++++++++------- .../main/native/include/DataLogCSVWriter.h | 14 +-- .../src/main/native/include/LogLoader.h | 5 +- 6 files changed, 95 insertions(+), 59 deletions(-) diff --git a/datalog-export/CMakeLists.txt b/datalog-export/CMakeLists.txt index b726259b880..47cbd76bece 100644 --- a/datalog-export/CMakeLists.txt +++ b/datalog-export/CMakeLists.txt @@ -4,9 +4,19 @@ include(CompileWarnings) include(GenResources) configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) -generate_resources(src/main/native/resources generated/main/cpp DLT dlt datalog-export_resources_src) +generate_resources( + src/main/native/resources + generated/main/cpp + DLT + dlt + datalog-export_resources_src +) -file(GLOB datalog-export_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) +file( + GLOB datalog-export_src + src/main/native/cpp/*.cpp + ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp +) if(WIN32) set(datalog-export_rc src/main/native/win/datalog-export.rc) @@ -31,4 +41,4 @@ if(WIN32) set_target_properties(datalog-export PROPERTIES WIN32_EXECUTABLE YES) elseif(APPLE) set_target_properties(datalog-export PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "datalog-export") -endif() \ No newline at end of file +endif() diff --git a/datalog-export/src/main/generate/WPILibVersion.cpp.in b/datalog-export/src/main/generate/WPILibVersion.cpp.in index d4bc735e071..cfe24411588 100644 --- a/datalog-export/src/main/generate/WPILibVersion.cpp.in +++ b/datalog-export/src/main/generate/WPILibVersion.cpp.in @@ -4,4 +4,4 @@ */ const char* GetWPILibVersion() { return "${wpilib_version}"; -} \ No newline at end of file +} diff --git a/datalog-export/src/main/native/cpp/LogLoader.cpp b/datalog-export/src/main/native/cpp/LogLoader.cpp index 9eb97de2f1a..a95202ec0a2 100644 --- a/datalog-export/src/main/native/cpp/LogLoader.cpp +++ b/datalog-export/src/main/native/cpp/LogLoader.cpp @@ -5,13 +5,14 @@ #include "LogLoader.h" #include +#include #include #include #include #include -#include "fmt/base.h" -#include "wpi/DataLogReader.h" +#include +#include #include #include #include @@ -24,7 +25,6 @@ LogLoader::LogLoader() {} LogLoader::~LogLoader() = default; void LogLoader::Load(std::string_view log_path) { - // Handle opening the file std::error_code ec; auto buf = wpi::MemoryBuffer::GetFile(log_path); @@ -38,7 +38,7 @@ void LogLoader::Load(std::string_view log_path) { m_error = "Not a valid datalog file"; return; } - unload(); // release the actual file, we have the data in the reader now + unload(); // release the actual file, we have the data in the reader now m_reader = std::make_unique(std::move(reader)); m_entryTree.clear(); @@ -52,23 +52,22 @@ void LogLoader::Load(std::string_view log_path) { // Summary info fmt::println("{}", fs::path{m_filename}.stem().string().c_str()); fmt::println("%u records, %u entries%s", m_reader->GetNumRecords(), - m_reader->GetNumEntries(), - m_reader->IsDone() ? "" : " (working)"); + m_reader->GetNumEntries(), + m_reader->IsDone() ? "" : " (working)"); if (!m_reader->IsDone()) { return; } } -std::vector LogLoader::GetRecords(std::string_view field_name) { +std::vector LogLoader::GetRecords( + std::string_view field_name) { std::vector record_list{}; const wpi::DataLogReaderEntry* entry = m_reader->GetEntry(field_name); - for (wpi::DataLogReaderRange range : entry->ranges) - { + for (wpi::DataLogReaderRange range : entry->ranges) { wpi::log::DataLogReader::iterator rangeReader = range.begin(); - while (!rangeReader->IsFinish()) - { + while (!rangeReader->IsFinish()) { record_list.push_back(*rangeReader); } } @@ -76,5 +75,9 @@ std::vector LogLoader::GetRecords(std::string_view fiel return record_list; } +std::vector datalogcli::LogLoader::GetAllRecords() { + std::vector record_list{}; - + //auto iter = m_reader- + return record_list; +} diff --git a/datalog-export/src/main/native/cpp/main.cpp b/datalog-export/src/main/native/cpp/main.cpp index 869a550391b..68b430773b1 100644 --- a/datalog-export/src/main/native/cpp/main.cpp +++ b/datalog-export/src/main/native/cpp/main.cpp @@ -1,29 +1,41 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + #include +#include #include #include +#include + +#include #include -#include "wpi/argparse.h" -#include "LogLoader.h" + #include "DataLogCSVWriter.h" +#include "DataLogJSONWriter.h" +#include "LogLoader.h" void export_json(fs::path log_path, fs::path output_path) { - -} - -void export_csv(fs::path log_path, fs::path output_path) { - + datalogcli::LogLoader loader{}; + loader.Load(log_path.string()); + std::vector records = loader.GetAllRecords(); + datalogcli::DataLogJSONWriter writer{}; + writer.ExportJSON(output_path, records); } -void write_json(std::vector records, fs::path output_path) { +void export_csv(fs::path log_path, fs::path output_path) {} -} +void write_json(std::vector records, + fs::path output_path) {} -void write_csv(std::vector records, fs::path output_path) { +void write_csv(std::vector records, + fs::path output_path) { datalogcli::DataLogCSVWriter writer{}; writer.ExportCsv(output_path.string(), 0); } -void extract_entry(std::string_view entry_name, fs::path log_path, fs::path output_path, bool use_json) { +void extract_entry(std::string_view entry_name, fs::path log_path, + fs::path output_path, bool use_json) { // represent entry as a list of records datalogcli::LogLoader loader{}; loader.Load(log_path.string()); @@ -32,23 +44,32 @@ void extract_entry(std::string_view entry_name, fs::path log_path, fs::path outp int main(int argc, char* argv[]) { wpi::ArgumentParser cli{"datalogcli"}; + std::string jsonMode{}; wpi::ArgumentParser export_json_command{"json"}; export_json_command.add_description( "Export a JSON representation of a DataLog file"); export_json_command.add_argument("log_file") .help("Path to the DataLog file to export"); - export_json_command.add_argument("json_file") + export_json_command.add_argument("export_path") .help( "Path of the JSON file to create with the exported data. If it " "exists, it will be overwritten."); + export_json_command.add_argument("-m", "--mode") + .help( + "How to format the exported JSON. 'direct' means that each record " + "will be directly converted to a JSON dictionary and exported, " + "including control records. 'direct_data' is the same as direct, but " + "control records will be ommitted. The default is 'direct_data'.") + .default_value("direct_data") + .store_into(jsonMode); wpi::ArgumentParser export_csv_command{"csv"}; export_csv_command.add_description( "Export a CSV representation of a DataLog file"); - export_csv_command.add_argument("-l", "--log-file").help("The DataLog file to export"); - export_csv_command.add_argument("-o", "--output-file") - .required() + export_csv_command.add_argument("log-file") + .help("The DataLog file to export"); + export_csv_command.add_argument("output-file") .help( "The CSV file to create with the exported data. If it " "exists, it will be overwritten."); @@ -64,13 +85,13 @@ int main(int argc, char* argv[]) { .required() .help("The DataLog file to extract from"); extract_entry_command.add_argument("--time-start") - .help("The timestamp to start extracting at"); + .help("The timestamp to start extracting at"); extract_entry_command.add_argument("-o", "--output") .required() .help( "The file to export the field and data to. It will be created or " "overwritten if it already exists"); - + cli.add_subparser(export_json_command); cli.add_subparser(export_csv_command); cli.add_subparser(extract_entry_command); @@ -87,16 +108,16 @@ int main(int argc, char* argv[]) { if (export_json_command) { // validate paths fs::path logPath{export_json_command.get("--log-file")}; - if (logPath.extension() != ".wpilog") - { + if (logPath.extension() != ".wpilog") { std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; return 1; } fs::path outputPath{export_json_command.get("--output")}; - if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") - { - std::cerr << "Only JSON and CSV are currently supported as output formats." << std::endl; + if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { + std::cerr + << "Only JSON and CSV are currently supported as output formats." + << std::endl; return 1; } @@ -104,16 +125,16 @@ int main(int argc, char* argv[]) { } else if (export_csv_command) { // validate paths fs::path logPath{export_csv_command.get("--log-file")}; - if (logPath.extension() != ".wpilog") - { + if (logPath.extension() != ".wpilog") { std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; return 1; } fs::path outputPath{export_csv_command.get("--output")}; - if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") - { - std::cerr << "Only JSON and CSV are currently supported as output formats." << std::endl; + if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { + std::cerr + << "Only JSON and CSV are currently supported as output formats." + << std::endl; return 1; } @@ -121,21 +142,22 @@ int main(int argc, char* argv[]) { } else if (extract_entry_command) { // validate paths fs::path logPath{extract_entry_command.get("--log-file")}; - if (logPath.extension() != ".wpilog") - { + if (logPath.extension() != ".wpilog") { std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; return 1; } fs::path outputPath{extract_entry_command.get("--output")}; - if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") - { - std::cerr << "Only JSON and CSV are currently supported as output formats." << std::endl; + if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { + std::cerr + << "Only JSON and CSV are currently supported as output formats." + << std::endl; return 1; } bool json{outputPath.extension() == ".json" ? true : false}; - extract_entry(extract_entry_command.get("--entry"), logPath, outputPath, json); + extract_entry(extract_entry_command.get("--entry"), logPath, outputPath, + json); } -} \ No newline at end of file +} diff --git a/datalog-export/src/main/native/include/DataLogCSVWriter.h b/datalog-export/src/main/native/include/DataLogCSVWriter.h index 1f8dca8b37b..aeb8e2bd021 100644 --- a/datalog-export/src/main/native/include/DataLogCSVWriter.h +++ b/datalog-export/src/main/native/include/DataLogCSVWriter.h @@ -7,11 +7,11 @@ #include namespace datalogcli { - class DataLogCSVWriter { - public: - explicit DataLogCSVWriter(); - ~DataLogCSVWriter(); +class DataLogCSVWriter { + public: + explicit DataLogCSVWriter(); + ~DataLogCSVWriter(); - void ExportCsv(std::string_view outputFolder, int style); - }; -} + void ExportCsv(std::string_view outputFolder, int style); +}; +} // namespace datalogcli diff --git a/datalog-export/src/main/native/include/LogLoader.h b/datalog-export/src/main/native/include/LogLoader.h index e3d684a4b72..099a6ef0139 100644 --- a/datalog-export/src/main/native/include/LogLoader.h +++ b/datalog-export/src/main/native/include/LogLoader.h @@ -8,8 +8,8 @@ #include #include #include -#include "wpi/DataLogReader.h" +#include #include namespace glass { @@ -45,8 +45,9 @@ class LogLoader { std::vector GetRecords(std::string_view field_name); - private: + std::vector GetAllRecords(); + private: std::string m_filename; std::unique_ptr m_reader; From 5468aee96bb714b314758c7630aef49de205101d Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 17:22:46 -0600 Subject: [PATCH 17/35] add json writer files --- .../src/main/native/cpp/DataLogJSONWriter.cpp | 20 ++++++++++++++++ .../src/main/native/cpp/LogLoader.cpp | 2 +- .../main/native/include/DataLogJSONWriter.h | 23 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 datalog-export/src/main/native/cpp/DataLogJSONWriter.cpp create mode 100644 datalog-export/src/main/native/include/DataLogJSONWriter.h diff --git a/datalog-export/src/main/native/cpp/DataLogJSONWriter.cpp b/datalog-export/src/main/native/cpp/DataLogJSONWriter.cpp new file mode 100644 index 00000000000..13a53ab2bdf --- /dev/null +++ b/datalog-export/src/main/native/cpp/DataLogJSONWriter.cpp @@ -0,0 +1,20 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "DataLogJSONWriter.h" + +#include + +#include +#include +#include + +void ExportJSON(fs::path outputPath, + std::vector records) { + // JSON structure + // List of blocks + // Each block is a direct transcription of a record + // this includes the entry id, timestamp, and data. If the record contains raw + // bytes, they will be represented as a base64 string. +} diff --git a/datalog-export/src/main/native/cpp/LogLoader.cpp b/datalog-export/src/main/native/cpp/LogLoader.cpp index a95202ec0a2..e0c685d9c72 100644 --- a/datalog-export/src/main/native/cpp/LogLoader.cpp +++ b/datalog-export/src/main/native/cpp/LogLoader.cpp @@ -78,6 +78,6 @@ std::vector LogLoader::GetRecords( std::vector datalogcli::LogLoader::GetAllRecords() { std::vector record_list{}; - //auto iter = m_reader- + // auto iter = m_reader- return record_list; } diff --git a/datalog-export/src/main/native/include/DataLogJSONWriter.h b/datalog-export/src/main/native/include/DataLogJSONWriter.h new file mode 100644 index 00000000000..c951876ebc9 --- /dev/null +++ b/datalog-export/src/main/native/include/DataLogJSONWriter.h @@ -0,0 +1,23 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include + +namespace datalogcli { +/** + * Helps with loading datalog files. + */ +class DataLogJSONWriter { + public: + void ExportJSON(fs::path exportPath, + std::vector records); + + private: +}; +} // namespace datalogcli From 125a5aaf273608ddb248dd8ae984f98ac103f33d Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 17:24:01 -0600 Subject: [PATCH 18/35] rename to sawmill --- CMakeLists.txt | 2 +- datalog-export/CMakeLists.txt | 44 ------------------- sawmill/CMakeLists.txt | 44 +++++++++++++++++++ .../src/main/generate/WPILibVersion.cpp.in | 0 .../src/main/native/cpp/DataLogCSVWriter.cpp | 0 .../src/main/native/cpp/DataLogJSONWriter.cpp | 0 .../src/main/native/cpp/LogLoader.cpp | 0 .../src/main/native/cpp/main.cpp | 0 .../main/native/include/DataLogCSVWriter.h | 0 .../main/native/include/DataLogJSONWriter.h | 2 + .../src/main/native/include/LogLoader.h | 0 11 files changed, 47 insertions(+), 45 deletions(-) delete mode 100644 datalog-export/CMakeLists.txt create mode 100644 sawmill/CMakeLists.txt rename {datalog-export => sawmill}/src/main/generate/WPILibVersion.cpp.in (100%) rename {datalog-export => sawmill}/src/main/native/cpp/DataLogCSVWriter.cpp (100%) rename {datalog-export => sawmill}/src/main/native/cpp/DataLogJSONWriter.cpp (100%) rename {datalog-export => sawmill}/src/main/native/cpp/LogLoader.cpp (100%) rename {datalog-export => sawmill}/src/main/native/cpp/main.cpp (100%) rename {datalog-export => sawmill}/src/main/native/include/DataLogCSVWriter.h (100%) rename {datalog-export => sawmill}/src/main/native/include/DataLogJSONWriter.h (90%) rename {datalog-export => sawmill}/src/main/native/include/LogLoader.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1eec148b21b..220e57e8946 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -300,7 +300,7 @@ if(WITH_NTCORE) add_subdirectory(ntcore) endif() -add_subdirectory(datalog-export) +add_subdirectory(sawmill) if(WITH_WPIMATH) if(WITH_JAVA) diff --git a/datalog-export/CMakeLists.txt b/datalog-export/CMakeLists.txt deleted file mode 100644 index 47cbd76bece..00000000000 --- a/datalog-export/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -project(datalog-export) - -include(CompileWarnings) -include(GenResources) - -configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) -generate_resources( - src/main/native/resources - generated/main/cpp - DLT - dlt - datalog-export_resources_src -) - -file( - GLOB datalog-export_src - src/main/native/cpp/*.cpp - ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp -) - -if(WIN32) - set(datalog-export_rc src/main/native/win/datalog-export.rc) -elseif(APPLE) - set(MACOSX_BUNDLE_ICON_FILE datalog-export.icns) - set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") -endif() - -add_executable( - datalog-export - ${datalog-export_src} - ${datalog-export_resources_src} - ${datalog-export_rc} - ${APP_ICON_MACOSX} -) - -target_include_directories(datalog-export PUBLIC src/main/native/include) - -target_link_libraries(datalog-export PRIVATE wpiutil) - -if(WIN32) - set_target_properties(datalog-export PROPERTIES WIN32_EXECUTABLE YES) -elseif(APPLE) - set_target_properties(datalog-export PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "datalog-export") -endif() diff --git a/sawmill/CMakeLists.txt b/sawmill/CMakeLists.txt new file mode 100644 index 00000000000..89bdd2b3081 --- /dev/null +++ b/sawmill/CMakeLists.txt @@ -0,0 +1,44 @@ +project(sawmill) + +include(CompileWarnings) +include(GenResources) + +configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) +generate_resources( + src/main/native/resources + generated/main/cpp + DLT + dlt + sawmill_resources_src +) + +file( + GLOB sawmill_src + src/main/native/cpp/*.cpp + ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp +) + +if(WIN32) + set(sawmill_rc src/main/native/win/sawmill.rc) +elseif(APPLE) + set(MACOSX_BUNDLE_ICON_FILE sawmill.icns) + set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") +endif() + +add_executable( + sawmill + ${sawmill_src} + ${sawmill_resources_src} + ${sawmill_rc} + ${APP_ICON_MACOSX} +) + +target_include_directories(sawmill PUBLIC src/main/native/include) + +target_link_libraries(sawmill PRIVATE wpiutil) + +if(WIN32) + set_target_properties(sawmill PROPERTIES WIN32_EXECUTABLE YES) +elseif(APPLE) + set_target_properties(sawmill PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "sawmill") +endif() diff --git a/datalog-export/src/main/generate/WPILibVersion.cpp.in b/sawmill/src/main/generate/WPILibVersion.cpp.in similarity index 100% rename from datalog-export/src/main/generate/WPILibVersion.cpp.in rename to sawmill/src/main/generate/WPILibVersion.cpp.in diff --git a/datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp similarity index 100% rename from datalog-export/src/main/native/cpp/DataLogCSVWriter.cpp rename to sawmill/src/main/native/cpp/DataLogCSVWriter.cpp diff --git a/datalog-export/src/main/native/cpp/DataLogJSONWriter.cpp b/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp similarity index 100% rename from datalog-export/src/main/native/cpp/DataLogJSONWriter.cpp rename to sawmill/src/main/native/cpp/DataLogJSONWriter.cpp diff --git a/datalog-export/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp similarity index 100% rename from datalog-export/src/main/native/cpp/LogLoader.cpp rename to sawmill/src/main/native/cpp/LogLoader.cpp diff --git a/datalog-export/src/main/native/cpp/main.cpp b/sawmill/src/main/native/cpp/main.cpp similarity index 100% rename from datalog-export/src/main/native/cpp/main.cpp rename to sawmill/src/main/native/cpp/main.cpp diff --git a/datalog-export/src/main/native/include/DataLogCSVWriter.h b/sawmill/src/main/native/include/DataLogCSVWriter.h similarity index 100% rename from datalog-export/src/main/native/include/DataLogCSVWriter.h rename to sawmill/src/main/native/include/DataLogCSVWriter.h diff --git a/datalog-export/src/main/native/include/DataLogJSONWriter.h b/sawmill/src/main/native/include/DataLogJSONWriter.h similarity index 90% rename from datalog-export/src/main/native/include/DataLogJSONWriter.h rename to sawmill/src/main/native/include/DataLogJSONWriter.h index c951876ebc9..060d7b040bb 100644 --- a/datalog-export/src/main/native/include/DataLogJSONWriter.h +++ b/sawmill/src/main/native/include/DataLogJSONWriter.h @@ -6,6 +6,8 @@ #include #include +#include +#include #include diff --git a/datalog-export/src/main/native/include/LogLoader.h b/sawmill/src/main/native/include/LogLoader.h similarity index 100% rename from datalog-export/src/main/native/include/LogLoader.h rename to sawmill/src/main/native/include/LogLoader.h From 8760e9ed9a16910479d2db327463b10e93edde7a Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 17:27:27 -0600 Subject: [PATCH 19/35] add getallrecords --- sawmill/src/main/native/cpp/LogLoader.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index e0c685d9c72..7f7ee77e3d4 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -78,6 +78,10 @@ std::vector LogLoader::GetRecords( std::vector datalogcli::LogLoader::GetAllRecords() { std::vector record_list{}; - // auto iter = m_reader- + for (wpi::log::DataLogRecord record : m_reader->GetReader()) + { + record_list.push_back(record); + } + return record_list; } From 58774b261751543145f5fb33c34056f4dcbb15b5 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 18:12:14 -0600 Subject: [PATCH 20/35] format --- sawmill/CMakeLists.txt | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/sawmill/CMakeLists.txt b/sawmill/CMakeLists.txt index 89bdd2b3081..4211bd45265 100644 --- a/sawmill/CMakeLists.txt +++ b/sawmill/CMakeLists.txt @@ -4,19 +4,9 @@ include(CompileWarnings) include(GenResources) configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) -generate_resources( - src/main/native/resources - generated/main/cpp - DLT - dlt - sawmill_resources_src -) - -file( - GLOB sawmill_src - src/main/native/cpp/*.cpp - ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp -) +generate_resources(src/main/native/resources generated/main/cpp DLT dlt sawmill_resources_src) + +file(GLOB sawmill_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) if(WIN32) set(sawmill_rc src/main/native/win/sawmill.rc) @@ -25,13 +15,7 @@ elseif(APPLE) set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") endif() -add_executable( - sawmill - ${sawmill_src} - ${sawmill_resources_src} - ${sawmill_rc} - ${APP_ICON_MACOSX} -) +add_executable(sawmill ${sawmill_src} ${sawmill_resources_src} ${sawmill_rc} ${APP_ICON_MACOSX}) target_include_directories(sawmill PUBLIC src/main/native/include) From 2d346d10ee3bcef2d730c9306f6e48c938b101cf Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 19:45:17 -0600 Subject: [PATCH 21/35] undo root cmakelists changes --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 220e57e8946..144ab8e1276 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,9 +85,9 @@ cmake_dependent_option( WITH_JAVA OFF ) -option(WITH_WPILIB "Build hal, wpilibc/j, and developerRobot (needs OpenCV)" OFF) +option(WITH_WPILIB "Build hal, wpilibc/j, and developerRobot (needs OpenCV)" ON) option(WITH_EXAMPLES "Build examples" OFF) -option(WITH_TESTS "Build unit tests (requires internet connection)" OFF) +option(WITH_TESTS "Build unit tests (requires internet connection)" ON) option(WITH_GUI "Build GUI items" ON) option(WITH_SIMULATION_MODULES "Build simulation modules" ON) option(WITH_PROTOBUF "Build protobuf support" ON) From cde5f442e407cf543a278e52f405f3d09a2acddb Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 20:41:37 -0600 Subject: [PATCH 22/35] rename namespace to sawmill --- sawmill/src/main/native/cpp/DataLogCSVWriter.cpp | 2 +- sawmill/src/main/native/cpp/LogLoader.cpp | 4 ++-- sawmill/src/main/native/cpp/main.cpp | 14 ++++++++------ sawmill/src/main/native/include/DataLogCSVWriter.h | 4 ++-- .../src/main/native/include/DataLogJSONWriter.h | 4 ++-- sawmill/src/main/native/include/LogLoader.h | 4 ++-- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp index 70396f235d1..83003acf248 100644 --- a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp +++ b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp @@ -110,7 +110,7 @@ InputFile::~InputFile() { } } -using namespace datalogcli; +using namespace sawmill; static wpi::mutex gExportMutex; static std::vector gExportErrors; diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index 7f7ee77e3d4..4d6b6fea0a1 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -18,7 +18,7 @@ #include #include -using namespace datalogcli; +using namespace sawmill; LogLoader::LogLoader() {} @@ -75,7 +75,7 @@ std::vector LogLoader::GetRecords( return record_list; } -std::vector datalogcli::LogLoader::GetAllRecords() { +std::vector sawmill::LogLoader::GetAllRecords() { std::vector record_list{}; for (wpi::log::DataLogRecord record : m_reader->GetReader()) diff --git a/sawmill/src/main/native/cpp/main.cpp b/sawmill/src/main/native/cpp/main.cpp index 68b430773b1..47e457a45c5 100644 --- a/sawmill/src/main/native/cpp/main.cpp +++ b/sawmill/src/main/native/cpp/main.cpp @@ -16,34 +16,36 @@ #include "LogLoader.h" void export_json(fs::path log_path, fs::path output_path) { - datalogcli::LogLoader loader{}; + sawmill::LogLoader loader{}; loader.Load(log_path.string()); std::vector records = loader.GetAllRecords(); - datalogcli::DataLogJSONWriter writer{}; + sawmill::DataLogJSONWriter writer{}; writer.ExportJSON(output_path, records); } -void export_csv(fs::path log_path, fs::path output_path) {} +void export_csv(fs::path log_path, fs::path output_path) { + +} void write_json(std::vector records, fs::path output_path) {} void write_csv(std::vector records, fs::path output_path) { - datalogcli::DataLogCSVWriter writer{}; + sawmill::DataLogCSVWriter writer{}; writer.ExportCsv(output_path.string(), 0); } void extract_entry(std::string_view entry_name, fs::path log_path, fs::path output_path, bool use_json) { // represent entry as a list of records - datalogcli::LogLoader loader{}; + sawmill::LogLoader loader{}; loader.Load(log_path.string()); std::vector records = loader.GetRecords(entry_name); } int main(int argc, char* argv[]) { - wpi::ArgumentParser cli{"datalogcli"}; + wpi::ArgumentParser cli{"sawmill"}; std::string jsonMode{}; wpi::ArgumentParser export_json_command{"json"}; diff --git a/sawmill/src/main/native/include/DataLogCSVWriter.h b/sawmill/src/main/native/include/DataLogCSVWriter.h index aeb8e2bd021..044f47450ef 100644 --- a/sawmill/src/main/native/include/DataLogCSVWriter.h +++ b/sawmill/src/main/native/include/DataLogCSVWriter.h @@ -6,7 +6,7 @@ #include -namespace datalogcli { +namespace sawmill { class DataLogCSVWriter { public: explicit DataLogCSVWriter(); @@ -14,4 +14,4 @@ class DataLogCSVWriter { void ExportCsv(std::string_view outputFolder, int style); }; -} // namespace datalogcli +} // namespace sawmill diff --git a/sawmill/src/main/native/include/DataLogJSONWriter.h b/sawmill/src/main/native/include/DataLogJSONWriter.h index 060d7b040bb..58fd5ff3147 100644 --- a/sawmill/src/main/native/include/DataLogJSONWriter.h +++ b/sawmill/src/main/native/include/DataLogJSONWriter.h @@ -11,7 +11,7 @@ #include -namespace datalogcli { +namespace sawmill { /** * Helps with loading datalog files. */ @@ -22,4 +22,4 @@ class DataLogJSONWriter { private: }; -} // namespace datalogcli +} // namespace sawmill diff --git a/sawmill/src/main/native/include/LogLoader.h b/sawmill/src/main/native/include/LogLoader.h index 099a6ef0139..010b74d1329 100644 --- a/sawmill/src/main/native/include/LogLoader.h +++ b/sawmill/src/main/native/include/LogLoader.h @@ -22,7 +22,7 @@ class DataLogReaderThread; class Logger; } // namespace wpi -namespace datalogcli { +namespace sawmill { /** * Helps with loading datalog files. */ @@ -66,4 +66,4 @@ class LogLoader { }; std::vector m_entryTree; }; -} // namespace datalogcli +} // namespace sawmill From 6c048b72ad91e31de3da6676d24576a2fc35ec33 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 21:44:19 -0600 Subject: [PATCH 23/35] parse data up front --- .../src/main/native/cpp/DataLogJSONWriter.cpp | 2 +- sawmill/src/main/native/cpp/LogLoader.cpp | 47 ++++++++++++++----- sawmill/src/main/native/cpp/main.cpp | 7 ++- .../src/main/native/include/DataLogExport.h | 14 ++++++ .../main/native/include/DataLogJSONWriter.h | 4 +- sawmill/src/main/native/include/LogLoader.h | 17 ++----- 6 files changed, 62 insertions(+), 29 deletions(-) create mode 100644 sawmill/src/main/native/include/DataLogExport.h diff --git a/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp b/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp index 13a53ab2bdf..a6eac1b0a02 100644 --- a/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp +++ b/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp @@ -11,7 +11,7 @@ #include void ExportJSON(fs::path outputPath, - std::vector records) { + std::vector records) { // JSON structure // List of blocks // Each block is a direct transcription of a record diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index 4d6b6fea0a1..d475d2faf51 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -24,10 +24,10 @@ LogLoader::LogLoader() {} LogLoader::~LogLoader() = default; -void LogLoader::Load(std::string_view log_path) { +void LogLoader::Load(fs::path logPath) { // Handle opening the file std::error_code ec; - auto buf = wpi::MemoryBuffer::GetFile(log_path); + auto buf = wpi::MemoryBuffer::GetFile(logPath.string()); if (ec) { m_error = fmt::format("Could not open file: {}", ec.message()); return; @@ -40,7 +40,6 @@ void LogLoader::Load(std::string_view log_path) { } unload(); // release the actual file, we have the data in the reader now m_reader = std::make_unique(std::move(reader)); - m_entryTree.clear(); // Handle Errors fmt::println("{}", m_error); @@ -58,9 +57,11 @@ void LogLoader::Load(std::string_view log_path) { if (!m_reader->IsDone()) { return; } + + } -std::vector LogLoader::GetRecords( +/*std::vector LogLoader::GetRecords( std::string_view field_name) { std::vector record_list{}; @@ -73,15 +74,35 @@ std::vector LogLoader::GetRecords( } return record_list; -} - -std::vector sawmill::LogLoader::GetAllRecords() { - std::vector record_list{}; - - for (wpi::log::DataLogRecord record : m_reader->GetReader()) - { - record_list.push_back(record); +}*/ + +std::vector sawmill::LogLoader::GetAllRecords() { + if (records.size() == 0) { + std::map> dataMap; + // get all records + for (wpi::log::DataLogRecord record : m_reader->GetReader()) + { + if (record.IsStart()) { + wpi::log::StartRecordData data; + if (record.GetStartData(&data)) { + // associate an entry id with a StartRecordData + dataMap[data.entry] = data; + } + } else if (record.IsFinish()) { + // remove the association + int entryId; + if (record.GetFinishEntry(&entryId)) + { + dataMap.erase(entryId); + } + } + int entryId = record.GetEntry(); + if (dataMap.contains(entryId)) + { + records.push_back(sawmill::DataLogRecord{dataMap[entryId], record}); + } + } } - return record_list; + return records; } diff --git a/sawmill/src/main/native/cpp/main.cpp b/sawmill/src/main/native/cpp/main.cpp index 47e457a45c5..633d4c74013 100644 --- a/sawmill/src/main/native/cpp/main.cpp +++ b/sawmill/src/main/native/cpp/main.cpp @@ -23,7 +23,10 @@ void export_json(fs::path log_path, fs::path output_path) { writer.ExportJSON(output_path, records); } -void export_csv(fs::path log_path, fs::path output_path) { +void export_csv(fs::path logPath, fs::path outputPaths) { + sawmill::LogLoader loader{}; + loader.Load(logPath); + // } @@ -140,7 +143,7 @@ int main(int argc, char* argv[]) { return 1; } - export_csv(logPath, outputPath); + //export_csv(logPath, outputPath); } else if (extract_entry_command) { // validate paths fs::path logPath{extract_entry_command.get("--log-file")}; diff --git a/sawmill/src/main/native/include/DataLogExport.h b/sawmill/src/main/native/include/DataLogExport.h new file mode 100644 index 00000000000..9db7668cbb9 --- /dev/null +++ b/sawmill/src/main/native/include/DataLogExport.h @@ -0,0 +1,14 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +namespace sawmill { + struct DataLogRecord { + const wpi::log::StartRecordData entryData; + wpi::log::DataLogRecord record; + }; +} \ No newline at end of file diff --git a/sawmill/src/main/native/include/DataLogJSONWriter.h b/sawmill/src/main/native/include/DataLogJSONWriter.h index 58fd5ff3147..e1efcfc603a 100644 --- a/sawmill/src/main/native/include/DataLogJSONWriter.h +++ b/sawmill/src/main/native/include/DataLogJSONWriter.h @@ -4,6 +4,8 @@ #pragma once +#include "DataLogExport.h" + #include #include #include @@ -18,7 +20,7 @@ namespace sawmill { class DataLogJSONWriter { public: void ExportJSON(fs::path exportPath, - std::vector records); + std::vector records); private: }; diff --git a/sawmill/src/main/native/include/LogLoader.h b/sawmill/src/main/native/include/LogLoader.h index 010b74d1329..f9c2daf479e 100644 --- a/sawmill/src/main/native/include/LogLoader.h +++ b/sawmill/src/main/native/include/LogLoader.h @@ -4,6 +4,8 @@ #pragma once +#include "DataLogExport.h" + #include #include #include @@ -41,11 +43,9 @@ class LogLoader { */ wpi::sig::Signal<> unload; - void Load(std::string_view log_path); - - std::vector GetRecords(std::string_view field_name); + void Load(fs::path logPath); - std::vector GetAllRecords(); + std::vector GetAllRecords(); private: std::string m_filename; @@ -57,13 +57,6 @@ class LogLoader { wpi::log::StartRecordData* entryData; - struct EntryTreeNode { - explicit EntryTreeNode(std::string_view name) : name{name} {} - std::string name; // name of just this node - std::string path; // full path if entry is nullptr - const wpi::DataLogReaderEntry* entry = nullptr; - std::vector children; // children, sorted by name - }; - std::vector m_entryTree; + std::vector records; }; } // namespace sawmill From 4588ea68a9def9b650a0f04655f8e1528a653c16 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 22:33:06 -0600 Subject: [PATCH 24/35] add entry map to loader --- sawmill/src/main/native/cpp/LogLoader.cpp | 25 +++++++++++---------- sawmill/src/main/native/include/LogLoader.h | 8 +++++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index d475d2faf51..a2e7e9a52ea 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -4,6 +4,8 @@ #include "LogLoader.h" +#include +#include #include #include #include @@ -57,8 +59,6 @@ void LogLoader::Load(fs::path logPath) { if (!m_reader->IsDone()) { return; } - - } /*std::vector LogLoader::GetRecords( @@ -78,31 +78,32 @@ void LogLoader::Load(fs::path logPath) { std::vector sawmill::LogLoader::GetAllRecords() { if (records.size() == 0) { - std::map> dataMap; // get all records - for (wpi::log::DataLogRecord record : m_reader->GetReader()) - { + for (wpi::log::DataLogRecord record : m_reader->GetReader()) { if (record.IsStart()) { wpi::log::StartRecordData data; if (record.GetStartData(&data)) { // associate an entry id with a StartRecordData dataMap[data.entry] = data; - } + } } else if (record.IsFinish()) { // remove the association int entryId; - if (record.GetFinishEntry(&entryId)) - { + if (record.GetFinishEntry(&entryId)) { dataMap.erase(entryId); } - } + } int entryId = record.GetEntry(); - if (dataMap.contains(entryId)) - { + if (dataMap.contains(entryId)) { records.push_back(sawmill::DataLogRecord{dataMap[entryId], record}); } } } - + return records; } + +std::map> +sawmill::LogLoader::GetEntryMap() { + return dataMap; +} diff --git a/sawmill/src/main/native/include/LogLoader.h b/sawmill/src/main/native/include/LogLoader.h index f9c2daf479e..dfa6258b84f 100644 --- a/sawmill/src/main/native/include/LogLoader.h +++ b/sawmill/src/main/native/include/LogLoader.h @@ -4,8 +4,6 @@ #pragma once -#include "DataLogExport.h" - #include #include #include @@ -14,6 +12,8 @@ #include #include +#include "DataLogExport.h" + namespace glass { class Storage; } // namespace glass @@ -47,6 +47,8 @@ class LogLoader { std::vector GetAllRecords(); + std::map> GetEntryMap(); + private: std::string m_filename; std::unique_ptr m_reader; @@ -57,6 +59,8 @@ class LogLoader { wpi::log::StartRecordData* entryData; + std::map> dataMap; + std::vector records; }; } // namespace sawmill From 6552add71a634fca88bf0ec59fd0d9ab65443480 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 22:33:38 -0600 Subject: [PATCH 25/35] use sawmill records --- sawmill/src/main/native/cpp/main.cpp | 9 ++++----- sawmill/src/main/native/include/DataLogExport.h | 11 ++++++----- sawmill/src/main/native/include/DataLogJSONWriter.h | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sawmill/src/main/native/cpp/main.cpp b/sawmill/src/main/native/cpp/main.cpp index 633d4c74013..ae98149c049 100644 --- a/sawmill/src/main/native/cpp/main.cpp +++ b/sawmill/src/main/native/cpp/main.cpp @@ -18,7 +18,7 @@ void export_json(fs::path log_path, fs::path output_path) { sawmill::LogLoader loader{}; loader.Load(log_path.string()); - std::vector records = loader.GetAllRecords(); + std::vector records = loader.GetAllRecords(); sawmill::DataLogJSONWriter writer{}; writer.ExportJSON(output_path, records); } @@ -26,8 +26,7 @@ void export_json(fs::path log_path, fs::path output_path) { void export_csv(fs::path logPath, fs::path outputPaths) { sawmill::LogLoader loader{}; loader.Load(logPath); - // - + // } void write_json(std::vector records, @@ -44,7 +43,7 @@ void extract_entry(std::string_view entry_name, fs::path log_path, // represent entry as a list of records sawmill::LogLoader loader{}; loader.Load(log_path.string()); - std::vector records = loader.GetRecords(entry_name); + //std::vector records = loader.GetRecords(entry_name); } int main(int argc, char* argv[]) { @@ -143,7 +142,7 @@ int main(int argc, char* argv[]) { return 1; } - //export_csv(logPath, outputPath); + // export_csv(logPath, outputPath); } else if (extract_entry_command) { // validate paths fs::path logPath{extract_entry_command.get("--log-file")}; diff --git a/sawmill/src/main/native/include/DataLogExport.h b/sawmill/src/main/native/include/DataLogExport.h index 9db7668cbb9..c82c5ef53e2 100644 --- a/sawmill/src/main/native/include/DataLogExport.h +++ b/sawmill/src/main/native/include/DataLogExport.h @@ -7,8 +7,9 @@ #include namespace sawmill { - struct DataLogRecord { - const wpi::log::StartRecordData entryData; - wpi::log::DataLogRecord record; - }; -} \ No newline at end of file +struct DataLogRecord { + const wpi::log::StartRecordData entryData; + wpi::log::DataLogRecord dataLogRecord; + int entryColumn = -1; +}; +} // namespace sawmill diff --git a/sawmill/src/main/native/include/DataLogJSONWriter.h b/sawmill/src/main/native/include/DataLogJSONWriter.h index e1efcfc603a..8a42037528b 100644 --- a/sawmill/src/main/native/include/DataLogJSONWriter.h +++ b/sawmill/src/main/native/include/DataLogJSONWriter.h @@ -4,15 +4,15 @@ #pragma once -#include "DataLogExport.h" - #include #include #include -#include +#include #include +#include "DataLogExport.h" + namespace sawmill { /** * Helps with loading datalog files. From 51f75aa6c7afbe42c058d7ec5bb53cf76b730fa7 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Tue, 7 Jan 2025 22:33:57 -0600 Subject: [PATCH 26/35] first attempt at csv writing --- .../src/main/native/cpp/DataLogCSVWriter.cpp | 315 ++++++++++++------ 1 file changed, 222 insertions(+), 93 deletions(-) diff --git a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp index 83003acf248..e011314bd2c 100644 --- a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp +++ b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp @@ -30,6 +30,7 @@ #include #include #include +#include namespace { struct InputFile { @@ -130,70 +131,69 @@ static void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str) { } } -static void ValueToCsv(wpi::raw_ostream& os, const Entry& entry, - const wpi::log::DataLogRecord& record) { - // handle systemTime specially - if (entry.name == "systemTime" && entry.type == "int64") { +static void ValueToCsv(wpi::raw_ostream& os, const sawmill::DataLogRecord& record) { + // systemTime needs special handling + if (record.entryData.name == "systemTime" && record.entryData.type == "int64") { int64_t val; - if (record.GetInteger(&val)) { + if (record.dataLogRecord.GetInteger(&val)) { std::time_t timeval = val / 1000000; wpi::print(os, "{:%Y-%m-%d %H:%M:%S}.{:06}", *std::localtime(&timeval), val % 1000000); return; } - } else if (entry.type == "double") { + } else if (record.entryData.type == "double") { double val; - if (record.GetDouble(&val)) { + if (record.dataLogRecord.GetDouble(&val)) { wpi::print(os, "{}", val); return; } - } else if (entry.type == "int64" || entry.type == "int") { + } else if (record.entryData.type == "int64" || record.entryData.type == "int") { // support "int" for compatibility with old NT4 datalogs int64_t val; - if (record.GetInteger(&val)) { + if (record.dataLogRecord.GetInteger(&val)) { wpi::print(os, "{}", val); return; } - } else if (entry.type == "string" || entry.type == "json") { + } else if (record.entryData.type == "string" || record.entryData.type == "json") { std::string_view val; - record.GetString(&val); + record.dataLogRecord.GetString(&val); os << '"'; PrintEscapedCsvString(os, val); os << '"'; return; - } else if (entry.type == "boolean") { + } else if (record.entryData.type == "boolean") { bool val; - if (record.GetBoolean(&val)) { + if (record.dataLogRecord.GetBoolean(&val)) { wpi::print(os, "{}", val); return; } - } else if (entry.type == "boolean[]") { + } else if (record.entryData.type == "boolean[]") { std::vector val; - if (record.GetBooleanArray(&val)) { + if (record.dataLogRecord.GetBooleanArray(&val)) { wpi::print(os, "{}", fmt::join(val, ";")); return; } - } else if (entry.type == "double[]") { + } else if (record.entryData.type == "double[]") { std::vector val; - if (record.GetDoubleArray(&val)) { + if (record.dataLogRecord.GetDoubleArray(&val)) { wpi::print(os, "{}", fmt::join(val, ";")); return; } - } else if (entry.type == "float[]") { + } else if (record.entryData.type == "float[]") { std::vector val; - if (record.GetFloatArray(&val)) { + if (record.dataLogRecord.GetFloatArray(&val)) { wpi::print(os, "{}", fmt::join(val, ";")); return; } - } else if (entry.type == "int64[]") { + } else if (record.entryData.type == "int64[]") { std::vector val; - if (record.GetIntegerArray(&val)) { + if (record.dataLogRecord.GetIntegerArray(&val)) { wpi::print(os, "{}", fmt::join(val, ";")); return; } - } else if (entry.type == "string[]") { + } else if (record.entryData.type == "string[]") { std::vector val; - if (record.GetStringArray(&val)) { + if (record.dataLogRecord.GetStringArray(&val)) { os << '"'; bool first = true; for (auto&& v : val) { @@ -210,86 +210,215 @@ static void ValueToCsv(wpi::raw_ostream& os, const Entry& entry, wpi::print(os, ""); } -static void ExportCsvFile(InputFile& f, wpi::raw_ostream& os, int style) { - // header - if (style == 0) { +// static void ValueToCsv(wpi::raw_ostream& os, const Entry& entry, +// const wpi::log::DataLogRecord& record) { +// // handle systemTime specially +// if (entry.name == "systemTime" && entry.type == "int64") { +// int64_t val; +// if (record.GetInteger(&val)) { +// std::time_t timeval = val / 1000000; +// wpi::print(os, "{:%Y-%m-%d %H:%M:%S}.{:06}", *std::localtime(&timeval), +// val % 1000000); +// return; +// } +// } else if (entry.type == "double") { +// double val; +// if (record.GetDouble(&val)) { +// wpi::print(os, "{}", val); +// return; +// } +// } else if (entry.type == "int64" || entry.type == "int") { +// // support "int" for compatibility with old NT4 datalogs +// int64_t val; +// if (record.GetInteger(&val)) { +// wpi::print(os, "{}", val); +// return; +// } +// } else if (entry.type == "string" || entry.type == "json") { +// std::string_view val; +// record.GetString(&val); +// os << '"'; +// PrintEscapedCsvString(os, val); +// os << '"'; +// return; +// } else if (entry.type == "boolean") { +// bool val; +// if (record.GetBoolean(&val)) { +// wpi::print(os, "{}", val); +// return; +// } +// } else if (entry.type == "boolean[]") { +// std::vector val; +// if (record.GetBooleanArray(&val)) { +// wpi::print(os, "{}", fmt::join(val, ";")); +// return; +// } +// } else if (entry.type == "double[]") { +// std::vector val; +// if (record.GetDoubleArray(&val)) { +// wpi::print(os, "{}", fmt::join(val, ";")); +// return; +// } +// } else if (entry.type == "float[]") { +// std::vector val; +// if (record.GetFloatArray(&val)) { +// wpi::print(os, "{}", fmt::join(val, ";")); +// return; +// } +// } else if (entry.type == "int64[]") { +// std::vector val; +// if (record.GetIntegerArray(&val)) { +// wpi::print(os, "{}", fmt::join(val, ";")); +// return; +// } +// } else if (entry.type == "string[]") { +// std::vector val; +// if (record.GetStringArray(&val)) { +// os << '"'; +// bool first = true; +// for (auto&& v : val) { +// if (!first) { +// os << ';'; +// } +// first = false; +// PrintEscapedCsvString(os, v); +// } +// os << '"'; +// return; +// } +// } +// wpi::print(os, ""); +// } + +void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, std::vector records, std::map> entryMap) { + // print header + if (style == 0) + { os << "Timestamp,Name,Value\n"; } else if (style == 1) { // scan for exported fields for this file to print header and assign columns os << "Timestamp"; int columnNum = 0; - for (auto&& entry : gEntries) { - if (entry.second->selected && - entry.second->inputFiles.find(&f) != entry.second->inputFiles.end()) { - os << ',' << '"'; - PrintEscapedCsvString(os, entry.first); - os << '"'; - entry.second->column = columnNum++; - } else { - entry.second->column = -1; - } + // TODO: Find a way to tie the entry data to a column number without modifying startrecorddata or iterating through every record + for (std::pair &entry : entryMap) + { + os << ',' << '"'; + PrintEscapedCsvString(os, entry.second.name); + os << '"'; + } - os << '\n'; } - - wpi::DenseMap nameMap; - for (wpi::log::DataLogRecord record : f.datalog->GetReader()) { - if (record.IsStart()) { - wpi::log::StartRecordData data; - if (record.GetStartData(&data)) { - auto it = gEntries.find(data.name); - if (it != gEntries.end() && it->second->selected) { - nameMap[data.entry] = it->second.get(); - } - } - } else if (record.IsFinish()) { - int entry; - if (record.GetFinishEntry(&entry)) { - nameMap.erase(entry); - } - } else if (!record.IsControl()) { - auto entryIt = nameMap.find(record.GetEntry()); - if (entryIt == nameMap.end()) { - continue; - } - Entry* entry = entryIt->second; - - if (style == 0) { - wpi::print(os, "{},\"", record.GetTimestamp() / 1000000.0); - PrintEscapedCsvString(os, entry->name); - os << '"' << ','; - ValueToCsv(os, *entry, record); - os << '\n'; - } else if (style == 1 && entry->column != -1) { - wpi::print(os, "{},", record.GetTimestamp() / 1000000.0); - for (int i = 0; i < entry->column; ++i) { - os << ','; - } - ValueToCsv(os, *entry, record); - os << '\n'; - } + + for (sawmill::DataLogRecord record : records) + { + // if this is a control record and we dont want to print those, skip + if (record.dataLogRecord.IsControl() && !printControlRecords) + { + continue; } - } -} -static void ExportCsv(std::string_view outputFolder, int style) { - fs::path outPath{outputFolder}; - for (auto&& f : gInputFiles) { - if (f.second->datalog) { - std::error_code ec; - auto of = fs::OpenFileForWrite( - outPath / fs::path{f.first}.replace_extension("csv"), ec, - fs::CD_CreateNew, fs::OF_Text); - if (ec) { - std::scoped_lock lock{gExportMutex}; - gExportErrors.emplace_back( - fmt::format("{}: {}", f.first, ec.message())); - ++gExportCount; - continue; + if (style == 0) + { + wpi::print(os, "{},\"", record.dataLogRecord.GetTimestamp() / 1000000.0); + PrintEscapedCsvString(os, record.entryData.name); + os << '"' << ','; + ValueToCsv(os, record); + os << '\n'; + } else if(style == 1) { + wpi::print(os, "{},", record.dataLogRecord.GetTimestamp() / 1000000.0); + for (int i = 0; i < record.entryColumn; ++i) { + os << ','; } - wpi::raw_fd_ostream os{fs::FileToFd(of, ec, fs::OF_Text), true}; - ExportCsvFile(*f.second, os, style); + ValueToCsv(os, record); + os << '\n'; } - ++gExportCount; + + } + + } + +// static void ExportCsvFile(InputFile& f, wpi::raw_ostream& os, int style) { +// // header +// if (style == 0) { +// os << "Timestamp,Name,Value\n"; +// } else if (style == 1) { +// // scan for exported fields for this file to print header and assign columns +// os << "Timestamp"; +// int columnNum = 0; +// for (auto&& entry : gEntries) { +// if (entry.second->selected && +// entry.second->inputFiles.find(&f) != entry.second->inputFiles.end()) { +// os << ',' << '"'; +// PrintEscapedCsvString(os, entry.first); +// os << '"'; +// entry.second->column = columnNum++; +// } else { +// entry.second->column = -1; +// } +// } +// os << '\n'; +// } + +// wpi::DenseMap nameMap; +// for (wpi::log::DataLogRecord record : f.datalog->GetReader()) { +// if (record.IsStart()) { +// wpi::log::StartRecordData data; +// if (record.GetStartData(&data)) { +// auto it = gEntries.find(data.name); +// if (it != gEntries.end() && it->second->selected) { +// nameMap[data.entry] = it->second.get(); +// } +// } +// } else if (record.IsFinish()) { +// int entry; +// if (record.GetFinishEntry(&entry)) { +// nameMap.erase(entry); +// } +// } else if (!record.IsControl()) { +// auto entryIt = nameMap.find(record.GetEntry()); +// if (entryIt == nameMap.end()) { +// continue; +// } +// Entry* entry = entryIt->second; + +// if (style == 0) { +// wpi::print(os, "{},\"", record.GetTimestamp() / 1000000.0); +// PrintEscapedCsvString(os, entry->name); +// os << '"' << ','; +// ValueToCsv(os, record); +// os << '\n'; +// } else if (style == 1 && entry->column != -1) { +// wpi::print(os, "{},", record.GetTimestamp() / 1000000.0); +// for (int i = 0; i < entry->column; ++i) { +// os << ','; +// } +// ValueToCsv(os, *entry, record); +// os << '\n'; +// } +// } +// } +// } + +// static void ExportCsv(std::string_view outputFolder, int style) { +// fs::path outPath{outputFolder}; +// for (auto&& f : gInputFiles) { +// if (f.second->datalog) { +// std::error_code ec; +// auto of = fs::OpenFileForWrite( +// outPath / fs::path{f.first}.replace_extension("csv"), ec, +// fs::CD_CreateNew, fs::OF_Text); +// if (ec) { +// std::scoped_lock lock{gExportMutex}; +// gExportErrors.emplace_back( +// fmt::format("{}: {}", f.first, ec.message())); +// ++gExportCount; +// continue; +// } +// wpi::raw_fd_ostream os{fs::FileToFd(of, ec, fs::OF_Text), true}; +// ExportCsvFile(*f.second, os, style); +// } +// ++gExportCount; +// } +// } From 2c2b92b9a961bc71d8138e64df5e04b6d34134c2 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Wed, 8 Jan 2025 10:19:24 -0600 Subject: [PATCH 27/35] use Entry struct from datalogtool --- sawmill/CMakeLists.txt | 5 + .../src/main/native/cpp/DataLogCSVWriter.cpp | 137 ++++++++---------- sawmill/src/main/native/cpp/LogLoader.cpp | 5 +- .../src/main/native/include/DataLogExport.h | 21 ++- sawmill/src/main/native/include/LogLoader.h | 6 +- 5 files changed, 88 insertions(+), 86 deletions(-) diff --git a/sawmill/CMakeLists.txt b/sawmill/CMakeLists.txt index 4211bd45265..19f73a63d8c 100644 --- a/sawmill/CMakeLists.txt +++ b/sawmill/CMakeLists.txt @@ -6,6 +6,11 @@ include(GenResources) configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) generate_resources(src/main/native/resources generated/main/cpp DLT dlt sawmill_resources_src) +# Generate compile_commands.json by default +if (NOT CMAKE_EXPORT_COMPILE_COMMANDS) + set(CMAKE_EXPORT_COMPILE_COMMANDS "YES" CACHE STRING "" FORCE) +endif() + file(GLOB sawmill_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) if(WIN32) diff --git a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp index e011314bd2c..8a5c21f3141 100644 --- a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp +++ b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp @@ -2,15 +2,12 @@ // Open Source Software; you can modify and/or share it under the terms of // the WPILib BSD license file in the root directory of this project. -#include "DataLogCSVWriter.h" #include #include #include -#include #include #include -#include #include #include #include @@ -32,84 +29,68 @@ #include #include -namespace { -struct InputFile { - explicit InputFile(std::unique_ptr datalog); +// namespace { +// struct InputFile { +// explicit InputFile(std::unique_ptr datalog); - InputFile(std::string_view filename, std::string_view status) - : filename{filename}, - stem{fs::path{filename}.stem().string()}, - status{status} {} +// InputFile(std::string_view filename, std::string_view status) +// : filename{filename}, +// stem{fs::path{filename}.stem().string()}, +// status{status} {} - ~InputFile(); +// ~InputFile(); - std::string filename; - std::string stem; - std::unique_ptr datalog; - std::string status; - bool highlight = false; -}; +// std::string filename; +// std::string stem; +// std::unique_ptr datalog; +// std::string status; +// bool highlight = false; +// }; +// } // namespace -struct Entry { - explicit Entry(const wpi::log::StartRecordData& srd) - : name{srd.name}, type{srd.type}, metadata{srd.metadata} {} - - std::string name; - std::string type; - std::string metadata; - std::set inputFiles; - bool typeConflict = false; - bool metadataConflict = false; - bool selected = true; - - // used only during export - int column = -1; -}; -} // namespace - -static std::map, std::less<>> - gInputFiles; +//static std::map, std::less<>> +// gInputFiles; static wpi::mutex gEntriesMutex; -static std::map, std::less<>> gEntries; +static std::map, std::less<>> gEntries; std::atomic_int gExportCount{0}; -InputFile::InputFile(std::unique_ptr datalog_) - : filename{datalog_->GetBufferIdentifier()}, - stem{fs::path{filename}.stem().string()}, - datalog{std::move(datalog_)} { - datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) { - std::scoped_lock lock{gEntriesMutex}; - auto it = gEntries.find(srd.name); - if (it == gEntries.end()) { - it = gEntries.emplace(srd.name, std::make_unique(srd)).first; - } else { - if (it->second->type != srd.type) { - it->second->typeConflict = true; - } - if (it->second->metadata != srd.metadata) { - it->second->metadataConflict = true; - } - } - it->second->inputFiles.emplace(this); - }); -} +// InputFile::InputFile(std::unique_ptr datalog_) +// : filename{datalog_->GetBufferIdentifier()}, +// stem{fs::path{filename}.stem().string()}, +// datalog{std::move(datalog_)} { +// datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) { +// std::scoped_lock lock{gEntriesMutex}; +// auto it = gEntries.find(srd.name); +// if (it == gEntries.end()) { +// it = gEntries.emplace(srd.name, std::make_unique(srd)).first; +// } else { +// if (it->second->type != srd.type) { +// it->second->typeConflict = true; +// } +// if (it->second->metadata != srd.metadata) { +// it->second->metadataConflict = true; +// } +// } +// it->second->inputFiles.emplace(this); +// }); +// } -InputFile::~InputFile() { - if (!datalog) { - return; - } - std::scoped_lock lock{gEntriesMutex}; - bool changed = false; - for (auto it = gEntries.begin(); it != gEntries.end();) { - it->second->inputFiles.erase(this); - if (it->second->inputFiles.empty()) { - it = gEntries.erase(it); - changed = true; - } else { - ++it; - } - } -} +// InputFile::~InputFile() { +// if (!datalog) { +// return; +// } +// std::scoped_lock lock{gEntriesMutex}; +// bool changed = false; +// for (auto it = gEntries.begin(); it != gEntries.end();) { +// it->second->inputFiles.erase(this); +// if (it->second->inputFiles.empty()) { +// it = gEntries.erase(it); +// changed = true; +// } else { +// ++it; +// } +// } +// } using namespace sawmill; @@ -290,7 +271,7 @@ static void ValueToCsv(wpi::raw_ostream& os, const sawmill::DataLogRecord& recor // wpi::print(os, ""); // } -void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, std::vector records, std::map> entryMap) { +void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, std::vector records, std::map> entryMap) { // print header if (style == 0) { @@ -299,14 +280,14 @@ void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, st // scan for exported fields for this file to print header and assign columns os << "Timestamp"; int columnNum = 0; - // TODO: Find a way to tie the entry data to a column number without modifying startrecorddata or iterating through every record - for (std::pair &entry : entryMap) + for (std::pair &entry : entryMap) { os << ',' << '"'; PrintEscapedCsvString(os, entry.second.name); os << '"'; - + entry.second.column = columnNum++; } + os << '\n'; } for (sawmill::DataLogRecord record : records) @@ -326,7 +307,7 @@ void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, st os << '\n'; } else if(style == 1) { wpi::print(os, "{},", record.dataLogRecord.GetTimestamp() / 1000000.0); - for (int i = 0; i < record.entryColumn; ++i) { + for (int i = 0; i < record.entryData.column; ++i) { os << ','; } ValueToCsv(os, record); diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index a2e7e9a52ea..193b7153b92 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -84,7 +83,7 @@ std::vector sawmill::LogLoader::GetAllRecords() { wpi::log::StartRecordData data; if (record.GetStartData(&data)) { // associate an entry id with a StartRecordData - dataMap[data.entry] = data; + dataMap[data.entry] = sawmill::Entry{data}; } } else if (record.IsFinish()) { // remove the association @@ -103,7 +102,7 @@ std::vector sawmill::LogLoader::GetAllRecords() { return records; } -std::map> +std::map> sawmill::LogLoader::GetEntryMap() { return dataMap; } diff --git a/sawmill/src/main/native/include/DataLogExport.h b/sawmill/src/main/native/include/DataLogExport.h index c82c5ef53e2..ca81eb07ac7 100644 --- a/sawmill/src/main/native/include/DataLogExport.h +++ b/sawmill/src/main/native/include/DataLogExport.h @@ -7,9 +7,26 @@ #include namespace sawmill { +struct Entry { + explicit Entry(const wpi::log::StartRecordData& srd) + : name{srd.name}, type{srd.type}, metadata{srd.metadata} {} + + std::string name; + std::string type; + std::string metadata; + //std::set inputFiles; + bool typeConflict = false; + bool metadataConflict = false; + bool selected = true; + + // used only during export + int column = -1; +}; + struct DataLogRecord { - const wpi::log::StartRecordData entryData; + const Entry entryData; wpi::log::DataLogRecord dataLogRecord; - int entryColumn = -1; }; + + } // namespace sawmill diff --git a/sawmill/src/main/native/include/LogLoader.h b/sawmill/src/main/native/include/LogLoader.h index dfa6258b84f..cdedbd1722e 100644 --- a/sawmill/src/main/native/include/LogLoader.h +++ b/sawmill/src/main/native/include/LogLoader.h @@ -6,11 +6,11 @@ #include #include -#include #include #include #include +#include #include "DataLogExport.h" @@ -47,7 +47,7 @@ class LogLoader { std::vector GetAllRecords(); - std::map> GetEntryMap(); + std::map> GetEntryMap(); private: std::string m_filename; @@ -59,7 +59,7 @@ class LogLoader { wpi::log::StartRecordData* entryData; - std::map> dataMap; + std::map> dataMap; std::vector records; }; From 974d4773315d8920cb3724c18486505773e7a3e6 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Wed, 8 Jan 2025 13:13:05 -0600 Subject: [PATCH 28/35] using namespace so dont need it in method definitions --- sawmill/src/main/native/cpp/LogLoader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index 193b7153b92..74c8b37911f 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -75,7 +75,7 @@ void LogLoader::Load(fs::path logPath) { return record_list; }*/ -std::vector sawmill::LogLoader::GetAllRecords() { +std::vector LogLoader::GetAllRecords() { if (records.size() == 0) { // get all records for (wpi::log::DataLogRecord record : m_reader->GetReader()) { @@ -103,6 +103,6 @@ std::vector sawmill::LogLoader::GetAllRecords() { } std::map> -sawmill::LogLoader::GetEntryMap() { +LogLoader::GetEntryMap() { return dataMap; } From e90a06b718dda85771fc1161c846becbc22bf302 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Wed, 8 Jan 2025 14:34:18 -0600 Subject: [PATCH 29/35] emplace instead of [] --- sawmill/src/main/native/cpp/LogLoader.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index 74c8b37911f..85fccce09e7 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "DataLogExport.h" #include #include @@ -83,7 +84,7 @@ std::vector LogLoader::GetAllRecords() { wpi::log::StartRecordData data; if (record.GetStartData(&data)) { // associate an entry id with a StartRecordData - dataMap[data.entry] = sawmill::Entry{data}; + dataMap.emplace(data.entry, sawmill::Entry{data}); } } else if (record.IsFinish()) { // remove the association @@ -94,11 +95,13 @@ std::vector LogLoader::GetAllRecords() { } int entryId = record.GetEntry(); if (dataMap.contains(entryId)) { - records.push_back(sawmill::DataLogRecord{dataMap[entryId], record}); + if (auto it = dataMap.find(entryId); it != dataMap.end()) { + records.emplace_back(it->second, record); + } } } } - + return records; } From 44c1cab2489d07a8ecb3a1244ee690266ec51ea7 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Wed, 8 Jan 2025 14:34:28 -0600 Subject: [PATCH 30/35] remove json temporarily --- sawmill/src/main/native/cpp/main.cpp | 26 +++++++++---------- .../main/native/include/DataLogCSVWriter.h | 4 +-- .../main/native/include/DataLogJSONWriter.h | 2 -- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/sawmill/src/main/native/cpp/main.cpp b/sawmill/src/main/native/cpp/main.cpp index ae98149c049..ad95759747d 100644 --- a/sawmill/src/main/native/cpp/main.cpp +++ b/sawmill/src/main/native/cpp/main.cpp @@ -15,13 +15,13 @@ #include "DataLogJSONWriter.h" #include "LogLoader.h" -void export_json(fs::path log_path, fs::path output_path) { - sawmill::LogLoader loader{}; - loader.Load(log_path.string()); - std::vector records = loader.GetAllRecords(); - sawmill::DataLogJSONWriter writer{}; - writer.ExportJSON(output_path, records); -} +// void export_json(fs::path log_path, fs::path output_path) { +// sawmill::LogLoader loader{}; +// loader.Load(log_path.string()); +// std::vector records = loader.GetAllRecords(); +// sawmill::DataLogJSONWriter writer{}; +// writer.ExportJSON(output_path, records); +// } void export_csv(fs::path logPath, fs::path outputPaths) { sawmill::LogLoader loader{}; @@ -32,11 +32,11 @@ void export_csv(fs::path logPath, fs::path outputPaths) { void write_json(std::vector records, fs::path output_path) {} -void write_csv(std::vector records, - fs::path output_path) { - sawmill::DataLogCSVWriter writer{}; - writer.ExportCsv(output_path.string(), 0); -} +// void write_csv(std::vector records, +// fs::path output_path) { +// sawmill::DataLogCSVWriter writer{}; +// writer.ExportCsv(output_path.string(), 0); +// } void extract_entry(std::string_view entry_name, fs::path log_path, fs::path output_path, bool use_json) { @@ -125,7 +125,7 @@ int main(int argc, char* argv[]) { return 1; } - export_json(logPath, outputPath); + //export_json(logPath, outputPath); } else if (export_csv_command) { // validate paths fs::path logPath{export_csv_command.get("--log-file")}; diff --git a/sawmill/src/main/native/include/DataLogCSVWriter.h b/sawmill/src/main/native/include/DataLogCSVWriter.h index 044f47450ef..b9d8f7c8d6c 100644 --- a/sawmill/src/main/native/include/DataLogCSVWriter.h +++ b/sawmill/src/main/native/include/DataLogCSVWriter.h @@ -4,13 +4,11 @@ #pragma once -#include +#include namespace sawmill { class DataLogCSVWriter { public: - explicit DataLogCSVWriter(); - ~DataLogCSVWriter(); void ExportCsv(std::string_view outputFolder, int style); }; diff --git a/sawmill/src/main/native/include/DataLogJSONWriter.h b/sawmill/src/main/native/include/DataLogJSONWriter.h index 8a42037528b..d9738f6a3c1 100644 --- a/sawmill/src/main/native/include/DataLogJSONWriter.h +++ b/sawmill/src/main/native/include/DataLogJSONWriter.h @@ -21,7 +21,5 @@ class DataLogJSONWriter { public: void ExportJSON(fs::path exportPath, std::vector records); - - private: }; } // namespace sawmill From 6cf8c9613a9f27f182f285eae1c28d25b7df6572 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Wed, 8 Jan 2025 14:52:28 -0600 Subject: [PATCH 31/35] start cli args refactor --- sawmill/CMakeLists.txt | 4 +- .../src/main/native/cpp/DataLogCSVWriter.cpp | 62 +++--- sawmill/src/main/native/cpp/LogLoader.cpp | 8 +- sawmill/src/main/native/cpp/main.cpp | 177 +++++++++--------- .../main/native/include/DataLogCSVWriter.h | 1 - .../src/main/native/include/DataLogExport.h | 5 +- sawmill/src/main/native/include/LogLoader.h | 2 + 7 files changed, 132 insertions(+), 127 deletions(-) diff --git a/sawmill/CMakeLists.txt b/sawmill/CMakeLists.txt index 19f73a63d8c..18f44f07a2b 100644 --- a/sawmill/CMakeLists.txt +++ b/sawmill/CMakeLists.txt @@ -7,8 +7,8 @@ configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) generate_resources(src/main/native/resources generated/main/cpp DLT dlt sawmill_resources_src) # Generate compile_commands.json by default -if (NOT CMAKE_EXPORT_COMPILE_COMMANDS) - set(CMAKE_EXPORT_COMPILE_COMMANDS "YES" CACHE STRING "" FORCE) +if(NOT CMAKE_EXPORT_COMPILE_COMMANDS) + set(CMAKE_EXPORT_COMPILE_COMMANDS "YES" CACHE STRING "" FORCE) endif() file(GLOB sawmill_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) diff --git a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp index 8a5c21f3141..ea174908482 100644 --- a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp +++ b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp @@ -2,7 +2,6 @@ // Open Source Software; you can modify and/or share it under the terms of // the WPILib BSD license file in the root directory of this project. - #include #include #include @@ -13,6 +12,7 @@ #include #include +#include #include #include #include @@ -27,7 +27,6 @@ #include #include #include -#include // namespace { // struct InputFile { @@ -48,17 +47,19 @@ // }; // } // namespace -//static std::map, std::less<>> -// gInputFiles; +// static std::map, std::less<>> +// gInputFiles; static wpi::mutex gEntriesMutex; -static std::map, std::less<>> gEntries; +static std::map, std::less<>> + gEntries; std::atomic_int gExportCount{0}; // InputFile::InputFile(std::unique_ptr datalog_) // : filename{datalog_->GetBufferIdentifier()}, // stem{fs::path{filename}.stem().string()}, // datalog{std::move(datalog_)} { -// datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) { +// datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) +// { // std::scoped_lock lock{gEntriesMutex}; // auto it = gEntries.find(srd.name); // if (it == gEntries.end()) { @@ -112,9 +113,11 @@ static void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str) { } } -static void ValueToCsv(wpi::raw_ostream& os, const sawmill::DataLogRecord& record) { +static void ValueToCsv(wpi::raw_ostream& os, + const sawmill::DataLogRecord& record) { // systemTime needs special handling - if (record.entryData.name == "systemTime" && record.entryData.type == "int64") { + if (record.entryData.name == "systemTime" && + record.entryData.type == "int64") { int64_t val; if (record.dataLogRecord.GetInteger(&val)) { std::time_t timeval = val / 1000000; @@ -128,14 +131,16 @@ static void ValueToCsv(wpi::raw_ostream& os, const sawmill::DataLogRecord& recor wpi::print(os, "{}", val); return; } - } else if (record.entryData.type == "int64" || record.entryData.type == "int") { + } else if (record.entryData.type == "int64" || + record.entryData.type == "int") { // support "int" for compatibility with old NT4 datalogs int64_t val; if (record.dataLogRecord.GetInteger(&val)) { wpi::print(os, "{}", val); return; } - } else if (record.entryData.type == "string" || record.entryData.type == "json") { + } else if (record.entryData.type == "string" || + record.entryData.type == "json") { std::string_view val; record.dataLogRecord.GetString(&val); os << '"'; @@ -271,17 +276,17 @@ static void ValueToCsv(wpi::raw_ostream& os, const sawmill::DataLogRecord& recor // wpi::print(os, ""); // } -void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, std::vector records, std::map> entryMap) { +void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, + std::vector records, + std::map> entryMap) { // print header - if (style == 0) - { + if (style == 0) { os << "Timestamp,Name,Value\n"; } else if (style == 1) { // scan for exported fields for this file to print header and assign columns os << "Timestamp"; int columnNum = 0; - for (std::pair &entry : entryMap) - { + for (std::pair& entry : entryMap) { os << ',' << '"'; PrintEscapedCsvString(os, entry.second.name); os << '"'; @@ -289,23 +294,20 @@ void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, st } os << '\n'; } - - for (sawmill::DataLogRecord record : records) - { + + for (sawmill::DataLogRecord record : records) { // if this is a control record and we dont want to print those, skip - if (record.dataLogRecord.IsControl() && !printControlRecords) - { + if (record.dataLogRecord.IsControl() && !printControlRecords) { continue; } - if (style == 0) - { + if (style == 0) { wpi::print(os, "{},\"", record.dataLogRecord.GetTimestamp() / 1000000.0); PrintEscapedCsvString(os, record.entryData.name); os << '"' << ','; ValueToCsv(os, record); os << '\n'; - } else if(style == 1) { + } else if (style == 1) { wpi::print(os, "{},", record.dataLogRecord.GetTimestamp() / 1000000.0); for (int i = 0; i < record.entryData.column; ++i) { os << ','; @@ -313,11 +315,7 @@ void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, st ValueToCsv(os, record); os << '\n'; } - - } - - } // static void ExportCsvFile(InputFile& f, wpi::raw_ostream& os, int style) { @@ -325,12 +323,12 @@ void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, st // if (style == 0) { // os << "Timestamp,Name,Value\n"; // } else if (style == 1) { -// // scan for exported fields for this file to print header and assign columns -// os << "Timestamp"; -// int columnNum = 0; -// for (auto&& entry : gEntries) { +// // scan for exported fields for this file to print header and assign +// columns os << "Timestamp"; int columnNum = 0; for (auto&& entry : +// gEntries) { // if (entry.second->selected && -// entry.second->inputFiles.find(&f) != entry.second->inputFiles.end()) { +// entry.second->inputFiles.find(&f) != +// entry.second->inputFiles.end()) { // os << ',' << '"'; // PrintEscapedCsvString(os, entry.first); // os << '"'; diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index 85fccce09e7..b868fe4a83d 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -11,7 +11,6 @@ #include #include #include -#include "DataLogExport.h" #include #include @@ -20,6 +19,8 @@ #include #include +#include "DataLogExport.h" + using namespace sawmill; LogLoader::LogLoader() {} @@ -101,11 +102,10 @@ std::vector LogLoader::GetAllRecords() { } } } - + return records; } -std::map> -LogLoader::GetEntryMap() { +std::map> LogLoader::GetEntryMap() { return dataMap; } diff --git a/sawmill/src/main/native/cpp/main.cpp b/sawmill/src/main/native/cpp/main.cpp index ad95759747d..9ad472d0a67 100644 --- a/sawmill/src/main/native/cpp/main.cpp +++ b/sawmill/src/main/native/cpp/main.cpp @@ -50,55 +50,60 @@ int main(int argc, char* argv[]) { wpi::ArgumentParser cli{"sawmill"}; std::string jsonMode{}; - wpi::ArgumentParser export_json_command{"json"}; - export_json_command.add_description( - "Export a JSON representation of a DataLog file"); - export_json_command.add_argument("log_file") - .help("Path to the DataLog file to export"); - export_json_command.add_argument("export_path") + wpi::ArgumentParser export_command{"export"}; + export_command.add_description( + "Export a DataLog file in a text-based format"); + export_command.add_argument("Type") + .help( + "The file type of the exported log, must be 'json' or 'csv'."); + export_command.add_argument("DataLog") + .help("A valid path to a .wpilog file to convert"); + export_command.add_argument("Output") .help( - "Path of the JSON file to create with the exported data. If it " - "exists, it will be overwritten."); - export_json_command.add_argument("-m", "--mode") - .help( - "How to format the exported JSON. 'direct' means that each record " - "will be directly converted to a JSON dictionary and exported, " - "including control records. 'direct_data' is the same as direct, but " - "control records will be ommitted. The default is 'direct_data'.") - .default_value("direct_data") - .store_into(jsonMode); - - wpi::ArgumentParser export_csv_command{"csv"}; - export_csv_command.add_description( - "Export a CSV representation of a DataLog file"); - export_csv_command.add_argument("log-file") - .help("The DataLog file to export"); - export_csv_command.add_argument("output-file") - .help( - "The CSV file to create with the exported data. If it " - "exists, it will be overwritten."); - - wpi::ArgumentParser extract_entry_command{"extract"}; - extract_entry_command.add_description( - "Extract the history of one entry from a DataLog file and store it in a " - "JSON or CSV file"); - extract_entry_command.add_argument("-e", "--entry") - .required() - .help("The entry to extract from the Log"); - extract_entry_command.add_argument("-l", "--log-file") - .required() - .help("The DataLog file to extract from"); - extract_entry_command.add_argument("--time-start") - .help("The timestamp to start extracting at"); - extract_entry_command.add_argument("-o", "--output") - .required() - .help( - "The file to export the field and data to. It will be created or " - "overwritten if it already exists"); - - cli.add_subparser(export_json_command); - cli.add_subparser(export_csv_command); - cli.add_subparser(extract_entry_command); + "A valid path to the location of the exported log file." + "All directories must exist, but if the file does not, a new file " + "will be created with the name of the original DataLog."); + // wpi::ArgumentParser export_json_command{"json"}; + // export_json_command.add_description( + // "Export a JSON representation of a DataLog file"); + // export_json_command.add_argument("log_file") + // .help("Path to the DataLog file to export"); + // export_json_command.add_argument("export_path") + // .help( + // "Path of the JSON file to create with the exported data. If it " + // "exists, it will be overwritten."); + // export_json_command.add_argument("-m", "--mode") + // .help( + // "How to format the exported JSON. 'direct' means that each record " + // "will be directly converted to a JSON dictionary and exported, " + // "including control records. 'direct_data' is the same as direct, but " + // "control records will be ommitted. The default is 'direct_data'.") + // .default_value("direct_data") + // .store_into(jsonMode); + + + + // wpi::ArgumentParser extract_entry_command{"extract"}; + // extract_entry_command.add_description( + // "Extract the history of one entry from a DataLog file and store it in a " + // "JSON or CSV file"); + // extract_entry_command.add_argument("-e", "--entry") + // .required() + // .help("The entry to extract from the Log"); + // extract_entry_command.add_argument("-l", "--log-file") + // .required() + // .help("The DataLog file to extract from"); + // extract_entry_command.add_argument("--time-start") + // .help("The timestamp to start extracting at"); + // extract_entry_command.add_argument("-o", "--output") + // .required() + // .help( + // "The file to export the field and data to. It will be created or " + // "overwritten if it already exists"); + + //cli.add_subparser(export_json_command); + cli.add_subparser(export_command); + //cli.add_subparser(extract_entry_command); try { cli.parse_args(argc, argv); @@ -109,32 +114,32 @@ int main(int argc, char* argv[]) { } // see which one was called - if (export_json_command) { - // validate paths - fs::path logPath{export_json_command.get("--log-file")}; - if (logPath.extension() != ".wpilog") { - std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; - return 1; - } - - fs::path outputPath{export_json_command.get("--output")}; - if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { - std::cerr - << "Only JSON and CSV are currently supported as output formats." - << std::endl; - return 1; - } + // if (export_json_command) { + // // validate paths + // fs::path logPath{export_json_command.get("--log-file")}; + // if (logPath.extension() != ".wpilog") { + // std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; + // return 1; + // } + + // fs::path outputPath{export_json_command.get("--output")}; + // if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { + // std::cerr + // << "Only JSON and CSV are currently supported as output formats." + // << std::endl; + // return 1; + // } //export_json(logPath, outputPath); - } else if (export_csv_command) { + /*} else*/ if (export_command) { // validate paths - fs::path logPath{export_csv_command.get("--log-file")}; + fs::path logPath{export_command.get("--log-file")}; if (logPath.extension() != ".wpilog") { std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; return 1; } - fs::path outputPath{export_csv_command.get("--output")}; + fs::path outputPath{export_command.get("--output")}; if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { std::cerr << "Only JSON and CSV are currently supported as output formats." @@ -143,25 +148,25 @@ int main(int argc, char* argv[]) { } // export_csv(logPath, outputPath); - } else if (extract_entry_command) { + } //else if (extract_entry_command) { // validate paths - fs::path logPath{extract_entry_command.get("--log-file")}; - if (logPath.extension() != ".wpilog") { - std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; - return 1; - } - - fs::path outputPath{extract_entry_command.get("--output")}; - if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { - std::cerr - << "Only JSON and CSV are currently supported as output formats." - << std::endl; - return 1; - } - - bool json{outputPath.extension() == ".json" ? true : false}; - - extract_entry(extract_entry_command.get("--entry"), logPath, outputPath, - json); - } + // fs::path logPath{extract_entry_command.get("--log-file")}; + // if (logPath.extension() != ".wpilog") { + // std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; + // return 1; + // } + + // fs::path outputPath{extract_entry_command.get("--output")}; + // if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { + // std::cerr + // << "Only JSON and CSV are currently supported as output formats." + // << std::endl; + // return 1; + // } + + // bool json{outputPath.extension() == ".json" ? true : false}; + + // extract_entry(extract_entry_command.get("--entry"), logPath, outputPath, + // json); + // } } diff --git a/sawmill/src/main/native/include/DataLogCSVWriter.h b/sawmill/src/main/native/include/DataLogCSVWriter.h index b9d8f7c8d6c..6dd4258de59 100644 --- a/sawmill/src/main/native/include/DataLogCSVWriter.h +++ b/sawmill/src/main/native/include/DataLogCSVWriter.h @@ -9,7 +9,6 @@ namespace sawmill { class DataLogCSVWriter { public: - void ExportCsv(std::string_view outputFolder, int style); }; } // namespace sawmill diff --git a/sawmill/src/main/native/include/DataLogExport.h b/sawmill/src/main/native/include/DataLogExport.h index ca81eb07ac7..beea19a83e8 100644 --- a/sawmill/src/main/native/include/DataLogExport.h +++ b/sawmill/src/main/native/include/DataLogExport.h @@ -4,6 +4,8 @@ #pragma once +#include + #include namespace sawmill { @@ -14,7 +16,7 @@ struct Entry { std::string name; std::string type; std::string metadata; - //std::set inputFiles; + // std::set inputFiles; bool typeConflict = false; bool metadataConflict = false; bool selected = true; @@ -28,5 +30,4 @@ struct DataLogRecord { wpi::log::DataLogRecord dataLogRecord; }; - } // namespace sawmill diff --git a/sawmill/src/main/native/include/LogLoader.h b/sawmill/src/main/native/include/LogLoader.h index cdedbd1722e..a0abee6b202 100644 --- a/sawmill/src/main/native/include/LogLoader.h +++ b/sawmill/src/main/native/include/LogLoader.h @@ -4,6 +4,8 @@ #pragma once +#include +#include #include #include #include From 9602c9bc6b19e0ccee6c98f666400ee354bc5c74 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Thu, 20 Feb 2025 14:10:24 -0500 Subject: [PATCH 32/35] add datalog dependency --- .../include/wpi/datalog/DataLogReader.h | 1 - .../include/wpi/datalog/DataLogReaderThread.h | 2 +- sawmill/CMakeLists.txt | 2 +- .../src/main/native/cpp/DataLogCSVWriter.cpp | 2 +- .../src/main/native/cpp/DataLogJSONWriter.cpp | 2 +- sawmill/src/main/native/cpp/LogLoader.cpp | 6 +- .../src/main/native/include/DataLogExport.h | 2 +- .../main/native/include/DataLogJSONWriter.h | 2 +- sawmill/src/main/native/include/LogLoader.h | 6 +- .../main/native/cpp/DataLogReaderThread.cpp | 136 ------------------ .../native/include/wpi/DataLogReaderThread.h | 115 --------------- 11 files changed, 12 insertions(+), 264 deletions(-) delete mode 100644 wpiutil/src/main/native/cpp/DataLogReaderThread.cpp delete mode 100644 wpiutil/src/main/native/include/wpi/DataLogReaderThread.h diff --git a/datalog/src/main/native/include/wpi/datalog/DataLogReader.h b/datalog/src/main/native/include/wpi/datalog/DataLogReader.h index 2a22725c7c4..8932ed85a26 100644 --- a/datalog/src/main/native/include/wpi/datalog/DataLogReader.h +++ b/datalog/src/main/native/include/wpi/datalog/DataLogReader.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include diff --git a/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h b/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h index b9a269f44f2..a110a251b26 100644 --- a/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h +++ b/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h @@ -4,7 +4,7 @@ #pragma once -#include +#include #include #include #include diff --git a/sawmill/CMakeLists.txt b/sawmill/CMakeLists.txt index 18f44f07a2b..a8582492752 100644 --- a/sawmill/CMakeLists.txt +++ b/sawmill/CMakeLists.txt @@ -24,7 +24,7 @@ add_executable(sawmill ${sawmill_src} ${sawmill_resources_src} ${sawmill_rc} ${A target_include_directories(sawmill PUBLIC src/main/native/include) -target_link_libraries(sawmill PRIVATE wpiutil) +target_link_libraries(sawmill PRIVATE wpiutil datalog) if(WIN32) set_target_properties(sawmill PROPERTIES WIN32_EXECUTABLE YES) diff --git a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp index ea174908482..57c4df2108a 100644 --- a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp +++ b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp b/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp index a6eac1b0a02..0a86f4e5027 100644 --- a/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp +++ b/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp @@ -6,7 +6,7 @@ #include -#include +#include #include #include diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp index b868fe4a83d..15f9799de4e 100644 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ b/sawmill/src/main/native/cpp/LogLoader.cpp @@ -13,8 +13,8 @@ #include #include -#include -#include +#include +#include #include #include #include @@ -42,7 +42,7 @@ void LogLoader::Load(fs::path logPath) { return; } unload(); // release the actual file, we have the data in the reader now - m_reader = std::make_unique(std::move(reader)); + m_reader = std::make_unique(std::move(reader)); // Handle Errors fmt::println("{}", m_error); diff --git a/sawmill/src/main/native/include/DataLogExport.h b/sawmill/src/main/native/include/DataLogExport.h index beea19a83e8..dae4994652c 100644 --- a/sawmill/src/main/native/include/DataLogExport.h +++ b/sawmill/src/main/native/include/DataLogExport.h @@ -6,7 +6,7 @@ #include -#include +#include namespace sawmill { struct Entry { diff --git a/sawmill/src/main/native/include/DataLogJSONWriter.h b/sawmill/src/main/native/include/DataLogJSONWriter.h index d9738f6a3c1..d376101bf10 100644 --- a/sawmill/src/main/native/include/DataLogJSONWriter.h +++ b/sawmill/src/main/native/include/DataLogJSONWriter.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include "DataLogExport.h" diff --git a/sawmill/src/main/native/include/LogLoader.h b/sawmill/src/main/native/include/LogLoader.h index a0abee6b202..b5602459126 100644 --- a/sawmill/src/main/native/include/LogLoader.h +++ b/sawmill/src/main/native/include/LogLoader.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include @@ -20,7 +20,7 @@ namespace glass { class Storage; } // namespace glass -namespace wpi { +namespace wpi::log { class DataLogReaderEntry; class DataLogReaderThread; class Logger; @@ -53,7 +53,7 @@ class LogLoader { private: std::string m_filename; - std::unique_ptr m_reader; + std::unique_ptr m_reader; std::string m_error; diff --git a/wpiutil/src/main/native/cpp/DataLogReaderThread.cpp b/wpiutil/src/main/native/cpp/DataLogReaderThread.cpp deleted file mode 100644 index bd5452a9b32..00000000000 --- a/wpiutil/src/main/native/cpp/DataLogReaderThread.cpp +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) FIRST and other WPILib contributors. -// Open Source Software; you can modify and/or share it under the terms of -// the WPILib BSD license file in the root directory of this project. - -<<<<<<<< HEAD:datalog/src/main/native/cpp/DataLogReaderThread.cpp -#include -#include -======== -#include "wpi/DataLogReaderThread.h" ->>>>>>>> ae195463f (apply datalog move to wpiutil):wpiutil/src/main/native/cpp/DataLogReaderThread.cpp - -#include -#include - -<<<<<<<< HEAD:datalog/src/main/native/cpp/DataLogReaderThread.cpp -#include "wpi/datalog/DataLogReaderThread.h" - -using namespace wpi::log; -======== -#include -#include - -using namespace wpi; ->>>>>>>> ae195463f (apply datalog move to wpiutil):wpiutil/src/main/native/cpp/DataLogReaderThread.cpp - -DataLogReaderThread::~DataLogReaderThread() { - if (m_thread.joinable()) { - m_active = false; - m_thread.join(); - } -} - -void DataLogReaderThread::ReadMain() { - wpi::SmallDenseMap< - int, std::pair>, 8> - schemaEntries; - - for (auto recordIt = m_reader.begin(), recordEnd = m_reader.end(); - recordIt != recordEnd; ++recordIt) { - auto& record = *recordIt; - if (!m_active) { - break; - } - ++m_numRecords; - if (record.IsStart()) { - DataLogReaderEntry data; - if (record.GetStartData(&data)) { - std::scoped_lock lock{m_mutex}; - auto& entryPtr = m_entriesById[data.entry]; - if (entryPtr) { - wpi::print("...DUPLICATE entry ID, overriding\n"); - } - auto [it, isNew] = m_entriesByName.emplace(data.name, data); - if (isNew) { - it->second.ranges.emplace_back(recordIt, recordEnd); - } - entryPtr = &it->second; - if (data.type == "structschema" || - data.type == "proto:FileDescriptorProto") { - schemaEntries.try_emplace(data.entry, entryPtr, - std::span{}); - } - sigEntryAdded(data); - } else { - wpi::print("Start(INVALID)\n"); - } - } else if (record.IsFinish()) { - int entry; - if (record.GetFinishEntry(&entry)) { - std::scoped_lock lock{m_mutex}; - auto it = m_entriesById.find(entry); - if (it == m_entriesById.end()) { - wpi::print("...ID not found\n"); - } else { - it->second->ranges.back().m_end = recordIt; - m_entriesById.erase(it); - } - } else { - wpi::print("Finish(INVALID)\n"); - } - } else if (record.IsSetMetadata()) { - wpi::log::MetadataRecordData data; - if (record.GetSetMetadataData(&data)) { - std::scoped_lock lock{m_mutex}; - auto it = m_entriesById.find(data.entry); - if (it == m_entriesById.end()) { - wpi::print("...ID not found\n"); - } else { - it->second->metadata = data.metadata; - } - } else { - wpi::print("SetMetadata(INVALID)\n"); - } - } else if (record.IsControl()) { - wpi::print("Unrecognized control record\n"); - } else { - auto it = schemaEntries.find(record.GetEntry()); - if (it != schemaEntries.end()) { - it->second.second = record.GetRaw(); - } - } - } - - // build schema databases - for (auto&& schemaPair : schemaEntries) { - auto name = schemaPair.second.first->name; - auto data = schemaPair.second.second; - if (data.empty()) { - continue; - } - if (auto strippedName = wpi::remove_prefix(name, "NT:")) { - name = *strippedName; - } - if (auto typeStr = wpi::remove_prefix(name, "/.schema/struct:")) { - std::string_view schema{reinterpret_cast(data.data()), - data.size()}; - std::string err; - auto desc = m_structDb.Add(*typeStr, schema, &err); - if (!desc) { - wpi::print("could not decode struct '{}' schema '{}': {}\n", name, - schema, err); - } - } else if (auto filename = wpi::remove_prefix(name, "/.schema/proto:")) { -#ifndef NO_PROTOBUF - // protobuf descriptor handling - if (!m_protoDb.Add(*filename, data)) { - wpi::print("could not decode protobuf '{}' filename '{}'\n", name, - *filename); - } -#endif - } - } - - sigDone(); - m_done = true; -} diff --git a/wpiutil/src/main/native/include/wpi/DataLogReaderThread.h b/wpiutil/src/main/native/include/wpi/DataLogReaderThread.h deleted file mode 100644 index b9a269f44f2..00000000000 --- a/wpiutil/src/main/native/include/wpi/DataLogReaderThread.h +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) FIRST and other WPILib contributors. -// Open Source Software; you can modify and/or share it under the terms of -// the WPILib BSD license file in the root directory of this project. - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "wpi/datalog/DataLogReader.h" - -#ifndef NO_PROTOBUF -#include -#endif - -namespace wpi::log { - -class DataLogReaderRange { - public: - DataLogReaderRange(wpi::log::DataLogReader::iterator begin, - wpi::log::DataLogReader::iterator end) - : m_begin{begin}, m_end{end} {} - - wpi::log::DataLogReader::iterator begin() const { return m_begin; } - wpi::log::DataLogReader::iterator end() const { return m_end; } - - wpi::log::DataLogReader::iterator m_begin; - wpi::log::DataLogReader::iterator m_end; -}; - -class DataLogReaderEntry : public wpi::log::StartRecordData { - public: - std::vector ranges; // ranges where this entry is valid -}; - -class DataLogReaderThread { - public: - explicit DataLogReaderThread(wpi::log::DataLogReader reader) - : m_reader{std::move(reader)}, m_thread{[this] { ReadMain(); }} {} - ~DataLogReaderThread(); - - bool IsDone() const { return m_done; } - std::string_view GetBufferIdentifier() const { - return m_reader.GetBufferIdentifier(); - } - unsigned int GetNumRecords() const { return m_numRecords; } - unsigned int GetNumEntries() const { - std::scoped_lock lock{m_mutex}; - return m_entriesByName.size(); - } - - // Passes Entry& to func - template - void ForEachEntryName(T&& func) { - std::scoped_lock lock{m_mutex}; - for (auto&& kv : m_entriesByName) { - func(kv.second); - } - } - - const DataLogReaderEntry* GetEntry(std::string_view name) const { - std::scoped_lock lock{m_mutex}; - auto it = m_entriesByName.find(name); - if (it == m_entriesByName.end()) { - return nullptr; - } - return &it->second; - } - - wpi::StructDescriptorDatabase& GetStructDatabase() { return m_structDb; } -#ifndef NO_PROTOBUF - wpi::ProtobufMessageDatabase& GetProtobufDatabase() { return m_protoDb; } -#endif - - const wpi::log::DataLogReader& GetReader() const { return m_reader; } - - // note: these are called on separate thread - wpi::sig::Signal_mt sigEntryAdded; - wpi::sig::Signal_mt<> sigDone; - - private: - void ReadMain(); - - wpi::log::DataLogReader m_reader; - mutable wpi::mutex m_mutex; - std::atomic_bool m_active{true}; - std::atomic_bool m_done{false}; - std::atomic m_numRecords{0}; - std::map> m_entriesByName; - wpi::DenseMap m_entriesById; - wpi::StructDescriptorDatabase m_structDb; -#ifndef NO_PROTOBUF - wpi::ProtobufMessageDatabase m_protoDb; -#endif - std::thread m_thread; -}; - -} // namespace wpi::log From c620d323971c583a1168d40b749bd11cc060fe20 Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Thu, 20 Feb 2025 14:14:19 -0500 Subject: [PATCH 33/35] remove resources stuff for now --- sawmill/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sawmill/CMakeLists.txt b/sawmill/CMakeLists.txt index a8582492752..32854998344 100644 --- a/sawmill/CMakeLists.txt +++ b/sawmill/CMakeLists.txt @@ -4,7 +4,7 @@ include(CompileWarnings) include(GenResources) configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) -generate_resources(src/main/native/resources generated/main/cpp DLT dlt sawmill_resources_src) +# generate_resources(src/main/native/resources generated/main/cpp DLT dlt sawmill_resources_src) # Generate compile_commands.json by default if(NOT CMAKE_EXPORT_COMPILE_COMMANDS) @@ -14,13 +14,13 @@ endif() file(GLOB sawmill_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) if(WIN32) - set(sawmill_rc src/main/native/win/sawmill.rc) + # set(sawmill_rc src/main/native/win/sawmill.rc) elseif(APPLE) - set(MACOSX_BUNDLE_ICON_FILE sawmill.icns) - set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") + # set(MACOSX_BUNDLE_ICON_FILE sawmill.icns) + # set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") endif() -add_executable(sawmill ${sawmill_src} ${sawmill_resources_src} ${sawmill_rc} ${APP_ICON_MACOSX}) +add_executable(sawmill ${sawmill_src}) #${sawmill_resources_src} ${sawmill_rc} ${APP_ICON_MACOSX}) target_include_directories(sawmill PUBLIC src/main/native/include) From 2c01df1861e5cffe663a35c796b8d4fcd64de52a Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Thu, 20 Feb 2025 14:34:26 -0500 Subject: [PATCH 34/35] fix datalog and sysid imports --- datalogtool/src/main/native/cpp/Exporter.cpp | 1 - sysid/src/main/native/cpp/view/LogLoader.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/datalogtool/src/main/native/cpp/Exporter.cpp b/datalogtool/src/main/native/cpp/Exporter.cpp index 7432c444b8d..2cf399b71c2 100644 --- a/datalogtool/src/main/native/cpp/Exporter.cpp +++ b/datalogtool/src/main/native/cpp/Exporter.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include diff --git a/sysid/src/main/native/cpp/view/LogLoader.cpp b/sysid/src/main/native/cpp/view/LogLoader.cpp index e5c56a9839f..b21bea46158 100644 --- a/sysid/src/main/native/cpp/view/LogLoader.cpp +++ b/sysid/src/main/native/cpp/view/LogLoader.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include From 0ae60719ced42896dc83651944cea6a94a782e7a Mon Sep 17 00:00:00 2001 From: DeltaDizzy Date: Thu, 20 Feb 2025 20:57:56 -0500 Subject: [PATCH 35/35] start on editor --- datalog/src/main/native/cpp/DataLogEditor.cpp | 27 ++ .../native/cpp/export/DataLogCSVExporter.cpp | 7 + .../native/cpp/export/DataLogExportUtils.cpp | 83 ++++ .../wpi/datalog/export/DataLogCSVExporter.h | 19 + .../wpi/datalog/export/DataLogEditor.h | 31 ++ .../wpi/datalog/export/DataLogExportUtils.h | 54 +++ .../src/main/native/cpp/DataLogCSVWriter.cpp | 403 ------------------ .../src/main/native/cpp/DataLogJSONWriter.cpp | 20 - sawmill/src/main/native/cpp/LogLoader.cpp | 111 ----- 9 files changed, 221 insertions(+), 534 deletions(-) create mode 100644 datalog/src/main/native/cpp/DataLogEditor.cpp create mode 100644 datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp create mode 100644 datalog/src/main/native/cpp/export/DataLogExportUtils.cpp create mode 100644 datalog/src/main/native/include/wpi/datalog/export/DataLogCSVExporter.h create mode 100644 datalog/src/main/native/include/wpi/datalog/export/DataLogEditor.h create mode 100644 datalog/src/main/native/include/wpi/datalog/export/DataLogExportUtils.h delete mode 100644 sawmill/src/main/native/cpp/DataLogCSVWriter.cpp delete mode 100644 sawmill/src/main/native/cpp/DataLogJSONWriter.cpp delete mode 100644 sawmill/src/main/native/cpp/LogLoader.cpp diff --git a/datalog/src/main/native/cpp/DataLogEditor.cpp b/datalog/src/main/native/cpp/DataLogEditor.cpp new file mode 100644 index 00000000000..cf305d25287 --- /dev/null +++ b/datalog/src/main/native/cpp/DataLogEditor.cpp @@ -0,0 +1,27 @@ +#include "wpi/datalog/export/DataLogEditor.h" +#include +#include +#include "wpi/datalog/DataLogReaderThread.h" +#include "wpi/datalog/export/DataLogExportUtils.h" + +namespace wpi::log { + DataLogEditor::DataLogEditor(const DataLogReaderThread& reader) { + for (auto record : reader.GetReader()) { + records.push_back(record); + } + } + + DataLogEditor DataLogEditor::ExtractEntries(const std::vector& entries) { + for (const auto& entry : entries) { + allowedEntries.emplace(entry.id); + } + return *this; + } + + DataLogEditor DataLogEditor::TrimToTime(uint64_t start, uint64_t end) { + endTime = end; + startTime = start; + return *this; + } +} + diff --git a/datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp b/datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp new file mode 100644 index 00000000000..a23909a2750 --- /dev/null +++ b/datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp @@ -0,0 +1,7 @@ +#include "wpi/datalog/export/DataLogCSVExporter.h" +#include +#include "wpi/datalog/export/DataLogExportUtils.h" + +namespace wpi::log::fileexport { + +} \ No newline at end of file diff --git a/datalog/src/main/native/cpp/export/DataLogExportUtils.cpp b/datalog/src/main/native/cpp/export/DataLogExportUtils.cpp new file mode 100644 index 00000000000..e925c8f3106 --- /dev/null +++ b/datalog/src/main/native/cpp/export/DataLogExportUtils.cpp @@ -0,0 +1,83 @@ +#include "wpi/datalog/export/DataLogExportUtils.h" +#include +#include +#include +#include +#include +#include +#include "fmt/format.h" +#include "wpi/MemoryBuffer.h" +#include "wpi/datalog/DataLogReader.h" +#include "wpi/datalog/DataLogReaderThread.h" +#include "wpi/mutex.h" +#include + +static wpi::mutex entriesMutex; +static std::map, + std::less<>> + entries; + +namespace wpi::log::fileexport { +InputFile::InputFile(std::unique_ptr datalog_) + : filename{datalog_->GetBufferIdentifier()}, + stem{fs::path{filename}.stem().string()}, + datalog{std::move(datalog_)} { + datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) { + // add this entry to the map + std::scoped_lock lock{entriesMutex}; + auto it = entries.find(srd.name); + if (it == entries.end()) { + // this entry isnt in the map, so lets add it + it = entries.emplace(srd.name, std::make_unique(srd)).first; + // if this is a gui make it rebuild the entry tree somehow lmfao + // maybe accept an optional callback??? + // TODO: RebuildEntryTree + } else { + // this entry IS already known, so lets make sure the start records match + if (it->second->type != srd.type) { + it->second->typeConflict = true; + } + if (it->second->metadata != srd.metadata) { + it->second->metadataConflict = true; + } + it->second->inputFiles.emplace(this); + } + }); +} + +InputFile::~InputFile() { + if (!datalog) { + return; + } + std::scoped_lock lock{entriesMutex}; + bool changed = false; + for (auto it = entries.begin(); it != entries.end();) { + it->second->inputFiles.erase(this); + if (it->second->inputFiles.empty()) { + it = entries.erase(it); + changed = true; + } else { + ++it; + } + } + if (changed) { + // TODO: rebuildentrytree + } +} + +std::unique_ptr LoadDataLog(std::string_view filename) { + auto fileBuffer = wpi::MemoryBuffer::GetFile(filename); + if (!fileBuffer) { + return std::make_unique( + filename, fmt::format("Could not open file: {}", fileBuffer.error())); + } + + wpi::log::DataLogReader reader{std::move(*fileBuffer)}; + if (!reader.IsValid()) { + return std::make_unique(filename, "Not a valid datalog file"); + } + + return std::make_unique( + std::make_unique(std::move(reader))); +} +} // namespace wpi::log::fileexport diff --git a/datalog/src/main/native/include/wpi/datalog/export/DataLogCSVExporter.h b/datalog/src/main/native/include/wpi/datalog/export/DataLogCSVExporter.h new file mode 100644 index 00000000000..97c4f8a00b4 --- /dev/null +++ b/datalog/src/main/native/include/wpi/datalog/export/DataLogCSVExporter.h @@ -0,0 +1,19 @@ +#pragma once +#include +#include +#include "wpi/datalog/DataLogReader.h" +#include "wpi/datalog/export/DataLogExportUtils.h" +#include "wpi/raw_ostream.h" + +namespace wpi::log::fileexport { +class DataLogCSVExporter { + public: + static void Export(std::string inputFilePath, std::string exportFilePath); + private: + void WriteValue(wpi::raw_ostream& os, const Entry& entry, + const wpi::log::DataLogRecord& record); + void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str); + fs::path exportFile; + fs::path inputFile; +}; +} // namespace wpi::log::fileexport \ No newline at end of file diff --git a/datalog/src/main/native/include/wpi/datalog/export/DataLogEditor.h b/datalog/src/main/native/include/wpi/datalog/export/DataLogEditor.h new file mode 100644 index 00000000000..a4e16226b4d --- /dev/null +++ b/datalog/src/main/native/include/wpi/datalog/export/DataLogEditor.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "wpi/datalog/DataLogReader.h" +#include "wpi/datalog/DataLogReaderThread.h" +#include "wpi/datalog/export/DataLogExportUtils.h" + +namespace wpi::log { +class DataLogEditor { + public: + explicit DataLogEditor(const DataLogReaderThread& reader); + + DataLogEditor ExtractEntries(const std::vector& entries); + DataLogEditor TrimToTime(uint64_t startTime, uint64_t endTime); + DataLogEditor RenameEntry(std::string_view currentName, std::string newName); + void ApplyEdits(); + + private: + std::vector records; + uint64_t startTime; + uint64_t endTime; + std::set allowedEntries; + std::map entryRenames; +}; +} // namespace wpi::log \ No newline at end of file diff --git a/datalog/src/main/native/include/wpi/datalog/export/DataLogExportUtils.h b/datalog/src/main/native/include/wpi/datalog/export/DataLogExportUtils.h new file mode 100644 index 00000000000..f201e2a74c2 --- /dev/null +++ b/datalog/src/main/native/include/wpi/datalog/export/DataLogExportUtils.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include "wpi/datalog/DataLogReaderThread.h" +#include "wpi/fs.h" + +namespace wpi::log::fileexport { +struct InputFile { + explicit InputFile(std::unique_ptr datalog); + + InputFile(std::string_view filename, std::string_view status) + : filename{filename}, + stem{fs::path{filename}.stem().string()}, + status{status} {} + + ~InputFile(); + + std::string filename; + std::string stem; + std::unique_ptr datalog; + std::string status; + bool highlight = false; +}; + +struct Entry { + explicit Entry(const wpi::log::StartRecordData& srd) + : name{srd.name}, type{srd.type}, metadata{srd.metadata} {} + + std::string name; + std::string type; + std::string metadata; + uint64_t id; + std::set inputFiles; + bool typeConflict = false; + bool metadataConflict = false; + bool selected = true; + + // used only during export + int column = -1; +}; + +struct EntryTreeNode { + explicit EntryTreeNode(std::string_view name) : name{name} {} + std::string name; // name of just this node + std::string path; // full path if entry is nullptr + Entry* entry = nullptr; + std::vector children; // children, sorted by name + int selected = 1; +}; + +std::unique_ptr LoadDataLog(std::string_view filename); +} // namespace wpi::log \ No newline at end of file diff --git a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp b/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp deleted file mode 100644 index 57c4df2108a..00000000000 --- a/sawmill/src/main/native/cpp/DataLogCSVWriter.cpp +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright (c) FIRST and other WPILib contributors. -// Open Source Software; you can modify and/or share it under the terms of -// the WPILib BSD license file in the root directory of this project. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// namespace { -// struct InputFile { -// explicit InputFile(std::unique_ptr datalog); - -// InputFile(std::string_view filename, std::string_view status) -// : filename{filename}, -// stem{fs::path{filename}.stem().string()}, -// status{status} {} - -// ~InputFile(); - -// std::string filename; -// std::string stem; -// std::unique_ptr datalog; -// std::string status; -// bool highlight = false; -// }; -// } // namespace - -// static std::map, std::less<>> -// gInputFiles; -static wpi::mutex gEntriesMutex; -static std::map, std::less<>> - gEntries; -std::atomic_int gExportCount{0}; - -// InputFile::InputFile(std::unique_ptr datalog_) -// : filename{datalog_->GetBufferIdentifier()}, -// stem{fs::path{filename}.stem().string()}, -// datalog{std::move(datalog_)} { -// datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) -// { -// std::scoped_lock lock{gEntriesMutex}; -// auto it = gEntries.find(srd.name); -// if (it == gEntries.end()) { -// it = gEntries.emplace(srd.name, std::make_unique(srd)).first; -// } else { -// if (it->second->type != srd.type) { -// it->second->typeConflict = true; -// } -// if (it->second->metadata != srd.metadata) { -// it->second->metadataConflict = true; -// } -// } -// it->second->inputFiles.emplace(this); -// }); -// } - -// InputFile::~InputFile() { -// if (!datalog) { -// return; -// } -// std::scoped_lock lock{gEntriesMutex}; -// bool changed = false; -// for (auto it = gEntries.begin(); it != gEntries.end();) { -// it->second->inputFiles.erase(this); -// if (it->second->inputFiles.empty()) { -// it = gEntries.erase(it); -// changed = true; -// } else { -// ++it; -// } -// } -// } - -using namespace sawmill; - -static wpi::mutex gExportMutex; -static std::vector gExportErrors; - -static void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str) { - auto s = str; - while (!s.empty()) { - std::string_view fragment; - std::tie(fragment, s) = wpi::split(s, '"'); - os << fragment; - if (!s.empty()) { - os << '"' << '"'; - } - } - if (wpi::ends_with(str, '"')) { - os << '"' << '"'; - } -} - -static void ValueToCsv(wpi::raw_ostream& os, - const sawmill::DataLogRecord& record) { - // systemTime needs special handling - if (record.entryData.name == "systemTime" && - record.entryData.type == "int64") { - int64_t val; - if (record.dataLogRecord.GetInteger(&val)) { - std::time_t timeval = val / 1000000; - wpi::print(os, "{:%Y-%m-%d %H:%M:%S}.{:06}", *std::localtime(&timeval), - val % 1000000); - return; - } - } else if (record.entryData.type == "double") { - double val; - if (record.dataLogRecord.GetDouble(&val)) { - wpi::print(os, "{}", val); - return; - } - } else if (record.entryData.type == "int64" || - record.entryData.type == "int") { - // support "int" for compatibility with old NT4 datalogs - int64_t val; - if (record.dataLogRecord.GetInteger(&val)) { - wpi::print(os, "{}", val); - return; - } - } else if (record.entryData.type == "string" || - record.entryData.type == "json") { - std::string_view val; - record.dataLogRecord.GetString(&val); - os << '"'; - PrintEscapedCsvString(os, val); - os << '"'; - return; - } else if (record.entryData.type == "boolean") { - bool val; - if (record.dataLogRecord.GetBoolean(&val)) { - wpi::print(os, "{}", val); - return; - } - } else if (record.entryData.type == "boolean[]") { - std::vector val; - if (record.dataLogRecord.GetBooleanArray(&val)) { - wpi::print(os, "{}", fmt::join(val, ";")); - return; - } - } else if (record.entryData.type == "double[]") { - std::vector val; - if (record.dataLogRecord.GetDoubleArray(&val)) { - wpi::print(os, "{}", fmt::join(val, ";")); - return; - } - } else if (record.entryData.type == "float[]") { - std::vector val; - if (record.dataLogRecord.GetFloatArray(&val)) { - wpi::print(os, "{}", fmt::join(val, ";")); - return; - } - } else if (record.entryData.type == "int64[]") { - std::vector val; - if (record.dataLogRecord.GetIntegerArray(&val)) { - wpi::print(os, "{}", fmt::join(val, ";")); - return; - } - } else if (record.entryData.type == "string[]") { - std::vector val; - if (record.dataLogRecord.GetStringArray(&val)) { - os << '"'; - bool first = true; - for (auto&& v : val) { - if (!first) { - os << ';'; - } - first = false; - PrintEscapedCsvString(os, v); - } - os << '"'; - return; - } - } - wpi::print(os, ""); -} - -// static void ValueToCsv(wpi::raw_ostream& os, const Entry& entry, -// const wpi::log::DataLogRecord& record) { -// // handle systemTime specially -// if (entry.name == "systemTime" && entry.type == "int64") { -// int64_t val; -// if (record.GetInteger(&val)) { -// std::time_t timeval = val / 1000000; -// wpi::print(os, "{:%Y-%m-%d %H:%M:%S}.{:06}", *std::localtime(&timeval), -// val % 1000000); -// return; -// } -// } else if (entry.type == "double") { -// double val; -// if (record.GetDouble(&val)) { -// wpi::print(os, "{}", val); -// return; -// } -// } else if (entry.type == "int64" || entry.type == "int") { -// // support "int" for compatibility with old NT4 datalogs -// int64_t val; -// if (record.GetInteger(&val)) { -// wpi::print(os, "{}", val); -// return; -// } -// } else if (entry.type == "string" || entry.type == "json") { -// std::string_view val; -// record.GetString(&val); -// os << '"'; -// PrintEscapedCsvString(os, val); -// os << '"'; -// return; -// } else if (entry.type == "boolean") { -// bool val; -// if (record.GetBoolean(&val)) { -// wpi::print(os, "{}", val); -// return; -// } -// } else if (entry.type == "boolean[]") { -// std::vector val; -// if (record.GetBooleanArray(&val)) { -// wpi::print(os, "{}", fmt::join(val, ";")); -// return; -// } -// } else if (entry.type == "double[]") { -// std::vector val; -// if (record.GetDoubleArray(&val)) { -// wpi::print(os, "{}", fmt::join(val, ";")); -// return; -// } -// } else if (entry.type == "float[]") { -// std::vector val; -// if (record.GetFloatArray(&val)) { -// wpi::print(os, "{}", fmt::join(val, ";")); -// return; -// } -// } else if (entry.type == "int64[]") { -// std::vector val; -// if (record.GetIntegerArray(&val)) { -// wpi::print(os, "{}", fmt::join(val, ";")); -// return; -// } -// } else if (entry.type == "string[]") { -// std::vector val; -// if (record.GetStringArray(&val)) { -// os << '"'; -// bool first = true; -// for (auto&& v : val) { -// if (!first) { -// os << ';'; -// } -// first = false; -// PrintEscapedCsvString(os, v); -// } -// os << '"'; -// return; -// } -// } -// wpi::print(os, ""); -// } - -void ExportCsvFile(wpi::raw_ostream& os, int style, bool printControlRecords, - std::vector records, - std::map> entryMap) { - // print header - if (style == 0) { - os << "Timestamp,Name,Value\n"; - } else if (style == 1) { - // scan for exported fields for this file to print header and assign columns - os << "Timestamp"; - int columnNum = 0; - for (std::pair& entry : entryMap) { - os << ',' << '"'; - PrintEscapedCsvString(os, entry.second.name); - os << '"'; - entry.second.column = columnNum++; - } - os << '\n'; - } - - for (sawmill::DataLogRecord record : records) { - // if this is a control record and we dont want to print those, skip - if (record.dataLogRecord.IsControl() && !printControlRecords) { - continue; - } - - if (style == 0) { - wpi::print(os, "{},\"", record.dataLogRecord.GetTimestamp() / 1000000.0); - PrintEscapedCsvString(os, record.entryData.name); - os << '"' << ','; - ValueToCsv(os, record); - os << '\n'; - } else if (style == 1) { - wpi::print(os, "{},", record.dataLogRecord.GetTimestamp() / 1000000.0); - for (int i = 0; i < record.entryData.column; ++i) { - os << ','; - } - ValueToCsv(os, record); - os << '\n'; - } - } -} - -// static void ExportCsvFile(InputFile& f, wpi::raw_ostream& os, int style) { -// // header -// if (style == 0) { -// os << "Timestamp,Name,Value\n"; -// } else if (style == 1) { -// // scan for exported fields for this file to print header and assign -// columns os << "Timestamp"; int columnNum = 0; for (auto&& entry : -// gEntries) { -// if (entry.second->selected && -// entry.second->inputFiles.find(&f) != -// entry.second->inputFiles.end()) { -// os << ',' << '"'; -// PrintEscapedCsvString(os, entry.first); -// os << '"'; -// entry.second->column = columnNum++; -// } else { -// entry.second->column = -1; -// } -// } -// os << '\n'; -// } - -// wpi::DenseMap nameMap; -// for (wpi::log::DataLogRecord record : f.datalog->GetReader()) { -// if (record.IsStart()) { -// wpi::log::StartRecordData data; -// if (record.GetStartData(&data)) { -// auto it = gEntries.find(data.name); -// if (it != gEntries.end() && it->second->selected) { -// nameMap[data.entry] = it->second.get(); -// } -// } -// } else if (record.IsFinish()) { -// int entry; -// if (record.GetFinishEntry(&entry)) { -// nameMap.erase(entry); -// } -// } else if (!record.IsControl()) { -// auto entryIt = nameMap.find(record.GetEntry()); -// if (entryIt == nameMap.end()) { -// continue; -// } -// Entry* entry = entryIt->second; - -// if (style == 0) { -// wpi::print(os, "{},\"", record.GetTimestamp() / 1000000.0); -// PrintEscapedCsvString(os, entry->name); -// os << '"' << ','; -// ValueToCsv(os, record); -// os << '\n'; -// } else if (style == 1 && entry->column != -1) { -// wpi::print(os, "{},", record.GetTimestamp() / 1000000.0); -// for (int i = 0; i < entry->column; ++i) { -// os << ','; -// } -// ValueToCsv(os, *entry, record); -// os << '\n'; -// } -// } -// } -// } - -// static void ExportCsv(std::string_view outputFolder, int style) { -// fs::path outPath{outputFolder}; -// for (auto&& f : gInputFiles) { -// if (f.second->datalog) { -// std::error_code ec; -// auto of = fs::OpenFileForWrite( -// outPath / fs::path{f.first}.replace_extension("csv"), ec, -// fs::CD_CreateNew, fs::OF_Text); -// if (ec) { -// std::scoped_lock lock{gExportMutex}; -// gExportErrors.emplace_back( -// fmt::format("{}: {}", f.first, ec.message())); -// ++gExportCount; -// continue; -// } -// wpi::raw_fd_ostream os{fs::FileToFd(of, ec, fs::OF_Text), true}; -// ExportCsvFile(*f.second, os, style); -// } -// ++gExportCount; -// } -// } diff --git a/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp b/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp deleted file mode 100644 index 0a86f4e5027..00000000000 --- a/sawmill/src/main/native/cpp/DataLogJSONWriter.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) FIRST and other WPILib contributors. -// Open Source Software; you can modify and/or share it under the terms of -// the WPILib BSD license file in the root directory of this project. - -#include "DataLogJSONWriter.h" - -#include - -#include -#include -#include - -void ExportJSON(fs::path outputPath, - std::vector records) { - // JSON structure - // List of blocks - // Each block is a direct transcription of a record - // this includes the entry id, timestamp, and data. If the record contains raw - // bytes, they will be represented as a base64 string. -} diff --git a/sawmill/src/main/native/cpp/LogLoader.cpp b/sawmill/src/main/native/cpp/LogLoader.cpp deleted file mode 100644 index 15f9799de4e..00000000000 --- a/sawmill/src/main/native/cpp/LogLoader.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) FIRST and other WPILib contributors. -// Open Source Software; you can modify and/or share it under the terms of -// the WPILib BSD license file in the root directory of this project. - -#include "LogLoader.h" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "DataLogExport.h" - -using namespace sawmill; - -LogLoader::LogLoader() {} - -LogLoader::~LogLoader() = default; - -void LogLoader::Load(fs::path logPath) { - // Handle opening the file - std::error_code ec; - auto buf = wpi::MemoryBuffer::GetFile(logPath.string()); - if (ec) { - m_error = fmt::format("Could not open file: {}", ec.message()); - return; - } - - wpi::log::DataLogReader reader{*std::move(buf)}; - if (!reader.IsValid()) { - m_error = "Not a valid datalog file"; - return; - } - unload(); // release the actual file, we have the data in the reader now - m_reader = std::make_unique(std::move(reader)); - - // Handle Errors - fmt::println("{}", m_error); - - if (!m_reader) { - return; - } - - // Summary info - fmt::println("{}", fs::path{m_filename}.stem().string().c_str()); - fmt::println("%u records, %u entries%s", m_reader->GetNumRecords(), - m_reader->GetNumEntries(), - m_reader->IsDone() ? "" : " (working)"); - - if (!m_reader->IsDone()) { - return; - } -} - -/*std::vector LogLoader::GetRecords( - std::string_view field_name) { - std::vector record_list{}; - - const wpi::DataLogReaderEntry* entry = m_reader->GetEntry(field_name); - for (wpi::DataLogReaderRange range : entry->ranges) { - wpi::log::DataLogReader::iterator rangeReader = range.begin(); - while (!rangeReader->IsFinish()) { - record_list.push_back(*rangeReader); - } - } - - return record_list; -}*/ - -std::vector LogLoader::GetAllRecords() { - if (records.size() == 0) { - // get all records - for (wpi::log::DataLogRecord record : m_reader->GetReader()) { - if (record.IsStart()) { - wpi::log::StartRecordData data; - if (record.GetStartData(&data)) { - // associate an entry id with a StartRecordData - dataMap.emplace(data.entry, sawmill::Entry{data}); - } - } else if (record.IsFinish()) { - // remove the association - int entryId; - if (record.GetFinishEntry(&entryId)) { - dataMap.erase(entryId); - } - } - int entryId = record.GetEntry(); - if (dataMap.contains(entryId)) { - if (auto it = dataMap.find(entryId); it != dataMap.end()) { - records.emplace_back(it->second, record); - } - } - } - } - - return records; -} - -std::map> LogLoader::GetEntryMap() { - return dataMap; -}