diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b88f9e010a..144ab8e1276 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -300,6 +300,8 @@ if(WITH_NTCORE) add_subdirectory(ntcore) endif() +add_subdirectory(sawmill) + if(WITH_WPIMATH) if(WITH_JAVA) set(WPIUNITS_DEP_REPLACE ${WPIUNITS_DEP_REPLACE_IMPL}) diff --git a/datalog/src/main/native/cpp/DataLogEditor.cpp b/datalog/src/main/native/cpp/DataLogEditor.cpp new file mode 100644 index 00000000000..cf305d25287 --- /dev/null +++ b/datalog/src/main/native/cpp/DataLogEditor.cpp @@ -0,0 +1,27 @@ +#include "wpi/datalog/export/DataLogEditor.h" +#include +#include +#include "wpi/datalog/DataLogReaderThread.h" +#include "wpi/datalog/export/DataLogExportUtils.h" + +namespace wpi::log { + DataLogEditor::DataLogEditor(const DataLogReaderThread& reader) { + for (auto record : reader.GetReader()) { + records.push_back(record); + } + } + + DataLogEditor DataLogEditor::ExtractEntries(const std::vector& entries) { + for (const auto& entry : entries) { + allowedEntries.emplace(entry.id); + } + return *this; + } + + DataLogEditor DataLogEditor::TrimToTime(uint64_t start, uint64_t end) { + endTime = end; + startTime = start; + return *this; + } +} + diff --git a/datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp b/datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp new file mode 100644 index 00000000000..a23909a2750 --- /dev/null +++ b/datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp @@ -0,0 +1,7 @@ +#include "wpi/datalog/export/DataLogCSVExporter.h" +#include +#include "wpi/datalog/export/DataLogExportUtils.h" + +namespace wpi::log::fileexport { + +} \ No newline at end of file diff --git a/datalog/src/main/native/cpp/export/DataLogExportUtils.cpp b/datalog/src/main/native/cpp/export/DataLogExportUtils.cpp new file mode 100644 index 00000000000..e925c8f3106 --- /dev/null +++ b/datalog/src/main/native/cpp/export/DataLogExportUtils.cpp @@ -0,0 +1,83 @@ +#include "wpi/datalog/export/DataLogExportUtils.h" +#include +#include +#include +#include +#include +#include +#include "fmt/format.h" +#include "wpi/MemoryBuffer.h" +#include "wpi/datalog/DataLogReader.h" +#include "wpi/datalog/DataLogReaderThread.h" +#include "wpi/mutex.h" +#include + +static wpi::mutex entriesMutex; +static std::map, + std::less<>> + entries; + +namespace wpi::log::fileexport { +InputFile::InputFile(std::unique_ptr datalog_) + : filename{datalog_->GetBufferIdentifier()}, + stem{fs::path{filename}.stem().string()}, + datalog{std::move(datalog_)} { + datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) { + // add this entry to the map + std::scoped_lock lock{entriesMutex}; + auto it = entries.find(srd.name); + if (it == entries.end()) { + // this entry isnt in the map, so lets add it + it = entries.emplace(srd.name, std::make_unique(srd)).first; + // if this is a gui make it rebuild the entry tree somehow lmfao + // maybe accept an optional callback??? + // TODO: RebuildEntryTree + } else { + // this entry IS already known, so lets make sure the start records match + if (it->second->type != srd.type) { + it->second->typeConflict = true; + } + if (it->second->metadata != srd.metadata) { + it->second->metadataConflict = true; + } + it->second->inputFiles.emplace(this); + } + }); +} + +InputFile::~InputFile() { + if (!datalog) { + return; + } + std::scoped_lock lock{entriesMutex}; + bool changed = false; + for (auto it = entries.begin(); it != entries.end();) { + it->second->inputFiles.erase(this); + if (it->second->inputFiles.empty()) { + it = entries.erase(it); + changed = true; + } else { + ++it; + } + } + if (changed) { + // TODO: rebuildentrytree + } +} + +std::unique_ptr LoadDataLog(std::string_view filename) { + auto fileBuffer = wpi::MemoryBuffer::GetFile(filename); + if (!fileBuffer) { + return std::make_unique( + filename, fmt::format("Could not open file: {}", fileBuffer.error())); + } + + wpi::log::DataLogReader reader{std::move(*fileBuffer)}; + if (!reader.IsValid()) { + return std::make_unique(filename, "Not a valid datalog file"); + } + + return std::make_unique( + std::make_unique(std::move(reader))); +} +} // namespace wpi::log::fileexport diff --git a/datalog/src/main/native/include/wpi/datalog/DataLogReader.h b/datalog/src/main/native/include/wpi/datalog/DataLogReader.h index 2a22725c7c4..8932ed85a26 100644 --- a/datalog/src/main/native/include/wpi/datalog/DataLogReader.h +++ b/datalog/src/main/native/include/wpi/datalog/DataLogReader.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include diff --git a/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h b/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h index 349b7001aec..a110a251b26 100644 --- a/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h +++ b/datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h @@ -4,6 +4,12 @@ #pragma once +#include +#include +#include +#include +#include + #include #include #include diff --git a/datalog/src/main/native/include/wpi/datalog/export/DataLogCSVExporter.h b/datalog/src/main/native/include/wpi/datalog/export/DataLogCSVExporter.h new file mode 100644 index 00000000000..97c4f8a00b4 --- /dev/null +++ b/datalog/src/main/native/include/wpi/datalog/export/DataLogCSVExporter.h @@ -0,0 +1,19 @@ +#pragma once +#include +#include +#include "wpi/datalog/DataLogReader.h" +#include "wpi/datalog/export/DataLogExportUtils.h" +#include "wpi/raw_ostream.h" + +namespace wpi::log::fileexport { +class DataLogCSVExporter { + public: + static void Export(std::string inputFilePath, std::string exportFilePath); + private: + void WriteValue(wpi::raw_ostream& os, const Entry& entry, + const wpi::log::DataLogRecord& record); + void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str); + fs::path exportFile; + fs::path inputFile; +}; +} // namespace wpi::log::fileexport \ No newline at end of file diff --git a/datalog/src/main/native/include/wpi/datalog/export/DataLogEditor.h b/datalog/src/main/native/include/wpi/datalog/export/DataLogEditor.h new file mode 100644 index 00000000000..a4e16226b4d --- /dev/null +++ b/datalog/src/main/native/include/wpi/datalog/export/DataLogEditor.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "wpi/datalog/DataLogReader.h" +#include "wpi/datalog/DataLogReaderThread.h" +#include "wpi/datalog/export/DataLogExportUtils.h" + +namespace wpi::log { +class DataLogEditor { + public: + explicit DataLogEditor(const DataLogReaderThread& reader); + + DataLogEditor ExtractEntries(const std::vector& entries); + DataLogEditor TrimToTime(uint64_t startTime, uint64_t endTime); + DataLogEditor RenameEntry(std::string_view currentName, std::string newName); + void ApplyEdits(); + + private: + std::vector records; + uint64_t startTime; + uint64_t endTime; + std::set allowedEntries; + std::map entryRenames; +}; +} // namespace wpi::log \ No newline at end of file diff --git a/datalog/src/main/native/include/wpi/datalog/export/DataLogExportUtils.h b/datalog/src/main/native/include/wpi/datalog/export/DataLogExportUtils.h new file mode 100644 index 00000000000..f201e2a74c2 --- /dev/null +++ b/datalog/src/main/native/include/wpi/datalog/export/DataLogExportUtils.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include "wpi/datalog/DataLogReaderThread.h" +#include "wpi/fs.h" + +namespace wpi::log::fileexport { +struct InputFile { + explicit InputFile(std::unique_ptr datalog); + + InputFile(std::string_view filename, std::string_view status) + : filename{filename}, + stem{fs::path{filename}.stem().string()}, + status{status} {} + + ~InputFile(); + + std::string filename; + std::string stem; + std::unique_ptr datalog; + std::string status; + bool highlight = false; +}; + +struct Entry { + explicit Entry(const wpi::log::StartRecordData& srd) + : name{srd.name}, type{srd.type}, metadata{srd.metadata} {} + + std::string name; + std::string type; + std::string metadata; + uint64_t id; + std::set inputFiles; + bool typeConflict = false; + bool metadataConflict = false; + bool selected = true; + + // used only during export + int column = -1; +}; + +struct EntryTreeNode { + explicit EntryTreeNode(std::string_view name) : name{name} {} + std::string name; // name of just this node + std::string path; // full path if entry is nullptr + Entry* entry = nullptr; + std::vector children; // children, sorted by name + int selected = 1; +}; + +std::unique_ptr LoadDataLog(std::string_view filename); +} // namespace wpi::log \ No newline at end of file diff --git a/sawmill/CMakeLists.txt b/sawmill/CMakeLists.txt new file mode 100644 index 00000000000..32854998344 --- /dev/null +++ b/sawmill/CMakeLists.txt @@ -0,0 +1,33 @@ +project(sawmill) + +include(CompileWarnings) +include(GenResources) + +configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) +# generate_resources(src/main/native/resources generated/main/cpp DLT dlt sawmill_resources_src) + +# Generate compile_commands.json by default +if(NOT CMAKE_EXPORT_COMPILE_COMMANDS) + set(CMAKE_EXPORT_COMPILE_COMMANDS "YES" CACHE STRING "" FORCE) +endif() + +file(GLOB sawmill_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) + +if(WIN32) + # set(sawmill_rc src/main/native/win/sawmill.rc) +elseif(APPLE) + # set(MACOSX_BUNDLE_ICON_FILE sawmill.icns) + # set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") +endif() + +add_executable(sawmill ${sawmill_src}) #${sawmill_resources_src} ${sawmill_rc} ${APP_ICON_MACOSX}) + +target_include_directories(sawmill PUBLIC src/main/native/include) + +target_link_libraries(sawmill PRIVATE wpiutil datalog) + +if(WIN32) + set_target_properties(sawmill PROPERTIES WIN32_EXECUTABLE YES) +elseif(APPLE) + set_target_properties(sawmill PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "sawmill") +endif() diff --git a/sawmill/src/main/generate/WPILibVersion.cpp.in b/sawmill/src/main/generate/WPILibVersion.cpp.in new file mode 100644 index 00000000000..cfe24411588 --- /dev/null +++ b/sawmill/src/main/generate/WPILibVersion.cpp.in @@ -0,0 +1,7 @@ +/** + * Autogenerated file! Do not manually edit this file. This version is regenerated + * any time the publish task is run, or when this file is deleted. + */ +const char* GetWPILibVersion() { + return "${wpilib_version}"; +} diff --git a/sawmill/src/main/native/cpp/main.cpp b/sawmill/src/main/native/cpp/main.cpp new file mode 100644 index 00000000000..9ad472d0a67 --- /dev/null +++ b/sawmill/src/main/native/cpp/main.cpp @@ -0,0 +1,172 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include +#include +#include +#include +#include + +#include +#include + +#include "DataLogCSVWriter.h" +#include "DataLogJSONWriter.h" +#include "LogLoader.h" + +// void export_json(fs::path log_path, fs::path output_path) { +// sawmill::LogLoader loader{}; +// loader.Load(log_path.string()); +// std::vector records = loader.GetAllRecords(); +// sawmill::DataLogJSONWriter writer{}; +// writer.ExportJSON(output_path, records); +// } + +void export_csv(fs::path logPath, fs::path outputPaths) { + sawmill::LogLoader loader{}; + loader.Load(logPath); + // +} + +void write_json(std::vector records, + fs::path output_path) {} + +// void write_csv(std::vector records, +// fs::path output_path) { +// sawmill::DataLogCSVWriter writer{}; +// writer.ExportCsv(output_path.string(), 0); +// } + +void extract_entry(std::string_view entry_name, fs::path log_path, + fs::path output_path, bool use_json) { + // represent entry as a list of records + sawmill::LogLoader loader{}; + loader.Load(log_path.string()); + //std::vector records = loader.GetRecords(entry_name); +} + +int main(int argc, char* argv[]) { + wpi::ArgumentParser cli{"sawmill"}; + std::string jsonMode{}; + + wpi::ArgumentParser export_command{"export"}; + export_command.add_description( + "Export a DataLog file in a text-based format"); + export_command.add_argument("Type") + .help( + "The file type of the exported log, must be 'json' or 'csv'."); + export_command.add_argument("DataLog") + .help("A valid path to a .wpilog file to convert"); + export_command.add_argument("Output") + .help( + "A valid path to the location of the exported log file." + "All directories must exist, but if the file does not, a new file " + "will be created with the name of the original DataLog."); + // wpi::ArgumentParser export_json_command{"json"}; + // export_json_command.add_description( + // "Export a JSON representation of a DataLog file"); + // export_json_command.add_argument("log_file") + // .help("Path to the DataLog file to export"); + // export_json_command.add_argument("export_path") + // .help( + // "Path of the JSON file to create with the exported data. If it " + // "exists, it will be overwritten."); + // export_json_command.add_argument("-m", "--mode") + // .help( + // "How to format the exported JSON. 'direct' means that each record " + // "will be directly converted to a JSON dictionary and exported, " + // "including control records. 'direct_data' is the same as direct, but " + // "control records will be ommitted. The default is 'direct_data'.") + // .default_value("direct_data") + // .store_into(jsonMode); + + + + // wpi::ArgumentParser extract_entry_command{"extract"}; + // extract_entry_command.add_description( + // "Extract the history of one entry from a DataLog file and store it in a " + // "JSON or CSV file"); + // extract_entry_command.add_argument("-e", "--entry") + // .required() + // .help("The entry to extract from the Log"); + // extract_entry_command.add_argument("-l", "--log-file") + // .required() + // .help("The DataLog file to extract from"); + // extract_entry_command.add_argument("--time-start") + // .help("The timestamp to start extracting at"); + // extract_entry_command.add_argument("-o", "--output") + // .required() + // .help( + // "The file to export the field and data to. It will be created or " + // "overwritten if it already exists"); + + //cli.add_subparser(export_json_command); + cli.add_subparser(export_command); + //cli.add_subparser(extract_entry_command); + + try { + cli.parse_args(argc, argv); + } catch (const std::exception& err) { + std::cerr << err.what() << std::endl; + std::cerr << cli; + return 1; + } + + // see which one was called + // if (export_json_command) { + // // validate paths + // fs::path logPath{export_json_command.get("--log-file")}; + // if (logPath.extension() != ".wpilog") { + // std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; + // return 1; + // } + + // fs::path outputPath{export_json_command.get("--output")}; + // if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { + // std::cerr + // << "Only JSON and CSV are currently supported as output formats." + // << std::endl; + // return 1; + // } + + //export_json(logPath, outputPath); + /*} else*/ if (export_command) { + // validate paths + fs::path logPath{export_command.get("--log-file")}; + if (logPath.extension() != ".wpilog") { + std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; + return 1; + } + + fs::path outputPath{export_command.get("--output")}; + if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { + std::cerr + << "Only JSON and CSV are currently supported as output formats." + << std::endl; + return 1; + } + + // export_csv(logPath, outputPath); + } //else if (extract_entry_command) { + // validate paths + // fs::path logPath{extract_entry_command.get("--log-file")}; + // if (logPath.extension() != ".wpilog") { + // std::cerr << "Please use a valid DataLog (.wpilog) file." << std::endl; + // return 1; + // } + + // fs::path outputPath{extract_entry_command.get("--output")}; + // if (outputPath.extension() != ".csv" || outputPath.extension() != ".json") { + // std::cerr + // << "Only JSON and CSV are currently supported as output formats." + // << std::endl; + // return 1; + // } + + // bool json{outputPath.extension() == ".json" ? true : false}; + + // extract_entry(extract_entry_command.get("--entry"), logPath, outputPath, + // json); + // } +} diff --git a/sawmill/src/main/native/include/DataLogCSVWriter.h b/sawmill/src/main/native/include/DataLogCSVWriter.h new file mode 100644 index 00000000000..6dd4258de59 --- /dev/null +++ b/sawmill/src/main/native/include/DataLogCSVWriter.h @@ -0,0 +1,14 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +namespace sawmill { +class DataLogCSVWriter { + public: + void ExportCsv(std::string_view outputFolder, int style); +}; +} // namespace sawmill diff --git a/sawmill/src/main/native/include/DataLogExport.h b/sawmill/src/main/native/include/DataLogExport.h new file mode 100644 index 00000000000..dae4994652c --- /dev/null +++ b/sawmill/src/main/native/include/DataLogExport.h @@ -0,0 +1,33 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include + +namespace sawmill { +struct Entry { + explicit Entry(const wpi::log::StartRecordData& srd) + : name{srd.name}, type{srd.type}, metadata{srd.metadata} {} + + std::string name; + std::string type; + std::string metadata; + // std::set inputFiles; + bool typeConflict = false; + bool metadataConflict = false; + bool selected = true; + + // used only during export + int column = -1; +}; + +struct DataLogRecord { + const Entry entryData; + wpi::log::DataLogRecord dataLogRecord; +}; + +} // namespace sawmill diff --git a/sawmill/src/main/native/include/DataLogJSONWriter.h b/sawmill/src/main/native/include/DataLogJSONWriter.h new file mode 100644 index 00000000000..d376101bf10 --- /dev/null +++ b/sawmill/src/main/native/include/DataLogJSONWriter.h @@ -0,0 +1,25 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include + +#include +#include + +#include "DataLogExport.h" + +namespace sawmill { +/** + * Helps with loading datalog files. + */ +class DataLogJSONWriter { + public: + void ExportJSON(fs::path exportPath, + std::vector records); +}; +} // namespace sawmill diff --git a/sawmill/src/main/native/include/LogLoader.h b/sawmill/src/main/native/include/LogLoader.h new file mode 100644 index 00000000000..b5602459126 --- /dev/null +++ b/sawmill/src/main/native/include/LogLoader.h @@ -0,0 +1,68 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "DataLogExport.h" + +namespace glass { +class Storage; +} // namespace glass + +namespace wpi::log { +class DataLogReaderEntry; +class DataLogReaderThread; +class Logger; +} // namespace wpi + +namespace sawmill { +/** + * Helps with loading datalog files. + */ +class LogLoader { + public: + /** + * Creates a log loader + */ + explicit LogLoader(); + + ~LogLoader(); + + /** + * Signal called when the current file is unloaded (invalidates any + * LogEntry*). + */ + wpi::sig::Signal<> unload; + + void Load(fs::path logPath); + + std::vector GetAllRecords(); + + std::map> GetEntryMap(); + + private: + std::string m_filename; + std::unique_ptr m_reader; + + std::string m_error; + + std::string m_filter; + + wpi::log::StartRecordData* entryData; + + std::map> dataMap; + + std::vector records; +}; +} // namespace sawmill