Sawmill: A CLI tool for exporting and trimming WPILib DataLogs

CMakeLists.txt

@@ -300,6 +300,8 @@ if(WITH_NTCORE)
     add_subdirectory(ntcore)
 endif()
 
+add_subdirectory(sawmill)
+
 if(WITH_WPIMATH)
     if(WITH_JAVA)
         set(WPIUNITS_DEP_REPLACE ${WPIUNITS_DEP_REPLACE_IMPL})

datalog/src/main/native/cpp/DataLogEditor.cpp

@@ -0,0 +1,27 @@
+#include "wpi/datalog/export/DataLogEditor.h"
+#include <cstdint>
+#include <vector>
+#include "wpi/datalog/DataLogReaderThread.h"
+#include "wpi/datalog/export/DataLogExportUtils.h"
+
+namespace wpi::log {
+  DataLogEditor::DataLogEditor(const DataLogReaderThread& reader) {
+    for (auto record : reader.GetReader()) {
+      records.push_back(record);
+    }
+  }
+
+  DataLogEditor DataLogEditor::ExtractEntries(const std::vector<fileexport::Entry>& entries) {
+    for (const auto& entry : entries) {
+      allowedEntries.emplace(entry.id);
+    }
+    return *this;
+  }
+
+  DataLogEditor DataLogEditor::TrimToTime(uint64_t start, uint64_t end) {
+    endTime = end;
+    startTime = start;
+    return *this;
+  }
+}
+

datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp

@@ -0,0 +1,7 @@
+#include "wpi/datalog/export/DataLogCSVExporter.h"
+#include <memory>
+#include "wpi/datalog/export/DataLogExportUtils.h"
+
+namespace wpi::log::fileexport {
+
+}

datalog/src/main/native/cpp/export/DataLogExportUtils.cpp

@@ -0,0 +1,83 @@
+#include "wpi/datalog/export/DataLogExportUtils.h"
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <string_view>
+#include <utility>
+#include "fmt/format.h"
+#include "wpi/MemoryBuffer.h"
+#include "wpi/datalog/DataLogReader.h"
+#include "wpi/datalog/DataLogReaderThread.h"
+#include "wpi/mutex.h"
+#include <wpi/fs.h>
+
+static wpi::mutex entriesMutex;
+static std::map<std::string, std::unique_ptr<wpi::log::fileexport::Entry>,
+                std::less<>>
+    entries;
+
+namespace wpi::log::fileexport {
+InputFile::InputFile(std::unique_ptr<wpi::log::DataLogReaderThread> datalog_)
+    : filename{datalog_->GetBufferIdentifier()},
+      stem{fs::path{filename}.stem().string()},
+      datalog{std::move(datalog_)} {
+  datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) {
+    // add this entry to the map
+    std::scoped_lock lock{entriesMutex};
+    auto it = entries.find(srd.name);
+    if (it == entries.end()) {
+      // this entry isnt in the map, so lets add it
+      it = entries.emplace(srd.name, std::make_unique<Entry>(srd)).first;
+      // if this is a gui make it rebuild the entry tree somehow lmfao
+      // maybe accept an optional callback???
+      // TODO: RebuildEntryTree
+    } else {
+      // this entry IS already known, so lets make sure the start records match
+      if (it->second->type != srd.type) {
+        it->second->typeConflict = true;
+      }
+      if (it->second->metadata != srd.metadata) {
+        it->second->metadataConflict = true;
+      }
+      it->second->inputFiles.emplace(this);
+    }
+  });
+}
+
+InputFile::~InputFile() {
+  if (!datalog) {
+    return;
+  }
+  std::scoped_lock lock{entriesMutex};
+  bool changed = false;
+  for (auto it = entries.begin(); it != entries.end();) {
+    it->second->inputFiles.erase(this);
+    if (it->second->inputFiles.empty()) {
+      it = entries.erase(it);
+      changed = true;
+    } else {
+      ++it;
+    }
+  }
+  if (changed) {
+    // TODO: rebuildentrytree
+  }
+}
+
+std::unique_ptr<InputFile> LoadDataLog(std::string_view filename) {
+  auto fileBuffer = wpi::MemoryBuffer::GetFile(filename);
+  if (!fileBuffer) {
+    return std::make_unique<InputFile>(
+        filename, fmt::format("Could not open file: {}", fileBuffer.error()));
+  }
+
+  wpi::log::DataLogReader reader{std::move(*fileBuffer)};
+  if (!reader.IsValid()) {
+    return std::make_unique<InputFile>(filename, "Not a valid datalog file");
+  }
+
+  return std::make_unique<InputFile>(
+      std::make_unique<wpi::log::DataLogReaderThread>(std::move(reader)));
+}
+}  // namespace wpi::log::fileexport

datalog/src/main/native/include/wpi/datalog/DataLogReader.h

@@ -9,7 +9,6 @@
 #include <iterator>
 #include <memory>
 #include <span>
-#include <utility>
 #include <vector>
 
 #include <wpi/MemoryBuffer.h>

datalog/src/main/native/include/wpi/datalog/DataLogReaderThread.h

@@ -4,6 +4,12 @@
 
 #pragma once
 
+#include <wpi/datalog/DataLogReader.h>
+#include <wpi/DenseMap.h>
+#include <wpi/Signal.h>
+#include <wpi/mutex.h>
+#include <wpi/struct/DynamicStruct.h>
+
 #include <atomic>
 #include <functional>
 #include <map>

datalog/src/main/native/include/wpi/datalog/export/DataLogCSVExporter.h

@@ -0,0 +1,19 @@
+#pragma once
+#include <wpi/fs.h>
+#include <string>
+#include "wpi/datalog/DataLogReader.h"
+#include "wpi/datalog/export/DataLogExportUtils.h"
+#include "wpi/raw_ostream.h"
+
+namespace wpi::log::fileexport {
+class DataLogCSVExporter {
+ public:
+  static void Export(std::string inputFilePath, std::string exportFilePath);
+ private:
+  void WriteValue(wpi::raw_ostream& os, const Entry& entry,
+                  const wpi::log::DataLogRecord& record);
+  void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str);
+  fs::path exportFile;
+  fs::path inputFile;
+};
+}  // namespace wpi::log::fileexport

datalog/src/main/native/include/wpi/datalog/export/DataLogEditor.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <chrono>
+#include <cstdint>
+#include <map>
+#include <set>
+#include <string>
+#include <string_view>
+#include <vector>
+#include "wpi/datalog/DataLogReader.h"
+#include "wpi/datalog/DataLogReaderThread.h"
+#include "wpi/datalog/export/DataLogExportUtils.h"
+
+namespace wpi::log {
+class DataLogEditor {
+ public:
+  explicit DataLogEditor(const DataLogReaderThread& reader);
+
+  DataLogEditor ExtractEntries(const std::vector<fileexport::Entry>& entries);
+  DataLogEditor TrimToTime(uint64_t startTime, uint64_t endTime);
+  DataLogEditor RenameEntry(std::string_view currentName, std::string newName);
+  void ApplyEdits();
+
+ private:
+  std::vector<DataLogRecord> records;
+  uint64_t startTime;
+  uint64_t endTime;
+  std::set<uint64_t> allowedEntries;
+  std::map<std::string, std::string> entryRenames;
+};
+}  // namespace wpi::log

datalog/src/main/native/include/wpi/datalog/export/DataLogExportUtils.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <string_view>
+#include "wpi/datalog/DataLogReaderThread.h"
+#include "wpi/fs.h"
+
+namespace wpi::log::fileexport {
+struct InputFile {
+  explicit InputFile(std::unique_ptr<wpi::log::DataLogReaderThread> datalog);
+
+  InputFile(std::string_view filename, std::string_view status)
+      : filename{filename},
+        stem{fs::path{filename}.stem().string()},
+        status{status} {}
+
+  ~InputFile();
+
+  std::string filename;
+  std::string stem;
+  std::unique_ptr<wpi::log::DataLogReaderThread> datalog;
+  std::string status;
+  bool highlight = false;
+};
+
+struct Entry {
+  explicit Entry(const wpi::log::StartRecordData& srd)
+      : name{srd.name}, type{srd.type}, metadata{srd.metadata} {}
+
+  std::string name;
+  std::string type;
+  std::string metadata;
+  uint64_t id;
+  std::set<InputFile*> inputFiles;
+  bool typeConflict = false;
+  bool metadataConflict = false;
+  bool selected = true;
+
+  // used only during export
+  int column = -1;
+};
+
+struct EntryTreeNode {
+  explicit EntryTreeNode(std::string_view name) : name{name} {}
+  std::string name;  // name of just this node
+  std::string path;  // full path if entry is nullptr
+  Entry* entry = nullptr;
+  std::vector<EntryTreeNode> children;  // children, sorted by name
+  int selected = 1;
+};
+
+std::unique_ptr<InputFile> LoadDataLog(std::string_view filename);
+}  // namespace wpi::log

sawmill/CMakeLists.txt

@@ -0,0 +1,33 @@
+project(sawmill)
+
+include(CompileWarnings)
+include(GenResources)
+
+configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp)
+# generate_resources(src/main/native/resources generated/main/cpp DLT dlt sawmill_resources_src)
+
+# Generate compile_commands.json by default
+if(NOT CMAKE_EXPORT_COMPILE_COMMANDS)
+    set(CMAKE_EXPORT_COMPILE_COMMANDS "YES" CACHE STRING "" FORCE)
+endif()
+
+file(GLOB sawmill_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp)
+
+if(WIN32)
+    # set(sawmill_rc src/main/native/win/sawmill.rc)
+elseif(APPLE)
+    # set(MACOSX_BUNDLE_ICON_FILE sawmill.icns)
+    # set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources")
+endif()
+
+add_executable(sawmill ${sawmill_src}) #${sawmill_resources_src} ${sawmill_rc} ${APP_ICON_MACOSX})
+
+target_include_directories(sawmill PUBLIC src/main/native/include)
+
+target_link_libraries(sawmill PRIVATE wpiutil datalog)
+
+if(WIN32)
+    set_target_properties(sawmill PROPERTIES WIN32_EXECUTABLE YES)
+elseif(APPLE)
+    set_target_properties(sawmill PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "sawmill")
+endif()

sawmill/src/main/generate/WPILibVersion.cpp.in

@@ -0,0 +1,7 @@
+/**
+ * Autogenerated file! Do not manually edit this file. This version is regenerated
+ * any time the publish task is run, or when this file is deleted.
+ */
+const char* GetWPILibVersion() {
+  return "${wpilib_version}";
+}