Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sawmill: A CLI tool for exporting and trimming WPILib DataLogs #7656

Draft
wants to merge 35 commits into
base: 2027
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
2c6129f
no fmt :/
DeltaDizzy Sep 11, 2024
1bd3ee3
add argparse
DeltaDizzy Sep 11, 2024
9b79798
turn stuff back on
DeltaDizzy Sep 11, 2024
1b17243
add cli commands
DeltaDizzy Sep 11, 2024
dc46ca8
notional log reader impl
DeltaDizzy Sep 12, 2024
7871d70
rename to datalogcli
DeltaDizzy Jan 5, 2025
c1780ed
remove local argparse
DeltaDizzy Jan 5, 2025
6cb318e
move ehader to indclude directory
DeltaDizzy Jan 5, 2025
f328cb3
use wpilib argparse
DeltaDizzy Jan 5, 2025
f62505e
apply datalog move to wpiutil
DeltaDizzy Jan 5, 2025
d7728f6
extract regards and stub writer methods
DeltaDizzy Jan 5, 2025
297ef33
need copy constructor
DeltaDizzy Jan 5, 2025
9ca69f7
just make the loader in situ
DeltaDizzy Jan 5, 2025
01a65a1
rename and add csv header
DeltaDizzy Jan 5, 2025
e718bc3
make writer a class
DeltaDizzy Jan 5, 2025
c27a70f
format
DeltaDizzy Jan 7, 2025
5468aee
add json writer files
DeltaDizzy Jan 7, 2025
125a5aa
rename to sawmill
DeltaDizzy Jan 7, 2025
8760e9e
add getallrecords
DeltaDizzy Jan 7, 2025
58774b2
format
DeltaDizzy Jan 8, 2025
2d346d1
undo root cmakelists changes
DeltaDizzy Jan 8, 2025
cde5f44
rename namespace to sawmill
DeltaDizzy Jan 8, 2025
6c048b7
parse data up front
DeltaDizzy Jan 8, 2025
4588ea6
add entry map to loader
DeltaDizzy Jan 8, 2025
6552add
use sawmill records
DeltaDizzy Jan 8, 2025
51f75aa
first attempt at csv writing
DeltaDizzy Jan 8, 2025
2c2b92b
use Entry struct from datalogtool
DeltaDizzy Jan 8, 2025
974d477
using namespace so dont need it in method definitions
DeltaDizzy Jan 8, 2025
e90a06b
emplace instead of []
DeltaDizzy Jan 8, 2025
44c1cab
remove json temporarily
DeltaDizzy Jan 8, 2025
6cf8c96
start cli args refactor
DeltaDizzy Jan 8, 2025
9602c9b
add datalog dependency
DeltaDizzy Feb 20, 2025
c620d32
remove resources stuff for now
DeltaDizzy Feb 20, 2025
2c01df1
fix datalog and sysid imports
DeltaDizzy Feb 20, 2025
0ae6071
start on editor
DeltaDizzy Feb 21, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,8 @@ if(WITH_NTCORE)
add_subdirectory(ntcore)
endif()

add_subdirectory(sawmill)

if(WITH_WPIMATH)
if(WITH_JAVA)
set(WPIUNITS_DEP_REPLACE ${WPIUNITS_DEP_REPLACE_IMPL})
Expand Down
27 changes: 27 additions & 0 deletions datalog/src/main/native/cpp/DataLogEditor.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#include "wpi/datalog/export/DataLogEditor.h"
#include <cstdint>
#include <vector>
#include "wpi/datalog/DataLogReaderThread.h"
#include "wpi/datalog/export/DataLogExportUtils.h"

namespace wpi::log {
DataLogEditor::DataLogEditor(const DataLogReaderThread& reader) {
for (auto record : reader.GetReader()) {
records.push_back(record);
}
}

DataLogEditor DataLogEditor::ExtractEntries(const std::vector<fileexport::Entry>& entries) {
for (const auto& entry : entries) {
allowedEntries.emplace(entry.id);
}
return *this;
}

DataLogEditor DataLogEditor::TrimToTime(uint64_t start, uint64_t end) {
endTime = end;
startTime = start;
return *this;
}
}

7 changes: 7 additions & 0 deletions datalog/src/main/native/cpp/export/DataLogCSVExporter.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#include "wpi/datalog/export/DataLogCSVExporter.h"
#include <memory>
#include "wpi/datalog/export/DataLogExportUtils.h"

namespace wpi::log::fileexport {

}
83 changes: 83 additions & 0 deletions datalog/src/main/native/cpp/export/DataLogExportUtils.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#include "wpi/datalog/export/DataLogExportUtils.h"
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <string_view>
#include <utility>
#include "fmt/format.h"
#include "wpi/MemoryBuffer.h"
#include "wpi/datalog/DataLogReader.h"
#include "wpi/datalog/DataLogReaderThread.h"
#include "wpi/mutex.h"
#include <wpi/fs.h>

static wpi::mutex entriesMutex;
static std::map<std::string, std::unique_ptr<wpi::log::fileexport::Entry>,
std::less<>>
entries;

namespace wpi::log::fileexport {
InputFile::InputFile(std::unique_ptr<wpi::log::DataLogReaderThread> datalog_)
: filename{datalog_->GetBufferIdentifier()},
stem{fs::path{filename}.stem().string()},
datalog{std::move(datalog_)} {
datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) {
// add this entry to the map
std::scoped_lock lock{entriesMutex};
auto it = entries.find(srd.name);
if (it == entries.end()) {
// this entry isnt in the map, so lets add it
it = entries.emplace(srd.name, std::make_unique<Entry>(srd)).first;
// if this is a gui make it rebuild the entry tree somehow lmfao
// maybe accept an optional callback???
// TODO: RebuildEntryTree
} else {
// this entry IS already known, so lets make sure the start records match
if (it->second->type != srd.type) {
it->second->typeConflict = true;
}
if (it->second->metadata != srd.metadata) {
it->second->metadataConflict = true;
}
it->second->inputFiles.emplace(this);
}
});
}

InputFile::~InputFile() {
if (!datalog) {
return;
}
std::scoped_lock lock{entriesMutex};
bool changed = false;
for (auto it = entries.begin(); it != entries.end();) {
it->second->inputFiles.erase(this);
if (it->second->inputFiles.empty()) {
it = entries.erase(it);
changed = true;
} else {
++it;
}
}
if (changed) {
// TODO: rebuildentrytree
}
}

std::unique_ptr<InputFile> LoadDataLog(std::string_view filename) {
auto fileBuffer = wpi::MemoryBuffer::GetFile(filename);
if (!fileBuffer) {
return std::make_unique<InputFile>(
filename, fmt::format("Could not open file: {}", fileBuffer.error()));
}

wpi::log::DataLogReader reader{std::move(*fileBuffer)};
if (!reader.IsValid()) {
return std::make_unique<InputFile>(filename, "Not a valid datalog file");
}

return std::make_unique<InputFile>(
std::make_unique<wpi::log::DataLogReaderThread>(std::move(reader)));
}
} // namespace wpi::log::fileexport
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
#include <iterator>
#include <memory>
#include <span>
#include <utility>
#include <vector>

#include <wpi/MemoryBuffer.h>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@

#pragma once

#include <wpi/datalog/DataLogReader.h>
#include <wpi/DenseMap.h>
#include <wpi/Signal.h>
#include <wpi/mutex.h>
#include <wpi/struct/DynamicStruct.h>

#include <atomic>
#include <functional>
#include <map>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#pragma once
#include <wpi/fs.h>
#include <string>
#include "wpi/datalog/DataLogReader.h"
#include "wpi/datalog/export/DataLogExportUtils.h"
#include "wpi/raw_ostream.h"

namespace wpi::log::fileexport {
class DataLogCSVExporter {
public:
static void Export(std::string inputFilePath, std::string exportFilePath);
private:
void WriteValue(wpi::raw_ostream& os, const Entry& entry,
const wpi::log::DataLogRecord& record);
void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str);
fs::path exportFile;
fs::path inputFile;
};
} // namespace wpi::log::fileexport
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#pragma once

#include <chrono>
#include <cstdint>
#include <map>
#include <set>
#include <string>
#include <string_view>
#include <vector>
#include "wpi/datalog/DataLogReader.h"
#include "wpi/datalog/DataLogReaderThread.h"
#include "wpi/datalog/export/DataLogExportUtils.h"

namespace wpi::log {
class DataLogEditor {
public:
explicit DataLogEditor(const DataLogReaderThread& reader);

DataLogEditor ExtractEntries(const std::vector<fileexport::Entry>& entries);
DataLogEditor TrimToTime(uint64_t startTime, uint64_t endTime);
DataLogEditor RenameEntry(std::string_view currentName, std::string newName);
void ApplyEdits();

private:
std::vector<DataLogRecord> records;
uint64_t startTime;
uint64_t endTime;
std::set<uint64_t> allowedEntries;
std::map<std::string, std::string> entryRenames;
};
} // namespace wpi::log
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#pragma once

#include <memory>
#include <string>
#include <string_view>
#include "wpi/datalog/DataLogReaderThread.h"
#include "wpi/fs.h"

namespace wpi::log::fileexport {
struct InputFile {
explicit InputFile(std::unique_ptr<wpi::log::DataLogReaderThread> datalog);

InputFile(std::string_view filename, std::string_view status)
: filename{filename},
stem{fs::path{filename}.stem().string()},
status{status} {}

~InputFile();

std::string filename;
std::string stem;
std::unique_ptr<wpi::log::DataLogReaderThread> datalog;
std::string status;
bool highlight = false;
};

struct Entry {
explicit Entry(const wpi::log::StartRecordData& srd)
: name{srd.name}, type{srd.type}, metadata{srd.metadata} {}

std::string name;
std::string type;
std::string metadata;
uint64_t id;
std::set<InputFile*> inputFiles;
bool typeConflict = false;
bool metadataConflict = false;
bool selected = true;

// used only during export
int column = -1;
};

struct EntryTreeNode {
explicit EntryTreeNode(std::string_view name) : name{name} {}
std::string name; // name of just this node
std::string path; // full path if entry is nullptr
Entry* entry = nullptr;
std::vector<EntryTreeNode> children; // children, sorted by name
int selected = 1;
};

std::unique_ptr<InputFile> LoadDataLog(std::string_view filename);
} // namespace wpi::log
33 changes: 33 additions & 0 deletions sawmill/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
project(sawmill)

include(CompileWarnings)
include(GenResources)

configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp)
# generate_resources(src/main/native/resources generated/main/cpp DLT dlt sawmill_resources_src)

# Generate compile_commands.json by default
if(NOT CMAKE_EXPORT_COMPILE_COMMANDS)
set(CMAKE_EXPORT_COMPILE_COMMANDS "YES" CACHE STRING "" FORCE)
endif()

file(GLOB sawmill_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp)

if(WIN32)
# set(sawmill_rc src/main/native/win/sawmill.rc)
elseif(APPLE)
# set(MACOSX_BUNDLE_ICON_FILE sawmill.icns)
# set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources")
endif()

add_executable(sawmill ${sawmill_src}) #${sawmill_resources_src} ${sawmill_rc} ${APP_ICON_MACOSX})

target_include_directories(sawmill PUBLIC src/main/native/include)

target_link_libraries(sawmill PRIVATE wpiutil datalog)

if(WIN32)
set_target_properties(sawmill PROPERTIES WIN32_EXECUTABLE YES)
elseif(APPLE)
set_target_properties(sawmill PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "sawmill")
endif()
7 changes: 7 additions & 0 deletions sawmill/src/main/generate/WPILibVersion.cpp.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
/**
* Autogenerated file! Do not manually edit this file. This version is regenerated
* any time the publish task is run, or when this file is deleted.
*/
const char* GetWPILibVersion() {
return "${wpilib_version}";
}
Loading
Loading