XsensDataReader Improvements, Delimiter Detection, and Tests

Author: alexbeattie42

CHANGELOG.md

@@ -8,6 +8,9 @@ This is not a comprehensive list of changes but rather a hand-curated collection
 
 v4.6
 =====
+- Improvements for the `XsensDataReader`. Add a configuration option to XSensDataReaderSettings to specify a known data rate (sampling frequency). Automatically detect the delimiter used in the file. Support the new Xsens export rotations formats (Rotation Matrix, Quaternion, or Euler angles) values from Xsens files. Update the parser to handle the path separator for data_folder and fix a memory leak. Verify integrity and uniformity of all files. Add tests with additional new and old Xsens formats. (#4063)
+
+
 
 
 v4.5.2

OpenSim/Common/CommonUtilities.cpp

@@ -31,6 +31,7 @@
 #include <iomanip>
 #include <memory>
 #include <sstream>
+#include <unordered_map>
 
 #include <SimTKcommon/internal/Pathname.h>
 
@@ -69,6 +70,32 @@ std::string OpenSim::getFormattedDateTime(
     return ss.str();
 }
 
+std::string OpenSim::detectDelimiter(
+        const std::string& input, const std::vector<std::string>& delimiters) {
+
+    std::unordered_map<std::string, std::size_t> counts;
+
+    // Count occurrences of common delimiters in the input string
+    std::transform(delimiters.begin(), delimiters.end(),
+            std::inserter(counts, counts.end()), [&input](const std::string& d) {
+                std::size_t count = 0;
+                std::size_t pos = 0;
+                // Find all occurrences of delimiter in input string
+                while ((pos = input.find(d, pos)) != std::string::npos) {
+                    ++count;
+                    pos += d.length(); // Move past the current delimiter
+                }
+                return std::pair<std::string, std::size_t>(d, count);
+            });
+
+    // Find the delimiter with the highest frequency
+    auto maxElem = std::max_element(counts.begin(), counts.end(),
+            [](const auto& a, const auto& b) { return a.second < b.second; });
+
+    // If a delimiter is found, return it, otherwise return an empty string
+    return (maxElem != counts.end() && maxElem->second > 0) ? maxElem->first : "";
+}
+
 SimTK::Vector OpenSim::createVectorLinspace(
         int length, double start, double end) {
     SimTK::Vector v(length);

OpenSim/Common/CommonUtilities.h

@@ -27,14 +27,15 @@
 #include "Assertion.h"
 #include <algorithm>
 #include <cmath>
+#include <condition_variable>
 #include <functional>
 #include <iostream>
 #include <memory>
 #include <mutex>
 #include <numeric>
 #include <stack>
-#include <condition_variable>
 #include <utility>
+#include <vector>
 
 #include <SimTKcommon/internal/BigMatrix.h>
 
@@ -121,6 +122,25 @@ OSIMCOMMON_API
 SimTK::Vector createVector(std::initializer_list<SimTK::Real> elements);
 #endif
 
+/**
+ * @brief Detects the most likely string delimiter in an input string.
+ *
+ * This function identifies the most frequent delimiter from a predefined list
+ * of common delimiters based on occurrences in the input string.
+ *
+ * @param input The string to analyze.
+ * @param delimiters A vector of candidate delimiters.
+ *
+ * @return: The most likely delimiter (as an std::string),
+ * or an empty string if none was found.
+ *
+ * @note Supports single-character delimiters.
+ */
+ /// @ingroup commonutil
+OSIMCOMMON_API
+std::string detectDelimiter(
+        const std::string& input, const std::vector<std::string>& delimiters);
+
 /**
  * @brief Checks if the elements of a vector are uniformly spaced.
  *

OpenSim/Common/FileAdapter.h

@@ -166,9 +166,12 @@ class TableMissingHeader : public Exception {
 public:
     TableMissingHeader(const std::string& file,
                        size_t line,
-                       const std::string& func) :
+                       const std::string& func,
+                       const std::string& message = "") :
         Exception(file, line, func) {
         std::string msg = "Table does not have metadata for 'header'.";
+        if(!message.empty())
+            msg += " " + message;
 
         addMessage(msg);
     }

OpenSim/Common/Test/testCommonUtilities.cpp

@@ -261,3 +261,49 @@ TEST_CASE("isUniform tests with createVectorLinspace(SimTK::Vector) and createVe
         REQUIRE_THAT(result_simtk.second, Catch::Matchers::WithinAbs(rate, tol));
     }
 }
+
+TEST_CASE("detectDelimiter") {
+    const std::vector<std::string> delimiters = {",", ";", "|", "\t", ":", " "};
+    SECTION("Comma") {
+        std::string input = "a,b,c,d";
+        auto delim = detectDelimiter(input, delimiters);
+        REQUIRE(delim == ",");
+    }
+    
+    SECTION("Pipe") {
+        std::string input = "a|b|c|d";
+        auto delim = detectDelimiter(input, delimiters);
+        REQUIRE(delim == "|");
+    }
+    
+    SECTION("Tab") {
+        std::string input = "a\tb\tc\td";
+        auto delim = detectDelimiter(input, delimiters);
+        REQUIRE(delim == "\t");
+    }
+    
+    SECTION("Semicolon") {
+        std::string input = "a;b;c;d";
+        auto delim = detectDelimiter(input, delimiters);
+        REQUIRE(delim == ";");
+    }
+    
+    SECTION("Space") {
+        std::string input = "a b c d";
+        auto delim = detectDelimiter(input, delimiters);
+        REQUIRE(delim == " ");
+    }
+    
+    SECTION("No Valid Delimiter") {
+        std::string input = "abcd";
+        auto delim = detectDelimiter(input, delimiters);
+        REQUIRE(delim == "");
+    }
+    
+    SECTION("Delimiter Exclusion") {
+        std::vector<std::string> small_delimiters = {",", ";"};
+        std::string input = "a|b|c|d";
+        auto delim = detectDelimiter(input, small_delimiters);
+        REQUIRE(delim == "");
+    }    
+}