Merge pull request #2399 from opensim-org/speedUpDelimFileAdapter

Speed up DelimFileAdapter

Author: aseth1

OpenSim/Auxiliary/auxiliaryTestFunctions.h

@@ -27,7 +27,6 @@
 #include <OpenSim/Common/Function.h>
 #include <OpenSim/Common/LinearFunction.h>
 #include <OpenSim/Common/PropertyObjArray.h>
-#include "OpenSim/Common/STOFileAdapter.h"
 #include "getRSS.h"
 
 #include <fstream>
@@ -333,26 +332,6 @@ inline bool revertToVersionNumber1(const std::string& filenameOld,
     return changedVersion;
 }
 
-// Add number of rows (nRows) and number of columns (nColumns) to the header of
-// the STO file. Note that nColumns will include time, so it will be number of
-// columns in the matrix plus 1 (for time).
-inline void addNumRowsNumColumns(const std::string& filenameOld,
-                                 const std::string& filenameNew) {
-    auto table = OpenSim::STOFileAdapter_<double>::read(filenameOld);
-    std::regex endheader{ R"( *endheader *)" };
-    std::ifstream fileOld{ filenameOld };
-    std::ofstream fileNew{ filenameNew };
-    std::string line{};
-    while (std::getline(fileOld, line)) {
-        if (std::regex_match(line, endheader))
-            fileNew << "nRows="    << table.getNumRows()        << "\n"
-                    << "nColumns=" << table.getNumColumns() + 1 << "\n"
-                    << "endheader\n";
-        else
-            fileNew << line << "\n";
-    }
-}
-
 // Estimate the memory usage of a *creator* that heap allocates an object
 // of type C and returns a pointer to it. Creator can also perform any 
 // initialization before returning the pointer.

OpenSim/Common/C3DFileAdapter.cpp

@@ -136,23 +136,24 @@ C3DFileAdapter::extendRead(const std::string& fileName) const {
         }
 
         // Create the data
-        auto& marker_table = *new 
-            TimeSeriesTableVec3(marker_times, marker_matrix, marker_labels);
+        auto marker_table = 
+            std::make_shared<TimeSeriesTableVec3>(marker_times, 
+                                                  marker_matrix, 
+                                                  marker_labels);
 
-        marker_table.
+        marker_table->
             updTableMetaData().
             setValueForKey("DataRate",
                 std::to_string(acquisition->GetPointFrequency()));
 
-        marker_table.
+        marker_table->
             updTableMetaData().
             setValueForKey("Units",
                 acquisition->GetPointUnit());
 
-        marker_table.updTableMetaData().setValueForKey("events", event_table);
+        marker_table->updTableMetaData().setValueForKey("events", event_table);
 
-        tables.emplace(_markers,
-            std::shared_ptr<TimeSeriesTableVec3>(&marker_table));
+        tables.emplace(_markers, marker_table);
     }
 
     // This is probably the right way to get the raw forces data from force 

OpenSim/Common/DelimFileAdapter.h

@@ -319,14 +319,13 @@ DelimFileAdapter<T>::extendRead(const std::string& fileName) const {
                      FileIsEmpty,
                      fileName);
 
-    auto table = std::make_shared<TimeSeriesTable_<T>>();
-
     size_t line_num{};
     // All the lines until "endheader" is header.
     std::regex endheader{R"([ \t]*)" + _endHeaderString + R"([ \t]*)"};
     std::regex keyvalue{R"((.*)=(.*))"};
     std::string header{};
     std::string line{};
+    ValueArrayDictionary keyValuePairs;
     while(std::getline(in_stream, line)) {
         ++line_num;
 
@@ -361,7 +360,7 @@ DelimFileAdapter<T>::extendRead(const std::string& fileName) const {
                     // Discard OpenSim version number. Version number is added
                     // during writing.
                 } else {
-                    table->updTableMetaData().setValueForKey(key, value);
+                    keyValuePairs.setValueForKey(key, value);
                 }
                 continue;
             }
@@ -372,7 +371,7 @@ DelimFileAdapter<T>::extendRead(const std::string& fileName) const {
         else
             header += "\n" + line;
     }
-    table->updTableMetaData().setValueForKey("header", header);
+    keyValuePairs.setValueForKey("header", header);
 
     // Callable to get the next line in form of vector of tokens.
     auto nextLine = [&] {
@@ -400,39 +399,62 @@ DelimFileAdapter<T>::extendRead(const std::string& fileName) const {
                      _timeColumnLabel,
                      column_labels[0]);
     column_labels.erase(column_labels.begin());
-    // Set the column labels as metadata.
-    ValueArray<std::string> value_array{};
-    for(const auto& cl : column_labels)
-        value_array.upd().push_back(SimTK::Value<std::string>{cl});
-    typename TimeSeriesTable_<T>::DependentsMetaData dep_metadata{};
-    dep_metadata.setValueArrayForKey("labels", value_array);
-    table->setDependentsMetaData(dep_metadata);
 
     // Read the rows one at a time and fill up the time column container and
-    // the data container.
+    // the data container. Start with a reasonable initial capacity for
+    // tradeoff between a small file and larger files. 100 worked well for
+    // a 50 MB file with ~80000 lines.
+    std::vector<double> timeVec;
+    int initCapacity = 100;
+    int ncol = static_cast<int>(column_labels.size());
+    timeVec.reserve(initCapacity);
+    SimTK::Matrix_<T> matrix(initCapacity, ncol);
+    
+    // Initialize current row and capacity
+    int curCapacity = initCapacity;
+    int curRow = 0;
+
+    // Start looping through each line
     auto row = nextLine();
-    while(!row.empty()) {
+    while (!row.empty()) {
         ++line_num;
+        
+        // Double capacity if we reach the end of the containers.
+        // This is necessary until Simbody issue #401 is addressed.
+        if (curRow+1 > curCapacity) {
+            curCapacity *= 2;
+            timeVec.reserve(curCapacity);
+            matrix.resizeKeep(curCapacity, ncol);
+        }
 
         // Time is column 0.
-        double time = std::stod(row.front());
+        timeVec.push_back(std::stod(row.front()));
         row.erase(row.begin());
 
         auto row_vector = readElems(row);
 
         OPENSIM_THROW_IF(row_vector.size() != column_labels.size(),
-                         RowLengthMismatch,
-                         fileName,
-                         line_num,
-                         column_labels.size(),
-                         static_cast<size_t>(row_vector.size()));
-
-        // Column 1 is time.
-        table->appendRow(time, std::move(row_vector));
+            RowLengthMismatch,
+            fileName,
+            line_num,
+            column_labels.size(),
+            static_cast<size_t>(row_vector.size()));
+        
+        matrix.updRow(curRow) = std::move(row_vector);
 
         row = nextLine();
+        ++curRow;
     }
 
+    // Resize the matrix down to the correct number of rows.
+    // This is necessary until Simbody issue #401 is addressed.
+    matrix.resizeKeep(curRow, ncol);
+
+    // Create the table and update other metadata from above
+    auto table = 
+        std::make_shared<TimeSeriesTable_<T>>(timeVec, matrix, column_labels);
+    table->updTableMetaData() = keyValuePairs;
+
     OutputTables output_tables{};
     output_tables.emplace(tableString(), table);
 

OpenSim/Common/MarkerData.cpp

@@ -33,6 +33,7 @@
 #include "SimmMacros.h"
 #include "Storage.h"
 #include "OpenSim/Auxiliary/auxiliaryTestFunctions.h"
+#include "OpenSim/Common/STOFileAdapter.h"
 
 //=============================================================================
 // STATICS
@@ -41,6 +42,29 @@ using namespace std;
 using namespace OpenSim;
 using SimTK::Vec3;
 
+//=============================================================================
+// HELPER FUNCTIONS
+//=============================================================================
+// Add number of rows (nRows) and number of columns (nColumns) to the header of
+// the STO file. Note that nColumns will include time, so it will be number of
+// columns in the matrix plus 1 (for time).
+inline void addNumRowsNumColumns(const std::string& filenameOld,
+    const std::string& filenameNew) {
+    auto table = OpenSim::STOFileAdapter_<double>::read(filenameOld);
+    std::regex endheader{ R"( *endheader *)" };
+    std::ifstream fileOld{ filenameOld };
+    std::ofstream fileNew{ filenameNew };
+    std::string line{};
+    while (std::getline(fileOld, line)) {
+        if (std::regex_match(line, endheader))
+            fileNew << "nRows=" << table.getNumRows() << "\n"
+            << "nColumns=" << table.getNumColumns() + 1 << "\n"
+            << "endheader\n";
+        else
+            fileNew << line << "\n";
+    }
+}
+
 //=============================================================================
 // CONSTRUCTOR(S) AND DESTRUCTOR
 //=============================================================================

OpenSim/Common/Test/testC3DFileAdapter.cpp

@@ -22,6 +22,7 @@
 
 #include "OpenSim/Common/C3DFileAdapter.h"
 #include "OpenSim/Common/TRCFileAdapter.h"
+#include "OpenSim/Common/STOFileAdapter.h"
 #include <OpenSim/Auxiliary/auxiliaryTestFunctions.h>
 
 #include <vector>

OpenSim/Common/Test/testMarkerData.cpp

@@ -24,6 +24,7 @@
 #include <OpenSim/Common/Storage.h>
 #include <OpenSim/Common/MarkerData.h>
 #include <OpenSim/Auxiliary/auxiliaryTestFunctions.h>
+#include <OpenSim/Common/STOFileAdapter.h>
 
 #include <unordered_set>
 

OpenSim/Common/Test/testStorage.cpp

@@ -23,6 +23,7 @@
 #include <fstream>
 #include <OpenSim/Common/Storage.h>
 #include <OpenSim/Auxiliary/auxiliaryTestFunctions.h>
+#include <OpenSim/Common/STOFileAdapter.h>
 
 using namespace OpenSim;
 using namespace std;