tsv-filter: Add an output buffer. Performance enhancement with narrow inputs with match rates. (#115)

Author: jondegenhardt

tsv-filter/src/tsv-filter.d

@@ -787,6 +787,21 @@ void tsvFilter(in TsvFilterOptions cmdopt, in string[] inputFiles)
     import std.range;
     import tsvutil : throwIfWindowsNewlineOnUnix;
 
+    /* An output buffer. Improves performance on narrow files with high percentages of
+     * writes. Want responsive output if output is rare, so ensure the first matched
+     * line is written, and that writes separated by long stretches of non-matched lines
+     * are written.
+     */
+    enum bufferReserveSize = 11264;
+    enum bufferFlushSize = 10240;
+    enum maxInputLinesWithoutBufferFlush = 1024;
+    size_t inputLinesWithoutBufferFlush = maxInputLinesWithoutBufferFlush + 1;
+
+    auto outputBuffer = appender!(char[]);
+    outputBuffer.reserve(bufferReserveSize);
+
+    scope(exit) if (outputBuffer.data.length > 0) write(outputBuffer.data);
+
     /* Process each input file, one line at a time. */
     auto lineFields = new char[][](cmdopt.maxFieldIndex + 1);
     bool headerWritten = false;
@@ -801,7 +816,8 @@ void tsvFilter(in TsvFilterOptions cmdopt, in string[] inputFiles)
                 /* Header. Output on the first file, skip subsequent files. */
                 if (!headerWritten)
                 {
-                    writeln(line);
+                    outputBuffer.put(line);
+                    outputBuffer.put('\n');
                     headerWritten = true;
                 }
             }
@@ -840,11 +856,23 @@ void tsvFilter(in TsvFilterOptions cmdopt, in string[] inputFiles)
                  */
                 try
                 {
+                    inputLinesWithoutBufferFlush++;
                     bool passed = cmdopt.disjunct ?
                         cmdopt.tests.any!(x => x(lineFields)) :
                         cmdopt.tests.all!(x => x(lineFields));
                     if (cmdopt.invert) passed = !passed;
-                    if (passed) writeln(line);
+                    if (passed)
+                    {
+                        outputBuffer.put(line);
+                        outputBuffer.put('\n');
+                        if (outputBuffer.data.length >= bufferFlushSize ||
+                            inputLinesWithoutBufferFlush > maxInputLinesWithoutBufferFlush)
+                        {
+                            write(outputBuffer.data);
+                            outputBuffer.clear;
+                            inputLinesWithoutBufferFlush = 0;
+                        }
+                    }
                 }
                 catch (Exception exc)
                 {

tsv-filter/tests/gold/basic_tests_1.txt

@@ -1231,6 +1231,15 @@ f1	f2	f3
 3x2-r2	2002	3002
 3x1-r1	201	301
 
+====[seq 100000 | tsv-filter --or --eq 1:1000 --eq 1:1100 --eq 1:5000 --eq 1:10000 --ge 1:70000 | wc -l | tr -d ' ']====
+30005
+
+====[seq 100000 | tsv-filter --le 1:20 | wc -l | tr -d ' ']====
+20
+
+====[seq 100000 | tsv-filter --or --le 1:20 --eq 1:50000 --eq 1:50001 | wc -l | tr -d ' ']====
+22
+
 Help and Version printing 1
 -----------------
 

tsv-filter/tests/tests.sh

@@ -231,6 +231,19 @@ cat input_3x2.tsv | ${prog} --header --ge 2:23 >> ${basic_tests_1} 2>&1
 echo "" >> ${basic_tests_1}; echo "====[cat input_3x2.tsv | tsv-filter --header --ge 2:23 -- input_3x3.tsv - input_3x1.tsv]====" >> ${basic_tests_1}
 cat input_3x2.tsv | ${prog} --header --ge 2:23 -- input_3x3.tsv - input_3x1.tsv >> ${basic_tests_1} 2>&1
 
+## These tests are for the output buffering cases.
+echo "" >> ${basic_tests_1};
+echo "====[seq 100000 | tsv-filter --or --eq 1:1000 --eq 1:1100 --eq 1:5000 --eq 1:10000 --ge 1:70000 | wc -l | tr -d ' ']====" >> ${basic_tests_1}
+seq 100000 | ${prog} --or --eq 1:1000 --eq 1:1100 --eq 1:5000 --eq 1:10000 --ge 1:70000 | wc -l | tr -d ' ' >> ${basic_tests_1} 2>&1
+
+echo "" >> ${basic_tests_1};
+echo "====[seq 100000 | tsv-filter --le 1:20 | wc -l | tr -d ' ']====" >> ${basic_tests_1}
+seq 100000 | ${prog} --le 1:20 | wc -l | tr -d ' ' >> ${basic_tests_1} 2>&1
+
+echo "" >> ${basic_tests_1};
+echo "====[seq 100000 | tsv-filter --or --le 1:20 --eq 1:50000 --eq 1:50001 | wc -l | tr -d ' ']====" >> ${basic_tests_1}
+seq 100000 | ${prog} --or --le 1:20 --eq 1:50000 --eq 1:50001 | wc -l | tr -d ' ' >> ${basic_tests_1} 2>&1
+
 ## Help and Version printing
 
 echo "" >> ${basic_tests_1}