CdxWriter: add sort option

Author: ato

CHANGELOG.md

@@ -5,7 +5,9 @@
 ### New features
 
 - CdxRecord: surt(), format(), values() and toString()
-- CdxWriter: CDXJ output support
+- CdxWriter
+  - CDXJ output support
+  - sort option
 - HttpMessage: `Content-Encoding: zstd` support
 - HttpRequest: `Content-Encoding: chunked` support
 - WarcReader: [Zstandard compressed WARC Files](https://iipc.github.io/warc-specifications/specifications/warc-zstd/) support

src/org/netpreserve/jwarc/cdx/CdxWriter.java

@@ -10,6 +10,7 @@
 import java.io.Writer;
 import java.net.URI;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
@@ -25,6 +26,7 @@ public class CdxWriter implements Closeable {
     private boolean postAppend = false;
     private Predicate<WarcRecord> recordFilter;
     private Consumer<String> warningHandler;
+    private List<String> sortBuffer;
 
     public CdxWriter(Writer writer) {
         this.writer = writer;
@@ -58,8 +60,16 @@ public void write(WarcCaptureRecord capture, String filename,
             String rawUrlKey = capture.target() + (capture.target().contains("?") ? '&' : '?') + encodedRequest;
             urlKey = URIs.toNormalizedSurt(rawUrlKey);
         }
-        writer.write(format.format(capture, filename, position, length, urlKey));
-        writer.write('\n');
+        writeLine(format.format(capture, filename, position, length, urlKey));
+    }
+
+    private void writeLine(String line) throws IOException {
+        if (sortBuffer != null) {
+            sortBuffer.add(line);
+        } else {
+            writer.write(line);
+            writer.write('\n');
+        }
     }
 
     /**
@@ -166,8 +176,24 @@ public void onWarning(Consumer<String> warningHandler) {
         this.warningHandler = warningHandler;
     }
 
+    public void setSort(boolean sort) {
+        if (sort) {
+            sortBuffer = new ArrayList<>();
+        } else {
+            sortBuffer = null;
+        }
+    }
+
     @Override
     public void close() throws IOException {
+        if (sortBuffer != null) {
+            sortBuffer.sort(null);
+            for (String line : sortBuffer) {
+                writer.write(line);
+                writer.write('\n');
+            }
+            sortBuffer = null;
+        }
         writer.close();
     }
 }

src/org/netpreserve/jwarc/tools/CdxTool.java

@@ -31,6 +31,7 @@ public static void main(String[] args) throws IOException {
         boolean fullFilePath = false;
         boolean postAppend = false;
         boolean digestUnchanged = false;
+        boolean sort = false;
         Predicate<WarcRecord> filter = null;
         for (int i = 0; i < args.length; i++) {
             if (args[i].startsWith("-")) {
@@ -86,6 +87,10 @@ public static void main(String[] args) throws IOException {
                 case "--revisits-excluded":
                     filter = record -> !(record instanceof WarcRevisit);
                     break;
+                case "-s":
+                case "--sort":
+                    sort = true;
+                    break;
                 case "-w":
                 case "--warc-full-path":
                     fullFilePath = true;
@@ -112,6 +117,7 @@ public static void main(String[] args) throws IOException {
             cdxWriter.setFormat(format);
             cdxWriter.setPostAppend(postAppend);
             cdxWriter.setRecordFilter(filter);
+            cdxWriter.setSort(sort);
 
             if (printHeader) cdxWriter.writeHeaderLine();
             cdxWriter.process(files, fullFilePath);

test/org/netpreserve/jwarc/cdx/CdxWriterTest.java

@@ -28,26 +28,28 @@ public void test() throws IOException {
             HttpResponse httpResponse = new HttpResponse.Builder(404, "Not Found")
                     .body(MediaType.HTML, new byte[0])
                     .build();
-            warcWriter.write(new WarcResponse.Builder("http://example.org/")
-                    .date(Instant.parse("2022-03-01T12:44:34Z"))
+            warcWriter.write(new WarcRevisit.Builder("http://example.org/")
+                    .date(Instant.parse("2022-03-02T21:44:34Z"))
                     .body(httpResponse)
                     .payloadDigest("sha256", "b04af472c47a8b1b5059b3404caac0e1bfb5a3c07b329be66f65cfab5ee8d3f3")
                     .build());
-            warcWriter.write(new WarcRevisit.Builder("http://example.org/")
-                    .date(Instant.parse("2022-03-02T21:44:34Z"))
+            warcWriter.write(new WarcResponse.Builder("http://example.org/")
+                    .date(Instant.parse("2022-03-01T12:44:34Z"))
                     .body(httpResponse)
                     .payloadDigest("sha256", "b04af472c47a8b1b5059b3404caac0e1bfb5a3c07b329be66f65cfab5ee8d3f3")
                     .build());
         }
 
         StringWriter cdxBuffer = new StringWriter();
-        CdxWriter cdxWriter = new CdxWriter(cdxBuffer);
-        cdxWriter.setFormat(new CdxFormat.Builder().digestUnchanged().build());
-        cdxWriter.writeHeaderLine();
-        cdxWriter.process(Collections.singletonList(testWarcFile), true);
+        try (CdxWriter cdxWriter = new CdxWriter(cdxBuffer)) {
+            cdxWriter.setFormat(new CdxFormat.Builder().digestUnchanged().build());
+            cdxWriter.setSort(true);
+            cdxWriter.writeHeaderLine();
+            cdxWriter.process(Collections.singletonList(testWarcFile), true);
+        }
         assertEquals(" CDX N b a m s k r M S V g\n" +
-                "org,example)/ 20220301124434 http://example.org/ text/html 404 sha256:WBFPI4WEPKFRWUCZWNAEZKWA4G73LI6APMZJXZTPMXH2WXXI2PZQ==== - - 398 0 " + testWarcFile + "\n" +
-                "org,example)/ 20220302214434 http://example.org/ warc/revisit 404 sha256:WBFPI4WEPKFRWUCZWNAEZKWA4G73LI6APMZJXZTPMXH2WXXI2PZQ==== - - 397 398 " + testWarcFile + "\n",
+                "org,example)/ 20220301124434 http://example.org/ text/html 404 sha256:WBFPI4WEPKFRWUCZWNAEZKWA4G73LI6APMZJXZTPMXH2WXXI2PZQ==== - - 398 397 " + testWarcFile + "\n" +
+                "org,example)/ 20220302214434 http://example.org/ warc/revisit 404 sha256:WBFPI4WEPKFRWUCZWNAEZKWA4G73LI6APMZJXZTPMXH2WXXI2PZQ==== - - 397 0 " + testWarcFile + "\n",
                 cdxBuffer.toString());
     }