CDXJ format: omit status and mime when unknown

Matches webrecorder/cdxj-indexer.

Author: ato

src/org/netpreserve/jwarc/cdx/CdxFormat.java

@@ -119,7 +119,20 @@ private void formatJsonBlock(WarcCaptureRecord record, String filename, long pos
                         value = PYWB_REVISIT_MIMETYPE;
                     } else {
                         try {
-                            value = record.payload().map(p -> p.type().base()).orElse(MediaType.OCTET_STREAM).toString();
+                            if (record instanceof WarcResponse &&
+                                    record.contentType().equals(MediaType.HTTP_RESPONSE)) {
+                                value = ((WarcResponse) record).http().headers().first("Content-Type")
+                                        .map(s -> MediaType.parseLeniently(s).base().toString())
+                                        .orElse(null);
+                            } else if (record instanceof WarcResource) {
+                                value = record.headers().first("Content-Type")
+                                        .map(s -> MediaType.parseLeniently(s).base().toString())
+                                        .orElse(null);
+                            } else {
+                                value = record.payload()
+                                        .map(p -> p.type().base())
+                                        .map(Object::toString).orElse(null);
+                            }
                         } catch (IOException e) {
                             value = null;
                         }
@@ -130,7 +143,8 @@ private void formatJsonBlock(WarcCaptureRecord record, String filename, long pos
                     break;
                 case "status":
                     try {
-                        value = String.valueOf(statusCode(record));
+                        Integer status = statusCode(record);
+                        value = status == 0 ? null : String.valueOf(status);
                     } catch (IOException e) {
                         value = null;
                     }
@@ -184,7 +198,7 @@ private static int statusCode(WarcCaptureRecord record) throws IOException {
                 return ((WarcResponse) record).gemini().statusHttpEquivalent();
             }
         }
-        return 200;
+        return 0;
     }
 
     String formatField(byte fieldName, WarcCaptureRecord record, String filename, long position, long size, String urlkey) throws IOException {
@@ -223,17 +237,9 @@ String formatField(byte fieldName, WarcCaptureRecord record, String filename, lo
                     return "-";
                 }
             case RESPONSE_CODE:
-                if (record instanceof WarcResponse || record instanceof WarcRevisit) {
-                    if (record instanceof WarcRevisit) {
-                        return Integer.toString(((WarcRevisit) record).http().status());
-                    }
-                    else if (record.contentType().base().equals(MediaType.HTTP)) {
-                        return Integer.toString(((WarcResponse) record).http().status());
-                    } else if (record.contentType().base().equals(MediaType.GEMINI)) {
-                        return String.format("%02d", ((WarcResponse) record).gemini().statusHttpEquivalent());
-                    }
-                }
-                return Integer.toString(statusCode(record));
+                int status = statusCode(record);
+                if (status == 0) status = 200;
+                return Integer.toString(status);
             default:
                 throw new IllegalArgumentException("Unknown CDX field: " + (char) fieldName);
         }

test/org/netpreserve/jwarc/cdx/CdxFormatTest.java

@@ -1,12 +1,10 @@
 package org.netpreserve.jwarc.cdx;
 
 import org.junit.Test;
-import org.netpreserve.jwarc.HttpResponse;
-import org.netpreserve.jwarc.MediaType;
-import org.netpreserve.jwarc.WarcResponse;
-import org.netpreserve.jwarc.WarcRevisit;
+import org.netpreserve.jwarc.*;
 
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Instant;
@@ -129,4 +127,16 @@ public void testCdxj() throws Exception {
         assertEquals("org,example)/ 20220302214434 {\"url\": \"http://example.org/\", \"mime\": \"text/html\", \"status\": \"404\", \"digest\": \"sha1:AQLNJ7DOPHK477BWWC726H7Y5XBPBNF7\", \"length\": \"456\", \"offset\": \"123\", \"filename\": \"example.warc.gz\"}",
                 CdxFormat.CDXJ.format(response, path.getFileName().toString(), 123, 456));
     }
+
+    @Test
+    public void testCdxjResource() throws Exception {
+        Path path = Paths.get("/home/jwarc/resource.warc.gz");
+        WarcResource resource = new WarcResource.Builder(URI.create("http://example.org/"))
+                .date(Instant.parse("2022-03-02T21:44:34Z"))
+                .payloadDigest("sha1", "AQLNJ7DOPHK477BWWC726H7Y5XBPBNF7")
+                .body(null, new byte[0])
+                .build();
+        assertEquals("org,example)/ 20220302214434 {\"url\": \"http://example.org/\", \"digest\": \"sha1:AQLNJ7DOPHK477BWWC726H7Y5XBPBNF7\", \"length\": \"456\", \"offset\": \"123\", \"filename\": \"resource.warc.gz\"}",
+                CdxFormat.CDXJ.format(resource, path.getFileName().toString(), 123, 456));
+    }
 }