WaczWriter: add support for generating pages.jsonl

ato · ato · commit e1130464bacd · 2025-12-27T23:07:36.000+09:00
diff --git a/src/org/netpreserve/jwarc/JsonPagesWriter.java b/src/org/netpreserve/jwarc/JsonPagesWriter.java
@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (C) 2025 National Library of Australia and the jwarc contributors
+ */
+
+package org.netpreserve.jwarc;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Writes pages.jsonl records.
+ * @see <a href="https://specs.webrecorder.net/wacz/latest/#pages-jsonl">WACZ spec</a>
+ */
+public class JsonPagesWriter implements Closeable {
+    private final Writer writer;
+    private boolean headerWritten = false;
+
+    public JsonPagesWriter(Writer writer) throws IOException {
+        this.writer = writer;
+    }
+
+    public void process(WarcReader reader) throws IOException {
+        for (WarcRecord record : reader) {
+            if (record instanceof WarcResponse || record instanceof WarcResource) {
+                WarcCaptureRecord capture = (WarcCaptureRecord) record;
+                MediaType type = capture.payloadType();
+                if (type.base().equals(MediaType.HTML) || type.base().equals(MediaType.XHTML)) {
+                    addPage(capture.target(), capture.date().toString(), null);
+                }
+            }
+        }
+    }
+
+    /**
+     * Writes the mandatory header line for pages.jsonl.
+     */
+    private void writeHeader() throws IOException {
+        Map<String, Object> header = new LinkedHashMap<>();
+        header.put("format", "json-pages-1.0");
+        header.put("id", "pages");
+        header.put("title", "All Pages");
+        Json.write(writer, header);
+        writer.write("\n");
+    }
+
+    /**
+     * Adds a page entry to pages.jsonl.
+     *
+     * @param url   the URL of the page
+     * @param ts    the timestamp in RFC3339 format
+     * @param title an optional title for the page
+     */
+    public void addPage(String url, String ts, String title) throws IOException {
+        if (!headerWritten) {
+            writeHeader();
+            headerWritten = true;
+        }
+        Map<String, Object> page = new LinkedHashMap<>();
+        page.put("url", url);
+        page.put("ts", ts);
+        if (title != null) page.put("title", title);
+        // Add other optional fields (id, text, size) as needed
+        Json.write(writer, page);
+        writer.write("\n");
+    }
+
+    @Override
+    public void close() throws IOException {
+        writer.close();
+    }
+
+    public void flush() throws IOException {
+        writer.flush();
+    }
+}
diff --git a/src/org/netpreserve/jwarc/MediaType.java b/src/org/netpreserve/jwarc/MediaType.java
@@ -236,6 +236,7 @@ private static byte[] init__media_type_eof_actions_0()
     public static final MediaType JSON = MediaType.parse("application/json");
     public static MediaType HTML = MediaType.parse("text/html");
     public static MediaType HTML_UTF8 = MediaType.parse("text/html;charset=utf-8");
+    public static final MediaType XHTML = MediaType.parse("application/xhtml+xml");
     public static MediaType HTTP = MediaType.parse("application/http");
     public static MediaType HTTP_REQUEST = MediaType.parse("application/http;msgtype=request");
     public static MediaType HTTP_RESPONSE = MediaType.parse("application/http;msgtype=response");
diff --git a/src/org/netpreserve/jwarc/WaczWriter.java b/src/org/netpreserve/jwarc/WaczWriter.java
@@ -43,6 +43,9 @@ public class WaczWriter implements Closeable {
     private final MessageDigest messageDigest;
     private final FileChannel cdxChannel;
     private final CdxWriter cdxWriter;
+    private final FileChannel pagesChannel;
+    private final JsonPagesWriter pagesWriter;
+    private boolean autoPages = true;
     private boolean finished;
 
     public WaczWriter(Path path) throws IOException {
@@ -62,6 +65,11 @@ public void close() throws IOException {
         this.cdxWriter = new CdxWriter(new BufferedWriter(new OutputStreamWriter(cdxGzipStream, StandardCharsets.UTF_8)));
         cdxWriter.setFormat(CdxFormat.CDXJ);
         cdxWriter.setSort(true);
+
+        Path pagesTempFile = Files.createTempFile("jwarc-pages", ".jsonl");
+        this.pagesChannel = FileChannel.open(pagesTempFile, READ, WRITE, DELETE_ON_CLOSE);
+        this.pagesWriter = new JsonPagesWriter(new BufferedWriter(new OutputStreamWriter(Channels.newOutputStream(pagesChannel), StandardCharsets.UTF_8)));
+
         set("profile", "data-package");
         set("wacz_version", "1.1.1");
         try  {
@@ -117,6 +125,10 @@ private void writeResource(String path, SeekableByteChannel source) throws IOExc
             long start = source.position();
             cdxWriter.process(new WarcReader(source), filename);
             source.position(start);
+            if (autoPages) {
+                pagesWriter.process(new WarcReader(source));
+                source.position(start);
+            }
         }
 
         zip.putNextEntry(entry);
@@ -151,6 +163,17 @@ private static long calculateCrc32(InputStream stream) throws IOException {
         return crc.getValue();
     }
 
+    /**
+     * Adds a page entry to pages.jsonl.
+     *
+     * @param url   the URL of the page
+     * @param ts    the timestamp in RFC3339 format
+     * @param title an optional title for the page
+     */
+    public void addPage(String url, String ts, String title) throws IOException {
+        pagesWriter.addPage(url, ts, title);
+    }
+
     private void writeDatapackageJson() throws IOException {
         // https://specs.webrecorder.net/wacz/1.1.1/#datapackage-json
         zip.putNextEntry(new ZipEntry("datapackage.json"));
@@ -180,6 +203,12 @@ public void finish() throws IOException {
             cdxChannel.position(0);
             writeResource("indexes/index.cdx.gz", cdxChannel);
         }
+        pagesWriter.flush();
+        boolean pagesProvided = resources.stream().anyMatch(r -> "pages/pages.jsonl".equals(r.get("path")));
+        if (!pagesProvided && pagesChannel.size() > 0) {
+            pagesChannel.position(0);
+            writeResource("pages/pages.jsonl", pagesChannel);
+        }
         writeDatapackageJson();
         zip.finish();
         metadata.clear();
@@ -202,6 +231,11 @@ public void close() throws IOException {
             } catch (IOException e) {
                 // ignore
             }
+            try {
+                pagesChannel.close();
+            } catch (IOException e) {
+                // ignore
+            }
             zip.close();
         }
     }
@@ -215,4 +249,12 @@ public void close() throws IOException {
     public void set(String name, Object value) {
         metadata.put(name, value);
     }
+
+    /**
+     * Enables or disables automatic detection of pages from WARC records.
+     * If disabled, only pages added via {@link #addPage} will be included.
+     */
+    public void setAutoPages(boolean enabled) {
+        this.autoPages = enabled;
+    }
 }
diff --git a/test/org/netpreserve/jwarc/WaczWriterTest.java b/test/org/netpreserve/jwarc/WaczWriterTest.java
@@ -8,10 +8,12 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
@@ -71,6 +73,20 @@ public void test() throws Exception {
             assertEquals("archive/test.warc.gz", resource.get("path"));
             assertEquals("test.warc.gz", resource.get("name"));
             assertEquals("sha256:" + warcSha256, resource.get("hash"));
+            
+            ZipEntry pagesEntry = zipFile.getEntry("pages/pages.jsonl");
+            assertNotNull(pagesEntry);
+            try (InputStream is = zipFile.getInputStream(pagesEntry);
+                 BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
+                String header = reader.readLine();
+                assertNotNull(header);
+                assertTrue(header.contains("json-pages-1.0"));
+                String page = reader.readLine();
+                assertNotNull(page);
+                assertTrue(page.contains("http://example.org/"));
+                assertTrue(page.contains("\"url\":"));
+                assertTrue(page.contains("\"ts\":"));
+            }
 
             String datapackageDigest = sha256(zipFile.getInputStream(datapackageEntry));
             ZipEntry digestEntry = zipFile.getEntry("datapackage-digest.json");
@@ -81,6 +97,32 @@ public void test() throws Exception {
         }
     }
 
+    @Test
+    public void testManualPages() throws Exception {
+        Path waczFile = temporaryFolder.newFile("manual.wacz").toPath();
+        Path pagesFile = temporaryFolder.newFile("pages.jsonl").toPath();
+        Files.write(pagesFile, Arrays.asList("{\"format\": \"json-pages-1.0\", \"id\": \"pages\", \"title\": \"Manual Pages\"}",
+                "{\"url\": \"http://example.org/manual\", \"ts\": \"2023-01-01T00:00:00Z\"}"), StandardCharsets.UTF_8);
+
+        try (WaczWriter waczWriter = new WaczWriter(Files.newOutputStream(waczFile))) {
+            waczWriter.setAutoPages(false);
+            waczWriter.writeResource("pages/pages.jsonl", pagesFile);
+        }
+
+        try (ZipFile zipFile = new ZipFile(waczFile.toFile())) {
+            ZipEntry pagesEntry = zipFile.getEntry("pages/pages.jsonl");
+            assertNotNull(pagesEntry);
+            try (InputStream is = zipFile.getInputStream(pagesEntry);
+                 BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
+                String header = reader.readLine();
+                assertTrue(header.contains("Manual Pages"));
+                String page = reader.readLine();
+                assertTrue(page.contains("http://example.org/manual"));
+                assertNull(reader.readLine());
+            }
+        }
+    }
+
     public String sha256(Path path) throws Exception {
         try (InputStream stream = Files.newInputStream(path)) {
             return sha256(stream);
