HttpRequest: handle chunked encoding

Author: ato

src/org/netpreserve/jwarc/HttpRequest.java

@@ -87,14 +87,19 @@ private static HttpRequest parse(ReadableByteChannel channel, ByteBuffer buffer,
             copyTo.write(buffer.duplicate());
         }
         MessageHeaders headers = parser.headers();
-        Long contentLength;
-        try {
-            contentLength = headers.first("Content-Length").map(Long::parseLong).orElse(null);
-        } catch (NumberFormatException e) {
-            if (strict) throw new IOException("Invalid Content-Length header", e);
-            contentLength = null;
+        MessageBody body;
+        if (headers.contains("Transfer-Encoding", "chunked")) {
+            body = new ChunkedBody(channel, buffer);
+        } else {
+            Long contentLength;
+            try {
+                contentLength = headers.first("Content-Length").map(Long::parseLong).orElse(0L);
+            } catch (NumberFormatException e) {
+                if (strict) throw new IOException("Invalid Content-Length header", e);
+                contentLength = null;
+            }
+            body = LengthedBody.createFromContentLength(channel, buffer, contentLength);
         }
-        LengthedBody body = LengthedBody.createFromContentLength(channel, buffer, contentLength);
         HttpRequest request = new HttpRequest(parser.method(), parser.target(), parser.version(), headers, body);
         request.serializedHeader = headerBytes;
         return request;

test/org/netpreserve/jwarc/HttpRequestTest.java

@@ -25,6 +25,24 @@ public void serializeHeaderShouldPreserveExactly() throws IOException {
         assertEquals(header, new String(request.serializeHeader(), US_ASCII));
     }
 
+    @Test
+    public void chunkedEncodingRequestShouldDecodeBody() throws IOException {
+        String header = "POST /submit HTTP/1.1\r\n" +
+                "Host: example.org\r\n" +
+                "Transfer-Encoding: chunked\r\n\r\n";
+        String chunkedBody = "4\r\nWiki\r\n" +
+                "5\r\npedia\r\n" +
+                "0\r\n\r\n";
+        String message = header + chunkedBody;
+
+        HttpRequest request = HttpRequest.parse(Channels.newChannel(new ByteArrayInputStream(message.getBytes(US_ASCII))));
+
+        assertEquals("POST", request.method());
+        assertEquals("/submit", request.target());
+        assertEquals(Optional.of("chunked"), request.headers().first("Transfer-Encoding"));
+        assertEquals("Wikipedia", new String(IOUtils.readNBytes(request.body().stream(), 32), US_ASCII));
+    }
+
     @Test(expected = IllegalArgumentException.class)
     public void invalidVersionShouldThrow() {
         new HttpRequest.Builder("GET", "/").version(MessageVersion.WARC_1_0);