TIKA-4809: Migrate /meta onto the shared pipes-backed PipesParser

Author: tballison

tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java

@@ -18,7 +18,6 @@
 
 import static org.apache.tika.server.core.resource.TikaResource.fillMetadata;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 
@@ -37,13 +36,16 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.extractor.DocumentSelector;
+import org.apache.tika.config.EmbeddedLimits;
+import org.apache.tika.exception.TikaException;
 import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.language.detect.LanguageHandler;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
+import org.apache.tika.pipes.api.ParseMode;
+import org.apache.tika.sax.BasicContentHandlerFactory;
+import org.apache.tika.sax.ContentHandlerFactory;
+import org.apache.tika.server.core.TikaServerParseException;
 
 
 @Path("/meta")
@@ -63,8 +65,9 @@ public MetadataResource(TikaResource tikaResource) {
     public Response getMetadataFromMultipart(Attachment att, @Context UriInfo info) throws Exception {
         ParseContext context = tikaResource.createParseContext();
         try (TikaInputStream tis = TikaInputStream.get(att.getObject(InputStream.class))) {
+            tis.getPath(); // Spool to temp file for pipes-based parsing
             return Response
-                    .ok(parseMetadata(tis, Metadata.newInstance(context), att.getHeaders(), info))
+                    .ok(parseMetadata(tis, Metadata.newInstance(context), att.getHeaders(), context))
                     .build();
         }
     }
@@ -79,25 +82,14 @@ public Response getMetadataFromMultipart(Attachment att, @Context UriInfo info)
     @Path("config")
     public Response getMetadataWithConfig(
             List<Attachment> attachments,
-            @Context HttpHeaders httpHeaders,
-            @Context UriInfo info) throws Exception {
+            @Context HttpHeaders httpHeaders) throws Exception {
 
         // Load default context from config, then overlay with request config
         ParseContext context = tikaResource.createParseContext();
         Metadata metadata = Metadata.newInstance(context);
         try (TikaInputStream tis = tikaResource.setupMultipartConfig(attachments, metadata, context)) {
-            // No need to parse embedded docs for metadata-only extraction
-            context.set(DocumentSelector.class, metadata1 -> false);
-
-            Parser parser = tikaResource.createParser();
             TikaResource.logRequest(LOG, "/meta/config", metadata);
-            tikaResource.parse(parser, LOG, info.getPath(), tis, new LanguageHandler() {
-                public void endDocument() {
-                    metadata.set("language", getLanguage().getLanguage());
-                }
-            }, metadata, context);
-
-            return Response.ok(metadata).build();
+            return Response.ok(parseMetadata(tis, metadata, httpHeaders.getRequestHeaders(), context)).build();
         }
     }
 
@@ -107,19 +99,19 @@ public Response getMetadata(InputStream is, @Context HttpHeaders httpHeaders, @C
         ParseContext context = tikaResource.createParseContext();
         Metadata metadata = Metadata.newInstance(context);
         try (TikaInputStream tis = TikaInputStream.get(is)) {
+            tis.getPath(); // Spool to temp file for pipes-based parsing
             return Response
-                    .ok(parseMetadata(tis, metadata, httpHeaders.getRequestHeaders(), info))
+                    .ok(parseMetadata(tis, metadata, httpHeaders.getRequestHeaders(), context))
                     .build();
         }
     }
 
     /**
-     * Get a specific metadata field. If the input stream cannot be parsed, but a
-     * value was found for the given metadata field, then the value of the field
-     * is returned as part of a 200 OK response; otherwise a
-     * {@link javax.ws.rs.core.Response.Status#BAD_REQUEST} is generated. If the stream
-     * was successfully parsed but the specific metadata field was not found, then a
-     * {@link javax.ws.rs.core.Response.Status#NOT_FOUND} is returned.
+     * Get a specific metadata field. If the document parses successfully but the
+     * specific metadata field was not found, a
+     * {@link javax.ws.rs.core.Response.Status#NOT_FOUND} is returned. Unlike the other
+     * /meta endpoints, a bare field value has no envelope to embed a container-level
+     * exception in, so that case is thrown (422) instead.
      * <p/>
      * Note that this method handles multivalue fields and returns possibly more
      * metadata value than requested.
@@ -131,35 +123,29 @@ public Response getMetadata(InputStream is, @Context HttpHeaders httpHeaders, @C
      * @param httpHeaders httpheaders
      * @param info        info
      * @param field       the tika metadata field name
-     * @return one of {@link javax.ws.rs.core.Response.Status#OK},
-     * {@link javax.ws.rs.core.Response.Status#NOT_FOUND}, or
-     * {@link javax.ws.rs.core.Response.Status#BAD_REQUEST}
+     * @return one of {@link javax.ws.rs.core.Response.Status#OK} or
+     * {@link javax.ws.rs.core.Response.Status#NOT_FOUND}
      * @throws Exception
      */
     @PUT
     @Path("{field}")
     @Produces({"text/csv", "application/json", "text/plain"})
     public Response getMetadataField(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info, @PathParam("field") String field) throws Exception {
-
-        // use BAD request to indicate that we may not have had enough data to
-        // process the request
-        Response.Status defaultErrorResponse = Response.Status.BAD_REQUEST;
         ParseContext context = tikaResource.createParseContext();
-        Metadata metadata = Metadata.newInstance(context);
-        boolean success = false;
+        Metadata metadata;
         try (TikaInputStream tis = TikaInputStream.get(is)) {
-            parseMetadata(tis, metadata, httpHeaders.getRequestHeaders(), info);
-            // once we've parsed the document successfully, we should use NOT_FOUND
-            // if we did not see the field
-            defaultErrorResponse = Response.Status.NOT_FOUND;
-            success = true;
-        } catch (Exception e) {
-            LOG.warn("Failed to process field {}", field, e);
+            tis.getPath(); // Spool to temp file for pipes-based parsing
+            metadata = parseMetadata(tis, Metadata.newInstance(context), httpHeaders.getRequestHeaders(), context);
+        }
+
+        String containerException = metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION);
+        if (containerException != null && !containerException.isEmpty()) {
+            throw new TikaServerParseException(new TikaException(containerException));
         }
 
-        if (success == false || metadata.get(field) == null) {
+        if (metadata.get(field) == null) {
             return Response
-                    .status(defaultErrorResponse)
+                    .status(Response.Status.NOT_FOUND)
                     .entity("Failed to get metadata field " + field)
                     .build();
         }
@@ -175,21 +161,26 @@ public Response getMetadataField(InputStream is, @Context HttpHeaders httpHeader
                 .build();
     }
 
-    protected Metadata parseMetadata(TikaInputStream tis, Metadata metadata, MultivaluedMap<String, String> httpHeaders, UriInfo info)
-            throws IOException, TikaConfigException {
-        // Load default context from config (includes DigesterFactory from parse-context)
-        final ParseContext context = tikaResource.createParseContext();
-        Parser parser = tikaResource.createParser();
-        fillMetadata(parser, metadata, httpHeaders);
-        //no need to parse embedded docs
-        context.set(DocumentSelector.class, metadata1 -> false);
+    /**
+     * Parses via the shared pipes-backed PipesParser, stopping at the container document
+     * (EmbeddedLimits maxDepth=0) with content capture off ("ignore" handler) -- metadata
+     * only, matching /meta's contract. Set unconditionally so per-request config can't
+     * turn content capture back on. A container-level exception is embedded in
+     * CONTAINER_EXCEPTION here, not thrown; getMetadataField throws instead since it
+     * returns a bare scalar with nowhere to embed it.
+     */
+    protected Metadata parseMetadata(TikaInputStream tis, Metadata metadata, MultivaluedMap<String, String> httpHeaders, ParseContext context)
+            throws Exception {
+        fillMetadata(null, metadata, httpHeaders);
+        context.set(EmbeddedLimits.class, new EmbeddedLimits(0, false, EmbeddedLimits.UNLIMITED, false));
+        context.set(ContentHandlerFactory.class,
+                new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.IGNORE, -1));
 
         TikaResource.logRequest(LOG, "/meta", metadata);
-        tikaResource.parse(parser, LOG, info.getPath(), tis, new LanguageHandler() {
-            public void endDocument() {
-                metadata.set("language", getLanguage().getLanguage());
-            }
-        }, metadata, context);
-        return metadata;
+        List<Metadata> metadataList = tikaResource.parseWithPipes(tis, metadata, context, ParseMode.RMETA);
+        if (metadataList.isEmpty()) {
+            return Metadata.newInstance(context);
+        }
+        return metadataList.get(0);
     }
 }

tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java

@@ -113,7 +113,8 @@ protected Path getUnpackEmitterBasePath() {
     @Test
     public void testEncrypted() throws Exception {
         for (String path : PATHS) {
-            if ("/rmeta".equals(path)) {
+            // /rmeta and /meta embed a container exception at 200 instead of throwing 422.
+            if ("/rmeta".equals(path) || "/meta".equals(path)) {
                 continue;
             }
             // Use path-based routing for /tika
@@ -132,7 +133,8 @@ public void testEncrypted() throws Exception {
     @Test
     public void testNullPointerOnTika() throws Exception {
         for (String path : PATHS) {
-            if ("/rmeta".equals(path)) {
+            // Same as testEncrypted.
+            if ("/rmeta".equals(path) || "/meta".equals(path)) {
                 continue;
             }
             // Use path-based routing for /tika
@@ -170,10 +172,7 @@ public void testEmptyParser() throws Exception {
     }
 
 
-    //For now, make sure that non-complete document
-    //still returns BAD_REQUEST.  We may want to
-    //make MetadataResource return the same types of parse
-    //exceptions as the others...
+    // A truncated document isn't a process failure -- NOT_FOUND, not BAD_REQUEST.
     @Test
     public void testMeta() throws Exception {
         InputStream stream = ClassLoader.getSystemResourceAsStream(TEST_HELLO_WORLD);
@@ -183,7 +182,7 @@ public void testMeta() throws Exception {
                 .type("application/mock+xml")
                 .accept(MediaType.TEXT_PLAIN)
                 .put(copy(stream, 100));
-        assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus());
+        assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus());
         String msg = getStringFromInputStream((InputStream) response.getEntity());
         assertEquals("Failed to get metadata field Author", msg);
     }

tika-server/tika-server-standard/src/main/java/org/apache/tika/server/standard/resource/XMPMetadataResource.java

@@ -58,8 +58,9 @@ public Response getMetadataField(InputStream is, @Context HttpHeaders httpHeader
     public Response getMetadataFromMultipart(Attachment att, @Context UriInfo info) throws Exception {
         ParseContext context = new ParseContext();
         try (TikaInputStream tis = TikaInputStream.get(att.getObject(InputStream.class))) {
+            tis.getPath(); // Spool to temp file for pipes-based parsing
             return Response
-                    .ok(parseMetadata(tis, Metadata.newInstance(context), att.getHeaders(), info))
+                    .ok(parseMetadata(tis, Metadata.newInstance(context), att.getHeaders(), context))
                     .build();
         }
     }
@@ -70,8 +71,9 @@ public Response getMetadata(InputStream is, @Context HttpHeaders httpHeaders, @C
         ParseContext context = new ParseContext();
         Metadata metadata = Metadata.newInstance(context);
         try (TikaInputStream tis = TikaInputStream.get(is)) {
+            tis.getPath(); // Spool to temp file for pipes-based parsing
             return Response
-                    .ok(parseMetadata(tis, metadata, httpHeaders.getRequestHeaders(), info))
+                    .ok(parseMetadata(tis, metadata, httpHeaders.getRequestHeaders(), context))
                     .build();
         }
     }

tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/MetadataResourceTest.java

@@ -47,6 +47,7 @@
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.serialization.JsonMetadata;
 import org.apache.tika.server.core.CXFTestBase;
+import org.apache.tika.server.core.TikaServerParseExceptionMapper;
 import org.apache.tika.server.core.resource.MetadataResource;
 import org.apache.tika.server.core.writer.CSVMessageBodyWriter;
 import org.apache.tika.server.core.writer.JSONMessageBodyWriter;
@@ -73,6 +74,8 @@ protected void setUpResources(JAXRSServerFactoryBean sf) {
     @Override
     protected void setUpProviders(JAXRSServerFactoryBean sf) {
         List<Object> providers = new ArrayList<>();
+        // Needed by getMetadataField's TikaServerParseException throw.
+        providers.add(new TikaServerParseExceptionMapper(false));
         providers.add(new JSONMessageBodyWriter());
         providers.add(new CSVMessageBodyWriter());
         providers.add(new XMPMessageBodyWriter());
@@ -118,10 +121,13 @@ public void testPasswordProtected() throws Exception {
                 .accept("application/json")
                 .post(new MultipartBody(Arrays.asList(fileAtt)));
 
-        // Won't work, no password given - EncryptedDocumentException returns 422
-        assertEquals(500, response.getStatus());
+        // A failed decrypt isn't a process failure -- 200, exception on the metadata.
+        assertEquals(200, response.getStatus());
+        Metadata noPasswordMetadata = JsonMetadata.fromJson(new InputStreamReader((InputStream) response.getEntity(), UTF_8));
+        assertContains("org.apache.tika.exception.EncryptedDocumentException",
+                noPasswordMetadata.get(TikaCoreProperties.CONTAINER_EXCEPTION));
 
-        // Test 2: Wrong password - should fail
+        // Test 2: Wrong password - should fail the same way
         fileCd = new ContentDisposition("form-data; name=\"file\"; filename=\"test.xls\"");
         fileAtt = new Attachment("file",
                 ClassLoader.getSystemResourceAsStream(TikaResourceTest.TEST_PASSWORD_PROTECTED), fileCd);
@@ -142,7 +148,10 @@ public void testPasswordProtected() throws Exception {
                 .accept("application/json")
                 .post(new MultipartBody(Arrays.asList(fileAtt, wrongConfigAtt)));
 
-        assertEquals(500, response.getStatus());
+        assertEquals(200, response.getStatus());
+        Metadata wrongPasswordMetadata = JsonMetadata.fromJson(new InputStreamReader((InputStream) response.getEntity(), UTF_8));
+        assertContains("org.apache.tika.exception.EncryptedDocumentException",
+                wrongPasswordMetadata.get(TikaCoreProperties.CONTAINER_EXCEPTION));
 
         // Test 3: Correct password - should work
         fileCd = new ContentDisposition("form-data; name=\"file\"; filename=\"test.xls\"");
@@ -213,16 +222,17 @@ public void testGetField_XXX_NotFound() throws Exception {
     }
 
     @Test
-    public void testGetField_Author_TEXT_Partial_BAD_REQUEST() throws Exception {
-
+    public void testGetField_Author_TEXT_Partial_UNPROCESSABLE() throws Exception {
+        // Truncating at 8000 bytes corrupts the OLE2 structure enough that OfficeParser
+        // throws -- a real container exception, not just a missing field.
         InputStream stream = ClassLoader.getSystemResourceAsStream(TikaResourceTest.TEST_DOC);
 
         Response response = WebClient
                 .create(endPoint + META_PATH + "/Author")
                 .type("application/msword")
                 .accept(MediaType.TEXT_PLAIN)
                 .put(copy(stream, 8000));
-        assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus());
+        assertEquals(422, response.getStatus());
     }
 
     @Test