TIKA-4809: Migrate /meta onto the shared pipes-backed PipesParser, take 2

Author: tballison

tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java

@@ -430,7 +430,8 @@ static List<ResourceProvider> loadCoreProviders(TikaServerConfig tikaServerConfi
             // Lifecycle (shutdown/close) is owned by whoever built the shared parser,
             // not by PipesResource.
             PipesParsingHelper helper = tikaResource.getPipesParsingHelper();
-            resourceProviders.add(new SingletonResourceProvider(new PipesResource(helper.getPipesParser())));
+            resourceProviders.add(new SingletonResourceProvider(
+                    new PipesResource(helper.getPipesParser(), helper.isReturnStackTrace())));
         }
         resourceProviders.addAll(loadResourceServices(serverStatus));
         return resourceProviders;

tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java

@@ -34,6 +34,7 @@
 
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.pipes.api.FetchEmitTuple;
@@ -168,7 +169,9 @@ public List<Metadata> parse(TikaInputStream tis, Metadata metadata,
             PipesResult result = pipesParser.parse(tuple);
 
             // Process result
-            return processResult(result);
+            List<Metadata> metadataList = processResult(result);
+            redactExceptionDetail(metadataList);
+            return metadataList;
 
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
@@ -281,6 +284,40 @@ private List<Metadata> processResult(PipesResult result) {
         return Collections.emptyList();
     }
 
+    /**
+     * Trims CONTAINER_EXCEPTION/EMBEDDED_EXCEPTION to one line unless returnStackTrace is
+     * on -- unlike buildProcessFailureResponse's family, a 200 response has no other way
+     * to signal a per-document exception, so we can't omit these fields entirely.
+     */
+    private void redactExceptionDetail(List<Metadata> metadataList) {
+        if (returnStackTrace || metadataList == null) {
+            return;
+        }
+        for (Metadata m : metadataList) {
+            summarizeInPlace(m, TikaCoreProperties.CONTAINER_EXCEPTION);
+            summarizeInPlace(m, TikaCoreProperties.EMBEDDED_EXCEPTION);
+        }
+    }
+
+    private static void summarizeInPlace(Metadata m, Property property) {
+        String full = m.get(property);
+        if (full != null) {
+            m.set(property, summarizeStackTrace(full, false));
+        }
+    }
+
+    /**
+     * First line of a stack trace (the caught exception's own class + message); no-op if
+     * returnStackTrace.
+     */
+    public static String summarizeStackTrace(String fullTrace, boolean returnStackTrace) {
+        if (returnStackTrace || fullTrace == null || fullTrace.isBlank()) {
+            return fullTrace;
+        }
+        int newline = fullTrace.indexOf('\n');
+        return newline < 0 ? fullTrace : fullTrace.substring(0, newline);
+    }
+
     /**
      * Maps PipesResult status to HTTP response status.
      */
@@ -314,6 +351,14 @@ public PipesParser getPipesParser() {
         return pipesParser;
     }
 
+    /**
+     * Whether failure responses may include the (potentially stack-trace-bearing)
+     * {@code PipesResult} message. Mirrors {@code TikaServerConfig.isReturnStackTrace()}.
+     */
+    public boolean isReturnStackTrace() {
+        return returnStackTrace;
+    }
+
     /**
      * Gets the PipesConfig instance.
      */
@@ -442,18 +487,10 @@ public UnpackResult parseUnpack(TikaInputStream tis, Metadata metadata,
                 Metadata containerMetadata = metadataList.get(0);
                 String containerException = containerMetadata.get(TikaCoreProperties.CONTAINER_EXCEPTION);
                 if (containerException != null) {
-                    // Map exception type to HTTP status
-                    // 422 (Unprocessable Entity) for parse-related exceptions
-                    int status = 422; // Default for parse exceptions
-                    if (containerException.contains("EncryptedDocumentException") ||
-                            containerException.contains("TikaException") ||
-                            containerException.contains("NullPointerException") ||
-                            containerException.contains("IllegalStateException")) {
-                        status = 422;
-                    }
-                    // Build response with exception string as body for stack trace support
-                    Response response = Response.status(status)
-                            .entity(containerException)
+                    // 422 already signals failure, so (unlike redactExceptionDetail's
+                    // 200 family) the body can be omitted entirely when off.
+                    Response response = Response.status(422)
+                            .entity(returnStackTrace ? containerException : "")
                             .type("text/plain")
                             .build();
                     throw new WebApplicationException(response);

tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java

@@ -52,14 +52,18 @@ public class PipesResource {
     private static final Logger LOG = LoggerFactory.getLogger(PipesResource.class);
 
     private final PipesParser pipesParser;
+    private final boolean returnStackTrace;
 
     /**
      * @param pipesParser shared parser, also used by /tika, /rmeta, and /unpack.
      *                     Lifecycle (construction, shutdown) is owned by whoever
      *                     built it, not by this class.
+     * @param returnStackTrace whether parse_exception may include the full stack trace
+     *                         vs. just the first line.
      */
-    public PipesResource(PipesParser pipesParser) {
+    public PipesResource(PipesParser pipesParser, boolean returnStackTrace) {
         this.pipesParser = pipesParser;
+        this.returnStackTrace = returnStackTrace;
     }
 
 
@@ -130,7 +134,8 @@ private Response processTuple(FetchEmitTuple fetchEmitTuple) throws InterruptedE
     private Map<String, String> parseException(String msg, boolean emitted) {
         Map<String, String> statusMap = new HashMap<>();
         statusMap.put("status", "ok");
-        statusMap.put("parse_exception", msg);
+        // 200 response, so trim rather than omit -- same reasoning as redactExceptionDetail.
+        statusMap.put("parse_exception", PipesParsingHelper.summarizeStackTrace(msg, returnStackTrace));
         statusMap.put("emitted", Boolean.toString(emitted));
         return statusMap;
     }

tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java

@@ -515,7 +515,7 @@ public String getMessage() {
     @PUT
     @Consumes("*/*")
     @Produces("text/xml")
-    public StreamingOutput getXhtml(final InputStream is, @Context HttpHeaders httpHeaders)
+    public Response getXhtml(final InputStream is, @Context HttpHeaders httpHeaders)
             throws IOException {
         TikaInputStream tis = TikaInputStream.get(is);
         tis.getPath(); // Spool to temp file for pipes-based parsing
@@ -530,7 +530,7 @@ public StreamingOutput getXhtml(final InputStream is, @Context HttpHeaders httpH
     @Consumes("*/*")
     @Produces("text/plain")
     @Path("text")
-    public StreamingOutput getText(final InputStream is, @Context HttpHeaders httpHeaders)
+    public Response getText(final InputStream is, @Context HttpHeaders httpHeaders)
             throws IOException {
         TikaInputStream tis = TikaInputStream.get(is);
         tis.getPath(); // Spool to temp file for pipes-based parsing
@@ -545,7 +545,7 @@ public StreamingOutput getText(final InputStream is, @Context HttpHeaders httpHe
     @Consumes("*/*")
     @Produces("text/html")
     @Path("html")
-    public StreamingOutput getHtml(final InputStream is, @Context HttpHeaders httpHeaders)
+    public Response getHtml(final InputStream is, @Context HttpHeaders httpHeaders)
             throws IOException {
         TikaInputStream tis = TikaInputStream.get(is);
         tis.getPath(); // Spool to temp file for pipes-based parsing
@@ -560,7 +560,7 @@ public StreamingOutput getHtml(final InputStream is, @Context HttpHeaders httpHe
     @Consumes("*/*")
     @Produces("text/xml")
     @Path("xml")
-    public StreamingOutput getXml(final InputStream is, @Context HttpHeaders httpHeaders)
+    public Response getXml(final InputStream is, @Context HttpHeaders httpHeaders)
             throws IOException {
         TikaInputStream tis = TikaInputStream.get(is);
         tis.getPath(); // Spool to temp file for pipes-based parsing
@@ -575,7 +575,7 @@ public StreamingOutput getXml(final InputStream is, @Context HttpHeaders httpHea
     @Consumes("*/*")
     @Produces("text/plain")
     @Path("md")
-    public StreamingOutput getMarkdown(final InputStream is, @Context HttpHeaders httpHeaders)
+    public Response getMarkdown(final InputStream is, @Context HttpHeaders httpHeaders)
             throws IOException {
         TikaInputStream tis = TikaInputStream.get(is);
         tis.getPath(); // Spool to temp file for pipes-based parsing
@@ -634,7 +634,7 @@ public Metadata getJson(final InputStream is, @Context HttpHeaders httpHeaders,
     @Consumes("multipart/form-data")
     @Produces("text/xml")
     @Path("config")
-    public StreamingOutput postRaw(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
+    public Response postRaw(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
             throws IOException, TikaConfigException {
         ParseContext context = createParseContext();
         Metadata metadata = Metadata.newInstance(context);
@@ -657,7 +657,7 @@ public StreamingOutput postRaw(List<Attachment> attachments, @Context HttpHeader
     @Consumes("multipart/form-data")
     @Produces("text/plain")
     @Path("config/text")
-    public StreamingOutput postText(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
+    public Response postText(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
             throws IOException, TikaConfigException {
         ParseContext context = createParseContext();
         Metadata metadata = Metadata.newInstance(context);
@@ -679,7 +679,7 @@ public StreamingOutput postText(List<Attachment> attachments, @Context HttpHeade
     @Consumes("multipart/form-data")
     @Produces("text/html")
     @Path("config/html")
-    public StreamingOutput postHtml(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
+    public Response postHtml(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
             throws IOException, TikaConfigException {
         ParseContext context = createParseContext();
         Metadata metadata = Metadata.newInstance(context);
@@ -701,7 +701,7 @@ public StreamingOutput postHtml(List<Attachment> attachments, @Context HttpHeade
     @Consumes("multipart/form-data")
     @Produces("text/xml")
     @Path("config/xml")
-    public StreamingOutput postXml(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
+    public Response postXml(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
             throws IOException, TikaConfigException {
         ParseContext context = createParseContext();
         Metadata metadata = Metadata.newInstance(context);
@@ -723,7 +723,7 @@ public StreamingOutput postXml(List<Attachment> attachments, @Context HttpHeader
     @Consumes("multipart/form-data")
     @Produces("text/plain")
     @Path("config/md")
-    public StreamingOutput postMarkdown(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
+    public Response postMarkdown(List<Attachment> attachments, @Context HttpHeaders httpHeaders)
             throws IOException, TikaConfigException {
         ParseContext context = createParseContext();
         Metadata metadata = Metadata.newInstance(context);
@@ -760,7 +760,7 @@ public Metadata postJson(List<Attachment> attachments, @Context HttpHeaders http
     /**
      * Produces raw streaming output (text, html, xml, md) using pipes-based parsing.
      */
-    private StreamingOutput produceRawOutput(TikaInputStream tis, Metadata metadata,
+    private Response produceRawOutput(TikaInputStream tis, Metadata metadata,
                                               MultivaluedMap<String, String> httpHeaders,
                                               String handlerTypeName) throws IOException {
         fillMetadata(null, metadata, httpHeaders);
@@ -771,8 +771,11 @@ private StreamingOutput produceRawOutput(TikaInputStream tis, Metadata metadata,
 
     /**
      * Produces raw streaming output with a pre-configured ParseContext (for PUT endpoints).
+     * A container-level parse exception doesn't discard content already captured -- status
+     * is 422 (no field to embed the exception in, unlike the JSON endpoints), but the body
+     * still carries whatever content was actually extracted.
      */
-    private StreamingOutput produceRawOutputWithContext(TikaInputStream tis, Metadata metadata,
+    private Response produceRawOutputWithContext(TikaInputStream tis, Metadata metadata,
                                               ParseContext context,
                                               String handlerTypeName) throws IOException {
         logRequest(LOG, "/tika", metadata);
@@ -794,42 +797,45 @@ private StreamingOutput produceRawOutputWithContext(TikaInputStream tis, Metadat
 
         LOG.debug("produceRawOutput: parseWithPipes returned {} metadata objects", metadataList.size());
 
-        // For raw streaming endpoints, throw exception if there was a parse error
-        // (JSON endpoints return exceptions in metadata)
-        // Note: CONTAINER_EXCEPTION is extracted before the metadata filter runs,
-        // so it's available in the passback even though the filter strips it
-        if (!metadataList.isEmpty()) {
-            String exception = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
-            if (exception != null && !exception.isEmpty()) {
-                LOG.debug("produceRawOutput: parse exception: {}", exception);
-                // Wrap in TikaException so TikaServerParseExceptionMapper returns 422
-                throw new TikaServerParseException(new TikaException(exception));
-            }
-        }
-
-        // Extract content from result
+        // Extract content before checking for an exception -- content must not be
+        // discarded just because a container-level exception also occurred.
         String content = "";
+        boolean hasException = false;
+        String exceptionMessage = null;
         if (!metadataList.isEmpty()) {
             String extracted = metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT);
             LOG.debug("produceRawOutput: TIKA_CONTENT length={}", extracted != null ? extracted.length() : 0);
             if (extracted != null) {
                 content = extracted;
             }
+            exceptionMessage = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
+            hasException = exceptionMessage != null && !exceptionMessage.isEmpty();
+            if (hasException) {
+                LOG.debug("produceRawOutput: parse exception: {}", exceptionMessage);
+            }
+        }
+        // No separate field for the exception here, unlike JSON bodies -- append it,
+        // gated by returnStackTrace like TikaServerParseExceptionMapper.
+        if (hasException && pipesParsingHelper != null && pipesParsingHelper.isReturnStackTrace()) {
+            content = content.isEmpty() ? exceptionMessage : content + "\n" + exceptionMessage;
         }
         final String finalContent = content;
 
-        return outputStream -> {
+        StreamingOutput streamingOutput = outputStream -> {
             try (Writer writer = new OutputStreamWriter(outputStream, UTF_8)) {
                 writer.write(finalContent);
                 writer.flush();
             }
         };
+        return Response.status(hasException ? 422 : Response.Status.OK.getStatusCode())
+                .entity(streamingOutput)
+                .build();
     }
 
     /**
      * Produces raw streaming output with a pre-configured ParseContext (for POST endpoints).
      */
-    private StreamingOutput produceRawOutput(TikaInputStream tis, Metadata metadata,
+    private Response produceRawOutput(TikaInputStream tis, Metadata metadata,
                                               ParseContext context,
                                               String handlerTypeName) throws IOException {
         return produceRawOutputWithContext(tis, metadata, context, handlerTypeName);

tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java

@@ -215,7 +215,7 @@ public void setUp() throws Exception {
             pipesConfig.setEmitStrategy(new EmitStrategyConfig(EmitStrategy.PASSBACK_ALL));
             this.pipesParser = PipesParser.load(tikaJsonConfig, pipesConfig, this.pipesConfigPath);
             PipesParsingHelper pipesParsingHelper = new PipesParsingHelper(this.pipesParser, pipesConfig,
-                    inputTempDirectory, getUnpackEmitterBasePath(), false);
+                    inputTempDirectory, getUnpackEmitterBasePath(), isReturnStackTrace());
 
             tikaResource = new TikaResource(tika, new ServerStatus(), pipesParsingHelper, isAllowPerRequestConfig());
         } finally {
@@ -377,6 +377,14 @@ protected boolean isAllowPerRequestConfig() {
         return false;
     }
 
+    /**
+     * Mirrors TikaServerConfig.isReturnStackTrace(); defaults to false (production
+     * default). Override in tests that exercise exception-detail visibility.
+     */
+    protected boolean isReturnStackTrace() {
+        return false;
+    }
+
     protected InputStream getPipesConfigInputStream() throws IOException {
         if (getPipesInputPath() == null) {
             return null;

tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java

@@ -67,6 +67,12 @@ public class StackTraceTest extends CXFTestBase {
     @TempDir
     private static Path unpackTempDir;
 
+    @Override
+    protected boolean isReturnStackTrace() {
+        // Matches this class's own TikaServerParseExceptionMapper(true) below.
+        return true;
+    }
+
     @Override
     protected void setUpResources(JAXRSServerFactoryBean sf) {
         List<ResourceProvider> rCoreProviders = new ArrayList<>();

tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaPipesTest.java

@@ -153,7 +153,7 @@ protected void setUpResources(JAXRSServerFactoryBean sf) {
             PipesConfig pipesConfig = PipesConfig.load(tikaJsonConfig);
             pipesConfig.setEmitStrategy(new EmitStrategyConfig(EmitStrategy.EMIT_ALL));
             pipesParser = PipesParser.load(tikaJsonConfig, pipesConfig, tikaConfigPath);
-            pipesResource = new PipesResource(pipesParser);
+            pipesResource = new PipesResource(pipesParser, false);
             rCoreProviders.add(new SingletonResourceProvider(pipesResource));
         } catch (IOException | TikaConfigException e) {
             throw new RuntimeException(e);

tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java

@@ -109,7 +109,11 @@ public void testJsonNPE() throws Exception {
         assertEquals("Nikolai Lobachevsky", metadata.get("author"));
         assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE));
         assertContains("some content", metadata.get(TikaCoreProperties.TIKA_CONTENT));
-        assertContains("null pointer message", metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION));
+        // returnStackTrace defaults to false here, so CONTAINER_EXCEPTION is trimmed to
+        // the caught exception's own class + message -- the NPE detail underneath it is
+        // intentionally not exposed by default.
+        assertContains("TikaException", metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION));
+        assertNotFound("null pointer message", metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION));
     }
 
     @Test

tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java

@@ -145,7 +145,7 @@ protected void setUpResources(JAXRSServerFactoryBean sf) {
             PipesConfig pipesConfig = PipesConfig.load(tikaJsonConfig);
             pipesConfig.setEmitStrategy(new EmitStrategyConfig(EmitStrategy.EMIT_ALL));
             pipesParser = PipesParser.load(tikaJsonConfig, pipesConfig, tikaConfigPath);
-            pipesResource = new PipesResource(pipesParser);
+            pipesResource = new PipesResource(pipesParser, false);
             rCoreProviders.add(new SingletonResourceProvider(pipesResource));
         } catch (IOException | TikaConfigException e) {
             throw new RuntimeException(e);