[TIKA-4825] carry caller Content-Type across the pipes worker boundary as a detection hint (#3039)

Author: dschmidt

CHANGES.txt

@@ -1,5 +1,16 @@
 Release 4.1.0 - ???
 
+   * Pipes now carries the caller-supplied Content-Type across the worker's
+     fresh-metadata boundary as a soft detection hint, so every forked-parse
+     endpoint (/tika, /meta, /rmeta, /unpack, /async, /pipes, plus tika-grpc
+     and embedded PipesForkParser) can route on a client Content-Type, not
+     only on the filename. Detection keeps the hint only when it equals or
+     specializes the content-detected type (e.g. refining image/tiff to
+     image/x-canon-cr2); for bytes with no magic it can select any type,
+     matching the routing power the filename already had. The
+     CONTENT_TYPE_USER_OVERRIDE key is deliberately not carried, so the hint
+     cannot force an unrelated type (TIKA-4825).
+
    * RawTiffParser extracts the camera-generated JPEG previews embedded in
      TIFF-based raw images (Nikon NEF/NRW, Sony ARW/SRF/SR2, Pentax PEF/PTX,
      Adobe DNG and Canon CR2, including BigTIFF DNG containers) as thumbnail

docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc

@@ -428,6 +428,15 @@ Transport headers are unaffected: `Content-Disposition`,
 `Content-Type` and `Content-Length` still describe the payload and still
 influence detection.
 
+NOTE: The way `Content-Type` influences detection changed. In 3.x, parsing ran
+in-process and a request `Content-Type` acted as a hard override that forced the
+type. In 4.x, parsing runs in a forked worker and the header is carried across as
+a *soft* hint: detection keeps it only when it equals or specializes the type
+detected from the content, and otherwise ignores it (TIKA-4825). A 3.x client
+that forced an unrelated type onto arbitrary bytes (for example `text/plain`)
+will now see that type ignored in favor of content-based detection. Supply the
+correct `Content-Type` (or a filename) to refine within the detected hierarchy.
+
 === Pipes Configuration (for `/pipes` and `/async`)
 
 No pipes or fetcher configuration is required to start the server: the default-on endpoints

tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java

@@ -36,6 +36,7 @@
 import org.apache.tika.extractor.EmbeddedDocumentExtractor;
 import org.apache.tika.extractor.UnpackHandler;
 import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.metadata.writelimiter.MetadataWriteLimiterFactory;
@@ -487,18 +488,8 @@ protected ParseDataOrPipesResult parseFromTuple() throws TikaException, Interrup
         }
         // Use newMetadata() to apply any configured write limits
         Metadata metadata = localContext.newMetadata();
-        // Carry the caller-supplied resource name across the fresh-metadata boundary so
-        // detection, suffix selection, and the Frictionless manifest's name field see
-        // the logical filename rather than whatever the fetcher's path happens to be
-        // (e.g., a server-side spool prefix). TikaInputStream.get(path, metadata)
-        // already honors a pre-set RESOURCE_NAME_KEY.
-        Metadata tupleMetadata = fetchEmitTuple.getMetadata();
-        String suppliedName = tupleMetadata == null
-                ? null
-                : tupleMetadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
-        if (!StringUtils.isBlank(suppliedName)) {
-            metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, suppliedName);
-        }
+        // Carry the caller's resource name and Content-Type detection hints (see javadoc).
+        carryCallerHints(fetchEmitTuple.getMetadata(), metadata);
         FetchHandler.TisOrResult tisOrResult = fetchHandler.fetch(fetchEmitTuple, metadata, localContext);
         if (tisOrResult.pipesResult() != null) {
             return new ParseDataOrPipesResult(null, tisOrResult.pipesResult());
@@ -516,7 +507,33 @@ protected ParseDataOrPipesResult parseFromTuple() throws TikaException, Interrup
         }
     }
 
-
+    /**
+     * Carries the caller-supplied detection hints from the tuple metadata across the
+     * fresh-metadata boundary into the metadata used for fetch and detection.
+     * <p>
+     * Only the resource name and the {@code Content-Type} soft hint are carried.
+     * {@code Content-Type} is applied by {@code MimeTypes.detect} via {@code applyHint},
+     * which keeps it only when it equals or specializes the magic-detected type (e.g.
+     * {@code image/tiff} -&gt; {@code image/x-raw-nikon} for a NEF supplied without a
+     * filename). The {@code CONTENT_TYPE_USER_OVERRIDE} key is deliberately NOT carried:
+     * it short-circuits detection unconditionally and would let a caller force any type.
+     *
+     * @param tupleMetadata the caller-supplied metadata (may be null)
+     * @param target the fresh metadata used for fetch and detection
+     */
+    static void carryCallerHints(Metadata tupleMetadata, Metadata target) {
+        if (tupleMetadata == null) {
+            return;
+        }
+        String suppliedName = tupleMetadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
+        if (!StringUtils.isBlank(suppliedName)) {
+            target.set(TikaCoreProperties.RESOURCE_NAME_KEY, suppliedName);
+        }
+        String suppliedContentType = tupleMetadata.get(HttpHeaders.CONTENT_TYPE);
+        if (!StringUtils.isBlank(suppliedContentType)) {
+            target.set(HttpHeaders.CONTENT_TYPE, suppliedContentType);
+        }
+    }
 
     private ParseContext setupParseContext() throws TikaException, IOException {
         // ContentHandlerFactory and ParseMode are retrieved from ParseContext in ParseHandler.

tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/PipesWorkerCallerHintsTest.java

@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.core.server;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeTypes;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * Unit tests for {@link PipesWorker#carryCallerHints(Metadata, Metadata)}, which carries the
+ * caller-supplied detection hints across the worker's fresh-metadata boundary.
+ */
+public class PipesWorkerCallerHintsTest {
+
+    @Test
+    public void testCarriesResourceNameAndContentType() {
+        Metadata tuple = new Metadata();
+        tuple.set(TikaCoreProperties.RESOURCE_NAME_KEY, "photo.nef");
+        tuple.set(HttpHeaders.CONTENT_TYPE, "image/x-raw-nikon");
+
+        Metadata target = new Metadata();
+        PipesWorker.carryCallerHints(tuple, target);
+
+        assertEquals("photo.nef", target.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+        assertEquals("image/x-raw-nikon", target.get(HttpHeaders.CONTENT_TYPE));
+    }
+
+    /**
+     * The Content-Type is carried only as a soft hint. The unconditional override keys
+     * must never be carried, or a caller could force any type past detection.
+     */
+    @Test
+    public void testDoesNotCarryOverrides() {
+        Metadata tuple = new Metadata();
+        tuple.set(TikaCoreProperties.CONTENT_TYPE_USER_OVERRIDE, "image/x-raw-nikon");
+        tuple.set(TikaCoreProperties.CONTENT_TYPE_PARSER_OVERRIDE, "image/x-raw-nikon");
+
+        Metadata target = new Metadata();
+        PipesWorker.carryCallerHints(tuple, target);
+
+        assertNull(target.get(TikaCoreProperties.CONTENT_TYPE_USER_OVERRIDE));
+        assertNull(target.get(TikaCoreProperties.CONTENT_TYPE_PARSER_OVERRIDE));
+        assertNull(target.get(HttpHeaders.CONTENT_TYPE));
+    }
+
+    @Test
+    public void testNullTupleIsNoOp() {
+        Metadata target = new Metadata();
+        target.set(TikaCoreProperties.RESOURCE_NAME_KEY, "keep.me");
+        PipesWorker.carryCallerHints(null, target);
+        assertEquals("keep.me", target.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+    }
+
+    @Test
+    public void testBlankValuesNotCarried() {
+        Metadata tuple = new Metadata();
+        tuple.set(TikaCoreProperties.RESOURCE_NAME_KEY, "   ");
+        tuple.set(HttpHeaders.CONTENT_TYPE, "");
+
+        Metadata target = new Metadata();
+        PipesWorker.carryCallerHints(tuple, target);
+
+        assertNull(target.get(TikaCoreProperties.RESOURCE_NAME_KEY));
+        assertNull(target.get(HttpHeaders.CONTENT_TYPE));
+    }
+
+    //content that magic-detects as image/tiff (little-endian TIFF marker, no CR2 marker)
+    private static final byte[] TIFF_BYTES = {'I', 'I', 0x2A, 0x00, 0, 0, 0, 8};
+
+    private static MediaType detectWithCarriedContentType(String contentType) throws Exception {
+        Metadata tuple = new Metadata();
+        tuple.set(HttpHeaders.CONTENT_TYPE, contentType);
+        Metadata target = new Metadata();
+        PipesWorker.carryCallerHints(tuple, target);
+        try (TikaInputStream tis = TikaInputStream.get(TIFF_BYTES)) {
+            return MimeTypes.getDefaultMimeTypes().detect(tis, target, new ParseContext());
+        }
+    }
+
+    /**
+     * A carried Content-Type that specializes the content-detected type refines detection.
+     * image/x-canon-cr2 is a sub-class-of image/tiff, and these bytes lack the CR2 marker.
+     */
+    @Test
+    public void testSpecializingContentTypeRefinesDetection() throws Exception {
+        assertEquals(MediaType.image("x-canon-cr2"),
+                detectWithCarriedContentType("image/x-canon-cr2"));
+    }
+
+    /**
+     * Security boundary: a carried Content-Type that does NOT specialize the content-detected
+     * type is ignored, so a caller cannot force an unrelated type onto the document.
+     */
+    @Test
+    public void testNonSpecializingContentTypeIgnored() throws Exception {
+        assertEquals(MediaType.image("tiff"), detectWithCarriedContentType("audio/mpeg"));
+        assertEquals(MediaType.image("tiff"), detectWithCarriedContentType("not-a-media-type"));
+    }
+}

tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java

@@ -191,7 +191,11 @@ public void testEmptyParser() throws Exception {
     }
 
 
-    // A truncated document isn't a process failure -- NOT_FOUND, not BAD_REQUEST.
+    // Since TIKA-4825 the caller-supplied Content-Type is carried into detection, so an
+    // explicit application/mock+xml routes the (truncated) document to the mock parser,
+    // which cannot parse the incomplete XML. That container exception maps to 422 for the
+    // bare-field endpoint (which has no envelope to embed it in). Without a Content-Type the
+    // truncated bytes detect as generic XML and still yield NOT_FOUND -- see testMetaNoType.
     @Test
     public void testMeta() throws Exception {
         InputStream stream = ClassLoader.getSystemResourceAsStream(TEST_HELLO_WORLD);
@@ -201,6 +205,19 @@ public void testMeta() throws Exception {
                 .type("application/mock+xml")
                 .accept(MediaType.TEXT_PLAIN)
                 .put(copy(stream, 100));
+        assertEquals(422, response.getStatus());
+    }
+
+    // A truncated document with no forcing Content-Type isn't a process failure --
+    // NOT_FOUND (field missing), not BAD_REQUEST.
+    @Test
+    public void testMetaNoType() throws Exception {
+        InputStream stream = ClassLoader.getSystemResourceAsStream(TEST_HELLO_WORLD);
+
+        Response response = WebClient
+                .create(endPoint + "/meta" + "/Author")
+                .accept(MediaType.TEXT_PLAIN)
+                .put(copy(stream, 100));
         assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus());
         String msg = getStringFromInputStream((InputStream) response.getEntity());
         assertEquals("Failed to get metadata field Author", msg);