Merge branch 'main' into metafile-rendering

# Conflicts:
#	CHANGES.txt

Author: dschmidt

CHANGES.txt

@@ -11,6 +11,15 @@ Release 4.1.0 - unreleased
      OLE2 formats (a WMF) as a THUMBNAIL embedded document, as the OOXML
      parsers do with the docProps thumbnail (TIKA-4855).
 
+   * Raw camera formats are detected by content: RawTiffDetector tells
+     Nikon NEF/NRW, Pentax PEF/PTX, Sony ARW/SRF/SR2, Samsung SRW and Adobe
+     DNG from a plain TIFF by their image directory (DNGVersion, the vendor
+     Compression codes, or a CFA/LinearRaw image plus Make), and Fuji RAF,
+     Panasonic RW2, Minolta MRW and the remaining Olympus ORF byte orders
+     get magic entries. Streams without a file name used to be image/tiff.
+     image/x-raw-samsung (*.srw) is new and parsed by RawTiffParser
+     (TIKA-4861).
+
    * DWGReadParser emits the drawing's THUMBNAILIMAGE as a THUMBNAIL embedded
      document instead of INLINE (TIKA-4853).
 

tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml

@@ -7189,6 +7189,9 @@
 
   <mime-type type="image/x-raw-fuji">
     <_comment>Fuji raw image</_comment>
+    <magic priority="50">
+      <match value="FUJIFILMCCD-RAW" type="string" offset="0"/>
+    </magic>
     <glob pattern="*.raf"/>
   </mime-type>
 
@@ -7243,6 +7246,9 @@
 
   <mime-type type="image/x-raw-minolta">
     <_comment>Minolta raw image</_comment>
+    <magic priority="50">
+      <match value="\x00MRM" type="string" offset="0"/>
+    </magic>
     <glob pattern="*.mrw"/>
   </mime-type>
 
@@ -7255,8 +7261,11 @@
 
   <mime-type type="image/x-raw-olympus">
     <_comment>Olympus raw image</_comment>
+    <!-- IIRO and IIRS little endian, MMOR big endian -->
     <magic priority="50">
       <match offset="0" type="string" value="\x49\x49\x52\x4F"/>
+      <match offset="0" type="string" value="\x49\x49\x52\x53"/>
+      <match offset="0" type="string" value="\x4D\x4D\x4F\x52"/>
     </magic>
     <glob pattern="*.orf"/>
   </mime-type>
@@ -7268,6 +7277,12 @@
     <glob pattern="*.pef"/>
   </mime-type>
 
+  <mime-type type="image/x-raw-samsung">
+    <_comment>Samsung raw image</_comment>
+    <sub-class-of type="image/tiff" />
+    <glob pattern="*.srw"/>
+  </mime-type>
+
   <mime-type type="image/x-raw-sony">
     <_comment>Sony raw image</_comment>
     <sub-class-of type="image/tiff" />
@@ -7298,6 +7313,10 @@
 
   <mime-type type="image/x-raw-panasonic">
     <_comment>Panasonic raw image</_comment>
+    <!-- RW2: a TIFF-like header with its own magic number 0x55 -->
+    <magic priority="50">
+      <match value="IIU\x00" type="string" offset="0"/>
+    </magic>
     <glob pattern="*.raw"/>
     <glob pattern="*.rw2"/>
   </mime-type>

tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java

@@ -123,6 +123,22 @@ private void assertTypeByNameAndData(String dataFile, String name, String typeFr
         }
     }
 
+    /**
+     * TIFF-based raw camera formats, told apart by RawTiffDetector without a
+     * file name (TIKA-4861); a plain TIFF and a CR2 (fixed signature) are
+     * untouched.
+     */
+    @Test
+    public void testRawCameraFormatsByData() throws Exception {
+        assertTypeByData("testNEF.nef", "image/x-raw-nikon");
+        assertTypeByData("testARW.arw", "image/x-raw-sony");
+        assertTypeByData("testPEF.pef", "image/x-raw-pentax");
+        assertTypeByData("testDNG.dng", "image/x-raw-adobe");
+        assertTypeByData("testDNG_bigtiff.dng", "image/x-raw-adobe");
+        assertTypeByData("testCR2.cr2", "image/x-canon-cr2");
+        assertTypeByData("testTIFF.tif", "image/tiff");
+    }
+
     @Test
     public void testDetectOLE2() throws Exception {
 /*        // Microsoft office types known by POI

tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/detect/TestDetectorLoading.java

@@ -32,17 +32,19 @@ public void testBasic() throws Exception {
         //integration test - detectors should be sorted alphabetically by class name
         Detector detector = TikaLoader.loadDefault().loadDetectors();
         List<Detector> detectors = ((CompositeDetector) detector).getDetectors();
-        assertEquals(7, detectors.size());
+        assertEquals(8, detectors.size());
         // Sorted alphabetically by full class name (all are org.apache.tika.*)
         assertEquals("org.apache.tika.detect.apple.BPListDetector", detectors.get(0).getClass().getName());
         assertEquals("org.apache.tika.detect.gzip.GZipSpecializationDetector",
                 detectors.get(1).getClass().getName());
-        assertEquals("org.apache.tika.detect.microsoft.POIFSContainerDetector",
+        assertEquals("org.apache.tika.detect.image.RawTiffDetector",
                 detectors.get(2).getClass().getName());
-        assertEquals("org.apache.tika.detect.mkv.MatroskaDetector", detectors.get(3).getClass().getName());
-        assertEquals("org.apache.tika.detect.ogg.OggDetector", detectors.get(4).getClass().getName());
-        assertEquals("org.apache.tika.detect.ole.MiscOLEDetector", detectors.get(5).getClass().getName());
+        assertEquals("org.apache.tika.detect.microsoft.POIFSContainerDetector",
+                detectors.get(3).getClass().getName());
+        assertEquals("org.apache.tika.detect.mkv.MatroskaDetector", detectors.get(4).getClass().getName());
+        assertEquals("org.apache.tika.detect.ogg.OggDetector", detectors.get(5).getClass().getName());
+        assertEquals("org.apache.tika.detect.ole.MiscOLEDetector", detectors.get(6).getClass().getName());
         assertEquals("org.apache.tika.detect.zip.DefaultZipContainerDetector",
-                detectors.get(6).getClass().getName());
+                detectors.get(7).getClass().getName());
     }
 }

tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/mime/TestMimeTypes.java

@@ -16,6 +16,7 @@
  */
 package org.apache.tika.mime;
 
+import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_16BE;
 import static java.nio.charset.StandardCharsets.UTF_16LE;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -894,7 +895,17 @@ public void testRawDetection() throws Exception {
         assertTypeByName("image/x-raw-minolta", "x.mrw");
         assertTypeByName("image/x-raw-nikon", "x.nef");
         assertTypeByName("image/x-raw-nikon", "x.nrw");
+        assertTypeByName("image/x-raw-samsung", "x.srw");
+        //MimeTypes alone sees a TIFF; RawTiffDetector tells the raw formats apart,
+        //see TestContainerAwareDetector
         assertTypeByData("image/tiff", "testNEF.nef");
+        //formats with a fixed signature (TIKA-4861)
+        assertTypeByData("image/x-raw-fuji", "FUJIFILMCCD-RAW 0201FF393103".getBytes(US_ASCII));
+        assertTypeByData("image/x-raw-panasonic", new byte[]{'I', 'I', 'U', 0, 0x18, 0, 0, 0});
+        assertTypeByData("image/x-raw-minolta", new byte[]{0, 'M', 'R', 'M', 0, 0, 0, 0});
+        assertTypeByData("image/x-raw-olympus", new byte[]{'I', 'I', 'R', 'O', 8, 0, 0, 0});
+        assertTypeByData("image/x-raw-olympus", new byte[]{'I', 'I', 'R', 'S', 8, 0, 0, 0});
+        assertTypeByData("image/x-raw-olympus", new byte[]{'M', 'M', 'O', 'R', 0, 0, 0, 8});
         assertTypeByNameAndData("image/x-raw-nikon", "testNEF.nef");
         assertTypeByData("image/tiff", "testARW.arw");
         assertTypeByNameAndData("image/x-raw-sony", "testARW.arw");