TIKA-4518 (#2366)

* TIKA-4518 -- improve pst handling with -Z option

Author: tballison

CHANGES.txt

@@ -7,17 +7,11 @@ Release 4.0.0-BETA1 - ???
    * Headers are no longer injected into the body/content of MSG files (TIKA-4345). Please open
      a ticket if you need this behavior across email formats.
 
-   * Remove tika-batch (TIKA-4333).
+   * Removed several modules, including: tika-batch (TIKA-4333), snaps deployment (TIKA-4502),
+     dotnet (TIKA-4332), advanced media module (TIKA-4500), tika-dl module (TIKA-4499),
+     tika-fuzzing module (TIKA-4506).
 
-   * Remove snaps deployment (TIKA-4502).
-
-   * Removed the dotnet module (TIKA-4332).
-
-   * Removed the advanced media module (TIKA-4500).
-   
-   * Removed the tika-dl module (TIKA-4499).
-
-   * Removed the tika-fuzzing module (TIKA-4506).
+   * API changes in the EmbeddedStreamTranslator (TIKA-4518).
 
   OTHER CHANGES
 

tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java

@@ -1112,9 +1112,7 @@ public void parseEmbedded(TikaInputStream tis, ContentHandler contentHandler, Me
 
             try (OutputStream os = Files.newOutputStream(outputFile)) {
                 if (embeddedStreamTranslator.shouldTranslate(tis, metadata)) {
-                    try (InputStream translated = embeddedStreamTranslator.translate(tis, metadata)) {
-                        IOUtils.copy(translated, os);
-                    }
+                    embeddedStreamTranslator.translate(tis, metadata, os);
                 } else {
                     IOUtils.copy(tis, os);
                 }

tika-app/src/test/java/org/apache/tika/cli/TikaCLIAsyncTest.java

@@ -121,7 +121,7 @@ public void testAsync() throws Exception {
                 json++;
             }
         }
-        assertEquals(20, json);
+        assertEquals(21, json);
     }
 
     private void checkForPrettyPrint(File f) throws IOException {

tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java

@@ -26,6 +26,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.io.Reader;
 import java.net.URI;
 import java.nio.file.FileVisitResult;
 import java.nio.file.FileVisitor;
@@ -34,6 +35,7 @@
 import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.jetbrains.annotations.NotNull;
@@ -44,7 +46,11 @@
 import org.junit.jupiter.api.io.TempDir;
 
 import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.serialization.JsonMetadataList;
 import org.apache.tika.utils.ProcessUtils;
+import org.apache.tika.utils.StringUtils;
 
 /**
  * Tests the Tika's cli
@@ -285,6 +291,28 @@ public void testRUnpack() throws Exception {
         testRecursiveUnpack("testPDFPackage.pdf", expectedChildren, 2);
     }
 
+    @Test
+    public void testPSTRUnpack() throws Exception {
+        String[] expectedChildren = new String[]{"testPST.pst.json",
+                "testPST.pst-embed/00000007-First email.msg",
+                "testPST.pst-embed/00000001-Feature Generators.msg",
+                "testPST.pst-embed/00000008-First email.msg",
+                "testPST.pst-embed/00000004-[jira] [Resolved] (TIKA-1249) Vcard files detection.msg",
+                "testPST.pst-embed/00000003-Feature Generators.msg",
+                "testPST.pst-embed/00000002-putstatic\".msg",
+                "testPST.pst-embed/00000005-[jira] [Commented] (TIKA-1250) Process loops infintely processing a CHM file.msg",
+                "testPST.pst-embed/00000009-attachment.docx",
+                "testPST.pst-embed/00000006-[WEBINAR] - \"Introducing Couchbase Server 2.5\".msg"};
+        testRecursiveUnpack("testPST.pst", expectedChildren, 2);
+        try (Reader reader = Files.newBufferedReader(extractDir.resolve("testPST.pst.json"))) {
+            List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
+            for (Metadata m : metadataList) {
+                String content = m.get(TikaCoreProperties.TIKA_CONTENT);
+                assertFalse(StringUtils.isBlank(content));
+            }
+        }
+    }
+
 
     /**
      * Tests -l option of the cli
@@ -378,7 +406,6 @@ private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFil
                 .list();
         assertNotNull(jsonFile);
         assertEquals(expectedLength, jsonFile.length);
-        //assertEquals(fileNames.size(), expectedChildrenFileNames.length);
 
         for (String expectedChildName : expectedChildrenFileNames) {
             assertTrue(fileNames.contains(expectedChildName));

tika-app/src/test/resources/test-data/testPST.pst

@@ -17,18 +17,17 @@
 package org.apache.tika.extractor;
 
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.List;
 
 import org.apache.tika.config.ServiceLoader;
+import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.utils.ServiceLoaderUtils;
 
 /**
  * Loads EmbeddedStreamTranslators via service loading.  Tries to run each
- * in turn and returns the first non-null value.  If no translation has occurred,
- * this returns the original InputStream. If a translation has occurred, the
- * translator will consume the InputStream but not close it.
+ * in turn. If a translator accepts the stream, it will do the translation but not close the stream.
  */
 public class DefaultEmbeddedStreamTranslator implements EmbeddedStreamTranslator {
 
@@ -58,7 +57,7 @@ private DefaultEmbeddedStreamTranslator(List<EmbeddedStreamTranslator> translato
      * @throws IOException
      */
     @Override
-    public boolean shouldTranslate(InputStream inputStream, Metadata metadata) throws IOException {
+    public boolean shouldTranslate(TikaInputStream inputStream, Metadata metadata) throws IOException {
         for (EmbeddedStreamTranslator translator : translators) {
             if (translator.shouldTranslate(inputStream, metadata)) {
                 return true;
@@ -68,20 +67,20 @@ public boolean shouldTranslate(InputStream inputStream, Metadata metadata) throw
     }
 
     /**
-     * This will consume the InputStream and return a new stream of translated bytes.
+     * This will consume the InputStream and write the stream to the output stream
      * @param inputStream
      * @param metadata
+     * @param outputStream to write to
      * @return
      * @throws IOException
      */
     @Override
-    public InputStream translate(InputStream inputStream, Metadata metadata) throws IOException {
+    public void translate(TikaInputStream inputStream, Metadata metadata, OutputStream outputStream) throws IOException {
         for (EmbeddedStreamTranslator translator : translators) {
-            InputStream translated = translator.translate(inputStream, metadata);
-            if (translated != null) {
-                return translated;
+            if (translator.shouldTranslate(inputStream, metadata)) {
+                translator.translate(inputStream, metadata, outputStream);
+                return;
             }
         }
-        return inputStream;
     }
 }

tika-core/src/main/java/org/apache/tika/extractor/DefaultEmbeddedStreamTranslator.java

@@ -17,8 +17,9 @@
 package org.apache.tika.extractor;
 
 import java.io.IOException;
-import java.io.InputStream;
+import java.io.OutputStream;
 
+import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
 
 /**
@@ -30,9 +31,8 @@
  */
 public interface EmbeddedStreamTranslator {
 
-    boolean shouldTranslate(InputStream inputStream, Metadata metadata) throws IOException;
+    boolean shouldTranslate(TikaInputStream inputStream, Metadata metadata) throws IOException;
 
-    InputStream translate(InputStream inputStream,
-                          Metadata metadata) throws IOException;
+    void translate(TikaInputStream inputStream, Metadata metadata, OutputStream os) throws IOException;
 
 }

tika-core/src/main/java/org/apache/tika/extractor/EmbeddedStreamTranslator.java

@@ -21,6 +21,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
@@ -110,18 +111,27 @@ public void parseEmbedded(
         }
     }
 
-    private void parseWithBytes(TikaInputStream stream, ContentHandler handler, Metadata metadata)
-            throws TikaException, IOException, SAXException {
-        //TODO -- improve the efficiency of this so that we're not
-        //literally writing out a file per request
+    private void parseWithBytes(TikaInputStream tis, ContentHandler handler, Metadata metadata) throws TikaException, IOException, SAXException {
+
         Path tmp = Files.createTempFile("tika-tmp-", ".bin");
-        if (embeddedStreamTranslator.shouldTranslate(stream, metadata)) {
-            Files.copy(embeddedStreamTranslator.translate(stream, metadata), tmp, StandardCopyOption.REPLACE_EXISTING);
-        } else {
-            Files.copy(stream, tmp, StandardCopyOption.REPLACE_EXISTING);
-        }
-        try (TikaInputStream tmpTis = TikaInputStream.get(tmp)) {
-            parse(tmpTis, handler, metadata);
+        try {
+            //translate the stream or not
+            if (embeddedStreamTranslator.shouldTranslate(tis, metadata)) {
+                try (OutputStream os = Files.newOutputStream(tmp)) {
+                    embeddedStreamTranslator.translate(tis, metadata, os);
+                }
+            } else {
+                Files.copy(tis, tmp, StandardCopyOption.REPLACE_EXISTING);
+            }
+
+            //now do the parse
+            if (tis.getOpenContainer() != null) {
+                parse(tis, handler, metadata);
+            } else {
+                try (TikaInputStream tisTmp = TikaInputStream.get(tmp)) {
+                    parse(tisTmp, handler, metadata);
+                }
+            }
         } finally {
             try {
                 storeEmbeddedBytes(tmp, metadata);
@@ -139,6 +149,10 @@ private void parse(InputStream stream, ContentHandler handler, Metadata metadata
     }
 
     private void storeEmbeddedBytes(Path p, Metadata metadata) {
+        if (p == null) {
+            return;
+        }
+
         if (! embeddedBytesSelector.select(metadata)) {
             if (LOGGER.isDebugEnabled()) {
                 LOGGER.debug("skipping embedded bytes {} <-> {}",

tika-core/src/main/java/org/apache/tika/extractor/RUnpackExtractor.java

@@ -140,12 +140,16 @@ public static String getSuffixFromPath(String path) {
 
     public static String getSanitizedEmbeddedFileName(Metadata metadata,
                                                       String defaultExtension, int maxLength) {
-        String path = getEmbeddedPath(metadata);
+        String path = getEmbeddedName(metadata);
         //fName could be a full path or null
         if (StringUtils.isBlank(path)) {
             return null;
         }
         path = path.replaceAll("\u0000", " ");
+        if (path.startsWith("\"") && path.endsWith("\"")) {
+            path = path.substring(1, path.length() - 1);
+        }
+
         int prefixLength = getPrefixLength(path);
         if (prefixLength > 0) {
             path = path.substring(prefixLength);
@@ -173,6 +177,7 @@ public static String getSanitizedEmbeddedFileName(Metadata metadata,
         namePart = namePart.replaceAll("(\\.\\.)+", "_");
         namePart = namePart.replaceAll("[/\\\\]+", "_");
         namePart = namePart.replaceAll(":+", "_");
+        namePart = namePart.trim();
 
         if (StringUtils.isBlank(namePart)) {
             return null;
@@ -286,6 +291,7 @@ private static String removeProtocol(String path) {
         return path;
     }
 
+    //may return null
     private static String getEmbeddedPath(Metadata metadata) {
         //potentially look for other values in embedded path or original file name, etc...
         //maybe different fallback order?
@@ -304,6 +310,27 @@ private static String getEmbeddedPath(Metadata metadata) {
         return metadata.get(TikaCoreProperties.ORIGINAL_RESOURCE_NAME);
     }
 
+    //this tries for resource name first, and then backs off to path
+    private static String getEmbeddedName(Metadata metadata) {
+        //potentially look for other values in embedded path or original file name, etc...
+        //maybe different fallback order?
+        String path = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
+        if (! StringUtils.isBlank(path)) {
+            return path;
+        }
+        path = metadata.get(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID);
+        if (! StringUtils.isBlank(path)) {
+            return path;
+        }
+
+        path = metadata.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH);
+        if (! StringUtils.isBlank(path)) {
+            return path;
+        }
+
+        return metadata.get(TikaCoreProperties.ORIGINAL_RESOURCE_NAME);
+    }
+
     /**
      * Calculate the extension based on the {@link Metadata#CONTENT_TYPE} value.
      * On parse exception or null value, return the default value.

tika-core/src/main/java/org/apache/tika/io/FilenameUtils.java

@@ -151,6 +151,9 @@ public void testEmbeddedFileNames() throws Exception {
         assertEquals("brown fox.xlsx", sanitizeFilename("a:/the quick:brown fox.xlsx"));
         assertEquals("_the quick brown fox.xlsx", sanitizeFilename("C:\\a/b/c/..the quick brown fox.xlsx"));
         assertEquals("_the quick brown fox.xlsx", sanitizeFilename("~/a/b/c/.the quick brown fox.xlsx"));
+        assertEquals("the quick%3Ebrown fox.xlsx", sanitizeFilename("the quick>brown fox.xlsx"));
+        assertEquals("the quick\"brown fox.xlsx", sanitizeFilename("the quick\"brown fox.xlsx"));
+        assertEquals("the quick brown fox.xlsx", sanitizeFilename("\"the quick brown fox.xlsx\""));
 
         assertEquals("_.docx", sanitizeFilename("..................docx"));
         assertEquals("_.docx", sanitizeFilename("..docx"));
@@ -168,7 +171,7 @@ public void testEmbeddedFileNames() throws Exception {
     @Test
     public void testEmbeddedFilePaths() throws Exception {
         String n = "the quick brown fox.docx";
-        /*assertEquals(n, sanitizePath(n));
+        assertEquals(n, sanitizePath(n));
         assertEquals(n, sanitizePath(n.substring(0, n.length() - 5),
                 "application/vnd.openxmlformats-officedocument.wordprocessingml.document"));
         assertEquals(n, sanitizeFilename("the quick\u0000brown fox.docx"));
@@ -204,7 +207,7 @@ public void testEmbeddedFilePaths() throws Exception {
         assertNull(sanitizePath(""));
         assertNull(sanitizePath(null));
         assertNull(sanitizePath("/"));
-        assertNull(sanitizePath("~/"));*/
+        assertNull(sanitizePath("~/"));
         assertNull(sanitizePath("C:"));
         assertNull(sanitizePath("C:/"));
         assertNull(sanitizePath("C:\\"));
@@ -235,6 +238,7 @@ private Metadata getMetadata(String name, String contentType) {
 
     private Metadata getMetadata(String name) {
         Metadata metadata = new Metadata();
+        metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, name);
         metadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_PATH, name);
         return metadata;
     }