Revert "Revert "TIKA-4474 -- force spooling on ooxml"" (#2388)

* Revert "Revert "TIKA-4474 -- force spooling on ooxml (#2386)" (#2387)"

This reverts commit f00dbce.

* TIKA-4474 -- force spooling for xml checkstyle

* TIKA-4474 -- remove monster file from multithreaded tests

Author: tballison

tika-core/src/test/java/org/apache/tika/TikaTest.java

@@ -42,8 +42,10 @@
 
 import org.apache.commons.io.IOUtils;
 import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
 
 import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.WriteLimitReachedException;
 import org.apache.tika.extractor.EmbeddedResourceHandler;
 import org.apache.tika.io.FilenameUtils;
 import org.apache.tika.io.TikaInputStream;
@@ -569,10 +571,12 @@ protected String getText(String filePath, Parser parser, Metadata metadata,
     public String getText(InputStream is, Parser parser, ParseContext context, Metadata metadata)
             throws Exception {
         ContentHandler handler = new BodyContentHandler(1000000);
-        try {
+        try (is) {
             parser.parse(is, handler, metadata, context);
-        } finally {
-            is.close();
+        } catch (SAXException e) {
+            if (!WriteLimitReachedException.isWriteLimitReached(e)) {
+                throw e;
+            }
         }
         return handler.toString();
     }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java

@@ -16,15 +16,12 @@
  */
 package org.apache.tika.parser.microsoft.ooxml;
 
-import java.io.EOFException;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.util.Locale;
 
-import org.apache.commons.compress.archivers.zip.UnsupportedZipFeatureException;
-import org.apache.commons.io.input.CloseShieldInputStream;
 import org.apache.poi.extractor.ExtractorFactory;
 import org.apache.poi.ooxml.POIXMLDocument;
 import org.apache.poi.ooxml.extractor.POIXMLExtractorFactory;
@@ -64,7 +61,6 @@
 import org.apache.tika.parser.microsoft.ooxml.xps.XPSTextExtractor;
 import org.apache.tika.parser.microsoft.ooxml.xslf.XSLFEventBasedPowerPointExtractor;
 import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFEventBasedWordExtractor;
-import org.apache.tika.utils.RereadableInputStream;
 import org.apache.tika.zip.utils.ZipSalvager;
 
 /**
@@ -105,10 +101,10 @@ public static void parse(InputStream stream, ContentHandler baseHandler, Metadat
             OOXMLExtractor extractor = null;
 
             // Locate or Open the OPCPackage for the file
-            TikaInputStream tis = TikaInputStream.cast(stream);
-            if (tis != null && tis.getOpenContainer() instanceof OPCPackageWrapper) {
+            TikaInputStream tis = TikaInputStream.get(stream);
+            if (tis.getOpenContainer() instanceof OPCPackageWrapper) {
                 pkg = ((OPCPackageWrapper) tis.getOpenContainer()).getOPCPackage();
-            } else if (tis != null && tis.hasFile()) {
+            } else {
                 try {
                     pkg = OPCPackage.open(tis.getFile().getPath(), PackageAccess.READ);
                 } catch (InvalidOperationException e) {
@@ -117,44 +113,6 @@ public static void parse(InputStream stream, ContentHandler baseHandler, Metadat
                     pkg = OPCPackage.open(tmpRepairedCopy, PackageAccess.READ);
                 }
                 tis.setOpenContainer(new OPCPackageWrapper(pkg));
-            } else {
-                //OPCPackage slurps rris into memory so we can close rris
-                //without apparent problems
-                mustRevertPackage = true;
-                try (RereadableInputStream rereadableInputStream = new RereadableInputStream(stream,
-                        MAX_BUFFER_LENGTH, false)) {
-                    try {
-                        pkg = OPCPackage.open(CloseShieldInputStream.wrap(rereadableInputStream));
-                    } catch (UnsupportedZipFeatureException e) {
-                        if (e.getFeature() !=
-                                UnsupportedZipFeatureException.Feature.DATA_DESCRIPTOR) {
-                            throw e;
-                        }
-                        rereadableInputStream.rewind();
-                        tmpRepairedCopy = Files.createTempFile("tika-ooxml-repair-", "").toFile();
-                        ZipSalvager.salvageCopy(rereadableInputStream, tmpRepairedCopy, false);
-                        //if there isn't enough left to be opened as a package
-                        //throw an exception -- we may want to fall back to streaming
-                        //parsing
-                        pkg = OPCPackage.open(tmpRepairedCopy, PackageAccess.READ);
-                    } catch (IOException e) {
-                        if (e instanceof EOFException) {
-                            //keep going
-                        } else if (e instanceof IOException && e.getMessage() != null &&
-                                e.getMessage().contains("Truncated")) {
-                            //keep going
-                        } else {
-                            throw e;
-                        }
-                        rereadableInputStream.rewind();
-                        tmpRepairedCopy = Files.createTempFile("tika-ooxml-repair-", "").toFile();
-                        ZipSalvager.salvageCopy(rereadableInputStream, tmpRepairedCopy, false);
-                        //if there isn't enough left to be opened as a package
-                        //throw an exception -- we may want to fall back to streaming
-                        //parsing
-                        pkg = OPCPackage.open(tmpRepairedCopy, PackageAccess.READ);
-                    }
-                }
             }
 
             if (pkg != null) {

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLParserTest.java

@@ -1795,6 +1795,9 @@ public void testMultiThreaded() throws Exception {
         RecursiveParserWrapper wrapper = new RecursiveParserWrapper(AUTO_DETECT_PARSER);
         testMultiThreaded(wrapper, parseContexts, numThreads, numIterations, path -> {
             String pathName = path.getName().toLowerCase(Locale.ENGLISH);
+            if (pathName.equalsIgnoreCase("testRecordSizeExceeded.xlsx")) {
+                return false;
+            }
             int i = pathName.lastIndexOf(".");
             String ext = "";
             if (i > -1) {
@@ -1814,4 +1817,11 @@ public void testFeatureExtraction() throws Exception {
         assertEquals("true", m.get(Office.HAS_TRACK_CHANGES));
         assertEquals("true", m.get(Office.HAS_COMMENTS));
     }
+
+    @Test
+    public void testNoRecordSizeOverflow() throws Exception {
+        //TIKA-4474 -- test: files (passed as stream) no longer have limit on record size as they are spooled
+        String content = getText("testRecordSizeExceeded.xlsx");
+        assertContains("Repetitive content pattern 3 for compression test row 1", content);
+    }
 }