Skip to content

Commit b8ed3fb

Browse files
tballisontballison
authored
TIKA-4878: ole, ooxml (#3132)
1 parent 726bb2a commit b8ed3fb

7 files changed

Lines changed: 272 additions & 41 deletions

File tree

‎CHANGES.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
Release 4.1.0 - unreleased
22

3+
* Embedded objects in Office documents are re-opened from their container
4+
instead of cached: every OOXML part (pictures, media, attachments), the
5+
OLE 2.0 package inside an OOXML part, the CONTENTS entry of an OLE 2.0
6+
object in a binary Office file, embedded objects in .ppt, XPS page
7+
images and Word EMF icons. Digesting an embedded document rewinds it;
8+
the cached copy that made possible cost heap for the whole object and,
9+
past the cache budget or the 1 MB floor, a temp file. The container
10+
hands the bytes back on demand, so neither is needed (TIKA-4878).
11+
312
* PDF attachments, PDF XMP packets, 3D on-instantiate scripts and PST
413
attachments are re-opened from their document instead of cached when
514
the embedded-document extractor rewinds them (digesting does, for every

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
4343
import org.apache.tika.extractor.EmbeddedDocumentUtil;
4444
import org.apache.tika.io.BoundedInputStream;
45+
import org.apache.tika.io.TemporaryResources;
4546
import org.apache.tika.io.TikaInputStream;
4647
import org.apache.tika.metadata.HttpHeaders;
4748
import org.apache.tika.metadata.Metadata;
@@ -300,16 +301,19 @@ private void handleCompObj(DirectoryEntry parentDir, POIFSDocumentType type, Str
300301
}
301302

302303
int length = contentsEntry.getSize();
303-
DocumentInputStream inp = null;
304+
//open once now so a broken entry is recorded here, not mid-parse
304305
try {
305-
inp = new DocumentInputStream(contentsEntry);
306+
new DocumentInputStream(contentsEntry).close();
306307
} catch (SecurityException e) {
307308
throw e;
308309
} catch (Exception e) {
309310
EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
310311
return;
311312
}
312-
try (TikaInputStream tis = TikaInputStream.get(inp)) {
313+
//the entry is in the container already: re-open it on rewind instead of
314+
//caching a copy that a digest of a large object would spill to disk
315+
try (TikaInputStream tis = TikaInputStream.get(
316+
() -> new DocumentInputStream(contentsEntry), new TemporaryResources(), null)) {
313317
// Try to work out what it is
314318
MediaType mediaType = getDetector().detect(tis, metadata, context);
315319
String extension = type.getExtension();
@@ -327,8 +331,6 @@ private void handleCompObj(DirectoryEntry parentDir, POIFSDocumentType type, Str
327331
metadata.set(TikaCoreProperties.RESOURCE_NAME_EXTENSION_INFERRED, true);
328332
metadata.set(HttpHeaders.CONTENT_LENGTH, Integer.toString(length));
329333
parseEmbedded(parentDir, tis, xhtml, metadata, outputHtml);
330-
} finally {
331-
inp.close();
332334
}
333335
}
334336

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
package org.apache.tika.parser.microsoft;
1818

1919
import java.io.IOException;
20-
import java.io.InputStream;
2120
import java.util.ArrayList;
2221
import java.util.HashMap;
2322
import java.util.HashSet;
@@ -60,6 +59,7 @@
6059
import org.apache.tika.exception.EncryptedDocumentException;
6160
import org.apache.tika.exception.TikaException;
6261
import org.apache.tika.extractor.EmbeddedDocumentUtil;
62+
import org.apache.tika.io.TemporaryResources;
6363
import org.apache.tika.io.TikaInputStream;
6464
import org.apache.tika.metadata.Metadata;
6565
import org.apache.tika.metadata.Office;
@@ -232,7 +232,8 @@ private void handleShowEmbeddedResources(HSLFSlideShow ss, XHTMLContentHandler x
232232
i, getDetectedMediaType(d));
233233
inferredExtension = true;
234234
}
235-
try (TikaInputStream tis = TikaInputStream.get(d.getInputStream())) {
235+
try (TikaInputStream tis = TikaInputStream.get(d::getInputStream,
236+
new TemporaryResources(), null)) {
236237
if (FileMagic.valueOf(tis) == FileMagic.OLE2) {
237238
try (POIFSFileSystem pfs = new POIFSFileSystem(tis)) {
238239
//coz ppts can have empty pfs...shrug...
@@ -664,23 +665,24 @@ private void handleSlideEmbeddedResources(ShapeContainer shapeContainer,
664665
attributes.addAttribute("", "id", "id", "CDATA", objID);
665666
xhtml.startElement("div", attributes);
666667
xhtml.endElement("div");
667-
InputStream dataStream = null;
668+
//open once now so a broken record is recorded here, not mid-parse
668669
try {
669-
dataStream = data.getInputStream();
670+
data.getInputStream().close();
670671
} catch (Exception e) {
671672
EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
672673
continue;
673674
}
674-
handleDataStream(dataStream, objID, oleShape.getProgId(), xhtml);
675+
handleDataStream(data, objID, oleShape.getProgId(), xhtml);
675676
}
676677
}
677678
}
678679
}
679680

680-
private void handleDataStream(InputStream dataStream, String objID, String progId,
681+
private void handleDataStream(HSLFObjectData data, String objID, String progId,
681682
XHTMLContentHandler xhtml) {
682683
//TODO -- inject progId into the metadata of the embedded file
683-
try (TikaInputStream tis = TikaInputStream.get(dataStream)) {
684+
try (TikaInputStream tis = TikaInputStream.get(data::getInputStream,
685+
new TemporaryResources(), null)) {
684686
String mediaType = null;
685687
if ("Excel.Chart.8".equals(progId)) {
686688
mediaType = "application/vnd.ms-excel";

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java‎

Lines changed: 31 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
5353
import org.apache.tika.extractor.EmbeddedDocumentUtil;
5454
import org.apache.tika.io.FilenameUtils;
55+
import org.apache.tika.io.TemporaryResources;
5556
import org.apache.tika.io.TikaInputStream;
5657
import org.apache.tika.metadata.HttpHeaders;
5758
import org.apache.tika.metadata.Metadata;
@@ -204,30 +205,29 @@ private void handleThumbnail(ContentHandler handler, Metadata metadata) throws S
204205
if (tPart == null) {
205206
continue;
206207
}
207-
try (InputStream tStream = tPart.getInputStream()) {
208-
Metadata thumbnailMetadata = Metadata.newInstance(context);
209-
String thumbName = tPart.getPartName().getName();
210-
thumbnailMetadata.set(TikaCoreProperties.INTERNAL_PATH, thumbName);
211-
thumbnailMetadata.set(TikaCoreProperties.RESOURCE_NAME_KEY,
212-
FilenameUtils.getName(thumbName));
213-
214-
AttributesImpl attributes = new AttributesImpl();
215-
attributes.addAttribute(XHTML, "class", "class", "CDATA", "embedded");
216-
attributes.addAttribute(XHTML, "id", "id", "CDATA", thumbName);
217-
handler.startElement(XHTML, "div", "div", attributes);
218-
handler.endElement(XHTML, "div", "div");
219-
220-
thumbnailMetadata.set(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID, thumbName);
221-
thumbnailMetadata.set(HttpHeaders.CONTENT_TYPE, tPart.getContentType());
222-
thumbnailMetadata.set(TikaCoreProperties.TITLE, tPart.getPartName().getName());
223-
thumbnailMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE,
224-
TikaCoreProperties.EmbeddedResourceType.THUMBNAIL.name());
225-
226-
if (embeddedExtractor.shouldParseEmbedded(thumbnailMetadata, context)) {
227-
try (TikaInputStream tis = TikaInputStream.get(tStream)) {
228-
embeddedExtractor.parseEmbedded(tis,
229-
new EmbeddedContentHandler(handler), thumbnailMetadata, context, false);
230-
}
208+
Metadata thumbnailMetadata = Metadata.newInstance(context);
209+
String thumbName = tPart.getPartName().getName();
210+
thumbnailMetadata.set(TikaCoreProperties.INTERNAL_PATH, thumbName);
211+
thumbnailMetadata.set(TikaCoreProperties.RESOURCE_NAME_KEY,
212+
FilenameUtils.getName(thumbName));
213+
214+
AttributesImpl attributes = new AttributesImpl();
215+
attributes.addAttribute(XHTML, "class", "class", "CDATA", "embedded");
216+
attributes.addAttribute(XHTML, "id", "id", "CDATA", thumbName);
217+
handler.startElement(XHTML, "div", "div", attributes);
218+
handler.endElement(XHTML, "div", "div");
219+
220+
thumbnailMetadata.set(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID, thumbName);
221+
thumbnailMetadata.set(HttpHeaders.CONTENT_TYPE, tPart.getContentType());
222+
thumbnailMetadata.set(TikaCoreProperties.TITLE, tPart.getPartName().getName());
223+
thumbnailMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE,
224+
TikaCoreProperties.EmbeddedResourceType.THUMBNAIL.name());
225+
226+
if (embeddedExtractor.shouldParseEmbedded(thumbnailMetadata, context)) {
227+
try (TikaInputStream tis = TikaInputStream.get(tPart::getInputStream,
228+
new TemporaryResources(), null)) {
229+
embeddedExtractor.parseEmbedded(tis,
230+
new EmbeddedContentHandler(handler), thumbnailMetadata, context, false);
231231
}
232232
}
233233
}
@@ -400,7 +400,8 @@ private void handleEmbeddedOLE(PackagePart part, XHTMLContentHandler xhtml, Stri
400400
//OLE 2.0
401401
updateMetadata(metadata, embeddedPartMetadata);
402402

403-
tis = TikaInputStream.get(fs.createDocumentInputStream(packageEntryName));
403+
tis = TikaInputStream.get(() -> fs.createDocumentInputStream(packageEntryName),
404+
new TemporaryResources(), null);
404405
if (embeddedExtractor.shouldParseEmbedded(metadata, context)) {
405406
embeddedExtractor
406407
.parseEmbedded(tis, xhtml, metadata, context, true);
@@ -437,10 +438,10 @@ private void handleEmbeddedOLE(PackagePart part, XHTMLContentHandler xhtml, Stri
437438
} catch (IOException e) {
438439
EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata, context);
439440
} finally {
440-
fs.close();
441441
if (tis != null) {
442442
tis.close();
443443
}
444+
fs.close();
444445
}
445446
}
446447

@@ -506,7 +507,10 @@ protected void handleEmbeddedFile(PackagePart part, XHTMLContentHandler xhtml,
506507

507508
// Call the recursing handler
508509
if (embeddedExtractor.shouldParseEmbedded(metadata, context)) {
509-
try (TikaInputStream tis = TikaInputStream.get(part.getInputStream())) {
510+
//the part is in the package already: re-open it on rewind instead of
511+
//caching a copy that a digest of a large part would spill to disk
512+
try (TikaInputStream tis = TikaInputStream.get(part::getInputStream,
513+
new TemporaryResources(), null)) {
510514
embeddedExtractor
511515
.parseEmbedded(tis, xhtml, metadata, context, true);
512516
}

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.tika.exception.TikaException;
4141
import org.apache.tika.exception.WriteLimitReachedException;
4242
import org.apache.tika.extractor.EmbeddedDocumentUtil;
43+
import org.apache.tika.io.TemporaryResources;
4344
import org.apache.tika.io.TikaInputStream;
4445
import org.apache.tika.metadata.Metadata;
4546
import org.apache.tika.metadata.Office;
@@ -401,7 +402,8 @@ private void resolveEmfNames(PackagePart documentPart,
401402
continue;
402403
}
403404
if ("image/x-emf".equals(emfPart.getContentType())) {
404-
try (TikaInputStream tis = TikaInputStream.get(emfPart.getInputStream())) {
405+
try (TikaInputStream tis = TikaInputStream.get(emfPart::getInputStream,
406+
new TemporaryResources(), null)) {
405407
EMFParser p = new EMFParser();
406408
Metadata m = Metadata.newInstance(context);
407409
p.parse(tis, new org.apache.tika.sax.ToTextContentHandler(), m, context);

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xps/XPSExtractorDecorator.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.tika.exception.TikaException;
4040
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
4141
import org.apache.tika.extractor.EmbeddedDocumentUtil;
42+
import org.apache.tika.io.TemporaryResources;
4243
import org.apache.tika.io.TikaInputStream;
4344
import org.apache.tika.metadata.Metadata;
4445
import org.apache.tika.parser.ParseContext;
@@ -83,7 +84,9 @@ private static TikaInputStream getZipStream(String zipPath, ZipPackage zipPackag
8384
if (zipEntry == null) {
8485
throw new TikaException("Couldn't find required zip entry: " + zipPath);
8586
}
86-
return TikaInputStream.get(zipEntrySource.getInputStream(zipEntry));
87+
ZipArchiveEntry entry = zipEntry;
88+
return TikaInputStream.get(() -> zipEntrySource.getInputStream(entry),
89+
new TemporaryResources(), null);
8790
}
8891

8992
@Override

0 commit comments

Comments
 (0)