Skip to content

Commit da7e4d8

Browse files
tballisontballison
authored
TIKA-4878: zip and friends (#3134)
1 parent 298144e commit da7e4d8

14 files changed

Lines changed: 488 additions & 46 deletions

File tree

‎CHANGES.txt‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,22 @@
11
Release 4.1.0 - unreleased
22

3+
* Entries of ODF, EPUB, GeoGebra, WACZ, XLZ and iWork containers, mbox
4+
messages and the AppleSingle data fork are re-opened from their
5+
container on rewind instead of cached: digesting rewinds every
6+
embedded document, and the cached copy cost heap for the whole entry
7+
and, past the cache budget or the 1 MB floor, a temp file. AppleSingle
8+
no longer spools its data fork to a temp file on every parse, and
9+
GeoGebra no longer spools every embedded picture to detect it
10+
(TIKA-4878).
11+
312
* A declared Content-Length is no longer treated as a measurement: the
413
zip-bomb ratio counts only measured input bytes (a container-declared
514
size on an embedded document could inflate its denominator), and a
615
re-openable source no longer reserves cache budget or sizes its buffer
716
from the declared length (a lying one could push a small payload to
817
disk or churn the shared budget). Neither is in a release: the
918
exposure arrived with TIKA-4868 and TIKA-4873 (TIKA-4878).
19+
1020
* Embedded objects in Office documents are re-opened from their container
1121
instead of cached: every OOXML part (pictures, media, attachments), the
1222
OLE 2.0 package inside an OOXML part, the CONTENTS entry of an OLE 2.0
@@ -15,7 +25,7 @@ Release 4.1.0 - unreleased
1525
the cached copy that made possible cost heap for the whole object and,
1626
past the cache budget or the 1 MB floor, a temp file. The container
1727
hands the bytes back on demand, so neither is needed (TIKA-4878).
18-
28+
1929
* PDF attachments, PDF XMP packets, 3D on-instantiate scripts and PST
2030
attachments are re-opened from their document instead of cached when
2131
the embedded-document extractor rewinds them (digesting does, for every

‎tika-core/src/test/java/org/apache/tika/extractor/RewindRecordingExtractor.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.extractor;
18+
19+
import static org.junit.jupiter.api.Assertions.assertFalse;
20+
import static org.junit.jupiter.api.Assertions.assertTrue;
21+
22+
import java.io.IOException;
23+
import java.util.ArrayList;
24+
import java.util.Collections;
25+
import java.util.List;
26+
27+
import org.xml.sax.ContentHandler;
28+
29+
import org.apache.tika.io.TikaInputStream;
30+
import org.apache.tika.metadata.Metadata;
31+
import org.apache.tika.parser.ParseContext;
32+
33+
/**
34+
* Test double for parsers that hand embedded documents over. Rewinds each
35+
* embedded stream the way a digester does, then records whether that left the
36+
* stream backed by a temp file and how many bytes it still yields.
37+
* <p>
38+
* Assert on what this extractor was handed, not on a watched temp directory:
39+
* a parser owns each child's {@code TemporaryResources}, so a directory watch
40+
* passes whatever the parser did.
41+
*/
42+
public class RewindRecordingExtractor implements EmbeddedDocumentExtractor {
43+
44+
private final List<Boolean> spooled = new ArrayList<>();
45+
private final List<Integer> lengths = new ArrayList<>();
46+
47+
@Override
48+
public boolean shouldParseEmbedded(Metadata metadata, ParseContext context) {
49+
return true;
50+
}
51+
52+
@Override
53+
public void parseEmbedded(TikaInputStream stream, ContentHandler handler, Metadata metadata,
54+
ParseContext context, boolean outputHtml) throws IOException {
55+
stream.enableRewind();
56+
stream.readAllBytes();
57+
stream.rewind();
58+
spooled.add(stream.hasFile());
59+
lengths.add(stream.readAllBytes().length);
60+
}
61+
62+
/** Byte counts of the embedded documents seen, in order. */
63+
public List<Integer> lengths() {
64+
return Collections.unmodifiableList(lengths);
65+
}
66+
67+
public void assertSawLength(int length) {
68+
assertTrue(lengths.contains(length),
69+
"an embedded document of " + length + " bytes reached the extractor; saw "
70+
+ lengths);
71+
}
72+
73+
public void assertSawLengthAtLeast(int length) {
74+
assertTrue(lengths.stream().anyMatch(l -> l >= length),
75+
"an embedded document of at least " + length + " bytes reached the extractor; saw "
76+
+ lengths);
77+
}
78+
79+
public void assertNothingSpooled() {
80+
for (int i = 0; i < spooled.size(); i++) {
81+
assertFalse(spooled.get(i), "embedded stream " + i + " (" + lengths.get(i)
82+
+ " bytes) was spooled to disk to rewind instead of re-opened");
83+
}
84+
}
85+
}

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/apple/AppleSingleFileParser.java‎

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
import java.io.IOException;
2020
import java.io.InputStream;
21+
import java.nio.channels.Channels;
22+
import java.nio.channels.SeekableByteChannel;
2123
import java.nio.charset.StandardCharsets;
2224
import java.util.ArrayList;
2325
import java.util.Collections;
@@ -35,7 +37,9 @@
3537
import org.apache.tika.exception.TikaMemoryLimitException;
3638
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
3739
import org.apache.tika.extractor.EmbeddedDocumentUtil;
40+
import org.apache.tika.io.CacheMemoryBudget;
3841
import org.apache.tika.io.EndianUtils;
42+
import org.apache.tika.io.TemporaryResources;
3943
import org.apache.tika.io.TikaInputStream;
4044
import org.apache.tika.metadata.Metadata;
4145
import org.apache.tika.metadata.TikaCoreProperties;
@@ -86,36 +90,54 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
8690

8791
EmbeddedDocumentExtractor ex = EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
8892

93+
//the data fork is handed over as a region of this stream, which needs a
94+
//seekable view after the header has been read sequentially
95+
tis.enableRewind(context.get(CacheMemoryBudget.class));
8996
short numEntries = readThroughNumEntries(tis);
90-
long bytesRead = 26;
9197
List<FieldInfo> fieldInfoList = getSortedFieldInfoList(tis, numEntries);
92-
bytesRead += 12 * numEntries;
9398
Metadata embeddedMetadata = Metadata.newInstance(context);
94-
bytesRead = processFieldEntries(tis, fieldInfoList, embeddedMetadata, bytesRead);
99+
processFieldEntries(tis, fieldInfoList, embeddedMetadata, 26 + 12L * numEntries);
95100
FieldInfo contentFieldInfo = getContentFieldInfo(fieldInfoList);
96101
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context);
97102
xhtml.startDocument();
98-
if (contentFieldInfo != null) {
99-
long diff = contentFieldInfo.offset - bytesRead;
100-
IOUtils.skipFully(tis, diff);
101-
if (ex.shouldParseEmbedded(embeddedMetadata, context)) {
102-
// Use BoundedInputStream to limit bytes read, then spool to temp file
103-
// for complete isolation from parent stream (reset() goes to embedded start)
104-
BoundedInputStream bounded =
105-
BoundedInputStream.builder()
106-
.setInputStream(tis)
107-
.setMaxCount(contentFieldInfo.length)
108-
.get();
109-
try (TikaInputStream inner = TikaInputStream.get(bounded)) {
110-
inner.getPath();
111-
ex.parseEmbedded(inner, xhtml, embeddedMetadata, context, true);
112-
}
103+
if (contentFieldInfo != null && ex.shouldParseEmbedded(embeddedMetadata, context)) {
104+
//re-opened from the channel on rewind: a digest re-reads the fork in place
105+
//instead of the copy-and-spool that getPath() used to force on every parse
106+
long offset = contentFieldInfo.offset;
107+
long length = contentFieldInfo.length;
108+
try (TikaInputStream inner = TikaInputStream.get(() -> region(tis, offset, length),
109+
new TemporaryResources(), null)) {
110+
ex.parseEmbedded(inner, xhtml, embeddedMetadata, context, true);
113111
}
114112
}
115113
xhtml.endDocument();
116114

117115
}
118116

117+
/**
118+
* The data fork as a fresh stream over the parent's seekable channel: in memory
119+
* when the parent is, from its file when it has one. The offset and length are
120+
* the file's own claims; a region past the end simply reads as empty.
121+
*/
122+
private static InputStream region(TikaInputStream tis, long offset, long length)
123+
throws IOException {
124+
if (offset < 0 || length < 0) {
125+
throw new IOException("AppleSingle data fork out of range: offset=" + offset +
126+
" length=" + length);
127+
}
128+
SeekableByteChannel channel = tis.getSeekableByteChannel();
129+
try {
130+
channel.position(offset);
131+
return BoundedInputStream.builder()
132+
.setInputStream(Channels.newInputStream(channel))
133+
.setMaxCount(length)
134+
.get();
135+
} catch (IOException e) {
136+
channel.close();
137+
throw e;
138+
}
139+
}
140+
119141
private FieldInfo getContentFieldInfo(List<FieldInfo> fieldInfoList) {
120142
for (FieldInfo fieldInfo : fieldInfoList) {
121143
if (fieldInfo.entryId == 1) {

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,8 @@ private MediaType processZipFile(ZipFile zipFile, Metadata metadata,
155155
if (type == null) {
156156
type = IWork13DocumentType.detectIfPossible(entry);
157157
}
158-
try (TikaInputStream tis = TikaInputStream.get(zipFile.getInputStream(entry))) {
158+
try (TikaInputStream tis = TikaInputStream.get(() -> zipFile.getInputStream(entry),
159+
new TemporaryResources(), null)) {
159160
processZipEntry(entry, tis, metadata, xhtml, parseContext, embeddedDocumentExtractor);
160161
} catch (SecurityException e) {
161162
throw e;

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork18PackageParser.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.tika.exception.TikaException;
3636
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
3737
import org.apache.tika.extractor.EmbeddedDocumentUtil;
38+
import org.apache.tika.io.TemporaryResources;
3839
import org.apache.tika.io.TikaInputStream;
3940
import org.apache.tika.metadata.HttpHeaders;
4041
import org.apache.tika.metadata.Metadata;
@@ -92,8 +93,7 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
9293
type = IWork18DocumentType.detectIfPossible(entry);
9394
}
9495
if (isPreview(entry) && zipFile.canReadEntryData(entry)) {
95-
try (TikaInputStream previewStream =
96-
TikaInputStream.get(zipFile.getInputStream(entry))) {
96+
try (TikaInputStream previewStream = entryStream(zipFile, entry)) {
9797
handleThumbnail(entry, previewStream, xhtml, context);
9898
}
9999
}
@@ -127,6 +127,11 @@ private static boolean isPreview(ZipEntry entry) {
127127
String name = entry.getName();
128128
return name.equals("preview.jpg") || name.endsWith("/preview.jpg");
129129
}
130+
/** Re-opened from the zip on rewind rather than cached: the entry is in the container already. */
131+
private static TikaInputStream entryStream(ZipFile zipFile, ZipArchiveEntry entry) {
132+
return TikaInputStream.get(() -> zipFile.getInputStream(entry), new TemporaryResources(), null);
133+
}
134+
130135

131136
private static void handleThumbnail(ZipEntry entry, TikaInputStream previewStream,
132137
XHTMLContentHandler xhtml, ParseContext context)

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/test/java/org/apache/tika/parser/apple/AppleSingleFileParserNoTempFileTest.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.parser.apple;
18+
19+
import java.io.ByteArrayOutputStream;
20+
import java.nio.file.Files;
21+
import java.nio.file.Path;
22+
import java.util.Random;
23+
24+
import org.junit.jupiter.api.Test;
25+
import org.junit.jupiter.api.io.TempDir;
26+
import org.xml.sax.helpers.DefaultHandler;
27+
28+
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
29+
import org.apache.tika.extractor.RewindRecordingExtractor;
30+
import org.apache.tika.io.TikaInputStream;
31+
import org.apache.tika.metadata.Metadata;
32+
import org.apache.tika.parser.ParseContext;
33+
34+
/**
35+
* The data fork of an AppleSingle file is a region of the file itself. The
36+
* stream handed to the embedded-document extractor must re-read that region on
37+
* rewind, from memory when the parent is in memory and from the file when it
38+
* has one, rather than copy it to a temp file. The parser used to force that
39+
* copy on every parse, digester or not.
40+
*/
41+
public class AppleSingleFileParserNoTempFileTest {
42+
43+
private static final int PAYLOAD_LENGTH = 2 * 1024 * 1024;
44+
private static final int HEADER_LENGTH = 26 + 12;
45+
46+
@TempDir
47+
Path tempDir;
48+
49+
@Test
50+
public void testDataForkFromFileIsNotSpooled() throws Exception {
51+
Path file = tempDir.resolve("fork.as");
52+
Files.write(file, appleSingle());
53+
try (TikaInputStream tis = TikaInputStream.get(file, new Metadata())) {
54+
RewindRecordingExtractor extractor = parse(tis);
55+
extractor.assertSawLength(PAYLOAD_LENGTH);
56+
extractor.assertNothingSpooled();
57+
}
58+
}
59+
60+
@Test
61+
public void testDataForkFromMemoryIsNotSpooled() throws Exception {
62+
try (TikaInputStream tis = TikaInputStream.get(appleSingle())) {
63+
RewindRecordingExtractor extractor = parse(tis);
64+
extractor.assertSawLength(PAYLOAD_LENGTH);
65+
extractor.assertNothingSpooled();
66+
}
67+
}
68+
69+
private static RewindRecordingExtractor parse(TikaInputStream tis) throws Exception {
70+
RewindRecordingExtractor extractor = new RewindRecordingExtractor();
71+
ParseContext context = new ParseContext();
72+
context.set(EmbeddedDocumentExtractor.class, extractor);
73+
new AppleSingleFileParser().parse(tis, new DefaultHandler(), new Metadata(), context);
74+
return extractor;
75+
}
76+
77+
/** Header, one entry (the data fork, id 1) and the fork itself. */
78+
private static byte[] appleSingle() {
79+
ByteArrayOutputStream out = new ByteArrayOutputStream();
80+
int32(out, 0x00051600);
81+
int32(out, 0x00020000);
82+
out.writeBytes(new byte[16]);
83+
out.write(0);
84+
out.write(1);
85+
int32(out, 1);
86+
int32(out, HEADER_LENGTH);
87+
int32(out, PAYLOAD_LENGTH);
88+
byte[] payload = new byte[PAYLOAD_LENGTH];
89+
new Random(4878).nextBytes(payload);
90+
out.writeBytes(payload);
91+
return out.toByteArray();
92+
}
93+
94+
private static void int32(ByteArrayOutputStream out, int v) {
95+
out.write((v >>> 24) & 0xFF);
96+
out.write((v >>> 16) & 0xFF);
97+
out.write((v >>> 8) & 0xFF);
98+
out.write(v & 0xFF);
99+
}
100+
}

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.tika.exception.TikaException;
4141
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
4242
import org.apache.tika.extractor.EmbeddedDocumentUtil;
43+
import org.apache.tika.io.TemporaryResources;
4344
import org.apache.tika.io.TikaInputStream;
4445
import org.apache.tika.metadata.HttpHeaders;
4546
import org.apache.tika.metadata.KeyPrefix;
@@ -144,11 +145,13 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
144145
saveHeaderInMetadata(mailMetadata, item);
145146
}
146147

147-
TikaInputStream msgStream = TikaInputStream.get(message.toInputStream());
148-
message = null;
149-
150-
if (extractor.shouldParseEmbedded(mailMetadata, context)) {
151-
extractor.parseEmbedded(msgStream, xhtml, mailMetadata, context, true);
148+
//re-opened over the buffer on rewind rather than cached: the
149+
//message is in memory already, a digest must not copy it again
150+
try (TikaInputStream msgStream = TikaInputStream.get(message::toInputStream,
151+
new TemporaryResources(), null)) {
152+
if (extractor.shouldParseEmbedded(mailMetadata, context)) {
153+
extractor.parseEmbedded(msgStream, xhtml, mailMetadata, context, true);
154+
}
152155
}
153156

154157
if (tracking) {

0 commit comments

Comments
 (0)