Skip to content

Commit 4a563d6

Browse files
authored
Add a placeholder marker (#3122)
1 parent c726a6b commit 4a563d6

10 files changed

Lines changed: 224 additions & 8 deletions

File tree

CHANGES.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
Release 4.1.0 - unreleased
22

3+
* Placeholder streams -- the empty stand-ins parsers hand parseEmbedded
4+
for content that is never extracted -- report an unknown length rather
5+
than their own zero, and the macro-failure entry is registered without
6+
parsing its sentinel (TIKA-4874).
7+
38
* TikaInputStream.hasReliableLength() distinguishes measured lengths
49
from declared Content-Length hints, and one-shot streams now carry a
510
declared length without spooling; detection sizes its magic read only
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.io;
18+
19+
import java.io.IOException;
20+
import java.io.InputStream;
21+
import java.nio.channels.SeekableByteChannel;
22+
import java.nio.file.Path;
23+
24+
/**
25+
* Empty stand-in for content that is never extracted: its emptiness describes the
26+
* placeholder, not the document, so it reports an unknown length rather than zero.
27+
*
28+
* @see TikaInputStream#getPlaceholder()
29+
*/
30+
class PlaceholderSource extends InputStream implements TikaInputSource {
31+
32+
private final TemporaryResources tmp;
33+
private Path spilledPath;
34+
35+
PlaceholderSource(TemporaryResources tmp) {
36+
this.tmp = tmp;
37+
}
38+
39+
@Override
40+
public int read() {
41+
return -1;
42+
}
43+
44+
@Override
45+
public int read(byte[] b, int off, int len) {
46+
// InputStream contract: a zero-length read returns 0, even at EOF
47+
return len == 0 ? 0 : -1;
48+
}
49+
50+
@Override
51+
public long skip(long n) {
52+
return 0;
53+
}
54+
55+
@Override
56+
public int available() {
57+
return 0;
58+
}
59+
60+
@Override
61+
public void seekTo(long newPosition) throws IOException {
62+
if (newPosition != 0) {
63+
throw new IOException("Invalid seek position: " + newPosition + " (empty source)");
64+
}
65+
}
66+
67+
@Override
68+
public Path materializedPath() {
69+
return spilledPath;
70+
}
71+
72+
@Override
73+
public boolean hasPath() {
74+
return spilledPath != null;
75+
}
76+
77+
@Override
78+
public Path getPath(String suffix) throws IOException {
79+
if (spilledPath == null) {
80+
spilledPath = tmp.createTempFile(suffix);
81+
}
82+
return spilledPath;
83+
}
84+
85+
@Override
86+
public long getLength() {
87+
return -1;
88+
}
89+
90+
@Override
91+
public boolean hasReliableLength() {
92+
return false;
93+
}
94+
95+
@Override
96+
public boolean isPlaceholder() {
97+
return true;
98+
}
99+
100+
@Override
101+
public void enableRewind(CacheMemoryBudget budget) {
102+
// No-op: there is nothing to rewind
103+
}
104+
105+
@Override
106+
public SeekableByteChannel getSeekableByteChannel() {
107+
return new MemorySeekableByteChannel(new byte[0], 0);
108+
}
109+
110+
@Override
111+
public synchronized void mark(int readlimit) {
112+
}
113+
114+
@Override
115+
public synchronized void reset() {
116+
}
117+
118+
@Override
119+
public boolean markSupported() {
120+
return true;
121+
}
122+
}

tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,14 @@ interface TikaInputSource extends Closeable {
6969
*/
7070
boolean hasReliableLength();
7171

72+
/**
73+
* Whether this source stands in for content that is never extracted. Spooling one
74+
* measures nothing, so its unknown length must not cost a temp file to confirm.
75+
*/
76+
default boolean isPlaceholder() {
77+
return false;
78+
}
79+
7280
/**
7381
* Enables full rewind capability.
7482
* <p>

tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,18 @@ public static TikaInputStream get(byte[] data, Metadata metadata) {
149149
return new TikaInputStream(inputSource, tmp, ext);
150150
}
151151

152+
/**
153+
* An empty stream standing in for content that is never extracted -- a metadata-only
154+
* entry, a rendering carried as an open container. It reports an <em>unknown</em>
155+
* length, so nothing mistakes the placeholder's size for the document's. Pair it with
156+
* {@link org.apache.tika.parser.MetadataOnlyParse} to register an entry without
157+
* parsing it, unless an open container supplies the content.
158+
*/
159+
public static TikaInputStream getPlaceholder() {
160+
TemporaryResources tmp = new TemporaryResources();
161+
return new TikaInputStream(new PlaceholderSource(tmp), tmp, "");
162+
}
163+
152164
public static TikaInputStream get(Path path) throws IOException {
153165
return get(path, new Metadata());
154166
}
@@ -473,7 +485,7 @@ public long getLength() throws IOException {
473485
return -1;
474486
}
475487
long len = source.getLength();
476-
if (len == -1) {
488+
if (len == -1 && !source.isPlaceholder()) {
477489
// Force spill to get length
478490
getPath();
479491
len = source.getLength();

tika-core/src/main/java/org/apache/tika/renderer/RenderResult.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ public TikaInputStream getInputStream() throws IOException {
6565
if (result instanceof Path) {
6666
return TikaInputStream.get((Path)result, metadata);
6767
} else {
68-
TikaInputStream tis = TikaInputStream.get(new byte[0]);
68+
// the rendering rides in the open container, not the stream
69+
TikaInputStream tis = TikaInputStream.getPlaceholder();
6970
tis.setOpenContainer(result);
7071
return tis;
7172
}
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.io;
18+
19+
import static org.junit.jupiter.api.Assertions.assertEquals;
20+
import static org.junit.jupiter.api.Assertions.assertFalse;
21+
import static org.junit.jupiter.api.Assertions.assertTrue;
22+
23+
import java.nio.file.Files;
24+
25+
import org.junit.jupiter.api.Test;
26+
27+
public class PlaceholderStreamTest {
28+
29+
@Test
30+
public void testPlaceholderDeclaresNoLength() throws Exception {
31+
try (TikaInputStream tis = TikaInputStream.getPlaceholder()) {
32+
assertFalse(tis.hasLength(), "a placeholder's size describes nothing");
33+
assertEquals(-1, tis.read(), "placeholder is empty");
34+
}
35+
}
36+
37+
/** A real empty document is not a placeholder: zero is honest there. */
38+
@Test
39+
public void testGenuinelyEmptyStreamStillDeclaresZero() throws Exception {
40+
try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
41+
assertTrue(tis.hasLength());
42+
assertEquals(0, tis.getLength());
43+
}
44+
}
45+
46+
/** Spooling must not turn the placeholder's absent length into a zero. */
47+
@Test
48+
public void testSpoolingKeepsLengthUnknown() throws Exception {
49+
try (TikaInputStream tis = TikaInputStream.getPlaceholder()) {
50+
assertEquals(0, Files.size(tis.getPath()));
51+
assertFalse(tis.hasLength());
52+
}
53+
}
54+
55+
/** Measuring a placeholder must not cost a temp file: there is nothing to measure. */
56+
@Test
57+
public void testMeasuringCostsNoTempFile() throws Exception {
58+
try (TikaInputStream tis = TikaInputStream.getPlaceholder()) {
59+
assertEquals(-1, tis.getLength());
60+
assertFalse(tis.hasFile());
61+
}
62+
}
63+
}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/MetafileRendering.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ private static TikaInputStream pictureStream(TikaInputStream source) throws IOEx
119119
if (source != null && source.hasFile()) {
120120
return TikaInputStream.get(source.getPath());
121121
}
122-
return TikaInputStream.get(new byte[0]);
122+
return TikaInputStream.getPlaceholder();
123123
}
124124

125125
/**

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.apache.tika.metadata.Metadata;
5656
import org.apache.tika.metadata.TikaCoreProperties;
5757
import org.apache.tika.mime.MediaType;
58+
import org.apache.tika.parser.MetadataOnlyParse;
5859
import org.apache.tika.parser.ParseContext;
5960
import org.apache.tika.parser.PasswordProvider;
6061
import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;
@@ -130,9 +131,13 @@ public static void extractMacros(POIFSFileSystem fs, ContentHandler xhtml,
130131
m.set(HttpHeaders.CONTENT_TYPE, "text/x-vbasic");
131132
EmbeddedDocumentUtil.recordException(e, m, context);
132133
if (embeddedDocumentExtractor.shouldParseEmbedded(m, context)) {
133-
embeddedDocumentExtractor.parseEmbedded(
134-
//pass in space character so that we don't trigger a zero-byte exception
135-
TikaInputStream.get(new byte[]{'\u0020'}), xhtml, m, context, true);
134+
// the entry carries the exception, not content: register it without a parse
135+
try (TikaInputStream tis = TikaInputStream.getPlaceholder()) {
136+
context.set(MetadataOnlyParse.class, MetadataOnlyParse.INSTANCE);
137+
embeddedDocumentExtractor.parseEmbedded(tis, xhtml, m, context, true);
138+
} finally {
139+
context.set(MetadataOnlyParse.class, null);
140+
}
136141
}
137142
return;
138143
}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -640,7 +640,7 @@ private RenderResult renderCurrentPage(PDPage pdPage, TemporaryResources tmpReso
640640
new PageRangeRequest(getCurrentPageNo(), getCurrentPageNo());
641641
if (thisRenderer instanceof PDDocumentRenderer) {
642642
//do not do autocloseable. We need to leave the pdDocument open!
643-
TikaInputStream tis = TikaInputStream.get(new byte[0]);
643+
TikaInputStream tis = TikaInputStream.getPlaceholder();
644644
tis.setOpenContainer(pdDocument);
645645
return thisRenderer.render(tis, pageMetadata, context, pageRangeRequest)
646646
.getResults().get(0);

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,7 @@ protected void extractInlineImageMetadataOnly(PDImage pdImage, Metadata metadata
451451
metadata.set(TIFF.IMAGE_LENGTH, pdImage.getHeight());
452452
//TODO: what else can we extract from the PDImage without rendering?
453453
//Register the image's metadata entry without decoding it (marker skips the parse).
454-
try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
454+
try (TikaInputStream tis = TikaInputStream.getPlaceholder()) {
455455
parseContext.set(MetadataOnlyParse.class, MetadataOnlyParse.INSTANCE);
456456
embeddedDocumentExtractor.parseEmbedded(tis,
457457
new EmbeddedContentHandler(xhtml), metadata, parseContext, false);

0 commit comments

Comments
 (0)