diff --git a/CHANGES.txt b/CHANGES.txt index c11045fde98..4121b259aab 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,10 @@ Release 4.1.0 - unreleased + * Placeholder streams -- the empty stand-ins parsers hand parseEmbedded + for content that is never extracted -- report an unknown length rather + than their own zero, and the macro-failure entry is registered without + parsing its sentinel (TIKA-4874). + * Embedded documents carry their size: ParsingEmbeddedDocumentExtractor sets Content-Length from the stream where the stream knows it and the parser did not say, which never spools to measure one, and the raw diff --git a/tika-core/src/main/java/org/apache/tika/io/PlaceholderSource.java b/tika-core/src/main/java/org/apache/tika/io/PlaceholderSource.java new file mode 100644 index 00000000000..ed958d0b3b4 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/io/PlaceholderSource.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.io; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Path; + +/** + * Empty stand-in for content that is never extracted: its emptiness describes the + * placeholder, not the document, so it reports an unknown length rather than zero. + * + * @see TikaInputStream#getPlaceholder() + */ +class PlaceholderSource extends InputStream implements TikaInputSource { + + private final TemporaryResources tmp; + private Path spilledPath; + + PlaceholderSource(TemporaryResources tmp) { + this.tmp = tmp; + } + + @Override + public int read() { + return -1; + } + + @Override + public int read(byte[] b, int off, int len) { + // InputStream contract: a zero-length read returns 0, even at EOF + return len == 0 ? 0 : -1; + } + + @Override + public long skip(long n) { + return 0; + } + + @Override + public int available() { + return 0; + } + + @Override + public void seekTo(long newPosition) throws IOException { + if (newPosition != 0) { + throw new IOException("Invalid seek position: " + newPosition + " (empty source)"); + } + } + + @Override + public Path materializedPath() { + return spilledPath; + } + + @Override + public boolean hasPath() { + return spilledPath != null; + } + + @Override + public Path getPath(String suffix) throws IOException { + if (spilledPath == null) { + spilledPath = tmp.createTempFile(suffix); + } + return spilledPath; + } + + @Override + public long getLength() { + return -1; + } + + @Override + public boolean isPlaceholder() { + return true; + } + + @Override + public void enableRewind(CacheMemoryBudget budget) { + // No-op: there is nothing to rewind + } + + @Override + public SeekableByteChannel getSeekableByteChannel() { + return new MemorySeekableByteChannel(new byte[0], 0); + } + + @Override + public synchronized void mark(int readlimit) { + } + + @Override + public synchronized void reset() { + } + + @Override + public boolean markSupported() { + return true; + } +} diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java b/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java index 9e7b4a8d30a..78500097c80 100644 --- a/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java +++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java @@ -63,6 +63,14 @@ interface TikaInputSource extends Closeable { */ long getLength(); + /** + * Whether this source stands in for content that is never extracted. Spooling one + * measures nothing, so its unknown length must not cost a temp file to confirm. + */ + default boolean isPlaceholder() { + return false; + } + /** * Enables full rewind capability. *

diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java index b58cd548db9..670e2581dd1 100644 --- a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java +++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java @@ -158,6 +158,18 @@ public static TikaInputStream get(byte[] data, Metadata metadata) { return new TikaInputStream(inputSource, tmp, ext); } + /** + * An empty stream standing in for content that is never extracted -- a metadata-only + * entry, a rendering carried as an open container. It reports an unknown + * length, so nothing mistakes the placeholder's size for the document's. Pair it with + * {@link org.apache.tika.parser.MetadataOnlyParse} to register an entry without + * parsing it, unless an open container supplies the content. + */ + public static TikaInputStream getPlaceholder() { + TemporaryResources tmp = new TemporaryResources(); + return new TikaInputStream(new PlaceholderSource(tmp), tmp, ""); + } + public static TikaInputStream get(Path path) throws IOException { return get(path, new Metadata()); } @@ -453,7 +465,7 @@ public long getLength() throws IOException { return -1; } long len = source.getLength(); - if (len == -1) { + if (len == -1 && !source.isPlaceholder()) { // Force spill to get length getPath(); len = source.getLength(); diff --git a/tika-core/src/main/java/org/apache/tika/renderer/RenderResult.java b/tika-core/src/main/java/org/apache/tika/renderer/RenderResult.java index 25588c45bb9..db6ab42bdf6 100644 --- a/tika-core/src/main/java/org/apache/tika/renderer/RenderResult.java +++ b/tika-core/src/main/java/org/apache/tika/renderer/RenderResult.java @@ -65,7 +65,8 @@ public TikaInputStream getInputStream() throws IOException { if (result instanceof Path) { return TikaInputStream.get((Path)result, metadata); } else { - TikaInputStream tis = TikaInputStream.get(new byte[0]); + // the rendering rides in the open container, not the stream + TikaInputStream tis = TikaInputStream.getPlaceholder(); tis.setOpenContainer(result); return tis; } diff --git a/tika-core/src/test/java/org/apache/tika/io/PlaceholderStreamTest.java b/tika-core/src/test/java/org/apache/tika/io/PlaceholderStreamTest.java new file mode 100644 index 00000000000..8dd2bffbe89 --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/io/PlaceholderStreamTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.io; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; + +import org.junit.jupiter.api.Test; + +public class PlaceholderStreamTest { + + @Test + public void testPlaceholderDeclaresNoLength() throws Exception { + try (TikaInputStream tis = TikaInputStream.getPlaceholder()) { + assertFalse(tis.hasLength(), "a placeholder's size describes nothing"); + assertEquals(-1, tis.read(), "placeholder is empty"); + } + } + + /** A real empty document is not a placeholder: zero is honest there. */ + @Test + public void testGenuinelyEmptyStreamStillDeclaresZero() throws Exception { + try (TikaInputStream tis = TikaInputStream.get(new byte[0])) { + assertTrue(tis.hasLength()); + assertEquals(0, tis.getLength()); + } + } + + /** Spooling must not turn the placeholder's absent length into a zero. */ + @Test + public void testSpoolingKeepsLengthUnknown() throws Exception { + try (TikaInputStream tis = TikaInputStream.getPlaceholder()) { + assertEquals(0, Files.size(tis.getPath())); + assertFalse(tis.hasLength()); + } + } + + /** Measuring a placeholder must not cost a temp file: there is nothing to measure. */ + @Test + public void testMeasuringCostsNoTempFile() throws Exception { + try (TikaInputStream tis = TikaInputStream.getPlaceholder()) { + assertEquals(-1, tis.getLength()); + assertFalse(tis.hasFile()); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/MetafileRendering.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/MetafileRendering.java index d4efd0a12cd..0032c97a64e 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/MetafileRendering.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/MetafileRendering.java @@ -119,7 +119,7 @@ private static TikaInputStream pictureStream(TikaInputStream source) throws IOEx if (source != null && source.hasFile()) { return TikaInputStream.get(source.getPath()); } - return TikaInputStream.get(new byte[0]); + return TikaInputStream.getPlaceholder(); } /** diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java index 423a51f86be..925e688f58b 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OfficeParser.java @@ -55,6 +55,7 @@ import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.MetadataOnlyParse; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.PasswordProvider; import org.apache.tika.parser.microsoft.ooxml.OOXMLParser; @@ -130,9 +131,13 @@ public static void extractMacros(POIFSFileSystem fs, ContentHandler xhtml, m.set(HttpHeaders.CONTENT_TYPE, "text/x-vbasic"); EmbeddedDocumentUtil.recordException(e, m, context); if (embeddedDocumentExtractor.shouldParseEmbedded(m, context)) { - embeddedDocumentExtractor.parseEmbedded( - //pass in space character so that we don't trigger a zero-byte exception - TikaInputStream.get(new byte[]{'\u0020'}), xhtml, m, context, true); + // the entry carries the exception, not content: register it without a parse + try (TikaInputStream tis = TikaInputStream.getPlaceholder()) { + context.set(MetadataOnlyParse.class, MetadataOnlyParse.INSTANCE); + embeddedDocumentExtractor.parseEmbedded(tis, xhtml, m, context, true); + } finally { + context.set(MetadataOnlyParse.class, null); + } } return; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java index aa57face74f..da4a51ed430 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java @@ -640,7 +640,7 @@ private RenderResult renderCurrentPage(PDPage pdPage, TemporaryResources tmpReso new PageRangeRequest(getCurrentPageNo(), getCurrentPageNo()); if (thisRenderer instanceof PDDocumentRenderer) { //do not do autocloseable. We need to leave the pdDocument open! - TikaInputStream tis = TikaInputStream.get(new byte[0]); + TikaInputStream tis = TikaInputStream.getPlaceholder(); tis.setOpenContainer(pdDocument); return thisRenderer.render(tis, pageMetadata, context, pageRangeRequest) .getResults().get(0); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java index 1c04c23a44a..e4387802f9f 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/image/ImageGraphicsEngine.java @@ -451,7 +451,7 @@ protected void extractInlineImageMetadataOnly(PDImage pdImage, Metadata metadata metadata.set(TIFF.IMAGE_LENGTH, pdImage.getHeight()); //TODO: what else can we extract from the PDImage without rendering? //Register the image's metadata entry without decoding it (marker skips the parse). - try (TikaInputStream tis = TikaInputStream.get(new byte[0])) { + try (TikaInputStream tis = TikaInputStream.getPlaceholder()) { parseContext.set(MetadataOnlyParse.class, MetadataOnlyParse.INSTANCE); embeddedDocumentExtractor.parseEmbedded(tis, new EmbeddedContentHandler(xhtml), metadata, parseContext, false);