From 1645e95d9937cb26942b2b823c4508180e1875b5 Mon Sep 17 00:00:00 2001 From: tallison Date: Mon, 24 Aug 2026 20:39:12 -0400 Subject: [PATCH 1/3] TIKA-4835-spill-less --- CHANGES.txt | 17 ++++ .../org/apache/tika/digest/DigestHelper.java | 30 +++++-- .../apache/tika/digest/TranslatedBytes.java | 85 ++++++++++++++++++ .../tika/digest/TranslatedBytesTest.java | 86 +++++++++++++++++++ .../parser/image/ImageMetadataExtractor.java | 32 +++++-- .../apache/tika/parser/image/ImageXmp.java | 16 ++-- .../apache/tika/parser/image/JpegParser.java | 9 +- .../apache/tika/parser/image/TiffParser.java | 13 ++- .../apache/tika/parser/image/WebPParser.java | 8 +- .../image/ImageParsersNoTempFileTest.java | 86 +++++++++++++++++++ .../microsoft/POIFSContainerDetector.java | 57 +++++++++++- .../POIFSContainerDetectorNoTempFileTest.java | 71 +++++++++++++++ .../tika/parser/odf/OpenDocumentParser.java | 7 +- .../odf/OpenDocumentParserNoTempFileTest.java | 68 +++++++++++++++ .../org/apache/tika/parser/pdf/PDFParser.java | 11 ++- .../parser/pdf/PDFParserNoTempFileTest.java | 71 +++++++++++++++ 16 files changed, 632 insertions(+), 35 deletions(-) create mode 100644 tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java create mode 100644 tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java create mode 100644 tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java diff --git a/CHANGES.txt b/CHANGES.txt index eea5ad329bc..92f6a8e0602 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,22 @@ Release 4.1.0 - unreleased + * The JPEG, TIFF and WebP parsers no longer spool in-memory input to a temp + file to read metadata, and the OpenDocument parser no longer spools each + inline picture before detecting it: both now rewind the stream (governed + by the CacheMemoryBudget when one is set) instead of calling getFile(). + On a 20k-file corpus sample these sites accounted for ~70% of the temp + bytes 4.x wrote, the dominant driver of 4.x's batch slowdown vs 3.x on + spinning disks. ImageMetadataExtractor gains InputStream overloads of + parseJpeg/parseTiff/parseWebP; ImageXmp.extractJpeg/extractWebp now take + an InputStream. POIFSContainerDetector opens in-memory OLE2 objects from + memory (under the CacheMemoryBudget, falling back to the file path for + anything POI's stream loader rejects) instead of spooling every embedded + OLE2 object to read its entry names, and the digest of translated + embedded streams (DigestHelper) buffers the translated bytes in memory + up to the budget before spilling. PDFParser's incremental-update xref + scan reads in-memory input from memory instead of spooling it + (TIKA-4835). + * The Kafka pipes iterator no longer stops at the first empty poll. A newly subscribed consumer spends its first poll(s) joining the group and returns empty even when the topic has a backlog, so the iterator could enqueue zero diff --git a/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java b/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java index 359ebf9f271..27ba26db1bc 100644 --- a/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java +++ b/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java @@ -18,8 +18,6 @@ import java.io.IOException; import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; import org.apache.tika.extractor.DefaultEmbeddedStreamTranslator; import org.apache.tika.extractor.EmbeddedStreamTranslator; @@ -59,6 +57,9 @@ public class DigestHelper { * @param context parse context (should contain DigesterFactory, may contain SkipContainerDocumentDigest marker) * @throws IOException if an I/O error occurs */ + // Same per-object threshold as StreamCache when no budget is in the context. + private static final long DEFAULT_TRANSLATED_MEMORY_THRESHOLD = 1024 * 1024; + public static void maybeDigest(TikaInputStream tis, Metadata metadata, ParseContext context) throws IOException { @@ -84,16 +85,31 @@ public static void maybeDigest(TikaInputStream tis, // The translator consumes `tis` (e.g. OLE2), so enableRewind() before and rewind() // after -- otherwise the caller would see an exhausted stream. if (EMBEDDED_STREAM_TRANSLATOR.shouldTranslate(tis, metadata)) { - tis.enableRewind(context.get(CacheMemoryBudget.class)); + CacheMemoryBudget budget = context.get(CacheMemoryBudget.class); + tis.enableRewind(budget); + // Translated size is unknown up front; the source length bounds it, so reserve + // that from the budget (or use the per-object default) as the in-memory threshold. + long threshold = DEFAULT_TRANSLATED_MEMORY_THRESHOLD; + long reserved = 0; + if (budget != null && tis.hasLength()) { + long len = tis.getLength(); + if (len > 0 && budget.tryReserve(len) > 0) { + reserved = len; + threshold = len; + } + } try (TemporaryResources tmp = new TemporaryResources()) { - Path tmpBytes = tmp.createTempFile(); - try (OutputStream os = Files.newOutputStream(tmpBytes)) { + TranslatedBytes translated = new TranslatedBytes(tmp, threshold); + try (OutputStream os = translated) { EMBEDDED_STREAM_TRANSLATOR.translate(tis, metadata, os); } - try (TikaInputStream translated = TikaInputStream.get(tmpBytes)) { - digester.digest(translated, metadata, context); + try (TikaInputStream translatedStream = translated.toTikaInputStream()) { + digester.digest(translatedStream, metadata, context); } } finally { + if (reserved > 0) { + budget.release(reserved); + } tis.rewind(); } } else { diff --git a/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java b/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java new file mode 100644 index 00000000000..2cf7ff4a633 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.digest; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; + +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; + +/** + * Sink for a translated embedded stream: bytes stay in memory up to {@code threshold} + * and spill to a temp file (owned by {@code tmp}) past it, so the common small object is + * digested without touching disk. + */ +class TranslatedBytes extends OutputStream { + + private final TemporaryResources tmp; + private final long threshold; + private UnsynchronizedByteArrayOutputStream memory = + UnsynchronizedByteArrayOutputStream.builder().get(); + private long size; + private Path spillFile; + private OutputStream spill; + + TranslatedBytes(TemporaryResources tmp, long threshold) { + this.tmp = tmp; + this.threshold = threshold; + } + + @Override + public void write(int b) throws IOException { + write(new byte[]{(byte) b}, 0, 1); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (spill == null && size + len > threshold) { + spillFile = tmp.createTempFile(); + spill = Files.newOutputStream(spillFile); + memory.writeTo(spill); + memory = null; + } + if (spill != null) { + spill.write(b, off, len); + } else { + memory.write(b, off, len); + } + size += len; + } + + @Override + public void close() throws IOException { + if (spill != null) { + spill.close(); + } + } + + boolean isInMemory() { + return spill == null; + } + + /** The translated content; the caller closes it. */ + TikaInputStream toTikaInputStream() throws IOException { + return spill == null ? TikaInputStream.get(memory.toByteArray()) : TikaInputStream.get(spillFile); + } +} diff --git a/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java b/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java new file mode 100644 index 00000000000..82677164fce --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.digest; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; + +public class TranslatedBytesTest { + + @TempDir + Path tempDir; + + @Test + public void testStaysInMemoryUnderThreshold() throws Exception { + byte[] data = new byte[1000]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) i; + } + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TranslatedBytes sink = new TranslatedBytes(tmp, 1000); + sink.write(data, 0, 600); + sink.write(data, 600, 400); + sink.close(); + assertTrue(sink.isInMemory()); + try (TikaInputStream tis = sink.toTikaInputStream()) { + assertArrayEquals(data, tis.readAllBytes()); + } + try (Stream files = Files.list(tempDir)) { + assertEquals(0, files.count()); + } + } + } + + @Test + public void testSpillsPastThreshold() throws Exception { + byte[] data = new byte[5000]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i * 7); + } + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TranslatedBytes sink = new TranslatedBytes(tmp, 1000); + sink.write(data, 0, 800); // in memory + sink.write(data, 800, 4200); // crosses the threshold: memory flushed to the file + sink.close(); + assertFalse(sink.isInMemory()); + try (Stream files = Files.list(tempDir)) { + assertEquals(1, files.count()); + } + try (TikaInputStream tis = sink.toTikaInputStream()) { + assertArrayEquals(data, tis.readAllBytes()); + } + } + // the temp file belongs to tmp and is gone once it closes + try (Stream files = Files.list(tempDir)) { + assertEquals(0, files.count()); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java index 3272f8d72f3..6511fd7f3fc 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java @@ -145,6 +145,16 @@ public void parseJpeg(File file) throws IOException, SAXException, TikaException } } + public void parseJpeg(InputStream stream) throws IOException, SAXException, TikaException { + try { + com.drew.metadata.Metadata jpegMetadata = + JpegMetadataReader.readMetadata(stream, JPEG_READERS_NO_XMP); + handle(jpegMetadata); + } catch (JpegProcessingException | MetadataException e) { + throw new TikaException("Can't read JPEG metadata", e); + } + } + public void parseTiff(File file) throws IOException, SAXException, TikaException { try { com.drew.metadata.Metadata tiffMetadata = TiffMetadataReader.readMetadata(file); @@ -154,14 +164,26 @@ public void parseTiff(File file) throws IOException, SAXException, TikaException } } + public void parseTiff(InputStream stream) throws IOException, SAXException, TikaException { + try { + com.drew.metadata.Metadata tiffMetadata = TiffMetadataReader.readMetadata(stream); + handle(tiffMetadata); + } catch (MetadataException | TiffProcessingException e) { + throw new TikaException("Can't read TIFF metadata", e); + } + } + public void parseWebP(File file) throws IOException, TikaException { + try { + handle(WebpMetadataReader.readMetadata(file)); + } catch (RiffProcessingException | MetadataException e) { + throw new TikaException("Can't process Riff data", e); + } + } + public void parseWebP(InputStream stream) throws IOException, TikaException { try { - com.drew.metadata.Metadata webPMetadata = new com.drew.metadata.Metadata(); - webPMetadata = WebpMetadataReader.readMetadata(file); - handle(webPMetadata); - } catch (IOException e) { - throw e; + handle(WebpMetadataReader.readMetadata(stream)); } catch (RiffProcessingException | MetadataException e) { throw new TikaException("Can't process Riff data", e); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java index af88c136592..179df5c708c 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java @@ -17,8 +17,6 @@ package org.apache.tika.parser.image; import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -28,6 +26,7 @@ import com.drew.imaging.jpeg.JpegSegmentData; import com.drew.imaging.jpeg.JpegSegmentReader; import com.drew.imaging.jpeg.JpegSegmentType; +import com.drew.lang.StreamReader; import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; import org.xml.sax.SAXException; @@ -82,11 +81,11 @@ static void extractRaw(byte[] xmp, Metadata metadata, ParseContext context) { } /** JPEG: read APP1 segments, reassemble Extended XMP, parse each resulting packet. */ - static void extractJpeg(File file, Metadata metadata, ParseContext context) { + static void extractJpeg(InputStream stream, Metadata metadata, ParseContext context) { try { Iterable app1; try { - JpegSegmentData data = JpegSegmentReader.readSegments(file, + JpegSegmentData data = JpegSegmentReader.readSegments(new StreamReader(stream), Collections.singletonList(JpegSegmentType.APP1)); app1 = data.getSegments(JpegSegmentType.APP1); } catch (JpegProcessingException e) { @@ -106,9 +105,9 @@ static void extractJpeg(File file, Metadata metadata, ParseContext context) { } /** WebP: pull the raw packet out of the RIFF {@code "XMP "} chunk and parse it. */ - static void extractWebp(File file, Metadata metadata, ParseContext context) { + static void extractWebp(InputStream stream, Metadata metadata, ParseContext context) { try { - byte[] xmp = readRiffChunk(file, "XMP "); + byte[] xmp = readRiffChunk(stream, "XMP "); if (xmp != null) { new XmpExtractor().extract(xmp, metadata, context); } @@ -123,8 +122,9 @@ static void extractWebp(File file, Metadata metadata, ParseContext context) { private static final long MAX_CHUNK = 64L * 1024 * 1024; /** Return the payload of the first top-level RIFF chunk with the given FourCC, or null. */ - private static byte[] readRiffChunk(File file, String fourCC) throws IOException { - try (InputStream in = new BufferedInputStream(new FileInputStream(file))) { + private static byte[] readRiffChunk(InputStream stream, String fourCC) throws IOException { + InputStream in = stream instanceof BufferedInputStream ? stream : new BufferedInputStream(stream); + { byte[] head = new byte[12]; if (IOUtils.read(in, head, 0, 12) < 12 || head[0] != 'R' || head[1] != 'I' || head[2] != 'F' || head[3] != 'F' || head[8] != 'W' || head[9] != 'E' || diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java index a96311857f5..22a7b0e0828 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java @@ -26,6 +26,7 @@ import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -54,9 +55,13 @@ void extractMetadata(InputStream stream, ContentHandler contentHandler, Metadata TemporaryResources tmp = new TemporaryResources(); try { TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata); + // Two sequential passes over the bytes; rewind instead of spooling to a file so + // in-memory embedded images never touch disk. + tis.enableRewind(parseContext.get(CacheMemoryBudget.class)); // XMP first so it is canonical; the metadata-extractor handlers (IPTC/EXIF) fill gaps. - ImageXmp.extractJpeg(tis.getFile(), metadata, parseContext); - new ImageMetadataExtractor(metadata).parseJpeg(tis.getFile()); + ImageXmp.extractJpeg(tis, metadata, parseContext); + tis.rewind(); + new ImageMetadataExtractor(metadata).parseJpeg(tis); } finally { tmp.dispose(); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java index 65664bff10f..a1c194954cd 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java @@ -26,6 +26,7 @@ import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -54,10 +55,18 @@ void extractMetadata(InputStream stream, ContentHandler contentHandler, Metadata TemporaryResources tmp = new TemporaryResources(); try { TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata); - tis.getFile(); // spool so tis is fully re-readable below + tis.enableRewind(parseContext.get(CacheMemoryBudget.class)); // XMP first so it is canonical; metadata-extractor (IPTC/EXIF) fills gaps. ImageXmp.scanAndExtract(tis, metadata, parseContext); - new ImageMetadataExtractor(metadata).parseTiff(tis.getFile()); + tis.rewind(); + ImageMetadataExtractor extractor = new ImageMetadataExtractor(metadata); + // File-backed input keeps drewnoakes' random-access file reader; in-memory input + // is read from the stream rather than spooled to disk. + if (tis.hasFile()) { + extractor.parseTiff(tis.getFile()); + } else { + extractor.parseTiff(tis); + } } finally { tmp.dispose(); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java index 1af3b5e5a34..a426d1c492c 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java @@ -25,6 +25,7 @@ import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; @@ -50,8 +51,11 @@ public Set getSupportedTypes(ParseContext context) { public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { // XMP first (canonical), then EXIF/etc. from metadata-extractor as fallback. - ImageXmp.extractWebp(tis.getFile(), metadata, context); - new ImageMetadataExtractor(metadata).parseWebP(tis.getFile()); + // Rewind between the two passes instead of spooling to a file. + tis.enableRewind(context.get(CacheMemoryBudget.class)); + ImageXmp.extractWebp(tis, metadata, context); + tis.rewind(); + new ImageMetadataExtractor(metadata).parseWebP(tis); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context); xhtml.startDocument(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java new file mode 100644 index 00000000000..031e9a9b1b5 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.image; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TIFF; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; + +/** + * Embedded images usually arrive already in memory; the image parsers must not spool + * them to disk just to read metadata (they used to call getFile() for every pass). + */ +public class ImageParsersNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testJpeg() throws Exception { + assertNoTempFile(new JpegParser(), "/test-documents/testJPEG_EXIF.jpg", TIFF.IMAGE_WIDTH.getName()); + } + + @Test + public void testTiff() throws Exception { + assertNoTempFile(new TiffParser(), "/test-documents/testTIFF.tif", TIFF.IMAGE_WIDTH.getName()); + } + + @Test + public void testWebP() throws Exception { + assertNoTempFile(new WebPParser(), "/test-documents/testWebp_Alpha_Lossless.webp", + ImageMetadataExtractor.UNKNOWN_IMG_NS + "Image Width"); + } + + private void assertNoTempFile(Parser parser, String resource, String widthKey) throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream(resource)) { + bytes = is.readAllBytes(); + } + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + // a stream-backed, non-file TikaInputStream whose only spill target is tempDir + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + parser.parse(tis, new DefaultHandler(), metadata, context); + // temp files live until tmp closes, so any spool would be visible right here + try (Stream files = Files.list(tempDir)) { + assertEquals(0, files.count(), "parser spooled an in-memory image to disk"); + } + } + assertNotNull(metadata.get(widthKey), "metadata was extracted"); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java index 356dcba2f15..007289db6b3 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Collections; @@ -44,6 +46,7 @@ import org.apache.tika.annotation.TikaComponent; import org.apache.tika.detect.Detector; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; @@ -554,7 +557,18 @@ private static Set getTopLevelNames(DirectoryNode root) { } - private Set getTopLevelNames(TikaInputStream stream) throws IOException { + // In-memory POIFS copies the whole object into heap; above this, spool as before. + private static final long MAX_IN_MEMORY_POIFS = 64L * 1024 * 1024; + private static final long DEFAULT_IN_MEMORY_POIFS = 1024 * 1024; + + private Set getTopLevelNames(TikaInputStream stream, ParseContext context) + throws IOException { + if (!stream.hasFile()) { + Set names = getTopLevelNamesInMemory(stream, context); + if (names != null) { + return names; + } + } // Force the document stream to a (possibly temporary) file // so we don't modify the current position of the stream. Path file = stream.getPath(); @@ -583,16 +597,51 @@ private Set getTopLevelNames(TikaInputStream stream) throws IOException } } + /** + * Opens the OLE2 container from memory instead of spooling it to a temp file. Returns + * null when the object is too large for the in-memory limit or POI cannot load it from a + * stream (the caller then spools, as before). + */ + private Set getTopLevelNamesInMemory(TikaInputStream stream, ParseContext context) + throws IOException { + CacheMemoryBudget budget = context == null ? null : context.get(CacheMemoryBudget.class); + long limit = budget == null ? DEFAULT_IN_MEMORY_POIFS : + Math.min(MAX_IN_MEMORY_POIFS, budget.getMaxBytes()); + // the channel is served from memory while the content fits the cache/budget + try (SeekableByteChannel channel = stream.getSeekableByteChannel()) { + long size = channel.size(); + if (size > limit) { + return null; + } + if (budget != null) { + if (budget.tryReserve(size) == 0) { + return null; + } + stream.addCloseableResource(() -> budget.release(size)); + } + POIFSFileSystem fs = new POIFSFileSystem(Channels.newInputStream(channel)); + stream.setOpenContainer(fs); + return getTopLevelNames(fs.getRoot()); + } catch (SecurityException e) { + throw e; + } catch (IOException | RuntimeException e) { + // POI's stream loader is stricter than its file loader (e.g. truncated objects); + // fall back to the file path so behavior is unchanged for anything it rejects + return null; + } + } + public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException { // Check if we have access to the document if (tis == null) { return MediaType.OCTET_STREAM; } - return handleTikaStream(tis, metadata); + return handleTikaStream(tis, metadata, parseContext); } - private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata) throws IOException { + private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata, ParseContext context) + throws IOException { //try for an open container Set names = tryOpenContainerOnTikaInputStream(tis, metadata); @@ -604,7 +653,7 @@ private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata) throw // If OLE, spool to disk if (names == null) { // spool to disk and try detection - names = getTopLevelNames(tis); + names = getTopLevelNames(tis, context); } // Detect based on the names (as available) diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java new file mode 100644 index 00000000000..b00f2e57e7b --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect.microsoft; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; + +/** + * Detection used to spool every in-memory OLE2 object to a temp file to read its + * top-level entry names; it must now open the container from memory. + */ +public class POIFSContainerDetectorNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testNoTempFileForInMemoryInput() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) { + bytes = is.readAllBytes(); + } + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + MediaType type = new POIFSContainerDetector().detect(tis, metadata, context); + assertEquals(MediaType.application("msword"), type); + try (Stream files = Files.list(tempDir)) { + assertEquals(0, files.count(), "detector spooled an in-memory OLE2 object to disk"); + } + assertTrue(tis.getOpenContainer() instanceof POIFSFileSystem, "open container kept for the parser"); + assertEquals(0, tis.getPosition(), "detection must not move the stream"); + assertEquals(0xd0, tis.read(), "stream still readable from the start"); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java index c500689ab45..f3db7c4af6e 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java @@ -45,6 +45,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; @@ -320,14 +321,14 @@ private void handleZipArchiveEntry(ZipArchiveEntry entry, TikaInputStream tisZip if (embeddedName.contains("Pictures/")) { embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, TikaCoreProperties.EmbeddedResourceType.INLINE.toString()); - //spool - tisZip.getFile(); + // rewind (not spool) so detection leaves the entry re-readable in memory + tisZip.enableRewind(context.get(CacheMemoryBudget.class)); MediaType embeddedMimeType = EmbeddedDocumentUtil.getDetector(context) .detect(tisZip, embeddedMetadata, context); if (embeddedMimeType != null) { embeddedMetadata.set(HttpHeaders.CONTENT_TYPE, embeddedMimeType.toString()); } - tisZip.reset(); + tisZip.rewind(); // Tag the picture with the draw:page indices it // appears on (set populated by scanPicturePages). // A null lookup means "not referenced by any diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java new file mode 100644 index 00000000000..78c56e5e9e5 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.odf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Inline pictures used to be spooled to a temp file before detection; with rewind + * support the whole document parses from memory. + */ +public class OpenDocumentParserNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testNoTempFileForInMemoryInput() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testODTEmbedded.odt")) { + bytes = is.readAllBytes(); + } + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + BodyContentHandler handler = new BodyContentHandler(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + new OpenDocumentParser().parse(tis, handler, metadata, context); + try (Stream files = Files.list(tempDir)) { + assertEquals(0, files.count(), "ODF parse spooled an in-memory entry to disk"); + } + } + assertTrue(handler.toString().length() > 0, "content was extracted"); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java index c61f33cc45b..d9451b0f5f0 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -315,8 +317,13 @@ private void scanXRefOffsets(PDFParserConfig localConfig, List xRefOffsets = new ArrayList<>(); //TODO -- can we use the PDFBox parser's RandomAccessRead //so that we don't have to reopen from file? - try (RandomAccessRead ra = - new RandomAccessReadBufferedFile(tikaInputStream.getFile())) { + // In-memory input is scanned from memory (as the main parse already does) rather + // than spooled to a file just for this pass. + try (SeekableByteChannel channel = tikaInputStream.hasFile() ? null : + tikaInputStream.getSeekableByteChannel(); + RandomAccessRead ra = channel == null ? + new RandomAccessReadBufferedFile(tikaInputStream.getFile()) : + new RandomAccessReadBuffer(Channels.newInputStream(channel))) { StartXRefScanner xRefScanner = new StartXRefScanner(ra); xRefOffsets.addAll(xRefScanner.scan()); } catch (IOException e) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java new file mode 100644 index 00000000000..9529fee4d4d --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.pdf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PDF; +import org.apache.tika.parser.ParseContext; + +/** + * The incremental-update xref scan used to spool in-memory PDFs to a file; it must scan + * from memory like the main parse does. + */ +public class PDFParserNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testXRefScanNoTempFileForInMemoryInput() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testPDF_incrementalUpdates.pdf")) { + bytes = is.readAllBytes(); + } + PDFParserConfig config = new PDFParserConfig(); + config.setExtractIncrementalUpdateInfo(true); + ParseContext context = new ParseContext(); + context.set(PDFParserConfig.class, config); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + new PDFParser().parse(tis, new DefaultHandler(), metadata, context); + try (Stream files = Files.list(tempDir)) { + assertEquals(0, files.count(), "xref scan spooled an in-memory PDF to disk"); + } + } + assertNotNull(metadata.get(PDF.PDF_INCREMENTAL_UPDATE_COUNT), "incremental update info was extracted"); + } +} From 60ddd5e741981cc0ce1a5eaab2a69b69fc7a61be Mon Sep 17 00:00:00 2001 From: tallison Date: Tue, 25 Aug 2026 13:38:16 -0400 Subject: [PATCH 2/3] TIKA-4835 -- improve POIFS and translated bytes handling --- .../org/apache/tika/digest/DigestHelper.java | 22 ++++---- .../apache/tika/digest/TranslatedBytes.java | 53 ++++++++++++++++--- .../tika/digest/TranslatedBytesTest.java | 35 +++++++++++- .../microsoft/POIFSContainerDetector.java | 3 ++ 4 files changed, 91 insertions(+), 22 deletions(-) diff --git a/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java b/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java index 27ba26db1bc..6f81a27baee 100644 --- a/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java +++ b/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java @@ -87,19 +87,15 @@ public static void maybeDigest(TikaInputStream tis, if (EMBEDDED_STREAM_TRANSLATOR.shouldTranslate(tis, metadata)) { CacheMemoryBudget budget = context.get(CacheMemoryBudget.class); tis.enableRewind(budget); - // Translated size is unknown up front; the source length bounds it, so reserve - // that from the budget (or use the per-object default) as the in-memory threshold. - long threshold = DEFAULT_TRANSLATED_MEMORY_THRESHOLD; - long reserved = 0; - if (budget != null && tis.hasLength()) { - long len = tis.getLength(); - if (len > 0 && budget.tryReserve(len) > 0) { - reserved = len; - threshold = len; - } + // Translated size is unknown up front (translation may inflate), so the sink + // starts at the source length / per-object default and grows from the budget. + long initial = DEFAULT_TRANSLATED_MEMORY_THRESHOLD; + if (tis.hasLength() && tis.getLength() > initial) { + initial = tis.getLength(); } + TranslatedBytes translated = null; try (TemporaryResources tmp = new TemporaryResources()) { - TranslatedBytes translated = new TranslatedBytes(tmp, threshold); + translated = new TranslatedBytes(tmp, budget, initial); try (OutputStream os = translated) { EMBEDDED_STREAM_TRANSLATOR.translate(tis, metadata, os); } @@ -107,8 +103,8 @@ public static void maybeDigest(TikaInputStream tis, digester.digest(translatedStream, metadata, context); } } finally { - if (reserved > 0) { - budget.release(reserved); + if (translated != null) { + translated.release(); } tis.rewind(); } diff --git a/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java b/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java index 2cf7ff4a633..706f93c133e 100644 --- a/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java +++ b/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java @@ -23,27 +23,43 @@ import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; /** - * Sink for a translated embedded stream: bytes stay in memory up to {@code threshold} - * and spill to a temp file (owned by {@code tmp}) past it, so the common small object is - * digested without touching disk. + * Sink for a translated embedded stream: bytes stay in memory while the shared + * {@link CacheMemoryBudget} (or the per-object default without one) allows, and spill to a + * temp file owned by {@code tmp} past that, so the common small object is digested without + * touching disk. Translation can inflate (compressed OLE payloads), so the reservation grows + * on demand rather than being fixed to the source length. */ class TranslatedBytes extends OutputStream { + private static final long GROW_CHUNK = 1024 * 1024; + private final TemporaryResources tmp; - private final long threshold; + private final CacheMemoryBudget budget; + private long threshold; + private long reserved; private UnsynchronizedByteArrayOutputStream memory = UnsynchronizedByteArrayOutputStream.builder().get(); private long size; private Path spillFile; private OutputStream spill; - TranslatedBytes(TemporaryResources tmp, long threshold) { + /** + * @param budget shared budget, or null for a fixed {@code initialThreshold} + * @param initialThreshold bytes allowed in memory before asking the budget for more + */ + TranslatedBytes(TemporaryResources tmp, CacheMemoryBudget budget, long initialThreshold) { this.tmp = tmp; - this.threshold = threshold; + this.budget = budget; + this.threshold = initialThreshold; + if (budget != null) { + reserved = budget.tryReserve(initialThreshold) > 0 ? initialThreshold : 0; + threshold = reserved; + } } @Override @@ -53,7 +69,7 @@ public void write(int b) throws IOException { @Override public void write(byte[] b, int off, int len) throws IOException { - if (spill == null && size + len > threshold) { + if (spill == null && size + len > threshold && !grow(size + len)) { spillFile = tmp.createTempFile(); spill = Files.newOutputStream(spillFile); memory.writeTo(spill); @@ -67,6 +83,21 @@ public void write(byte[] b, int off, int len) throws IOException { size += len; } + // Extends the in-memory allowance from the budget in whole chunks; false => spill. + private boolean grow(long needed) { + if (budget == null) { + return false; + } + while (threshold < needed) { + if (budget.tryReserve(GROW_CHUNK) == 0) { + return false; + } + reserved += GROW_CHUNK; + threshold += GROW_CHUNK; + } + return true; + } + @Override public void close() throws IOException { if (spill != null) { @@ -74,6 +105,14 @@ public void close() throws IOException { } } + /** Returns the reservation to the budget; call once the digest is done with the bytes. */ + void release() { + if (budget != null && reserved > 0) { + budget.release(reserved); + reserved = 0; + } + } + boolean isInMemory() { return spill == null; } diff --git a/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java b/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java index 82677164fce..6f8d5bf0c67 100644 --- a/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java +++ b/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; @@ -44,7 +45,7 @@ public void testStaysInMemoryUnderThreshold() throws Exception { } try (TemporaryResources tmp = new TemporaryResources()) { tmp.setTemporaryFileDirectory(tempDir); - TranslatedBytes sink = new TranslatedBytes(tmp, 1000); + TranslatedBytes sink = new TranslatedBytes(tmp, null, 1000); sink.write(data, 0, 600); sink.write(data, 600, 400); sink.close(); @@ -66,7 +67,7 @@ public void testSpillsPastThreshold() throws Exception { } try (TemporaryResources tmp = new TemporaryResources()) { tmp.setTemporaryFileDirectory(tempDir); - TranslatedBytes sink = new TranslatedBytes(tmp, 1000); + TranslatedBytes sink = new TranslatedBytes(tmp, null, 1000); sink.write(data, 0, 800); // in memory sink.write(data, 800, 4200); // crosses the threshold: memory flushed to the file sink.close(); @@ -83,4 +84,34 @@ public void testSpillsPastThreshold() throws Exception { assertEquals(0, files.count()); } } + + @Test + public void testGrowsFromBudgetInsteadOfSpilling() throws Exception { + byte[] data = new byte[3 * 1024 * 1024]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i * 13); + } + CacheMemoryBudget budget = new CacheMemoryBudget(64L * 1024 * 1024); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TranslatedBytes sink = new TranslatedBytes(tmp, budget, 1024 * 1024); + sink.write(data, 0, data.length); // 3x the initial threshold + sink.close(); + assertTrue(sink.isInMemory(), "should have grown its reservation, not spilled"); + try (TikaInputStream tis = sink.toTikaInputStream()) { + assertArrayEquals(data, tis.readAllBytes()); + } + sink.release(); + } + // a budget too small to grow into => spill + CacheMemoryBudget tiny = new CacheMemoryBudget(1024 * 1024 + 1); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TranslatedBytes sink = new TranslatedBytes(tmp, tiny, 1024 * 1024); + sink.write(data, 0, data.length); + sink.close(); + assertFalse(sink.isInMemory()); + sink.release(); + } + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java index 007289db6b3..9930e96d0cd 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java @@ -607,6 +607,9 @@ private Set getTopLevelNamesInMemory(TikaInputStream stream, ParseContex CacheMemoryBudget budget = context == null ? null : context.get(CacheMemoryBudget.class); long limit = budget == null ? DEFAULT_IN_MEMORY_POIFS : Math.min(MAX_IN_MEMORY_POIFS, budget.getMaxBytes()); + // attach the budget first: without it the drain below caches only the per-object + // default (1MB) in memory and spills the rest before we can even check the size + stream.enableRewind(budget); // the channel is served from memory while the content fits the cache/budget try (SeekableByteChannel channel = stream.getSeekableByteChannel()) { long size = channel.size(); From ee9a7a3792864cde2b614ddcf7e26018a205d196 Mon Sep 17 00:00:00 2001 From: tallison Date: Tue, 25 Aug 2026 18:31:05 -0400 Subject: [PATCH 3/3] TIKA-4835 -- keep drewnoakes file tags for file-backed images; fix pdf --- CHANGES.txt | 7 ++- .../parser/microsoft/rtf/RTFParserTest.java | 4 +- .../apache/tika/parser/image/JpegParser.java | 9 ++- .../apache/tika/parser/image/WebPParser.java | 7 ++- .../org/apache/tika/parser/pdf/PDFParser.java | 59 +++++++++++++++---- 5 files changed, 70 insertions(+), 16 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d608ea22973..27042f85b17 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -14,8 +14,11 @@ Release 4.1.0 - unreleased OLE2 object to read its entry names, and the digest of translated embedded streams (DigestHelper) buffers the translated bytes in memory up to the budget before spilling. PDFParser's incremental-update xref - scan reads in-memory input from memory instead of spooling it - (TIKA-4835). + scan reads in-memory input from memory instead of spooling it. One + visible metadata change: embedded (in-memory) JPEG/TIFF/WebP images no + longer carry metadata-extractor's file-system tags (img:File Name, + img:File Size, img:File Modified Date), which described the temp file, + not the image; real files keep them (TIKA-4835). * tika-server and tika-async-cli now start from a config that contains // or /* */ comments, as the configuration docs have always said they diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java index 96ba2b3c7b4..9b4fe1b5ea7 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java @@ -145,7 +145,9 @@ public void testRegularImages() throws Exception { //need flexibility for if tesseract is installed or not //TODO -- fix this test. It is too fragile. - assertTrue(meta_jpg.names().length >= 52 && meta_jpg.names().length <= 60); + // in-memory embedded images no longer carry metadata-extractor's temp-file + // name/size/date tags (TIKA-4835), hence the lower bound + assertTrue(meta_jpg.names().length >= 49 && meta_jpg.names().length <= 60); assertTrue(meta_jpg_exif.names().length >= 100 && meta_jpg_exif.names().length <= 130); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java index 22a7b0e0828..536749f74dd 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java @@ -61,7 +61,14 @@ void extractMetadata(InputStream stream, ContentHandler contentHandler, Metadata // XMP first so it is canonical; the metadata-extractor handlers (IPTC/EXIF) fill gaps. ImageXmp.extractJpeg(tis, metadata, parseContext); tis.rewind(); - new ImageMetadataExtractor(metadata).parseJpeg(tis); + ImageMetadataExtractor extractor = new ImageMetadataExtractor(metadata); + // A real file keeps metadata-extractor's file-system tags (name/size/date); + // in-memory input reads from the stream instead of being spooled to disk. + if (tis.hasFile()) { + extractor.parseJpeg(tis.getFile()); + } else { + extractor.parseJpeg(tis); + } } finally { tmp.dispose(); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java index a426d1c492c..66262178b5e 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java @@ -55,7 +55,12 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata tis.enableRewind(context.get(CacheMemoryBudget.class)); ImageXmp.extractWebp(tis, metadata, context); tis.rewind(); - new ImageMetadataExtractor(metadata).parseWebP(tis); + ImageMetadataExtractor extractor = new ImageMetadataExtractor(metadata); + if (tis.hasFile()) { + extractor.parseWebP(tis.getFile()); + } else { + extractor.parseWebP(tis); + } XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context); xhtml.startDocument(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java index d9451b0f5f0..9178abf0141 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java @@ -20,8 +20,6 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.channels.Channels; -import java.nio.channels.SeekableByteChannel; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -31,6 +29,7 @@ import java.util.Set; import javax.xml.stream.XMLStreamException; +import org.apache.commons.io.input.CloseShieldInputStream; import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; import org.apache.pdfbox.Loader; import org.apache.pdfbox.cos.COSArray; @@ -68,6 +67,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.AccessPermissions; import org.apache.tika.metadata.HttpHeaders; @@ -183,6 +183,10 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata context.set(OCRPageCounter.class, new OCRPageCounter()); try { if (shouldSpool(localConfig)) { + // Later stages re-read the document (xref scan, renderer, parse). Keep the + // content re-readable from the cache instead of spooling it to a file; + // getPath() still spools on demand for anything that truly needs a file. + ensureRereadable(tis, context); context.set(PDFRenderingState.class, new PDFRenderingState(tis)); } @@ -317,15 +321,25 @@ private void scanXRefOffsets(PDFParserConfig localConfig, List xRefOffsets = new ArrayList<>(); //TODO -- can we use the PDFBox parser's RandomAccessRead //so that we don't have to reopen from file? - // In-memory input is scanned from memory (as the main parse already does) rather - // than spooled to a file just for this pass. - try (SeekableByteChannel channel = tikaInputStream.hasFile() ? null : - tikaInputStream.getSeekableByteChannel(); - RandomAccessRead ra = channel == null ? - new RandomAccessReadBufferedFile(tikaInputStream.getFile()) : - new RandomAccessReadBuffer(Channels.newInputStream(channel))) { - StartXRefScanner xRefScanner = new StartXRefScanner(ra); - xRefOffsets.addAll(xRefScanner.scan()); + // In-memory input is scanned from memory rather than spooled to a file for this + // pass. Rewind support is enabled first so the later parse (and a renderer's + // getPath(), if rendering is on) can re-read the content from the cache: the old + // getFile() spool used to provide that as a side effect. + boolean fileBacked = tikaInputStream.hasFile(); + try { + if (!fileBacked) { + tikaInputStream.enableRewind(parseContext.get(CacheMemoryBudget.class)); + } + try (RandomAccessRead ra = fileBacked ? + new RandomAccessReadBufferedFile(tikaInputStream.getFile()) : + new RandomAccessReadBuffer(CloseShieldInputStream.wrap(tikaInputStream))) { + StartXRefScanner xRefScanner = new StartXRefScanner(ra); + xRefOffsets.addAll(xRefScanner.scan()); + } finally { + if (!fileBacked) { + tikaInputStream.rewind(); + } + } } catch (IOException e) { //swallow } @@ -487,11 +501,32 @@ private void renderPagesBeforeParse(TikaInputStream tstream, } } + /** Enables rewind on stream-backed input at position 0; falls back to a spool otherwise. */ + private static void ensureRereadable(TikaInputStream tis, ParseContext context) throws IOException { + if (tis.hasFile()) { + return; + } + try { + tis.enableRewind(context.get(CacheMemoryBudget.class)); + } catch (IOException e) { + tis.getPath(); + } + } + + private static void rewindQuietly(TikaInputStream tis) { + try { + tis.rewind(); + } catch (IOException e) { + // not rewindable (no rewind enabled and not file-backed): leave as is + } + } + private RenderResults renderPDF(TikaInputStream tstream, ParseContext parseContext, PDFParserConfig localConfig) throws IOException, TikaException { Metadata metadata = Metadata.newInstance(parseContext); metadata.set(TikaCoreProperties.TYPE, MEDIA_TYPE.toString()); + rewindQuietly(tstream); return renderer.render( tstream, metadata, parseContext, PageRangeRequest.RENDER_ALL); } @@ -515,6 +550,8 @@ protected PDDocument getPDDocument(TikaInputStream tis, String password, } finally { tis.removeCloseShield(); } + // the renderer re-reads the stream after this load + rewindQuietly(tis); } return pdDocument; } catch (IOException e) {