Skip to content

TIKA-4696 improve inline tagging #2711

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tballison merged 2 commits into from
Mar 23, 2026
Merged

TIKA-4696 improve inline tagging #2711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e16183b
improve tagging of inline images
tballison Mar 23, 2026
509502f
improve tagging of inline images
tballison Mar 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions tika-core/src/main/java/org/apache/tika/metadata/MAPI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,4 +77,18 @@ public interface MAPI {
Property ATTACH_MIME = Property.internalText(PREFIX_MAPI_ATTACH_META + "mime");
Property ATTACH_LANGUAGE = Property.internalText(PREFIX_MAPI_ATTACH_META + "language");

/**
* PidTagAttachFlags (0x3714) — indicates which body formats might reference this attachment.
* Bit 1 (0x1) = ATT_INVISIBLE_IN_HTML
* Bit 2 (0x2) = ATT_INVISIBLE_IN_RTF
* Bit 3 (0x4) = ATT_RENDERED_IN_BODY
*/
Property ATTACH_FLAGS = Property.internalInteger(PREFIX_MAPI_ATTACH_META + "flags");

/**
* PidTagAttachmentHidden (0x7FFE) — indicates whether this attachment is hidden from the end
* user. Inline images typically have this set to true.
*/
Property ATTACH_HIDDEN = Property.internalBoolean(PREFIX_MAPI_ATTACH_META + "hidden");

}
154 changes: 140 additions & 14 deletions ...ser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,11 @@

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.UnsupportedCharsetException;
Expand Down Expand Up @@ -56,7 +59,10 @@
import org.apache.poi.hsmf.datatypes.StringChunk;
import org.apache.poi.hsmf.datatypes.Types;
import org.apache.poi.hsmf.exceptions.ChunkNotFoundException;
import org.apache.poi.poifs.filesystem.DirectoryEntry;
import org.apache.poi.poifs.filesystem.DirectoryNode;
import org.apache.poi.poifs.filesystem.DocumentEntry;
import org.apache.poi.poifs.filesystem.DocumentInputStream;
import org.apache.poi.util.CodePageUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -79,6 +85,7 @@
import org.apache.tika.parser.html.JSoupParser;
import org.apache.tika.parser.mailcommons.MailDateParser;
import org.apache.tika.parser.microsoft.msg.ExtendedMetadataExtractor;
import org.apache.tika.parser.microsoft.msg.RTFEncapsulatedHTMLExtractor;
import org.apache.tika.parser.microsoft.rtf.RTFParser;
import org.apache.tika.parser.txt.CharsetDetector;
import org.apache.tika.parser.txt.CharsetMatch;
Expand Down Expand Up @@ -173,6 +180,7 @@ private static void loadMessageClasses() {
private static Pattern HEADER_KEY_PAT =
Pattern.compile("\\A([\\x21-\\x39\\x3B-\\x7E]+):(.*?)\\Z");

private final DirectoryNode root;
private final MAPIMessage msg;
private final ParseContext parseContext;
private final boolean extractAllAlternatives;
Expand All @@ -181,6 +189,7 @@ private static void loadMessageClasses() {

public OutlookExtractor(DirectoryNode root, Metadata metadata, ParseContext context) throws TikaException {
super(context, metadata);
this.root = root;
this.parseContext = context;
this.extractAllAlternatives =
context.get(OfficeParserConfig.class).isExtractAllAlternativesFromMSG();
Expand Down Expand Up @@ -317,18 +326,7 @@ private void _parse(XHTMLContentHandler xhtml) throws TikaException, SAXExceptio

private void updateAttachmentMetadata(AttachmentChunks attachment, Metadata metadata,
Set<String> contentIdNames) {
StringChunk contentIdChunk = attachment.getAttachContentId();
if (contentIdChunk != null) {
String contentId = contentIdChunk.getValue();
if (! StringUtils.isBlank(contentId)) {
contentId = contentId.trim();
if (contentIdNames.contains(contentId)) {
metadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE_KEY,
TikaCoreProperties.EmbeddedResourceType.INLINE.name());
}
metadata.set(MAPI.ATTACH_CONTENT_ID, contentId);
}
}
// Extract string-based metadata from POI's named chunk getters
addStringChunkToMetadata(MAPI.ATTACH_LONG_PATH_NAME, attachment.getAttachLongPathName(), metadata);
addStringChunkToMetadata(MAPI.ATTACH_LONG_FILE_NAME, attachment.getAttachLongFileName(), metadata);
addStringChunkToMetadata(MAPI.ATTACH_FILE_NAME, attachment.getAttachFileName(), metadata);
Expand All @@ -337,6 +335,129 @@ private void updateAttachmentMetadata(AttachmentChunks attachment, Metadata meta
addStringChunkToMetadata(MAPI.ATTACH_EXTENSION, attachment.getAttachExtension(), metadata);
addStringChunkToMetadata(MAPI.ATTACH_MIME, attachment.getAttachMimeTag(), metadata);
addStringChunkToMetadata(MAPI.ATTACH_LANGUAGE, attachment.getAttachLanguage(), metadata);

// Extract fixed properties from the attachment's __properties_version1.0 stream
// POI's AttachmentChunks doesn't parse this stream, so we read it directly.
Map<Integer, Long> attachProps = readAttachmentProperties(attachment.getPOIFSName());
Long attachFlags = attachProps.get(PID_TAG_ATTACH_FLAGS);
if (attachFlags != null) {
metadata.set(MAPI.ATTACH_FLAGS, attachFlags.intValue());
}
Long attachHidden = attachProps.get(PID_TAG_ATTACHMENT_HIDDEN);
if (attachHidden != null) {
metadata.set(MAPI.ATTACH_HIDDEN, attachHidden.intValue() != 0);
}

// Determine inline vs attachment
String contentId = null;
StringChunk contentIdChunk = attachment.getAttachContentId();
if (contentIdChunk != null) {
String rawCid = contentIdChunk.getValue();
if (!StringUtils.isBlank(rawCid)) {
contentId = rawCid.trim();
metadata.set(MAPI.ATTACH_CONTENT_ID, contentId);
}
}

if (contentId != null && contentIdNames.contains(contentId)) {
// Layer 1: CID referenced in the message body — high confidence inline
metadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE_KEY,
TikaCoreProperties.EmbeddedResourceType.INLINE.name());
} else if (contentId != null
&& attachFlags != null
&& (attachFlags & ATT_RENDERED_IN_BODY) != 0
&& isInlineableMimeType(metadata.get(MAPI.ATTACH_MIME))) {
// Layer 2: MAPI says rendered in body + image MIME type — the CID regex
// missed it (e.g. encapsulated RTF with stripped img tags)
metadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE_KEY,
TikaCoreProperties.EmbeddedResourceType.INLINE.name());
}
}

private static final Set<String> INLINEABLE_MIME_TYPES = Set.of(
"application/x-ms-wmz",
"application/x-ms-emz",
"application/x-msmetafile",
"image/x-wmf",
"image/x-emf",
"image/wmf",
"image/emf"
);

/**
* Returns true for MIME types that are safe to label as INLINE.
* We gate on this to avoid marking PDFs, DOCX, etc. as inline — downstream
* consumers use INLINE to decide what to index separately.
*/
private static boolean isInlineableMimeType(String mimeType) {
if (StringUtils.isBlank(mimeType)) {
return false;
}
String lower = mimeType.toLowerCase(Locale.ROOT).trim();
return lower.startsWith("image/") || INLINEABLE_MIME_TYPES.contains(lower);
}

// PidTagAttachFlags (0x3714) — bit flags indicating which body formats reference this
private static final int PID_TAG_ATTACH_FLAGS = 0x3714;
// Bit 2 = ATT_RENDERED_IN_BODY: this attachment is referenced by the body
private static final int ATT_RENDERED_IN_BODY = 0x4;
// PidTagAttachmentHidden (0x7FFE) — boolean, true if hidden from end user (inline images)
private static final int PID_TAG_ATTACHMENT_HIDDEN = 0x7FFE;

/**
* Read fixed MAPI properties from the __properties_version1.0 stream inside an
* attachment storage. POI's {@link AttachmentChunks} does not parse this stream.
*
* <p>The stream format is: 8-byte header, followed by 16-byte property entries.
* Each entry: 2 bytes property type, 2 bytes property ID, 4 bytes flags,
* 8 bytes value (inline for fixed-size types).</p>
*
* @param poifsName the OLE2 directory name for this attachment
* (e.g. "__attach_version1.0_#00000000")
* @return map of property ID to value for fixed-size integer/boolean properties
*/
private Map<Integer, Long> readAttachmentProperties(String poifsName) {
Map<Integer, Long> result = new HashMap<>();
try {
DirectoryEntry attachDir = (DirectoryEntry) root.getEntry(poifsName);
DocumentEntry propsEntry =
(DocumentEntry) attachDir.getEntry("__properties_version1.0");
byte[] data;
try (InputStream dis = new DocumentInputStream(propsEntry)) {
data = dis.readAllBytes();
}
if (data.length < 8) {
return result;
}
ByteBuffer buf = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN);
int offset = 8; // skip 8-byte header
while (offset + 16 <= data.length) {
int propType = buf.getShort(offset) & 0xFFFF;
int propId = buf.getShort(offset + 2) & 0xFFFF;
long value;
switch (propType) {
case 0x0003: // PtypInteger32
value = buf.getInt(offset + 8) & 0xFFFFFFFFL;
result.put(propId, value);
break;
case 0x000B: // PtypBoolean
value = buf.getShort(offset + 8) & 0xFFFF;
result.put(propId, value);
break;
case 0x0014: // PtypInteger64
value = buf.getLong(offset + 8);
result.put(propId, value);
break;
default:
// skip variable-length, binary, time and other types
break;
}
offset += 16;
}
} catch (Exception e) {
LOGGER.debug("Could not read attachment properties for {}", poifsName, e);
}
return result;
}

private void addStringChunkToMetadata(Property property, StringChunk stringChunk, Metadata metadata) {
Expand Down Expand Up @@ -534,8 +655,13 @@ private void _handleBestBodyChunk(Chunk htmlChunk, Chunk rtfChunk, Chunk textChu
}

private void extractContentIdNamesFromRtf(byte[] data, Metadata metadata, Set<String> contentIdNames) {
//for now, hope that there's encapsulated html
//TODO: check for encapsulated html. If it doesn't exist, handle RTF specifically
// Try to de-encapsulate the HTML from the RTF first
String html = RTFEncapsulatedHTMLExtractor.extract(data);
if (html != null) {
extractContentIdNamesFromHtml(html.getBytes(UTF_8), metadata, contentIdNames);
return;
}
// Fall back to scanning the raw RTF bytes for cid: references
extractContentIdNamesFromHtml(data, metadata, contentIdNames);
}

Expand Down
Loading
Loading