Skip to content

TIKA-4752-follow-up #2871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tballison merged 6 commits into from
Jun 5, 2026
Merged

TIKA-4752-follow-up #2871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
652ffb2
TIKA-4752 -- improve zip name detection
tballison Jun 5, 2026
eff4f45
merge main
tballison Jun 5, 2026
723b2f9
TIKA-4752 -- follow up
tballison Jun 5, 2026
94784ea
TIKA-4752 -- clean up
tballison Jun 5, 2026
1c44798
Potential fix for pull request finding
tballison Jun 5, 2026
51797a4
Merge branch 'main' into TIKA-4752-follow-up
tballison Jun 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion tika-core/src/main/java/org/apache/tika/detect/MetadataCharsetDetector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@
* HTTP/MIME Content-Type header (e.g. {@code text/html; charset=UTF-8}).</li>
* <li>{@link TikaCoreProperties#CONTENT_TYPE_HINT} — the {@code charset} parameter
* of a content-type a source <em>claimed</em> for the bytes (e.g. an HTML
* {@code <meta>} tag, or a zip entry's UTF-8 (EFS) flag). A hint, not a verdict.</li>
* {@code <meta>} tag, or a zip entry's UTF-8 (EFS) flag). A hint, not a verdict.
* This key is only consulted when {@link MetadataCharsetDetector} is included in
* the active {@link org.apache.tika.detect.EncodingDetector} chain.</li>
* <li>{@link Metadata#CONTENT_ENCODING} — a bare charset label set by parsers
* such as {@code RFC822Parser}, which splits Content-Type into a bare
* media-type key and a separate charset key.</li>
Expand Down
133 changes: 65 additions & 68 deletions ...ser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;

import org.apache.tika.detect.CharsetSupersets;
import org.apache.tika.detect.DefaultEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.detect.EncodingResult;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
Expand All @@ -81,14 +82,11 @@
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.html.HtmlEncodingDetector;
import org.apache.tika.parser.html.JSoupParser;
import org.apache.tika.parser.mailcommons.MailDateParser;
import org.apache.tika.parser.microsoft.msg.ExtendedMetadataExtractor;
import org.apache.tika.parser.microsoft.rtf.RTFParser;
import org.apache.tika.parser.microsoft.rtf.jflex.RTFHtmlDecapsulator;
import org.apache.tika.parser.txt.CharsetDetector;
import org.apache.tika.parser.txt.CharsetMatch;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
Expand Down Expand Up @@ -183,7 +181,7 @@ private static void loadMessageClasses() {
private final DirectoryNode root;
private final MAPIMessage msg;
private final ParseContext parseContext;
HtmlEncodingDetector detector = new HtmlEncodingDetector();
private static final EncodingDetector DEFAULT_ENCODING_DETECTOR = new DefaultEncodingDetector();


public OutlookExtractor(DirectoryNode root, Metadata metadata, ParseContext context) throws TikaException {
Expand Down Expand Up @@ -848,92 +846,91 @@ private void guess7BitEncoding(MAPIMessage msg) {
return;
}

// A declared charset (message codepage, else a Content-Type header) is a hint,
// not a verdict -- the detector evaluates it against the raw body bytes below.
String declared = declaredCharset(msg, mainChunks);

// Detect on the raw body bytes -- the HTML binary chunk if present, else the
// text body. (msg.getHtmlBody() is an already-decoded String, so detecting its
// re-encoded bytes would just report the re-encoding charset.)
byte[] body = null;
ByteChunk htmlBinary = mainChunks.getHtmlBodyChunkBinary();
if (htmlBinary != null && htmlBinary.getValue() != null) {
body = htmlBinary.getValue();
} else if (mainChunks.getTextBodyChunk() != null) {
body = mainChunks.getTextBodyChunk().getRawValue();
}

EncodingDetector encodingDetector = context.get(EncodingDetector.class);
if (encodingDetector == null) {
encodingDetector = DEFAULT_ENCODING_DETECTOR;
}

if (body != null && body.length > 0) {
Metadata metadata = new Metadata();
if (declared != null) {
metadata.set(TikaCoreProperties.CONTENT_TYPE_HINT,
"text/plain; charset=" + declared);
}
try (TikaInputStream tis = TikaInputStream.get(body)) {
List<EncodingResult> results = encodingDetector.detect(tis, metadata, context);
if (!results.isEmpty() && results.get(0).getConfidence() > 0.35f
&& tryToSet7BitEncoding(msg, results.get(0).getDecodeAs().name())) {
return;
}
} catch (IOException e) {
//swallow
}
}

// No body to adjudicate against (or detection abstained): trust the declaration.
if (declared != null) {
tryToSet7BitEncoding(msg, declared);
}
}

/**
* The charset a 7-bit message declares for itself: its codepage property
* (MESSAGE_CODEPAGE / INTERNET_CPID), else a {@code charset} on a Content-Type
* header. A hint for the detector, not a verdict. {@code null} if none.
*/
private static String declaredCharset(MAPIMessage msg, Chunks mainChunks) {
Map<MAPIProperty, List<PropertyValue>> props = mainChunks.getProperties();
if (props != null) {
// First choice is a codepage property
for (MAPIProperty prop : new MAPIProperty[]{MAPIProperty.MESSAGE_CODEPAGE, MAPIProperty.INTERNET_CPID}) {
for (MAPIProperty prop : new MAPIProperty[]{MAPIProperty.MESSAGE_CODEPAGE,
MAPIProperty.INTERNET_CPID}) {
List<PropertyValue> val = props.get(prop);
if (val != null && val.size() > 0) {
int codepage = ((PropertyValue.LongPropertyValue) val.get(0)).getValue();
String encoding = null;
try {
encoding = CodePageUtil.codepageToEncoding(codepage, true);
} catch (UnsupportedEncodingException e) {
//swallow
}
if (tryToSet7BitEncoding(msg, encoding)) {
return;
String encoding = CodePageUtil.codepageToEncoding(codepage, true);
if (encoding != null && Charset.isSupported(encoding)) {
return encoding;
}
} catch (UnsupportedEncodingException | IllegalArgumentException e) {
//swallow, try the next source
}
}
}
}

// Second choice is a charset on a content type header
try {
String[] headers = msg.getHeaders();
if (headers != null && headers.length > 0) {
// Look for a content type with a charset
Pattern p = Pattern.compile("Content-Type:.*?charset=[\"']?([^;'\"]+)[\"']?", Pattern.CASE_INSENSITIVE);

if (headers != null) {
Pattern p = Pattern.compile(
"Content-Type:.*?charset=[\"']?([^;'\"]+)[\"']?", Pattern.CASE_INSENSITIVE);
for (String header : headers) {
if (header.startsWith("Content-Type")) {
Matcher m = p.matcher(header);
if (m.matches()) {
// Found it! Tell all the string chunks
String charset = m.group(1);
if (tryToSet7BitEncoding(msg, charset)) {
return;
}
return m.group(1);
}
}
}
}
} catch (ChunkNotFoundException e) {
//swallow
}

// Nothing suitable in the headers, try HTML
// TODO: do we need to replicate this in Tika? If we wind up
// parsing the html version of the email, this is duplicative??
// Or do we need to reset the header strings based on the html
// meta header if there is no other information?
try {
String html = msg.getHtmlBody();
if (html != null && html.length() > 0) {
Charset charset = null;
try (TikaInputStream tis = TikaInputStream.get(html.getBytes(UTF_8))) {
List<EncodingResult> encResults =
detector.detect(tis, EMPTY_METADATA, context);
charset = encResults.isEmpty() ? null : encResults.get(0).getDecodeAs();
} catch (IOException e) {
//swallow
}
if (charset != null && tryToSet7BitEncoding(msg, charset.name())) {
return;
}
}
} catch (ChunkNotFoundException e) {
//swallow
}

//absolute last resort, try charset detector
StringChunk text = mainChunks.getTextBodyChunk();
if (text != null) {
CharsetDetector detector = new CharsetDetector();
detector.setText(text.getRawValue());
CharsetMatch match = detector.detect();
if (match != null && match.getConfidence() > 35) {
String charsetName = match.getName();
try {
charsetName = CharsetSupersets.decodeAs(Charset.forName(charsetName)).name();
} catch (IllegalArgumentException e) {
//ICU name not a resolvable Java charset; use as-is
}
if (tryToSet7BitEncoding(msg, charsetName)) {
return;
}
}
}
return null;
}

private boolean tryToSet7BitEncoding(MAPIMessage msg, String charsetName) {
Expand Down
14 changes: 10 additions & 4 deletions ...les/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

import org.apache.tika.config.TikaComponent;
import org.apache.tika.detect.CharsetSupersets;
import org.apache.tika.detect.DefaultEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.detect.EncodingResult;
import org.apache.tika.exception.TikaException;
Expand All @@ -40,7 +41,6 @@
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.txt.Icu4jEncodingDetector;
import org.apache.tika.sax.XHTMLContentHandler;

/**
Expand All @@ -57,6 +57,7 @@ public class DBFParser implements Parser {
private static final int ROWS_TO_BUFFER_FOR_CHARSET_DETECTION = 10;
private static final int MAX_CHARS_FOR_CHARSET_DETECTION = 20000;
private static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
private static final EncodingDetector DEFAULT_ENCODING_DETECTOR = new DefaultEncodingDetector();

private static final Set<MediaType> SUPPORTED_TYPES =
Collections.singleton(MediaType.application("x-dbf"));
Expand Down Expand Up @@ -140,11 +141,16 @@ private Charset getCharset(List<DBFRow> firstRows, DBFFileHeader header, ParseCo
}
byte[] bytes = bos.toByteArray();
if (bytes.length > 20) {
EncodingDetector detector = new Icu4jEncodingDetector();
EncodingDetector detector = parseContext.get(EncodingDetector.class);
if (detector == null) {
detector = DEFAULT_ENCODING_DETECTOR;
}
try (TikaInputStream tis = TikaInputStream.get(bytes)) {
List<EncodingResult> results =
detector.detect(TikaInputStream.get(bytes), new Metadata(), parseContext);
charset = results.isEmpty() ? null : results.get(0).getCharset();
detector.detect(tis, new Metadata(), parseContext);
if (!results.isEmpty() && results.get(0).getCharset() != null) {
charset = results.get(0).getCharset();
}
}
}
return charset;
Expand Down
Loading