TIKA-4751 - decode as (#2867)

Author: tballison

tika-core/src/main/java/org/apache/tika/detect/AutoDetectReader.java

@@ -100,13 +100,12 @@ private static Charset detect(TikaInputStream tis, Metadata metadata,
         // Ask all given detectors for the character encoding
         List<EncodingResult> results = detector.detect(tis, metadata, new ParseContext());
         if (!results.isEmpty()) {
-            Charset detected = results.get(0).getCharset();
-            Charset superset = CharsetSupersets.supersetOf(detected);
-            if (superset != null) {
-                metadata.set(TikaCoreProperties.DECODED_CHARSET, superset.name());
-                return superset;
+            EncodingResult result = results.get(0);
+            Charset decodeAs = result.getDecodeAs();
+            if (!decodeAs.equals(result.getCharset())) {
+                metadata.set(TikaCoreProperties.DECODED_CHARSET, decodeAs.name());
             }
-            return detected;
+            return decodeAs;
         }
 
         // Try determining the encoding based on hints in document metadata.

tika-core/src/main/java/org/apache/tika/detect/CharsetSupersets.java

@@ -88,4 +88,15 @@ public static Charset supersetOf(Charset detected) {
             return null;
         }
     }
+
+    /**
+     * The charset to decode with: {@link #supersetOf(Charset) superset} of
+     * {@code detected} if one exists, else {@code detected} (null only if
+     * {@code detected} is null). For bare-{@link Charset} callers; detection
+     * results can use {@link EncodingResult#getDecodeAs()}.
+     */
+    public static Charset decodeAs(Charset detected) {
+        Charset superset = supersetOf(detected);
+        return superset != null ? superset : detected;
+    }
 }

tika-core/src/main/java/org/apache/tika/detect/EncodingResult.java

@@ -126,10 +126,24 @@ public EncodingResult(Charset charset, float confidence, String label,
         this.resultType = resultType;
     }
 
+    /**
+     * The detected charset. For <em>decoding</em> bytes prefer {@link #getDecodeAs()};
+     * this is the charset to <em>report</em> (Content-Type / detected-encoding).
+     */
     public Charset getCharset() {
         return charset;
     }
 
+    /**
+     * The charset to decode with: {@link #getCharset()} widened to its superset when
+     * one exists (e.g. GBK &rarr; GB18030), else the detected charset unchanged.
+     *
+     * @see CharsetSupersets
+     */
+    public Charset getDecodeAs() {
+        return CharsetSupersets.decodeAs(charset);
+    }
+
     /**
      * Detection confidence in {@code [0.0, 1.0]}.  Meaningful for ranking
      * among {@link ResultType#STATISTICAL} candidates.  For

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java

@@ -163,6 +163,8 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
         List<EncodingResult> encResults = encodingDetector.detect(tis, metadata, context);
         Charset charset = encResults.isEmpty() ? DEFAULT_CHARSET
                 : encResults.get(0).getCharset();
+        Charset decodeAs = encResults.isEmpty() ? DEFAULT_CHARSET
+                : encResults.get(0).getDecodeAs();
         String previous = metadata.get(Metadata.CONTENT_TYPE);
         MediaType contentType = null;
         if (previous == null || previous.startsWith("text/html")) {
@@ -195,7 +197,7 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
         tis.setCloseShield();
         Document document;
         try {
-            document = Jsoup.parse(tis, charset.name(), "",
+            document = Jsoup.parse(tis, decodeAs.name(), "",
                     Parser.htmlParser().tagSet(tagSet));
         } finally {
             tis.removeCloseShield();

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java

@@ -68,6 +68,7 @@
 import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
+import org.apache.tika.detect.CharsetSupersets;
 import org.apache.tika.detect.EncodingResult;
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.extractor.EmbeddedDocumentUtil;
@@ -903,7 +904,7 @@ private void guess7BitEncoding(MAPIMessage msg) {
                 try (TikaInputStream tis = TikaInputStream.get(html.getBytes(UTF_8))) {
                     List<EncodingResult> encResults =
                             detector.detect(tis, EMPTY_METADATA, context);
-                    charset = encResults.isEmpty() ? null : encResults.get(0).getCharset();
+                    charset = encResults.isEmpty() ? null : encResults.get(0).getDecodeAs();
                 } catch (IOException e) {
                     //swallow
                 }
@@ -921,9 +922,16 @@ private void guess7BitEncoding(MAPIMessage msg) {
             CharsetDetector detector = new CharsetDetector();
             detector.setText(text.getRawValue());
             CharsetMatch match = detector.detect();
-            if (match != null && match.getConfidence() > 35 &&
-                    tryToSet7BitEncoding(msg, match.getName())) {
-                return;
+            if (match != null && match.getConfidence() > 35) {
+                String charsetName = match.getName();
+                try {
+                    charsetName = CharsetSupersets.decodeAs(Charset.forName(charsetName)).name();
+                } catch (IllegalArgumentException e) {
+                    //ICU name not a resolvable Java charset; use as-is
+                }
+                if (tryToSet7BitEncoding(msg, charsetName)) {
+                    return;
+                }
             }
         }
     }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/dbf/DBFParser.java

@@ -30,6 +30,7 @@
 import org.xml.sax.SAXException;
 
 import org.apache.tika.config.TikaComponent;
+import org.apache.tika.detect.CharsetSupersets;
 import org.apache.tika.detect.EncodingDetector;
 import org.apache.tika.detect.EncodingResult;
 import org.apache.tika.exception.TikaException;
@@ -89,14 +90,16 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
 
         Charset charset = getCharset(firstRows, header, context);
         metadata.set(Metadata.CONTENT_ENCODING, charset.toString());
+        //report detected (above); decode with its superset
+        Charset decodeAs = CharsetSupersets.decodeAs(charset);
 
         XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context);
         xhtml.startDocument();
         xhtml.startElement("table");
         xhtml.startElement("thead");
         for (DBFColumnHeader col : header.getCols()) {
             xhtml.startElement("th");
-            xhtml.characters(col.getName(charset));
+            xhtml.characters(col.getName(decodeAs));
             xhtml.endElement("th");
         }
         xhtml.endElement("thead");
@@ -106,12 +109,12 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
         //now write cached rows
         while (firstRows.size() > 0) {
             DBFRow cachedRow = firstRows.remove(0);
-            writeRow(cachedRow, charset, xhtml);
+            writeRow(cachedRow, decodeAs, xhtml);
         }
 
         //now continue with rest
         while (row != null) {
-            writeRow(row, charset, xhtml);
+            writeRow(row, decodeAs, xhtml);
             row = reader.next();
         }
         xhtml.endElement("tbody");

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java

@@ -565,7 +565,7 @@ private String detectEntryName(ZipArchiveEntry entry, Metadata parentMetadata,
             try (TikaInputStream detectStream = TikaInputStream.get(entryName)) {
                 List<EncodingResult> encResults =
                         getEncodingDetector().detect(detectStream, parentMetadata, context);
-                Charset candidate = encResults.isEmpty() ? null : encResults.get(0).getCharset();
+                Charset candidate = encResults.isEmpty() ? null : encResults.get(0).getDecodeAs();
                 if (candidate != null) {
                     return new String(entry.getRawName(), candidate);
                 }