Skip to content

Commit d1e81e3

Browse files
authored
1 parent d66da4f commit d1e81e3

14 files changed

Lines changed: 672 additions & 425 deletions

File tree

tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,7 @@ private CjkDecodeValidator() {
6868
* little legacy evidence (legacy high bytes < {@link #MIN_HIGH_BYTES})
6969
*/
7070
public static double strippedFailureRate(byte[] bytes, Charset cjkCharset) {
71-
Charset decodeAs = CharsetSupersets.supersetOf(cjkCharset);
72-
if (decodeAs == null) {
73-
decodeAs = cjkCharset;
74-
}
71+
Charset decodeAs = CharsetSupersets.decodeAs(cjkCharset);
7572
CharsetDecoder dec = decodeAs.newDecoder()
7673
.onMalformedInput(CodingErrorAction.REPORT)
7774
.onUnmappableCharacter(CodingErrorAction.REPORT);

tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CosineFamilyArbiter.java

Lines changed: 0 additions & 241 deletions
This file was deleted.

tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/ScoredCandidate.java

Lines changed: 0 additions & 56 deletions
This file was deleted.

0 commit comments

Comments
 (0)