Skip to content

Commit d4091f5

Browse files
tballisontballison
authored
TIKA-4810 -- Restore tolerated-UTF-8 structural promotion, gated on evidence volume
2 parents e9fb967 + 68f04d0 commit d4091f5

3 files changed

Lines changed: 223 additions & 10 deletions

File tree

‎tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,12 @@ public class MojibusterEncodingDetector implements EncodingDetector {
174174
*/
175175
private static final int UTF8_MAX_TOLERATED_ERRORS = 1;
176176

177+
/** Minimum valid multi-byte UTF-8 sequences before a tolerated (not clean)
178+
* probe is promoted to STRUCTURAL — else a short filename could false-
179+
* positive on a single coincidental error (mirrors {@link
180+
* CjkDecodeValidator#MIN_HIGH_BYTES}). */
181+
private static final int MIN_TOLERATED_UTF8_SEQUENCES = 30;
182+
177183
/** Windows-1252: the WHATWG-canonical default for unlabeled Western content. */
178184
private static final String WIN1252 = "windows-1252";
179185

@@ -355,16 +361,18 @@ public List<EncodingResult> detect(byte[] probe, Metadata metadata) {
355361
}
356362
}
357363
LOG.trace("mojibuster utf8Check={} tolerated={}", utf8, utf8Tolerated);
358-
// Emit a structural UTF-8 candidate only when the grammar is definitively
359-
// clean (LIKELY_UTF8). When the probe is NOT_UTF8 but within the error
360-
// tolerance (utf8Tolerated), NB's UTF-8 result is already kept as a
361-
// STATISTICAL candidate (see NOT_UTF8 disqualifier above) — promoting it
362-
// to STRUCTURAL here would cause the "return only top-1 STRUCTURAL" path
363-
// to short-circuit JunkFilter, preventing it from comparing UTF-8 against
364-
// windows-1252. For short probes a single bad byte in otherwise-ASCII
365-
// content is more likely a genuine Latin-1/windows-1252 byte than a
366-
// corrupt UTF-8 sequence; JunkFilter has enough signal to arbitrate.
367-
if (utf8 == StructuralEncodingRules.Utf8Result.LIKELY_UTF8) {
364+
// Promote on LIKELY_UTF8, or on tolerated errors backed by abundant
365+
// evidence (evidenceTolerated). Bare tolerance isn't enough: on a short
366+
// probe (e.g. a zip entry name, routed here by ZipParser) NB already
367+
// covers a real UTF-8 case as STATISTICAL, so a lone tolerated error is
368+
// more likely a coincidentally-valid legacy string. But on a long,
369+
// genuinely-UTF-8 document NB can return an empty pool for some scripts
370+
// (TIKA-4810) — without this, one stray legacy byte costs the whole
371+
// document its STRUCTURAL proof and JunkFilter has nothing to prefer
372+
// over the declared charset.
373+
boolean evidenceTolerated = utf8Tolerated
374+
&& StructuralEncodingRules.countUtf8Sequences(probe) >= MIN_TOLERATED_UTF8_SEQUENCES;
375+
if (utf8 == StructuralEncodingRules.Utf8Result.LIKELY_UTF8 || evidenceTolerated) {
368376
pool.add(new EncodingResult(
369377
java.nio.charset.StandardCharsets.UTF_8,
370378
UTF8_STRUCTURAL_CONF, "UTF-8",

‎tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -894,6 +894,81 @@ public static int countUtf8Errors(byte[] bytes, int offset, int length) {
894894
return errors;
895895
}
896896

897+
/** Counts complete, valid multi-byte UTF-8 sequences — companion to
898+
* {@link #countUtf8Errors}, same walk, opposite tally. Gauges how much
899+
* genuine UTF-8 evidence a probe carries independent of its error count. */
900+
public static int countUtf8Sequences(byte[] bytes) {
901+
return countUtf8Sequences(bytes, 0, bytes.length);
902+
}
903+
904+
public static int countUtf8Sequences(byte[] bytes, int offset, int length) {
905+
int sequences = 0;
906+
int i = offset;
907+
int end = offset + length;
908+
while (i < end) {
909+
int b = bytes[i] & 0xFF;
910+
if (b < 0x80) {
911+
i++;
912+
continue;
913+
}
914+
int seqLen;
915+
if (b >= 0xF8) {
916+
i++;
917+
continue;
918+
} else if (b >= 0xF0) {
919+
seqLen = 4;
920+
} else if (b >= 0xE0) {
921+
seqLen = 3;
922+
} else if (b >= 0xC0) {
923+
seqLen = 2;
924+
} else {
925+
i++;
926+
continue;
927+
}
928+
if (seqLen == 2 && b <= 0xC1) {
929+
i++;
930+
continue;
931+
}
932+
int kEnd = Math.min(seqLen, end - i);
933+
if (kEnd < seqLen) {
934+
break;
935+
}
936+
boolean bad = false;
937+
for (int k = 1; k < seqLen; k++) {
938+
int cb = bytes[i + k] & 0xFF;
939+
if (cb < 0x80 || cb > 0xBF) {
940+
bad = true;
941+
break;
942+
}
943+
}
944+
if (bad) {
945+
i += seqLen;
946+
continue;
947+
}
948+
if (seqLen == 3) {
949+
int cp = ((b & 0x0F) << 12)
950+
| ((bytes[i + 1] & 0xFF) & 0x3F) << 6
951+
| ((bytes[i + 2] & 0xFF) & 0x3F);
952+
if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF)) {
953+
i += seqLen;
954+
continue;
955+
}
956+
} else if (seqLen == 4) {
957+
int cp = ((b & 0x07) << 18)
958+
| ((bytes[i + 1] & 0xFF) & 0x3F) << 12
959+
| ((bytes[i + 2] & 0xFF) & 0x3F) << 6
960+
| ((bytes[i + 3] & 0xFF) & 0x3F);
961+
if (cp < 0x10000 || cp > 0x10FFFF) {
962+
i += seqLen;
963+
continue;
964+
}
965+
}
966+
sequences++;
967+
i += seqLen;
968+
}
969+
return sequences;
970+
}
971+
897972
// -----------------------------------------------------------------------
898973
// Result type
899974
// -----------------------------------------------------------------------

‎tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.ml.chardetect;
18+
19+
import static org.junit.jupiter.api.Assertions.assertFalse;
20+
import static org.junit.jupiter.api.Assertions.assertTrue;
21+
22+
import java.io.ByteArrayOutputStream;
23+
import java.io.IOException;
24+
import java.nio.charset.Charset;
25+
import java.nio.charset.StandardCharsets;
26+
import java.util.List;
27+
28+
import org.junit.jupiter.api.Test;
29+
30+
import org.apache.tika.detect.EncodingResult;
31+
32+
/**
33+
* TIKA-4810: commit 360b3d354 (2026-06-10) dropped the {@code || utf8Tolerated}
34+
* branch that promoted a tolerated (near-clean) probe to STRUCTURAL UTF-8,
35+
* assuming NB's statistical layer always covers the fallback. It doesn't (a
36+
* real Bengali news page's NB pool came back empty) — but restoring the branch
37+
* unconditionally would re-open a false positive on short zip entry names
38+
* (9-30 bytes, routed through this detector by {@code ZipParser}), which is
39+
* why it was narrowed in the first place.
40+
*/
41+
public class ToleratedUtf8StructuralRegressionTest {
42+
43+
private static final String BENGALI_SENTENCE =
44+
"সেমিতে ক্রোয়েশিয়া টাইব্রেকারে রাশিয়াকে হারিয়ে ফাইনালে উঠেছে। ";
45+
46+
private static MojibusterEncodingDetector newDetector() {
47+
try {
48+
return new MojibusterEncodingDetector();
49+
} catch (Exception e) {
50+
throw new RuntimeException(e);
51+
}
52+
}
53+
54+
@Test
55+
public void longDocumentWithOneStrayByteIsStillUtf8() throws IOException {
56+
byte[] probe = buildProbe(30);
57+
List<EncodingResult> results = newDetector().detect(probe);
58+
boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
59+
"UTF-8".equals(r.getCharset().name())
60+
&& r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
61+
assertTrue(hasStructuralUtf8,
62+
"A long, overwhelmingly UTF-8 document with a single tolerated "
63+
+ "error byte must still yield a STRUCTURAL UTF-8 candidate; "
64+
+ "results were: " + results);
65+
}
66+
67+
/** Zip-entry-name-shaped probe: must not be promoted on tolerance alone. */
68+
@Test
69+
public void shortProbeWithOneStrayByteIsNotPromoted() throws IOException {
70+
ByteArrayOutputStream bo = new ByteArrayOutputStream();
71+
bo.write(0xA9); // raw © byte: invalid as a UTF-8 lead
72+
bo.writeBytes("café-Köln.txt".getBytes(StandardCharsets.UTF_8));
73+
byte[] probe = bo.toByteArray();
74+
75+
List<EncodingResult> results = newDetector().detect(probe);
76+
boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
77+
"UTF-8".equals(r.getCharset().name())
78+
&& r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
79+
assertFalse(hasStructuralUtf8,
80+
"A short probe shaped like a zip entry name must not be promoted "
81+
+ "to STRUCTURAL UTF-8 on a single tolerated error alone; "
82+
+ "results were: " + results);
83+
}
84+
85+
/** Real GBK filename from attachment_name_diffs.xlsx; must stay GB18030. */
86+
@Test
87+
public void chineseGbkFilenameIsNotPromotedToUtf8() {
88+
byte[] probe = "说明.txt".getBytes(Charset.forName("GBK"));
89+
List<EncodingResult> results = newDetector().detect(probe);
90+
boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
91+
"UTF-8".equals(r.getCharset().name())
92+
&& r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
93+
assertFalse(hasStructuralUtf8,
94+
"A short GBK filename must not be promoted to STRUCTURAL UTF-8 "
95+
+ "on a single tolerated error alone; results were: " + results);
96+
assertTrue(results.stream().anyMatch(r -> r.getCharset().name().startsWith("GB")),
97+
"Expected a GB18030/GBK candidate; results were: " + results);
98+
}
99+
100+
/** Real windows-1252 filename from attachment_name_diffs.xlsx. */
101+
@Test
102+
public void sauteFilenameIsNotPromotedToUtf8() {
103+
byte[] probe = "Sauté.txt".getBytes(Charset.forName("windows-1252"));
104+
List<EncodingResult> results = newDetector().detect(probe);
105+
boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
106+
"UTF-8".equals(r.getCharset().name())
107+
&& r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
108+
assertFalse(hasStructuralUtf8,
109+
"A short windows-1252 filename must not be promoted to STRUCTURAL "
110+
+ "UTF-8 on a single tolerated error alone; results were: " + results);
111+
}
112+
113+
/** Declared-windows-1252 HTML page, genuinely UTF-8, one stray raw © byte. */
114+
private static byte[] buildProbe(int repeatCount) throws IOException {
115+
StringBuilder body = new StringBuilder();
116+
for (int i = 0; i < repeatCount; i++) {
117+
body.append(BENGALI_SENTENCE);
118+
}
119+
ByteArrayOutputStream bo = new ByteArrayOutputStream();
120+
bo.writeBytes(("<html><head><meta http-equiv=\"Content-Type\" "
121+
+ "content=\"text/html; charset=windows-1252\">")
122+
.getBytes(StandardCharsets.US_ASCII));
123+
bo.writeBytes("<meta name=\"copyright\" content=\"".getBytes(StandardCharsets.US_ASCII));
124+
bo.write(0xA9); // raw © byte: invalid as a UTF-8 lead
125+
bo.writeBytes(" 2013\"></head><body><title>".getBytes(StandardCharsets.US_ASCII));
126+
bo.writeBytes(body.toString().getBytes(StandardCharsets.UTF_8));
127+
bo.writeBytes("</title></body></html>".getBytes(StandardCharsets.US_ASCII));
128+
return bo.toByteArray();
129+
}
130+
}

0 commit comments

Comments
 (0)