Skip to content

Commit 31c6651

Browse files
tballisontballison
committed
TIKA-4810 -- Restore tolerated-UTF-8 structural promotion, gated on evidence volume
1 parent 3f784e9 commit 31c6651

3 files changed

Lines changed: 288 additions & 10 deletions

File tree

‎tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,18 @@ public class MojibusterEncodingDetector implements EncodingDetector {
174174
*/
175175
private static final int UTF8_MAX_TOLERATED_ERRORS = 1;
176176

177+
/**
178+
* Minimum count of complete, valid multi-byte UTF-8 sequences required before
179+
* a tolerated (NOT_UTF8-but-within-error-budget) probe is promoted to a
180+
* STRUCTURAL UTF-8 candidate. Tolerance alone isn't enough evidence at any
181+
* length — 1 error in a 20-byte zip entry name is a 5% error rate, easily a
182+
* coincidentally-valid legacy-encoded string, not corrupted UTF-8. Requiring
183+
* substantial genuine multi-byte evidence (mirrors {@link
184+
* CjkDecodeValidator#MIN_HIGH_BYTES}) separates that short-probe false-positive
185+
* risk from the long-document case this tolerance mechanism exists for.
186+
*/
187+
private static final int MIN_TOLERATED_UTF8_SEQUENCES = 30;
188+
177189
/** Windows-1252: the WHATWG-canonical default for unlabeled Western content. */
178190
private static final String WIN1252 = "windows-1252";
179191

@@ -355,16 +367,24 @@ public List<EncodingResult> detect(byte[] probe, Metadata metadata) {
355367
}
356368
}
357369
LOG.trace("mojibuster utf8Check={} tolerated={}", utf8, utf8Tolerated);
358-
// Emit a structural UTF-8 candidate only when the grammar is definitively
359-
// clean (LIKELY_UTF8). When the probe is NOT_UTF8 but within the error
360-
// tolerance (utf8Tolerated), NB's UTF-8 result is already kept as a
361-
// STATISTICAL candidate (see NOT_UTF8 disqualifier above) — promoting it
362-
// to STRUCTURAL here would cause the "return only top-1 STRUCTURAL" path
363-
// to short-circuit JunkFilter, preventing it from comparing UTF-8 against
364-
// windows-1252. For short probes a single bad byte in otherwise-ASCII
365-
// content is more likely a genuine Latin-1/windows-1252 byte than a
366-
// corrupt UTF-8 sequence; JunkFilter has enough signal to arbitrate.
367-
if (utf8 == StructuralEncodingRules.Utf8Result.LIKELY_UTF8) {
370+
// Emit a structural UTF-8 candidate when the grammar is definitively clean
371+
// (LIKELY_UTF8), OR when it's tolerated AND backed by abundant genuine
372+
// multi-byte evidence (evidenceTolerated below). Bare tolerance is not
373+
// promoted: on a short probe (e.g. a zip entry name — ZipParser routes
374+
// entry-name bytes through this same detector) NB's UTF-8 result is
375+
// already kept as a STATISTICAL candidate (see NOT_UTF8 disqualifier
376+
// above), and a single tolerated error there is more likely a
377+
// coincidentally-valid legacy-encoded string than corrupted UTF-8 — regr-
378+
// ession-tested in ToleratedUtf8StructuralRegressionTest. On a long,
379+
// overwhelmingly-UTF-8 document a single stray legacy byte (e.g. a raw
380+
// 0xA9 copyright sign) must not cost the whole document its STRUCTURAL
381+
// proof: NB can come back with an empty pool for some scripts, leaving
382+
// nothing for JunkFilter to prefer over the declared charset — real-world
383+
// regression from commit 360b3d354 (2026-06-10), which dropped this
384+
// branch entirely on the assumption that an NB fallback always exists.
385+
boolean evidenceTolerated = utf8Tolerated
386+
&& StructuralEncodingRules.countUtf8Sequences(probe) >= MIN_TOLERATED_UTF8_SEQUENCES;
387+
if (utf8 == StructuralEncodingRules.Utf8Result.LIKELY_UTF8 || evidenceTolerated) {
368388
pool.add(new EncodingResult(
369389
java.nio.charset.StandardCharsets.UTF_8,
370390
UTF8_STRUCTURAL_CONF, "UTF-8",

‎tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -894,6 +894,89 @@ public static int countUtf8Errors(byte[] bytes, int offset, int length) {
894894
return errors;
895895
}
896896

897+
/**
898+
* Counts complete, valid multi-byte UTF-8 sequences in the sample —
899+
* companion to {@link #countUtf8Errors}, same walk, opposite tally. Used
900+
* to gauge how much genuine UTF-8 evidence a probe carries independent of
901+
* its error count: a probe with one tolerated error and hundreds of valid
902+
* sequences is overwhelmingly UTF-8; a probe with one tolerated error and
903+
* two or three valid sequences (a short filename, say) is not distinguishable
904+
* from a coincidentally-valid legacy-encoded string.
905+
*
906+
* @return number of complete, well-formed multi-byte UTF-8 sequences
907+
*/
908+
public static int countUtf8Sequences(byte[] bytes) {
909+
return countUtf8Sequences(bytes, 0, bytes.length);
910+
}
911+
912+
public static int countUtf8Sequences(byte[] bytes, int offset, int length) {
913+
int sequences = 0;
914+
int i = offset;
915+
int end = offset + length;
916+
while (i < end) {
917+
int b = bytes[i] & 0xFF;
918+
if (b < 0x80) {
919+
i++;
920+
continue;
921+
}
922+
int seqLen;
923+
if (b >= 0xF8) {
924+
i++;
925+
continue;
926+
} else if (b >= 0xF0) {
927+
seqLen = 4;
928+
} else if (b >= 0xE0) {
929+
seqLen = 3;
930+
} else if (b >= 0xC0) {
931+
seqLen = 2;
932+
} else {
933+
i++;
934+
continue;
935+
}
936+
if (seqLen == 2 && b <= 0xC1) {
937+
i++;
938+
continue;
939+
}
940+
int kEnd = Math.min(seqLen, end - i);
941+
if (kEnd < seqLen) {
942+
break;
943+
}
944+
boolean bad = false;
945+
for (int k = 1; k < seqLen; k++) {
946+
int cb = bytes[i + k] & 0xFF;
947+
if (cb < 0x80 || cb > 0xBF) {
948+
bad = true;
949+
break;
950+
}
951+
}
952+
if (bad) {
953+
i += seqLen;
954+
continue;
955+
}
956+
if (seqLen == 3) {
957+
int cp = ((b & 0x0F) << 12)
958+
| ((bytes[i + 1] & 0xFF) & 0x3F) << 6
959+
| ((bytes[i + 2] & 0xFF) & 0x3F);
960+
if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF)) {
961+
i += seqLen;
962+
continue;
963+
}
964+
} else if (seqLen == 4) {
965+
int cp = ((b & 0x07) << 18)
966+
| ((bytes[i + 1] & 0xFF) & 0x3F) << 12
967+
| ((bytes[i + 2] & 0xFF) & 0x3F) << 6
968+
| ((bytes[i + 3] & 0xFF) & 0x3F);
969+
if (cp < 0x10000 || cp > 0x10FFFF) {
970+
i += seqLen;
971+
continue;
972+
}
973+
}
974+
sequences++;
975+
i += seqLen;
976+
}
977+
return sequences;
978+
}
979+
897980
// -----------------------------------------------------------------------
898981
// Result type
899982
// -----------------------------------------------------------------------

‎tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.ml.chardetect;
18+
19+
import static org.junit.jupiter.api.Assertions.assertFalse;
20+
import static org.junit.jupiter.api.Assertions.assertTrue;
21+
22+
import java.io.ByteArrayOutputStream;
23+
import java.io.IOException;
24+
import java.nio.charset.Charset;
25+
import java.nio.charset.StandardCharsets;
26+
import java.util.List;
27+
28+
import org.junit.jupiter.api.Test;
29+
30+
import org.apache.tika.detect.EncodingResult;
31+
32+
/**
33+
* Regression test for a real-world failure: a genuinely UTF-8 HTML page whose
34+
* only single-byte "legacy" artifact (a stray {@code &copy;} written as raw
35+
* {@code 0xA9} rather than an entity) sits before the bulk of the document's
36+
* real multi-byte content. {@link StructuralEncodingRules#checkUtf8} correctly
37+
* reports {@code NOT_UTF8} for the whole probe (one malformed lead byte), and
38+
* the tolerance mechanism in {@link MojibusterEncodingDetector} is supposed to
39+
* recognize this as "essentially UTF-8" when there's abundant genuine
40+
* multi-byte evidence.
41+
*
42+
* <p>Commit 360b3d354 ("merge conflict and flaky test", 2026-06-10) dropped the
43+
* {@code || utf8Tolerated} branch that used to promote this case to a
44+
* STRUCTURAL UTF-8 candidate, on the assumption that the NB statistical layer
45+
* would independently propose UTF-8 as a fallback. That assumption doesn't
46+
* hold for every script/corpus (verified against a real Bengali-language news
47+
* page): NB's own candidate pool can come back completely empty, leaving
48+
* Mojibuster with nothing but the {@code windows-1252} "give up" default —
49+
* silent, complete mojibake on an otherwise-clean UTF-8 document.</p>
50+
*
51+
* <p>The companion {@link #shortProbeWithOneStrayByteIsNotPromoted()} test
52+
* guards the reason that branch was narrowed in the first place: zip entry
53+
* names are typically 9-30 bytes, and {@link
54+
* org.apache.tika.parser.pkg.ZipParser} runs them through this same detector
55+
* (see {@code ZipParser#isDetectCharsetsInEntryNames}). A single coincidental
56+
* error byte in a short, genuinely-legacy-encoded filename must NOT be enough
57+
* to promote it to STRUCTURAL UTF-8 — that would re-open the false-positive
58+
* this detector is relied on to avoid for filenames.</p>
59+
*/
60+
public class ToleratedUtf8StructuralRegressionTest {
61+
62+
private static final String BENGALI_SENTENCE =
63+
"সেমিতে ক্রোয়েশিয়া টাইব্রেকারে রাশিয়াকে হারিয়ে ফাইনালে উঠেছে। ";
64+
65+
private static MojibusterEncodingDetector newDetector() {
66+
try {
67+
return new MojibusterEncodingDetector();
68+
} catch (Exception e) {
69+
throw new RuntimeException(e);
70+
}
71+
}
72+
73+
/**
74+
* Long document, abundant genuine multi-byte UTF-8 evidence, exactly one
75+
* tolerated error byte before it. Must still be recognized as UTF-8.
76+
*/
77+
@Test
78+
public void longDocumentWithOneStrayByteIsStillUtf8() throws IOException {
79+
byte[] probe = buildProbe(30);
80+
List<EncodingResult> results = newDetector().detect(probe);
81+
boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
82+
"UTF-8".equals(r.getCharset().name())
83+
&& r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
84+
assertTrue(hasStructuralUtf8,
85+
"A long, overwhelmingly UTF-8 document with a single tolerated "
86+
+ "error byte must still yield a STRUCTURAL UTF-8 candidate; "
87+
+ "results were: " + results);
88+
}
89+
90+
/**
91+
* Short probe (the zip-entry-name shape), exactly one error byte, only a
92+
* handful of genuine multi-byte sequences. Must NOT be promoted to
93+
* STRUCTURAL UTF-8 on the strength of tolerance alone — that's the
94+
* false-positive TIKA-4752-era filename detection depends on avoiding.
95+
*/
96+
@Test
97+
public void shortProbeWithOneStrayByteIsNotPromoted() throws IOException {
98+
// ~20 bytes: one legacy high byte + a couple of genuine multi-byte
99+
// UTF-8 chars — the shape of a real (short) zip entry name, not a
100+
// full document.
101+
ByteArrayOutputStream bo = new ByteArrayOutputStream();
102+
bo.write(0xA9); // stray legacy byte, invalid as a UTF-8 lead
103+
bo.writeBytes("café-Köln.txt".getBytes(StandardCharsets.UTF_8));
104+
byte[] probe = bo.toByteArray();
105+
106+
List<EncodingResult> results = newDetector().detect(probe);
107+
boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
108+
"UTF-8".equals(r.getCharset().name())
109+
&& r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
110+
assertFalse(hasStructuralUtf8,
111+
"A short probe shaped like a zip entry name must not be promoted "
112+
+ "to STRUCTURAL UTF-8 on a single tolerated error alone; "
113+
+ "results were: " + results);
114+
}
115+
116+
/**
117+
* Real embedded-file-name regression from {@code attachment_name_diffs.xlsx}
118+
* (commoncrawl3/5D/5DXWH7R4A5Q6VAWBAMBSUZM5PNEVAE63): a GBK zip entry name
119+
* ({@code 说明.txt}) must stay GB18030, not get pulled toward STRUCTURAL
120+
* UTF-8 by tolerance — the same false-positive risk as the Latin case,
121+
* CJK-flavored.
122+
*/
123+
@Test
124+
public void chineseGbkFilenameIsNotPromotedToUtf8() {
125+
byte[] probe = "说明.txt".getBytes(Charset.forName("GBK"));
126+
List<EncodingResult> results = newDetector().detect(probe);
127+
boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
128+
"UTF-8".equals(r.getCharset().name())
129+
&& r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
130+
assertFalse(hasStructuralUtf8,
131+
"A short GBK filename must not be promoted to STRUCTURAL UTF-8 "
132+
+ "on a single tolerated error alone; results were: " + results);
133+
assertTrue(results.stream().anyMatch(r -> r.getCharset().name().startsWith("GB")),
134+
"Expected a GB18030/GBK candidate; results were: " + results);
135+
}
136+
137+
/**
138+
* Real embedded-file-name regression from {@code attachment_name_diffs.xlsx}
139+
* (bug_trackers/MOZILLA/240463-316268/MOZILLA-296795-4.zip): a windows-1252
140+
* zip entry name ({@code Sauté.txt}) must stay legacy SBCS, not get promoted
141+
* to STRUCTURAL UTF-8 by tolerance.
142+
*/
143+
@Test
144+
public void sauteFilenameIsNotPromotedToUtf8() {
145+
byte[] probe = "Sauté.txt".getBytes(Charset.forName("windows-1252"));
146+
List<EncodingResult> results = newDetector().detect(probe);
147+
boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
148+
"UTF-8".equals(r.getCharset().name())
149+
&& r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
150+
assertFalse(hasStructuralUtf8,
151+
"A short windows-1252 filename must not be promoted to STRUCTURAL "
152+
+ "UTF-8 on a single tolerated error alone; results were: " + results);
153+
}
154+
155+
/** HTML wrapper + {@code repeatCount} copies of a real Bengali sentence,
156+
* with a single raw {@code 0xA9} (not a UTF-8 encoded {@code ©}) planted
157+
* in a meta tag before the real content — matches the real-world
158+
* failure exactly (declared windows-1252, genuinely UTF-8 body). */
159+
private static byte[] buildProbe(int repeatCount) throws IOException {
160+
StringBuilder body = new StringBuilder();
161+
for (int i = 0; i < repeatCount; i++) {
162+
body.append(BENGALI_SENTENCE);
163+
}
164+
ByteArrayOutputStream bo = new ByteArrayOutputStream();
165+
bo.writeBytes(("<html><head><meta http-equiv=\"Content-Type\" "
166+
+ "content=\"text/html; charset=windows-1252\">")
167+
.getBytes(StandardCharsets.US_ASCII));
168+
bo.writeBytes("<meta name=\"copyright\" content=\"".getBytes(StandardCharsets.US_ASCII));
169+
bo.write(0xA9); // stray legacy byte, invalid as a UTF-8 lead
170+
bo.writeBytes(" 2013\"></head><body><title>".getBytes(StandardCharsets.US_ASCII));
171+
bo.writeBytes(body.toString().getBytes(StandardCharsets.UTF_8));
172+
bo.writeBytes("</title></body></html>".getBytes(StandardCharsets.US_ASCII));
173+
return bo.toByteArray();
174+
}
175+
}

0 commit comments

Comments
 (0)