+// e.g. [ARABIC, ARMENIAN, BENGALI, CYRILLIC, DEVANAGARI, GEORGIAN,
+// GREEK, GUJARATI, GURMUKHI, HAN, HANGUL, HEBREW, HIRAGANA,
+// KANNADA, KHMER, LAO, LATIN, MALAYALAM, MYANMAR, ORIYA,
+// SINHALA, TAMIL, TELUGU, THAANA, THAI, TIBETAN, ...]
+----
+
+If the dominant script of an input is not in this set, `score()` returns a
+`TextQualityScore` where `isUnknown()` is true and no z-score is available.
+
+== Thresholds and operating points
+
+There is no universally correct threshold. The right cutoff depends on your
+content and tolerance for false positives (flagging good text as junk).
+
+**Starting points:**
+
+* **Trigger re-OCR**: z < −2.0 (catches ~95% of severe corruption while flagging
+ ~2–5% of legitimate text on average, more for short strings).
+* **Charset tiebreaking**: prefer the candidate with the higher z-score when
+ `delta() > 1.0`; abstain if `delta() < 0.5`.
+* **Training data filtering**: z < −1.5 to remove mojibake and bot-generated
+ noise from NLP corpora.
+
+For short text (under ~50 UTF-8 bytes), use `getCiLow()` rather than `getZScore()`
+for threshold decisions, since the confidence interval widens substantially.
+
+== Limitations
+
+* **Script coverage**: only scripts with a trained model can be scored. Unknown
+ scripts return `isUnknown() = true`.
+* **Short text**: scoring is unreliable below ~15 UTF-8 bytes. The model needs
+ at least a few bigrams to produce a stable estimate.
+* **Closely related charsets in the same script pool**: the LATIN model is trained
+ across hundreds of languages, which dilutes the signal for closely related
+ Western European and Baltic encodings (e.g., cp1252 vs. cp1257 on Lithuanian
+ text). The winner is usually correct, but delta may be small (< 0.5).
+* **Deliberately obfuscated text**: content designed to look like natural language
+ (e.g. by adversarial padding) is not detected.
+
+== Further reading
+
+For training methodology, model format, evaluation harness, and guidance on
+improving the model, see
+xref:advanced/junk-detection-build.adoc[Building the Junk Detector].
diff --git a/tika-core/src/main/java/org/apache/tika/quality/TextQualityComparison.java b/tika-core/src/main/java/org/apache/tika/quality/TextQualityComparison.java
new file mode 100644
index 00000000000..8c054b0ef75
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/quality/TextQualityComparison.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.quality;
+
+/**
+ * Result of comparing two candidate strings for text quality via
+ * {@link TextQualityDetector#compare}.
+ *
+ * A typical use is charset-decoding arbitration: given raw bytes decoded
+ * two different ways (e.g. cp1251 vs cp1252), pass each decoded string with a
+ * label and let the detector pick the cleaner one.
+ *
+ *
The {@code delta} field is the absolute difference between the two z-scores.
+ * A delta near zero means the model is uncertain; larger values indicate
+ * confident discrimination. As a rough guide: delta > 1.0 is useful signal,
+ * delta > 3.0 is confident.
+ */
+public final class TextQualityComparison {
+
+ private final String winner;
+ private final float delta;
+ private final TextQualityScore scoreA;
+ private final TextQualityScore scoreB;
+ private final String labelA;
+ private final String labelB;
+
+ public TextQualityComparison(String winner, float delta,
+ TextQualityScore scoreA, TextQualityScore scoreB,
+ String labelA, String labelB) {
+ this.winner = winner;
+ this.delta = delta;
+ this.scoreA = scoreA;
+ this.scoreB = scoreB;
+ this.labelA = labelA;
+ this.labelB = labelB;
+ }
+
+ /**
+ * Returns {@code "A"} if candidate A is cleaner, {@code "B"} otherwise.
+ * Check {@link #delta()} to gauge confidence.
+ */
+ public String winner() {
+ return winner;
+ }
+
+ /**
+ * Absolute difference in z-scores between the two candidates.
+ * Small delta = uncertain; large delta = confident.
+ */
+ public float delta() {
+ return delta;
+ }
+
+ /** Quality score for candidate A. */
+ public TextQualityScore scoreA() {
+ return scoreA;
+ }
+
+ /** Quality score for candidate B. */
+ public TextQualityScore scoreB() {
+ return scoreB;
+ }
+
+ /** Label supplied for candidate A (e.g. a charset name or encoding description). */
+ public String labelA() {
+ return labelA;
+ }
+
+ /** Label supplied for candidate B. */
+ public String labelB() {
+ return labelB;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(java.util.Locale.ROOT,
+ "TextQualityComparison[winner=%s(%s) delta=%.3f A=%s B=%s]",
+ winner, winner.equals("A") ? labelA : labelB,
+ delta, scoreA, scoreB);
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/quality/TextQualityDetector.java b/tika-core/src/main/java/org/apache/tika/quality/TextQualityDetector.java
new file mode 100644
index 00000000000..d832b5a169d
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/quality/TextQualityDetector.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.quality;
+
+/**
+ * Scores a string for text quality and arbitrates between two candidate strings.
+ *
+ *
Implementations are expected to be immutable and thread-safe after construction.
+ *
+ *
Implementations are registered via the standard Java {@link java.util.ServiceLoader}
+ * mechanism: place the fully-qualified class name in
+ * {@code META-INF/services/org.apache.tika.quality.TextQualityDetector}.
+ *
+ *
Typical usage:
+ *
{@code
+ * TextQualityDetector detector = ServiceLoader.load(TextQualityDetector.class)
+ * .findFirst().orElseThrow();
+ *
+ * // Score a string
+ * TextQualityScore score = detector.score(text);
+ * if (score.getZScore() < -2.0) { ... flag or re-process ... }
+ *
+ * // Arbitrate between two charset decodings
+ * TextQualityComparison cmp = detector.compare("cp1252", decodedAsCp1252,
+ * "cp1251", decodedAsCp1251);
+ * String winner = cmp.winner(); // "A" or "B"
+ * }
+ */
+public interface TextQualityDetector {
+
+ /**
+ * Scores the given string for text quality.
+ *
+ * @param text the string to score; must not be null
+ * @return a {@link TextQualityScore}; check {@link TextQualityScore#isUnknown()}
+ * if the input is empty or the script is not covered by the model
+ */
+ TextQualityScore score(String text);
+
+ /**
+ * Compares two candidate strings and returns which is higher-quality (cleaner text).
+ *
+ * A common use case is charset-decoding arbitration: given raw bytes decoded
+ * via two different charsets, pass each decoded string here with a human-readable
+ * label (e.g. the charset name) and the detector will pick the one that looks
+ * more like natural language.
+ *
+ * @param labelA human-readable label for candidate A (e.g. {@code "cp1252"})
+ * @param candidateA first candidate string
+ * @param labelB human-readable label for candidate B (e.g. {@code "cp1251"})
+ * @param candidateB second candidate string
+ * @return a {@link TextQualityComparison} with the winning label and confidence delta
+ */
+ TextQualityComparison compare(String labelA, String candidateA,
+ String labelB, String candidateB);
+}
diff --git a/tika-core/src/main/java/org/apache/tika/quality/TextQualityScore.java b/tika-core/src/main/java/org/apache/tika/quality/TextQualityScore.java
new file mode 100644
index 00000000000..8eea89ddec4
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/quality/TextQualityScore.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.quality;
+
+/**
+ * Result of scoring a string for text quality via a {@link TextQualityDetector}.
+ *
+ *
{@code zScore} is the primary output: how many standard deviations below
+ * typical clean text this string scores on its dominant script's model.
+ * Negative means worse than average clean text; more negative means worse.
+ *
+ *
{@code pClean} is a probability estimate in [0,1] that this is clean text.
+ *
+ *
{@code ciLow} / {@code ciHigh} are the 95% confidence interval bounds on
+ * {@code zScore}. For short strings these bounds are wide; for long strings
+ * they narrow. Prefer {@code ciLow < threshold} over {@code zScore < threshold}
+ * when triggering actions, to reduce false positives on short strings.
+ */
+public final class TextQualityScore {
+
+ /** Sentinel z-score returned when scoring could not be run (e.g. null or empty input). */
+ public static final float UNKNOWN = Float.NaN;
+
+ private final float zScore;
+ private final float pClean;
+ private final float ciLow;
+ private final float ciHigh;
+ private final String dominantScript;
+
+ public TextQualityScore(float zScore, float pClean,
+ float ciLow, float ciHigh,
+ String dominantScript) {
+ this.zScore = zScore;
+ this.pClean = pClean;
+ this.ciLow = ciLow;
+ this.ciHigh = ciHigh;
+ this.dominantScript = dominantScript;
+ }
+
+ /** Z-score relative to clean text for the detected script. 0 = average clean; negative = worse. */
+ public float getZScore() {
+ return zScore;
+ }
+
+ /** Probability in [0,1] that this string is clean text. */
+ public float getPClean() {
+ return pClean;
+ }
+
+ /** Lower bound of the 95% confidence interval on zScore. */
+ public float getCiLow() {
+ return ciLow;
+ }
+
+ /** Upper bound of the 95% confidence interval on zScore. */
+ public float getCiHigh() {
+ return ciHigh;
+ }
+
+ /** Name of the dominant Unicode script detected, e.g. "LATIN", "CYRILLIC", "ARABIC". */
+ public String getDominantScript() {
+ return dominantScript;
+ }
+
+ /** True if scoring could not be performed (e.g. empty or unsupported-script input). */
+ public boolean isUnknown() {
+ return Float.isNaN(zScore);
+ }
+
+ @Override
+ public String toString() {
+ if (isUnknown()) {
+ return "TextQualityScore[UNKNOWN script=" + dominantScript + "]";
+ }
+ return String.format(java.util.Locale.ROOT,
+ "TextQualityScore[z=%.3f p=%.3f ci=(%.3f,%.3f) script=%s]",
+ zScore, pClean, ciLow, ciHigh, dominantScript);
+ }
+}
diff --git a/tika-ml/pom.xml b/tika-ml/pom.xml
index 3af2057aa81..5c9cf03af1b 100644
--- a/tika-ml/pom.xml
+++ b/tika-ml/pom.xml
@@ -34,6 +34,7 @@
tika-ml-core
tika-ml-chardetect
+ tika-ml-junkdetect
diff --git a/tika-ml/tika-ml-junkdetect/pom.xml b/tika-ml/tika-ml-junkdetect/pom.xml
new file mode 100644
index 00000000000..672e49195a0
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/pom.xml
@@ -0,0 +1,153 @@
+
+
+
+
+ tika-ml
+ org.apache.tika
+ ${revision}
+
+ 4.0.0
+
+ tika-ml-junkdetect
+ Apache Tika ML junk detector — runtime and training tools
+
+ Language-agnostic text quality scorer that discriminates between clean UTF-8 text and
+ mojibake, reversed text, wrong-codec decodings, and other corruption forms.
+ Provides a standalone "languageyness" score suitable for re-OCR triggering and
+ charset-decoding arbitration.
+
+ Runtime classes (JunkDetector, ScriptDetector, feature extractors) and bundled model
+ resources live here. Training and evaluation CLI tools live in the tools subpackage.
+
+
+
+
+ org.apache.tika
+ tika-core
+ ${revision}
+
+
+ org.apache.tika
+ tika-ml-core
+ ${revision}
+
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ test
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ org.apache.tika.ml.junkdetect
+
+
+
+
+
+ org.apache.rat
+ apache-rat-plugin
+
+
+ **/*.bin
+ **/*.txt
+
+
+
+
+
+ de.thetaphi
+ forbiddenapis
+
+ true
+
+
+
+
+
+
+
+
+ train
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+ package
+ shade
+
+ true
+ tools
+
+
+ org.apache.tika.ml.junkdetect.tools.TrainJunkModel
+
+
+
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 3.0.0-rc1
+
+
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
new file mode 100644
index 00000000000..07aeb641648
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
@@ -0,0 +1,660 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.tika.quality.TextQualityComparison;
+import org.apache.tika.quality.TextQualityDetector;
+import org.apache.tika.quality.TextQualityScore;
+
+/**
+ * Language-agnostic text quality scorer. Discriminates clean UTF-8 text from
+ * mojibake, reversed text, wrong-codec decodings, and other corruption forms.
+ *
+ * Scoring combines up to three features, depending on the model version:
+ *
+ * - Byte-bigram log-probability — 256×256 table of log P(b|a) over
+ * consecutive byte pairs in the UTF-8 encoding.
+ * - Unicode named-block transition log-probability (version 2+) —
+ * N×N table of log P(block_b | block_a) where block IDs are the named
+ * {@link Character.UnicodeBlock} values (BASIC_LATIN, ARABIC,
+ * CJK_UNIFIED_IDEOGRAPHS, etc.).
+ * - Control-byte fraction (version 2+) — fraction of bytes in control
+ * ranges [0x01–0x08, 0x0B, 0x0C, 0x0E–0x1F, 0x7F].
+ *
+ *
+ * All features are calibrated (mu/sigma) on held-out dev text so their z-scores
+ * are on a common scale.
+ *
+ *
Features are combined by a per-script logistic regression classifier:
+ * {@code w1*z1 + w2*z2 + w3*z3 + w4*z4 + bias}, where weights are fit on
+ * clean vs. corrupted dev windows. The natural junk threshold is 0 (positive
+ * logit = clean); use a negative threshold for conservative detection
+ * (e.g., {@code score < -1}).
+ *
+ * Instances are immutable and thread-safe after construction.
+ *
+ *
Typical usage:
+ *
{@code
+ * JunkDetector detector = JunkDetector.loadFromClasspath();
+ * TextQualityScore score = detector.score("some text");
+ * if (score.getZScore() < 0) { ... flag as junk ... }
+ *
+ * // Arbitrate between two charset decodings
+ * TextQualityComparison result = detector.compare("cp1252", ascp1252, "cp1251", ascp1251);
+ * String winner = result.winner(); // "A" or "B"
+ * }
+ */
+public final class JunkDetector implements TextQualityDetector {
+
+ /** Classpath resource path for the bundled production model. */
+ public static final String DEFAULT_MODEL_RESOURCE =
+ "org/apache/tika/ml/junkdetect/junkdetect.bin";
+
+ static final String MAGIC = "JUNKDET1";
+
+ private final int modelVersion;
+
+ // Feature 1: byte bigrams (all versions)
+ private final Map tables; // script → float[65536] log-prob
+ private final Map calibrations; // script → float[2] {mu, sigma}
+
+ // Feature 2: named-block transitions (version 2+); null for v1 models
+ private final Map blockTables; // script → float[blockN*blockN]
+ private final Map blockCalibrations; // script → float[2] {mu, sigma}
+ private final int blockN; // block table dimension (0 for v1)
+
+ // Feature 3: control-byte fraction (version 2+); null for v1 models
+ private final Map controlCalibrations; // script → float[2] {mu, sigma}
+
+ // Feature combination: per-script linear classifier (version 3+); null for v1/v2 models
+ // float[numFeatures+1] = {w1, ..., wN, bias}; positive logit = clean
+ private final Map classifierWeights;
+
+ // Feature 4: global script-transition (version 4+); null for v1/v2/v3 models
+ // One global table: float[numScriptBuckets * numScriptBuckets] log P(script_b | script_a)
+ // Uses raw UnicodeScript names (not SCRIPT_MODEL_FALLBACK) to distinguish HIRAGANA/KATAKANA/HAN.
+ private final float[] scriptTransitionTable;
+ private final float[] scriptTransitionCalibration; // float[2] = {mu, sigma}
+ private final Map scriptBucketIndex; // raw UnicodeScript name → bucket ID
+ private final int numScriptBuckets; // 0 for v1/v2/v3
+
+ // Shared block index for v2+ models: UnicodeBlock → index [0, blockN-1)
+ // Index blockN-1 is the "unassigned" bucket (null UnicodeBlock).
+ private final Map blockIndex;
+
+ private JunkDetector(int modelVersion,
+ Map tables,
+ Map calibrations,
+ Map blockTables,
+ Map blockCalibrations,
+ int blockN,
+ Map controlCalibrations,
+ Map classifierWeights,
+ Map blockIndex,
+ float[] scriptTransitionTable,
+ float[] scriptTransitionCalibration,
+ Map scriptBucketIndex,
+ int numScriptBuckets) {
+ this.modelVersion = modelVersion;
+ this.tables = Collections.unmodifiableMap(tables);
+ this.calibrations = Collections.unmodifiableMap(calibrations);
+ this.blockTables = blockTables != null
+ ? Collections.unmodifiableMap(blockTables) : null;
+ this.blockCalibrations = blockCalibrations != null
+ ? Collections.unmodifiableMap(blockCalibrations) : null;
+ this.blockN = blockN;
+ this.controlCalibrations = controlCalibrations != null
+ ? Collections.unmodifiableMap(controlCalibrations) : null;
+ this.classifierWeights = classifierWeights != null
+ ? Collections.unmodifiableMap(classifierWeights) : null;
+ this.blockIndex = blockIndex;
+ this.scriptTransitionTable = scriptTransitionTable;
+ this.scriptTransitionCalibration = scriptTransitionCalibration;
+ this.scriptBucketIndex = scriptBucketIndex != null
+ ? Collections.unmodifiableMap(scriptBucketIndex) : null;
+ this.numScriptBuckets = numScriptBuckets;
+ }
+
+ // -----------------------------------------------------------------------
+ // Factory methods
+ // -----------------------------------------------------------------------
+
+ /**
+ * Loads the bundled model from the classpath.
+ *
+ * @throws IOException if the model resource is missing or malformed
+ */
+ public static JunkDetector loadFromClasspath() throws IOException {
+ InputStream is = JunkDetector.class.getClassLoader()
+ .getResourceAsStream(DEFAULT_MODEL_RESOURCE);
+ if (is == null) {
+ throw new IOException("Model resource not found on classpath: "
+ + DEFAULT_MODEL_RESOURCE);
+ }
+ try (InputStream wrapped = is) {
+ return load(wrapped);
+ }
+ }
+
+ /**
+ * Loads a model from the given file path. The file may be gzipped or raw.
+ */
+ public static JunkDetector loadFromPath(Path path) throws IOException {
+ try (InputStream is = Files.newInputStream(path)) {
+ return load(is);
+ }
+ }
+
+ /**
+ * Loads a model from an {@link InputStream}. Gzip-detection is automatic.
+ * Supports model versions 1 through 5.
+ */
+ public static JunkDetector load(InputStream rawIs) throws IOException {
+ byte[] peek = rawIs.readNBytes(2);
+ InputStream rest = new java.io.SequenceInputStream(
+ new java.io.ByteArrayInputStream(peek), rawIs);
+ InputStream in;
+ if (peek.length >= 2 && (peek[0] & 0xFF) == 0x1f && (peek[1] & 0xFF) == 0x8b) {
+ in = new GZIPInputStream(rest);
+ } else {
+ in = rest;
+ }
+
+ try (DataInputStream dis = new DataInputStream(in)) {
+ byte[] magic = dis.readNBytes(8);
+ if (!new String(magic, StandardCharsets.UTF_8).equals(MAGIC)) {
+ throw new IOException("Not a JunkDetector model file (bad magic)");
+ }
+ int version = dis.readUnsignedByte();
+ if (version != 5) {
+ throw new IOException("Unsupported model version: " + version
+ + ". Only version 5 is supported. Retrain the model with TrainJunkModel.");
+ }
+
+ int numScripts = dis.readInt();
+
+ // Block names (v5): stored in model for JVM-independence
+ int blockN = dis.readUnsignedShort();
+ String[] blockNames = new String[blockN - 1];
+ for (int i = 0; i < blockN - 1; i++) {
+ int nameLen = dis.readUnsignedShort();
+ blockNames[i] = new String(dis.readNBytes(nameLen), StandardCharsets.UTF_8);
+ }
+ Map blockIndex = buildBlockIndexFromNames(blockNames);
+
+ // Global script-transition section
+ int numScriptBuckets = dis.readUnsignedByte();
+ Map scriptBucketIndex = new LinkedHashMap<>(numScriptBuckets * 2);
+ for (int i = 0; i < numScriptBuckets; i++) {
+ int nameLen = dis.readUnsignedShort();
+ String bucketName = new String(dis.readNBytes(nameLen), StandardCharsets.UTF_8);
+ scriptBucketIndex.put(bucketName, i);
+ }
+ float[] scriptTransitionTable = readFloatTable(dis, numScriptBuckets * numScriptBuckets);
+ float[] scriptTransitionCalibration = new float[]{dis.readFloat(), dis.readFloat()};
+
+ Map tables = new HashMap<>(numScripts * 2);
+ Map calibrations = new HashMap<>(numScripts * 2);
+ Map blockTables = new HashMap<>(numScripts * 2);
+ Map blockCalibrations = new HashMap<>(numScripts * 2);
+ Map controlCalibrations = new HashMap<>(numScripts * 2);
+ Map classifierWeights = new HashMap<>(numScripts * 2);
+
+ for (int s = 0; s < numScripts; s++) {
+ int nameLen = dis.readUnsignedShort();
+ String script = new String(dis.readNBytes(nameLen), StandardCharsets.UTF_8);
+
+ // Feature 1: byte bigrams
+ calibrations.put(script, new float[]{dis.readFloat(), dis.readFloat()});
+ tables.put(script, readFloatTable(dis, 65536));
+
+ // Feature 2: named-block transitions
+ blockCalibrations.put(script, new float[]{dis.readFloat(), dis.readFloat()});
+ blockTables.put(script, readFloatTable(dis, blockN * blockN));
+
+ // Feature 3: control-byte fraction
+ controlCalibrations.put(script, new float[]{dis.readFloat(), dis.readFloat()});
+
+ // Classifier weights: num_features (1 byte) + num_features floats + 1 bias
+ int numFeatures = dis.readUnsignedByte();
+ float[] weights = new float[numFeatures + 1]; // last = bias
+ for (int j = 0; j <= numFeatures; j++) {
+ weights[j] = dis.readFloat();
+ }
+ classifierWeights.put(script, weights);
+ }
+
+ return new JunkDetector(version, tables, calibrations,
+ blockTables, blockCalibrations, blockN,
+ controlCalibrations, classifierWeights, blockIndex,
+ scriptTransitionTable, scriptTransitionCalibration,
+ scriptBucketIndex, numScriptBuckets);
+ }
+ }
+
+ private static float[] readFloatTable(DataInputStream dis, int size) throws IOException {
+ byte[] tableBytes = dis.readNBytes(size * 4);
+ float[] table = new float[size];
+ ByteBuffer buf = ByteBuffer.wrap(tableBytes).order(ByteOrder.BIG_ENDIAN);
+ buf.asFloatBuffer().get(table);
+ return table;
+ }
+
+ /**
+ * Builds the stable ordered mapping from {@link Character.UnicodeBlock} to index.
+ * This must produce the same ordering as {@link TrainJunkModel#buildBlockIndex()}.
+ * Used for v2/v3/v4 models only; v5+ models store block names in the file.
+ */
+ static Map buildBlockIndex() {
+ LinkedHashMap index = new LinkedHashMap<>();
+ for (int cp = 0; cp <= 0x10FFFF; cp++) {
+ Character.UnicodeBlock b = Character.UnicodeBlock.of(cp);
+ if (b != null) index.putIfAbsent(b, index.size());
+ }
+ return Collections.unmodifiableMap(index);
+ }
+
+ /**
+ * Builds a block index from an ordered array of block names stored in a v5+ model.
+ * Resolves each name via {@link Character.UnicodeBlock#forName(String)}.
+ * Throws {@link IOException} if any name is not recognised by the current JVM —
+ * this means the model was trained on a newer JVM; retrain on the minimum
+ * supported JVM (Java 17) to produce a compatible model.
+ *
+ * @param blockNames ordered array of block names (index = position in block table)
+ * @return unmodifiable map from UnicodeBlock to table index
+ */
+ static Map buildBlockIndexFromNames(String[] blockNames)
+ throws IOException {
+ Map index = new HashMap<>(blockNames.length * 2);
+ for (int i = 0; i < blockNames.length; i++) {
+ try {
+ Character.UnicodeBlock b = Character.UnicodeBlock.forName(blockNames[i]);
+ index.put(b, i);
+ } catch (IllegalArgumentException e) {
+ throw new IOException("Unicode block not known to this JVM: " + blockNames[i]
+ + ". Model was trained on a newer JVM; retrain on Java 17.", e);
+ }
+ }
+ return Collections.unmodifiableMap(index);
+ }
+
+ // -----------------------------------------------------------------------
+ // TextQualityDetector implementation
+ // -----------------------------------------------------------------------
+
+ /**
+ * {@inheritDoc}
+ *
+ * The text is split into contiguous runs of the same Unicode script.
+ * Each run is scored against its own script model. Logits are combined
+ * as a byte-count-weighted average, so mixed-script text (e.g. half
+ * LATIN, half HAN) is scored fairly without arbitrarily picking one script.
+ * COMMON, INHERITED, and UNKNOWN codepoints (spaces, punctuation, digits)
+ * are attached to the preceding script run.
+ */
+ @Override
+ public TextQualityScore score(String text) {
+ if (text == null || text.isEmpty()) {
+ return unknownScore("UNKNOWN");
+ }
+ return scoreText(text);
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ *
Each candidate is scored independently via {@link #score(String)}.
+ * The candidate with the higher score wins.
+ *
+ *
An UNKNOWN score (script not in model) is treated as neutral (0) rather
+ * than {@code -∞}. This prevents a garbled-but-recognisable decoding from
+ * beating a correct decoding whose script happens to be unknown to the model —
+ * for example, a pure-katakana zip entry name decoded as Shift-JIS (UNKNOWN)
+ * vs. the same bytes decoded as UTF-8 (garbled LATIN, negative z-score).
+ */
+ @Override
+ public TextQualityComparison compare(String labelA, String candidateA,
+ String labelB, String candidateB) {
+ TextQualityScore scoreA = score(candidateA);
+ TextQualityScore scoreB = score(candidateB);
+
+ // UNKNOWN = "no evidence" = 0, not -∞. A text whose script is not in the
+ // model is assumed to be neutral, not junk.
+ float zA = scoreA.isUnknown() ? 0f : scoreA.getZScore();
+ float zB = scoreB.isUnknown() ? 0f : scoreB.getZScore();
+
+ String winner = zA >= zB ? "A" : "B";
+ float delta = Math.abs(zA - zB);
+
+ return new TextQualityComparison(winner, delta, scoreA, scoreB, labelA, labelB);
+ }
+
+ /** Returns the set of script names this model knows about. */
+ public Set knownScripts() {
+ return tables.keySet();
+ }
+
+ /** Returns the version of the loaded model (1, 2, or 3). */
+ public int getModelVersion() {
+ return modelVersion;
+ }
+
+ // -----------------------------------------------------------------------
+ // Internal scoring
+ // -----------------------------------------------------------------------
+
+ private TextQualityScore scoreText(String text) {
+ List runs = buildScriptRuns(text);
+
+ // Global z4: script-transition feature over the whole input string.
+ // Computed before chunking because it captures document-level script mixing.
+ float z4 = computeScriptTransitionZ(text);
+
+ // Score each run against its own model; aggregate weighted by byte count.
+ float totalBytes = 0;
+ float weightedLogit = 0;
+ String dominantScript = null;
+ int maxBytes = 0;
+ int totalBigramCount = 0;
+ float[] dominantCal1 = null;
+
+ for (ScriptRun run : runs) {
+ if (!tables.containsKey(run.script)) {
+ continue; // skip scripts not in model; treat as neutral, not junk
+ }
+ byte[] runUtf8 = run.text.getBytes(StandardCharsets.UTF_8);
+ if (runUtf8.length < 2) {
+ continue; // too short to score
+ }
+ float logit = scoreChunk(runUtf8, run.text, run.script, z4);
+ int n = runUtf8.length;
+ weightedLogit += logit * n;
+ totalBytes += n;
+ totalBigramCount += n - 1;
+ if (n > maxBytes) {
+ maxBytes = n;
+ dominantScript = run.script;
+ dominantCal1 = calibrations.get(run.script);
+ }
+ }
+
+ if (totalBytes == 0 || dominantScript == null) {
+ String label = runs.isEmpty() ? "LATIN" : runs.get(0).script;
+ return unknownScore(label);
+ }
+
+ float zScore = weightedLogit / totalBytes;
+
+ float uncertainty = (dominantCal1 != null && totalBigramCount > 0)
+ ? (float) (1.96 * dominantCal1[1] / Math.sqrt(totalBigramCount)) : 0f;
+ float ciLow = zScore - uncertainty;
+ float ciHigh = zScore + uncertainty;
+ float pClean = (float) (1.0 / (1.0 + Math.exp(-zScore)));
+
+ return new TextQualityScore(zScore, pClean, ciLow, ciHigh, dominantScript);
+ }
+
+ /**
+ * Scores a single script-homogeneous chunk and returns its logit.
+ * Positive = clean, negative = junk. Returns 0 (neutral) if the chunk
+ * has no model or is too short.
+ */
+ private float scoreChunk(byte[] utf8, String text, String script, float z4) {
+ float[] bigramTable = tables.get(script);
+ if (bigramTable == null || utf8.length < 2) {
+ return 0f;
+ }
+
+ // Feature 1: byte-bigram mean log-prob
+ double bigramSum = 0;
+ int bigramCount = 0;
+ for (int i = 0; i + 1 < utf8.length; i++) {
+ bigramSum += bigramTable[((utf8[i] & 0xFF) << 8) | (utf8[i + 1] & 0xFF)];
+ bigramCount++;
+ }
+ float meanBigramLogProb = (float) (bigramSum / bigramCount);
+ float[] cal1 = calibrations.get(script);
+ float z1 = (meanBigramLogProb - cal1[0]) / cal1[1];
+
+ // Feature 2: named-block transition mean log-prob
+ float z2 = 0f;
+ float[] blockTable = blockTables.get(script);
+ if (blockTable != null) {
+ int nullId = blockN - 1;
+ int prev = -1;
+ double blockSum = 0;
+ int blockCount = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ Character.UnicodeBlock b = Character.UnicodeBlock.of(cp);
+ int blockId = b != null ? blockIndex.getOrDefault(b, nullId) : nullId;
+ if (prev >= 0) {
+ blockSum += blockTable[prev * blockN + blockId];
+ blockCount++;
+ }
+ prev = blockId;
+ i += Character.charCount(cp);
+ }
+ if (blockCount > 0) {
+ float meanBlockLogProb = (float) (blockSum / blockCount);
+ float[] cal2 = blockCalibrations.get(script);
+ z2 = cal2 != null ? (meanBlockLogProb - cal2[0]) / cal2[1] : 0f;
+ }
+ }
+
+ // Feature 3: control-byte fraction (stored as −fraction, so higher = cleaner)
+ long controlCount = 0;
+ for (byte b : utf8) {
+ if (isControlByte(b & 0xFF)) controlCount++;
+ }
+ float controlScore = -(float) controlCount / utf8.length;
+ float[] cal3 = controlCalibrations.get(script);
+ float z3 = cal3 != null ? (controlScore - cal3[0]) / cal3[1] : 0f;
+
+ // Per-script linear classifier: w1*z1 + w2*z2 + w3*z3 + w4*z4 + bias
+ float[] cw = classifierWeights.get(script);
+ if (cw != null) {
+ int nFeat = cw.length - 1; // bias is last
+ float logit = cw[nFeat]; // bias
+ if (nFeat >= 1) logit += cw[0] * z1;
+ if (nFeat >= 2) logit += cw[1] * z2;
+ if (nFeat >= 3) logit += cw[2] * z3;
+ if (nFeat >= 4) logit += cw[3] * z4;
+ return logit;
+ }
+ return (z1 + z2 + z3 + z4) / 4.0f; // fallback: equal weight
+ }
+
+ /**
+ * Computes the global script-transition z-score for the whole input string.
+ * Uses raw {@link Character.UnicodeScript} values — NOT {@link #SCRIPT_MODEL_FALLBACK} —
+ * so that HIRAGANA, KATAKANA, and HAN remain distinct, preserving the
+ * characteristic script-mixing pattern of Japanese text.
+ *
+ * Returns 0 if the string has fewer than two non-neutral codepoints.
+ */
+ private float computeScriptTransitionZ(String text) {
+ if (scriptTransitionTable == null || scriptBucketIndex == null
+ || scriptTransitionCalibration == null || numScriptBuckets == 0) {
+ return 0f;
+ }
+ int otherBucket = numScriptBuckets - 1;
+ int prev = -1;
+ double sum = 0;
+ int count = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ continue;
+ }
+ int bucket = scriptBucketIndex.getOrDefault(s.name(), otherBucket);
+ if (prev >= 0) {
+ sum += scriptTransitionTable[prev * numScriptBuckets + bucket];
+ count++;
+ }
+ prev = bucket;
+ }
+ if (count == 0) {
+ return 0f;
+ }
+ float mean = (float) (sum / count);
+ return (mean - scriptTransitionCalibration[0]) / scriptTransitionCalibration[1];
+ }
+
+ /**
+ * Splits text into maximal runs of the same Unicode script.
+ * COMMON, INHERITED, and UNKNOWN codepoints (spaces, punctuation, digits)
+ * are attached to the preceding script run so that inter-word bigrams are
+ * preserved within each run. Any leading COMMON characters are prepended
+ * to the first non-COMMON run.
+ */
+ private List buildScriptRuns(String text) {
+ List runs = new ArrayList<>();
+ String currentScript = null;
+ StringBuilder currentText = new StringBuilder();
+ StringBuilder leadingCommon = new StringBuilder();
+
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ if (currentScript != null) {
+ currentText.appendCodePoint(cp);
+ } else {
+ leadingCommon.appendCodePoint(cp);
+ }
+ continue;
+ }
+
+ String scriptName = SCRIPT_MODEL_FALLBACK.getOrDefault(s.name(), s.name());
+
+ if (!scriptName.equals(currentScript)) {
+ if (currentScript != null && currentText.length() > 0) {
+ runs.add(new ScriptRun(currentScript, currentText.toString()));
+ }
+ currentScript = scriptName;
+ currentText = new StringBuilder();
+ if (leadingCommon.length() > 0) {
+ currentText.append(leadingCommon);
+ leadingCommon.setLength(0);
+ }
+ }
+ currentText.appendCodePoint(cp);
+ }
+
+ if (currentScript != null && currentText.length() > 0) {
+ runs.add(new ScriptRun(currentScript, currentText.toString()));
+ }
+ return runs;
+ }
+
+ private static final class ScriptRun {
+ final String script;
+ final String text;
+ ScriptRun(String script, String text) {
+ this.script = script;
+ this.text = text;
+ }
+ }
+
+ /**
+ * Returns true if the byte value is a control character that should not appear
+ * in natural-language UTF-8 text: {@code [0x01–0x08, 0x0B, 0x0C, 0x0E–0x1F, 0x7F]}.
+ *
+ * Excluded: 0x00 (null), 0x09 (tab), 0x0A (newline), 0x0D (carriage return)
+ * — all appear legitimately in text.
+ */
+ static boolean isControlByte(int b) {
+ return (b >= 0x01 && b <= 0x08)
+ || b == 0x0B || b == 0x0C
+ || (b >= 0x0E && b <= 0x1F)
+ || b == 0x7F;
+ }
+
+ private static TextQualityScore unknownScore(String script) {
+ return new TextQualityScore(TextQualityScore.UNKNOWN, Float.NaN,
+ Float.NaN, Float.NaN, script);
+ }
+
+ /**
+ * Maps Unicode scripts that share a trained model with a related script.
+ * Japanese kana (HIRAGANA, KATAKANA) map to HAN because the HAN model is
+ * trained on mixed Japanese text containing all three writing systems, so
+ * its byte-bigram and block-transition tables cover kana sequences.
+ */
+ private static final Map SCRIPT_MODEL_FALLBACK = Map.of(
+ "HIRAGANA", "HAN",
+ "KATAKANA", "HAN"
+ );
+
+ /**
+ * Detects the dominant Unicode script of the given text by histogramming
+ * {@link Character.UnicodeScript} over all codepoints, excluding COMMON,
+ * INHERITED, and UNKNOWN pseudo-scripts. Returns "LATIN" for ASCII-only text.
+ *
+ * Script names are mapped through {@link #SCRIPT_MODEL_FALLBACK} so that
+ * scripts without dedicated models fall back to a related trained model
+ * (e.g. KATAKANA and HIRAGANA both use the HAN model).
+ */
+ static String detectDominantScript(String text) {
+ Map counts = new HashMap<>();
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s != Character.UnicodeScript.COMMON
+ && s != Character.UnicodeScript.INHERITED
+ && s != Character.UnicodeScript.UNKNOWN) {
+ counts.merge(s, 1, Integer::sum);
+ }
+ i += Character.charCount(cp);
+ }
+ if (counts.isEmpty()) {
+ return "LATIN";
+ }
+ String name = counts.entrySet().stream()
+ .max(Map.Entry.comparingByValue())
+ .map(e -> e.getKey().name())
+ .orElse("LATIN");
+ return SCRIPT_MODEL_FALLBACK.getOrDefault(name, name);
+ }
+}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java
new file mode 100644
index 00000000000..27a5436d5e4
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java
@@ -0,0 +1,646 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect.tools;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.Normalizer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * Builds per-script positive training data for the junk detector from MADLAD-400
+ * and Wikipedia sentence files.
+ *
+ * Script groups are derived entirely from the data: for each language directory
+ * the dominant Unicode script is detected by histogramming {@link Character.UnicodeScript}
+ * over a sample of sentences (COMMON, INHERITED, and UNKNOWN pseudo-scripts excluded).
+ * Languages that share the same dominant script are pooled. No script groups are
+ * hardcoded.
+ *
+ *
The total byte budget is distributed across script groups proportionally to
+ * each group's empirical byte-bigram entropy, measured from a small sample.
+ * Scripts with high entropy (e.g. CJK, which has thousands of distinct 3-byte
+ * codepoints) receive a proportionally larger allocation than low-entropy scripts
+ * (e.g. Arabic, whose UTF-8 high bytes cluster in a narrow 0xD8-0xDB range).
+ * This ensures every script's bigram table is estimated with comparable statistical
+ * quality regardless of character-set size.
+ *
+ *
Within each script group the byte budget is distributed evenly across its
+ * member languages, ensuring diversity (no single language dominates).
+ *
+ *
Input format ({@code sentences_madlad.txt} and {@code sentences_wikipedia.txt}):
+ * {@code lineNum TAB text}, UTF-8. MADLAD records contain literal {@code \n} escape
+ * sequences as sub-sentence separators (full scraped documents); Wikipedia records
+ * are individual sentences. Both are split/cleaned to sentence-level strings.
+ *
+ *
Output:
+ *
+ * output-dir/
+ * {script}.train.gz — 80% split, one NFC-normalised sentence per line
+ * {script}.dev.gz — 10% split, used for calibration (mu/sigma)
+ * {script}.test.gz — 10% split, held out for final evaluation only
+ * manifest.tsv — per-script stats: entropy, budget, bytes written, languages
+ *
+ *
+ * Usage:
+ *
+ * java BuildJunkTrainingData \
+ * --data-dir ~/datasets/madlad/data \
+ * --output-dir ~/datasets/madlad/junkdetect \
+ * [--total-budget-bytes 50000000]
+ *
+ */
+public class BuildJunkTrainingData {
+
+ // -----------------------------------------------------------------------
+ // Defaults
+ // -----------------------------------------------------------------------
+
+ /** Lines read per language to determine dominant script. */
+ private static final int DEFAULT_SCRIPT_SAMPLE_LINES = 2_000;
+
+ /**
+ * UTF-8 bytes loaded per script group for entropy estimation.
+ * Budget is spread evenly across languages in the group.
+ * 200KB is enough to observe the bigram distribution reliably.
+ */
+ private static final long ENTROPY_SAMPLE_BYTES = 200_000L;
+
+ /**
+ * Total UTF-8 byte budget across all script groups. Divided proportionally
+ * by bigram entropy after the sampling phase. 50MB gives ~1–3MB per script
+ * on average across 34 groups; scale up for production runs.
+ */
+ private static final long DEFAULT_TOTAL_BUDGET_BYTES = 50_000_000L;
+
+ /** Minimum UTF-8 byte length for a sentence to pass the quality filter. */
+ private static final int DEFAULT_MIN_BYTES = 50;
+
+ /** Maximum fraction of codepoints that may be ASCII punctuation/digits. */
+ private static final double DEFAULT_MAX_PUNC_FRAC = 0.30;
+
+ /** Fraction of sentences written to each split (train / dev / test = 80/10/10). */
+ private static final double TRAIN_FRAC = 0.80;
+ private static final double DEV_FRAC = 0.10;
+ // remaining (1 - TRAIN_FRAC - DEV_FRAC) goes to the test split
+
+ /**
+ * Minimum number of sentences that must land in the dev split for a script to be
+ * included in the model. Scripts below this floor have too few samples to reliably
+ * estimate calibration statistics (mu/sigma), which produces noisy z-scores and
+ * inflated false positive rates. With DEV_FRAC=0.10 the effective minimum total
+ * sentence count is minDevSentences / DEV_FRAC (default: 5,000 total sentences).
+ */
+ private static final int DEFAULT_MIN_DEV_SENTENCES = 500;
+
+ // -----------------------------------------------------------------------
+ // Entry point
+ // -----------------------------------------------------------------------
+
+ public static void main(String[] args) throws IOException {
+ Path dataDir = Paths.get(System.getProperty("user.home"), "datasets", "madlad", "data");
+ Path outputDir = Paths.get(System.getProperty("user.home"), "datasets", "madlad", "junkdetect");
+ int scriptSampleLines = DEFAULT_SCRIPT_SAMPLE_LINES;
+ long totalBudgetBytes = DEFAULT_TOTAL_BUDGET_BYTES;
+ int minBytes = DEFAULT_MIN_BYTES;
+ double maxPuncFrac = DEFAULT_MAX_PUNC_FRAC;
+ int seed = 42;
+ boolean dryRun = false;
+ int minDevSentences = DEFAULT_MIN_DEV_SENTENCES;
+
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--data-dir":
+ dataDir = Paths.get(args[++i]);
+ break;
+ case "--output-dir":
+ outputDir = Paths.get(args[++i]);
+ break;
+ case "--script-sample-lines":
+ scriptSampleLines = Integer.parseInt(args[++i]);
+ break;
+ case "--total-budget-bytes":
+ totalBudgetBytes = Long.parseLong(args[++i]);
+ break;
+ case "--min-bytes":
+ minBytes = Integer.parseInt(args[++i]);
+ break;
+ case "--max-punc-frac":
+ maxPuncFrac = Double.parseDouble(args[++i]);
+ break;
+ case "--seed":
+ seed = Integer.parseInt(args[++i]);
+ break;
+ case "--min-dev-sentences":
+ minDevSentences = Integer.parseInt(args[++i]);
+ break;
+ case "--dry-run":
+ dryRun = true;
+ break;
+ default:
+ System.err.println("Unknown argument: " + args[i]);
+ printUsage();
+ System.exit(1);
+ }
+ }
+
+ System.out.println("=== BuildJunkTrainingData ===");
+ System.out.println(" data-dir: " + dataDir);
+ System.out.println(" output-dir: " + outputDir);
+ System.out.printf( " total-budget-bytes: %,d (%.1f MB)%n",
+ totalBudgetBytes, totalBudgetBytes / 1_000_000.0);
+ System.out.printf( " min-bytes: %d%n", minBytes);
+ System.out.printf( " max-punc-frac: %.2f%n", maxPuncFrac);
+ System.out.printf( " min-dev-sentences: %d (min total ≈ %d)%n",
+ minDevSentences, (int)(minDevSentences / DEV_FRAC));
+ System.out.println(" dry-run: " + dryRun);
+
+ if (!Files.isDirectory(dataDir)) {
+ System.err.println("ERROR: data-dir not found: " + dataDir);
+ System.exit(1);
+ }
+
+ // -----------------------------------------------------------------------
+ // Phase 1: Detect dominant script per language, group languages
+ // -----------------------------------------------------------------------
+
+ System.out.println("\n--- Phase 1: Detecting dominant script per language ---");
+
+ Map> scriptGroups = new TreeMap<>();
+ Map langToScript = new LinkedHashMap<>();
+
+ try (var dirStream = Files.list(dataDir)) {
+ List langDirs = dirStream.filter(Files::isDirectory).sorted().toList();
+ for (Path langDir : langDirs) {
+ String lang = langDir.getFileName().toString();
+ String script = detectDominantScript(langDir, scriptSampleLines);
+ langToScript.put(lang, script);
+ scriptGroups.computeIfAbsent(script, k -> new ArrayList<>()).add(langDir);
+ System.out.printf(" %-12s → %s%n", lang, script);
+ }
+ }
+ System.out.printf("%n → %d languages, %d script groups%n",
+ langToScript.size(), scriptGroups.size());
+
+ // -----------------------------------------------------------------------
+ // Phase 2: Load small sample per script, compute byte-bigram entropy
+ // -----------------------------------------------------------------------
+
+ System.out.println("\n--- Phase 2: Estimating byte-bigram entropy per script ---");
+
+ Map scriptEntropy = new TreeMap<>();
+ for (Map.Entry> entry : scriptGroups.entrySet()) {
+ String script = entry.getKey();
+ List langDirs = entry.getValue();
+
+ long perLangSampleBytes = Math.max(ENTROPY_SAMPLE_BYTES / langDirs.size(), 2_000L);
+ List sample = new ArrayList<>();
+ for (Path langDir : langDirs) {
+ loadSentences(langDir, perLangSampleBytes, minBytes, maxPuncFrac, sample);
+ }
+
+ double entropy = computeBigramEntropy(sample);
+ scriptEntropy.put(script, entropy);
+ System.out.printf(" %-20s H=%.3f bits (%d sentences)%n",
+ script, entropy, sample.size());
+ }
+
+ // -----------------------------------------------------------------------
+ // Phase 3: Allocate byte budget proportional to entropy
+ // -----------------------------------------------------------------------
+
+ System.out.println("\n--- Phase 3: Allocating byte budget ---");
+
+ double totalEntropy = scriptEntropy.values().stream()
+ .mapToDouble(Double::doubleValue).sum();
+
+ Map scriptBudget = new TreeMap<>();
+ for (Map.Entry e : scriptEntropy.entrySet()) {
+ long budget = (long) (totalBudgetBytes * e.getValue() / totalEntropy);
+ scriptBudget.put(e.getKey(), budget);
+ System.out.printf(" %-20s H=%.3f → %,d bytes (%.1f MB)%n",
+ e.getKey(), e.getValue(), budget, budget / 1_000_000.0);
+ }
+
+ if (dryRun) {
+ System.out.println("\nDry-run: stopping before writing files.");
+ return;
+ }
+
+ // -----------------------------------------------------------------------
+ // Phase 4: Collect data, write train/dev splits
+ // -----------------------------------------------------------------------
+
+ Files.createDirectories(outputDir);
+ System.out.println("\n--- Phase 4a: Round 1 — collecting with initial budgets ---");
+
+ Random rng = new Random(seed);
+
+ // Collect sentences and actual byte counts for every script
+ Map> allSentences = new LinkedHashMap<>();
+ Map actualBytes = new LinkedHashMap<>();
+
+ for (Map.Entry budgetEntry : scriptBudget.entrySet()) {
+ String script = budgetEntry.getKey();
+ long budget = budgetEntry.getValue();
+ List langDirs = scriptGroups.get(script);
+
+ long perLangBytes = Math.max(budget / langDirs.size(), 1L);
+ List sentences = new ArrayList<>();
+ long totalBytesLoaded = 0;
+
+ for (Path langDir : langDirs) {
+ long remaining = budget - totalBytesLoaded;
+ if (remaining <= 0) break;
+ long langBytes = loadSentences(langDir,
+ Math.min(perLangBytes, remaining),
+ minBytes, maxPuncFrac, sentences);
+ totalBytesLoaded += langBytes;
+ if (langBytes > 0) {
+ System.out.printf(" %-12s %-20s +%,d bytes%n",
+ script, langDir.getFileName(), langBytes);
+ }
+ }
+ allSentences.put(script, sentences);
+ actualBytes.put(script, totalBytesLoaded);
+ }
+
+ // Compute surplus bytes from data-starved scripts (< 90% of budget used)
+ long surplus = 0;
+ for (Map.Entry e : scriptBudget.entrySet()) {
+ long budget = e.getValue();
+ long actual = actualBytes.getOrDefault(e.getKey(), 0L);
+ if (actual < budget * 0.9) {
+ surplus += (budget - actual);
+ }
+ }
+
+ // Round 2: redistribute surplus to saturated scripts proportional to entropy
+ if (surplus > 0) {
+ System.out.printf(
+ "\n--- Phase 4b: Redistributing %,d surplus bytes (%.1f MB) ---\n",
+ surplus, surplus / 1_000_000.0);
+
+ double saturatedEntropy = scriptBudget.entrySet().stream()
+ .filter(e -> actualBytes.getOrDefault(e.getKey(), 0L) >= e.getValue() * 0.9)
+ .mapToDouble(e -> scriptEntropy.getOrDefault(e.getKey(), 0.0))
+ .sum();
+
+ for (Map.Entry budgetEntry : scriptBudget.entrySet()) {
+ String script = budgetEntry.getKey();
+ long budget = budgetEntry.getValue();
+ long actual = actualBytes.getOrDefault(script, 0L);
+ if (actual < budget * 0.9) continue; // data-starved — skip
+
+ long extra = (long) (surplus
+ * scriptEntropy.getOrDefault(script, 0.0) / saturatedEntropy);
+ if (extra <= 0) continue;
+
+ long newBudget = budget + extra;
+ List langDirs = scriptGroups.get(script);
+ long perLangBytes = Math.max(newBudget / langDirs.size(), 1L);
+
+ List sentences = new ArrayList<>();
+ long totalBytesLoaded = 0;
+ for (Path langDir : langDirs) {
+ long remaining = newBudget - totalBytesLoaded;
+ if (remaining <= 0) break;
+ long langBytes = loadSentences(langDir,
+ Math.min(perLangBytes, remaining),
+ minBytes, maxPuncFrac, sentences);
+ totalBytesLoaded += langBytes;
+ }
+ if (!sentences.isEmpty()) {
+ allSentences.put(script, sentences);
+ actualBytes.put(script, totalBytesLoaded);
+ System.out.printf(" %-20s +%,d extra → %,d total bytes, %,d sentences%n",
+ script, extra, totalBytesLoaded, sentences.size());
+ }
+ }
+ }
+
+ // Write split files
+ System.out.println("\n--- Phase 4c: Writing train/dev/test splits ---");
+
+ // manifest columns: script, entropy, budget_bytes, written_bytes, sentences, train_bytes, languages
+ Map manifestStats = new TreeMap<>();
+
+ for (Map.Entry> e : allSentences.entrySet()) {
+ String script = e.getKey();
+ List sentences = e.getValue();
+
+ int expectedDevSize = (int) (sentences.size() * DEV_FRAC);
+ if (sentences.isEmpty() || expectedDevSize < minDevSentences) {
+ System.out.printf(
+ " SKIP %-20s — %,d sentences → dev=%d < min-dev-sentences=%d%n",
+ script, sentences.size(), expectedDevSize, minDevSentences);
+ manifestStats.put(script, new long[]{0, 0, 0, 0, 0});
+ continue;
+ }
+
+ Collections.shuffle(sentences, rng);
+
+ int nTrain = (int) (sentences.size() * TRAIN_FRAC);
+ int nDev = (int) (sentences.size() * DEV_FRAC);
+ List train = sentences.subList(0, nTrain);
+ List dev = sentences.subList(nTrain, nTrain + nDev);
+ List test = sentences.subList(nTrain + nDev, sentences.size());
+
+ String baseName = script.toLowerCase();
+ writeGzipped(outputDir.resolve(baseName + ".train.gz"), train);
+ writeGzipped(outputDir.resolve(baseName + ".dev.gz"), dev);
+ writeGzipped(outputDir.resolve(baseName + ".test.gz"), test);
+
+ long totalBytesLoaded = actualBytes.getOrDefault(script, 0L);
+ manifestStats.put(script,
+ new long[]{totalBytesLoaded, sentences.size(), nTrain, nDev, test.size()});
+ System.out.printf(
+ " WROTE %-12s — %,d bytes, %,d sentences (train=%,d dev=%,d test=%,d)%n",
+ script, totalBytesLoaded, sentences.size(),
+ nTrain, nDev, test.size());
+ }
+
+ // -----------------------------------------------------------------------
+ // Phase 5: Write manifest
+ // -----------------------------------------------------------------------
+
+ Path manifest = outputDir.resolve("manifest.tsv");
+ try (BufferedWriter w = Files.newBufferedWriter(manifest, StandardCharsets.UTF_8)) {
+ w.write("script\tentropy_bits\tbudget_bytes\twritten_bytes\tsentences"
+ + "\ttrain_sentences\tdev_sentences\ttest_sentences\tlanguages\n");
+ for (Map.Entry e : manifestStats.entrySet()) {
+ String script = e.getKey();
+ long[] stats = e.getValue();
+ double entropy = scriptEntropy.getOrDefault(script, 0.0);
+ long budget = scriptBudget.getOrDefault(script, 0L);
+ String langs = scriptGroups.get(script).stream()
+ .map(p -> p.getFileName().toString())
+ .reduce((a, b) -> a + "," + b).orElse("");
+ w.write(String.format("%s\t%.3f\t%d\t%d\t%d\t%d\t%d\t%d\t%s%n",
+ script, entropy, budget,
+ stats[0], stats[1], stats[2], stats[3], stats[4], langs));
+ }
+ }
+
+ System.out.println("\nWrote manifest: " + manifest);
+ System.out.println("Done.");
+ }
+
+ // -----------------------------------------------------------------------
+ // Script detection
+ // -----------------------------------------------------------------------
+
+ /**
+ * Detects the dominant Unicode script for a language by histogramming
+ * {@link Character.UnicodeScript} over a sample of its sentences.
+ * COMMON, INHERITED, and UNKNOWN pseudo-scripts are excluded from voting.
+ * Returns "COMMON" if no script reaches at least 1% of codepoints.
+ */
+ static String detectDominantScript(Path langDir, int sampleLines) {
+ Map counts = new HashMap<>();
+ long total = 0;
+
+ outer:
+ for (String filename : new String[]{"sentences_wikipedia.txt", "sentences_madlad.txt"}) {
+ Path file = langDir.resolve(filename);
+ if (!Files.exists(file)) {
+ continue;
+ }
+ try (BufferedReader r = new BufferedReader(
+ new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8))) {
+ String line;
+ int linesRead = 0;
+ while ((line = r.readLine()) != null && linesRead < sampleLines) {
+ String text = extractText(line);
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s != Character.UnicodeScript.COMMON
+ && s != Character.UnicodeScript.INHERITED
+ && s != Character.UnicodeScript.UNKNOWN) {
+ counts.merge(s, 1L, Long::sum);
+ total++;
+ }
+ i += Character.charCount(cp);
+ }
+ linesRead++;
+ }
+ } catch (IOException e) {
+ // Skip unreadable file; report COMMON if nothing else succeeds
+ }
+ if (total >= sampleLines * 10L) {
+ break outer; // sufficient signal
+ }
+ }
+
+ if (total == 0) {
+ return "COMMON";
+ }
+
+ // Plurality with a 1% floor to suppress spurious Latin wins on mixed text
+ Character.UnicodeScript best = Character.UnicodeScript.COMMON;
+ long bestCount = total / 100;
+ for (Map.Entry e : counts.entrySet()) {
+ if (e.getValue() > bestCount) {
+ bestCount = e.getValue();
+ best = e.getKey();
+ }
+ }
+ return best.name();
+ }
+
+ // -----------------------------------------------------------------------
+ // Entropy estimation
+ // -----------------------------------------------------------------------
+
+ /**
+ * Computes the empirical byte-bigram Shannon entropy (bits) of a list of
+ * UTF-8 sentences.
+ *
+ * All 256×256 = 65,536 consecutive byte pairs are counted; entropy is
+ * {@code -sum p(a,b) * log2(p(a,b))} over pairs with non-zero count.
+ * Maximum theoretical value is 16 bits (all pairs equally likely).
+ * Typical ranges: Latin ~8–11 bits, Arabic ~9–12, CJK ~13–15.
+ */
+ static double computeBigramEntropy(List sentences) {
+ long[] counts = new long[65536];
+ long total = 0;
+ for (String s : sentences) {
+ byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ for (int i = 0; i + 1 < bytes.length; i++) {
+ counts[((bytes[i] & 0xFF) << 8) | (bytes[i + 1] & 0xFF)]++;
+ total++;
+ }
+ }
+ if (total == 0) {
+ return 0.0;
+ }
+ double entropy = 0.0;
+ for (long c : counts) {
+ if (c > 0) {
+ double p = (double) c / total;
+ entropy -= p * (Math.log(p) / Math.log(2.0));
+ }
+ }
+ return entropy;
+ }
+
+ // -----------------------------------------------------------------------
+ // Sentence loading and filtering
+ // -----------------------------------------------------------------------
+
+ /**
+ * Loads filtered, NFC-normalised sentences from {@code langDir} until
+ * {@code maxBytes} UTF-8 bytes have been accumulated, and appends them
+ * to {@code result}.
+ *
+ * Reads {@code sentences_wikipedia.txt} before {@code sentences_madlad.txt}.
+ * MADLAD records contain literal {@code \n} escape sequences as sub-sentence
+ * separators (full scraped documents) and are split accordingly.
+ *
+ * @return total UTF-8 bytes of accepted sentences appended
+ */
+ static long loadSentences(Path langDir, long maxBytes, int minBytes,
+ double maxPuncFrac, List result) {
+ long bytesLoaded = 0;
+ for (String filename : new String[]{"sentences_wikipedia.txt", "sentences_madlad.txt"}) {
+ if (bytesLoaded >= maxBytes) {
+ break;
+ }
+ Path file = langDir.resolve(filename);
+ if (!Files.exists(file)) {
+ continue;
+ }
+ try (BufferedReader r = new BufferedReader(
+ new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8))) {
+ String line;
+ while ((line = r.readLine()) != null && bytesLoaded < maxBytes) {
+ String raw = extractText(line);
+ for (String part : raw.split("\\\\n")) {
+ String text = part.replace("\\r", "")
+ .replace("\\t", " ")
+ .strip()
+ .replaceAll("\\s+", " ");
+ if (text.isEmpty()) {
+ continue;
+ }
+ String filtered = filterSentence(text, minBytes, maxPuncFrac);
+ if (filtered != null) {
+ int sentBytes = filtered.getBytes(StandardCharsets.UTF_8).length;
+ result.add(filtered);
+ bytesLoaded += sentBytes;
+ if (bytesLoaded >= maxBytes) {
+ break;
+ }
+ }
+ }
+ }
+ } catch (IOException e) {
+ System.err.println("WARNING: could not read " + file + ": " + e.getMessage());
+ }
+ }
+ return bytesLoaded;
+ }
+
+ /**
+ * Applies quality filters to a single sentence and NFC-normalises it.
+ *
+ * @return the normalised sentence, or {@code null} if it should be discarded
+ */
+ static String filterSentence(String text, int minBytes, double maxPuncFrac) {
+ if (text.indexOf('\uFFFD') >= 0) {
+ return null;
+ }
+ text = Normalizer.normalize(text, Normalizer.Form.NFC);
+ if (text.getBytes(StandardCharsets.UTF_8).length < minBytes) {
+ return null;
+ }
+ int cpCount = 0;
+ int puncCount = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ cpCount++;
+ if (cp >= 0x21 && cp <= 0x7E && !Character.isLetter(cp)) {
+ puncCount++;
+ }
+ i += Character.charCount(cp);
+ }
+ if (cpCount > 0 && (double) puncCount / cpCount > maxPuncFrac) {
+ return null;
+ }
+ return text;
+ }
+
+ // -----------------------------------------------------------------------
+ // I/O helpers
+ // -----------------------------------------------------------------------
+
+ private static String extractText(String line) {
+ int tab = line.indexOf('\t');
+ String text = (tab >= 0) ? line.substring(tab + 1) : line;
+ return text.replace("\uFEFF", "");
+ }
+
+ private static void writeGzipped(Path path, List lines) throws IOException {
+ try (BufferedWriter w = new BufferedWriter(
+ new OutputStreamWriter(
+ new GZIPOutputStream(Files.newOutputStream(path)),
+ StandardCharsets.UTF_8))) {
+ for (String line : lines) {
+ w.write(line);
+ w.newLine();
+ }
+ }
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: BuildJunkTrainingData [options]");
+ System.err.println(" --data-dir MADLAD data root"
+ + " (default: ~/datasets/madlad/data)");
+ System.err.println(" --output-dir Output directory"
+ + " (default: ~/datasets/madlad/junkdetect)");
+ System.err.println(" --script-sample-lines N Lines per language for script"
+ + " detection (default: 2000)");
+ System.err.println(" --total-budget-bytes N Total UTF-8 bytes across all"
+ + " scripts (default: 50000000)");
+ System.err.println(" --min-bytes N Min UTF-8 bytes per sentence"
+ + " (default: 50)");
+ System.err.println(" --max-punc-frac F Max ASCII punct fraction"
+ + " (default: 0.30)");
+ System.err.println(" --min-dev-sentences N Min sentences in dev split for a"
+ + " script to be included (default: 500). Scripts below this floor"
+ + " have unreliable calibration and inflated FPR.");
+ System.err.println(" --seed N Random seed (default: 42)");
+ System.err.println(" --dry-run Detect scripts + show budget,"
+ + " skip file writing");
+ }
+}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkDetector.java
new file mode 100644
index 00000000000..6b6057fc34f
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/EvalJunkDetector.java
@@ -0,0 +1,777 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect.tools;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.charset.UnsupportedCharsetException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.tika.ml.junkdetect.JunkDetector;
+import org.apache.tika.quality.TextQualityComparison;
+import org.apache.tika.quality.TextQualityScore;
+
+/**
+ * Ablation evaluation for the junk detector.
+ *
+ * For each script's dev set, scores clean sentences alongside several corruption
+ * modes at various injection rates and string lengths. Computes per-cell Cohen's d
+ * (discrimination power) and TPR/FPR at a fixed z-score threshold.
+ *
+ *
Output files in {@code --output-dir}:
+ *
+ * - detail.tsv — one row per (script, distortion, rate, length):
+ * {@code script, distortion, param, length, n_clean, n_corrupt,
+ * mean_clean_z, mean_corrupt_z, cohens_d, fpr, tpr}
+ *
- summary.tsv — macro-averaged Cohen's d and FPR/TPR per
+ * (distortion, rate, length) across all scripts.
+ *
- compare.tsv — pairwise codec-comparison accuracy using the
+ * {@link JunkDetector#compare} API, stratified by string length.
+ * This is the primary metric for the charset-arbitration use case;
+ * larger mean delta = better discrimination at that length.
+ *
+ *
+ * Why char-remap is not in summary.tsv: The character-level wrong-codec
+ * substitution (e.g. CP1252→CP1255, replacing umlauts with Hebrew letters) is added
+ * to training at a 5% rate. At that rate it is too subtle to detect via the absolute
+ * {@link JunkDetector#score} API — z-score distributions barely separate (Cohen's d ≈ 0).
+ * The distortion trains the LR to distinguish subtly-wrong from correct decodings, which
+ * only manifests as larger pairwise deltas in {@link JunkDetector#compare}. Measuring it
+ * via summary.tsv would produce misleading d≈0 "failure" rows; see compare.tsv instead.
+ *
+ *
Cohen's d = (mean_clean_z − mean_corrupt_z) / pooled_std.
+ * Higher d = better discrimination. FPR = fraction of clean text falsely flagged;
+ * TPR = fraction of corrupted text correctly flagged. Both use threshold = −2.0.
+ *
+ *
To compare two model versions: run eval before and after, then diff the
+ * summary and compare TSVs. The "macro_d" column in summary.tsv and the
+ * "mean_delta" columns in compare.tsv are the headline metrics.
+ *
+ *
Usage:
+ *
+ * java EvalJunkDetector \
+ * --model /path/to/junkdetect.bin (default: classpath)
+ * --data-dir ~/datasets/madlad/junkdetect
+ * --output-dir /path/to/results (default: data-dir/eval)
+ * --split dev|test (default: dev)
+ * --samples 200
+ * --compare-n 200 (qualifying pairs per codec pair per length)
+ * --seed 42
+ * --lengths 5,9,15,30,50,100,200
+ * --compare-lengths 5,9,15,30,50
+ * --rates 0.01,0.05,0.10,0.25,0.50,0.90
+ * --threshold -2.0
+ *
+ */
+public class EvalJunkDetector {
+
+ public static void main(String[] args) throws Exception {
+ // Defaults
+ Path modelPath = null;
+ Path dataDir = Paths.get(System.getProperty("user.home"),
+ "datasets", "madlad", "junkdetect");
+ Path outputDir = null;
+ String split = "dev";
+ int samplesPerCell = 200;
+ int compareN = 200;
+ long seed = 42L;
+ int[] lengths = {5, 9, 15, 30, 50, 100, 200};
+ int[] compareLengths = {5, 9, 15, 30, 50};
+ double[] rates = {0.01, 0.05, 0.10, 0.25, 0.50, 0.90};
+ float threshold = -2.0f;
+
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--model":
+ modelPath = Paths.get(args[++i]);
+ break;
+ case "--data-dir":
+ dataDir = Paths.get(args[++i]);
+ break;
+ case "--output-dir":
+ outputDir = Paths.get(args[++i]);
+ break;
+ case "--split":
+ split = args[++i];
+ if (!split.equals("dev") && !split.equals("test")) {
+ System.err.println("--split must be 'dev' or 'test'");
+ System.exit(1);
+ }
+ break;
+ case "--samples":
+ samplesPerCell = Integer.parseInt(args[++i]);
+ break;
+ case "--compare-n":
+ compareN = Integer.parseInt(args[++i]);
+ break;
+ case "--seed":
+ seed = Long.parseLong(args[++i]);
+ break;
+ case "--lengths":
+ lengths = Arrays.stream(args[++i].split(","))
+ .mapToInt(Integer::parseInt).toArray();
+ break;
+ case "--compare-lengths":
+ compareLengths = Arrays.stream(args[++i].split(","))
+ .mapToInt(Integer::parseInt).toArray();
+ break;
+ case "--rates":
+ rates = Arrays.stream(args[++i].split(","))
+ .mapToDouble(Double::parseDouble).toArray();
+ break;
+ case "--threshold":
+ threshold = Float.parseFloat(args[++i]);
+ break;
+ default:
+ System.err.println("Unknown argument: " + args[i]);
+ System.exit(1);
+ }
+ }
+
+ if (outputDir == null) {
+ outputDir = dataDir.resolve("eval");
+ }
+ Files.createDirectories(outputDir);
+
+ JunkDetector detector = modelPath != null
+ ? JunkDetector.loadFromPath(modelPath)
+ : JunkDetector.loadFromClasspath();
+
+ System.err.println("=== EvalJunkDetector ===");
+ System.err.println(" data-dir: " + dataDir);
+ System.err.println(" output-dir: " + outputDir);
+ System.err.println(" split: " + split
+ + (split.equals("test") ? " [FINAL REPORTING MODE]" : ""));
+ System.err.println(" scripts in model: " + detector.knownScripts().size());
+ System.err.println(" threshold: " + threshold);
+
+ // Build wrong-codec remap tables for char-remap distortion
+ List> remapTables = new ArrayList<>();
+ for (String[] pair : TrainJunkModel.WRONG_CODEC_PAIRS) {
+ Map table = TrainJunkModel.buildRemapTable(pair[0], pair[1]);
+ if (!table.isEmpty()) remapTables.add(table);
+ }
+ System.err.println(" remap tables: " + remapTables.size());
+
+ String suffix = "." + split + ".gz";
+ List devFiles;
+ try (var stream = Files.list(dataDir)) {
+ devFiles = stream
+ .filter(p -> p.getFileName().toString().endsWith(suffix))
+ .sorted()
+ .collect(Collectors.toList());
+ }
+
+ if (devFiles.isEmpty()) {
+ System.err.println("ERROR: no *" + suffix + " files found in " + dataDir);
+ System.exit(1);
+ }
+
+ Path detailPath = outputDir.resolve("detail.tsv");
+ Path summaryPath = outputDir.resolve("summary.tsv");
+ Path comparePath = outputDir.resolve("compare.tsv");
+
+ List allRows = new ArrayList<>();
+
+ try (PrintWriter detail = new PrintWriter(
+ Files.newBufferedWriter(detailPath, StandardCharsets.UTF_8))) {
+
+ detail.println("script\tdistortion\tparam\tlength"
+ + "\tn_clean\tn_corrupt"
+ + "\tmean_clean_z\tmean_corrupt_z"
+ + "\tcohens_d\tfpr\ttpr");
+
+ for (Path devFile : devFiles) {
+ String filename = devFile.getFileName().toString();
+ String script = filename
+ .substring(0, filename.length() - suffix.length())
+ .toUpperCase();
+
+ System.err.printf("%n--- %s ---%n", script);
+
+ List sentences = loadSentences(devFile, samplesPerCell * 20);
+ if (sentences.size() < 10) {
+ System.err.printf(" Skipping — only %d sentences%n", sentences.size());
+ continue;
+ }
+
+ for (int len : lengths) {
+ List cleanZ = scoreClean(detector, sentences, len,
+ samplesPerCell, new Random(seed));
+
+ // --- injection ---
+ for (double rate : rates) {
+ List corruptZ = scoreWithInjection(detector, sentences, len,
+ rate, samplesPerCell, new Random(seed + 1));
+ Row row = new Row(script, "inject",
+ String.format("%.2f", rate), len,
+ cleanZ, corruptZ, threshold);
+ allRows.add(row);
+ detail.println(row.toTsv());
+ }
+
+ // --- codepoint reversal ---
+ {
+ List corruptZ = scoreReversed(detector, sentences, len,
+ samplesPerCell, new Random(seed + 2));
+ Row row = new Row(script, "char-reverse", "-", len,
+ cleanZ, corruptZ, threshold);
+ allRows.add(row);
+ detail.println(row.toTsv());
+ }
+
+ // --- byte shuffle ---
+ {
+ List corruptZ = scoreShuffled(detector, sentences, len,
+ samplesPerCell, new Random(seed + 3));
+ Row row = new Row(script, "byte-shuffle", "-", len,
+ cleanZ, corruptZ, threshold);
+ allRows.add(row);
+ detail.println(row.toTsv());
+ }
+
+ // --- wrong-codec: re-read UTF-8 bytes as ISO-8859-1 then re-encode ---
+ {
+ List corruptZ = scoreWrongCodec(detector, sentences, len,
+ samplesPerCell, new Random(seed + 4));
+ Row row = new Row(script, "wrong-codec", "latin1-as-utf8", len,
+ cleanZ, corruptZ, threshold);
+ allRows.add(row);
+ detail.println(row.toTsv());
+ }
+
+ // --- byte-swap ---
+ {
+ List corruptZ = scoreByteSwapped(detector, sentences, len,
+ samplesPerCell, new Random(seed + 5));
+ Row row = new Row(script, "byte-swap", "-", len,
+ cleanZ, corruptZ, threshold);
+ allRows.add(row);
+ detail.println(row.toTsv());
+ }
+
+ // char-remap distortion is evaluated only via compare.tsv (pairwise delta),
+ // not via absolute score() — see class Javadoc for rationale.
+
+ detail.flush();
+ }
+ }
+ }
+
+ writeSummary(summaryPath, allRows, lengths, rates, threshold);
+ writeCompareEval(detector, dataDir, suffix, comparePath, compareN, compareLengths, seed);
+
+ System.err.println("\nWrote " + detailPath);
+ System.err.println("Wrote " + summaryPath);
+ System.err.println("Wrote " + comparePath);
+ System.err.println("Done.");
+ }
+
+ // -----------------------------------------------------------------------
+ // Summary aggregation
+ // -----------------------------------------------------------------------
+
+ private static void writeSummary(Path summaryPath, List rows,
+ int[] lengths, double[] rates, float threshold)
+ throws IOException {
+ try (PrintWriter out = new PrintWriter(
+ Files.newBufferedWriter(summaryPath, StandardCharsets.UTF_8))) {
+
+ out.println("distortion\tparam\tlength\tn_scripts"
+ + "\tmacro_cohens_d\tmacro_fpr\tmacro_tpr");
+
+ List conditions = new ArrayList<>();
+ for (double rate : rates) {
+ conditions.add(new String[]{"inject", String.format("%.2f", rate)});
+ }
+ conditions.add(new String[]{"char-reverse", "-"});
+ conditions.add(new String[]{"byte-shuffle", "-"});
+ conditions.add(new String[]{"wrong-codec", "latin1-as-utf8"});
+ conditions.add(new String[]{"byte-swap", "-"});
+ // char-remap is intentionally excluded from summary: at 5% rate the character-level
+ // wrong-codec substitution is too subtle to detect via absolute score() — both the
+ // clean and corrupted strings score similarly. The right metric for char-remap is
+ // compare.tsv (pairwise delta), where it shows up strongly. Including it here would
+ // make d≈0 rows that look like failures but are actually expected.
+
+ for (String[] cond : conditions) {
+ String distortion = cond[0];
+ String param = cond[1];
+ for (int len : lengths) {
+ List matching = rows.stream()
+ .filter(r -> r.distortion.equals(distortion)
+ && r.param.equals(param)
+ && r.length == len)
+ .collect(Collectors.toList());
+ if (matching.isEmpty()) continue;
+
+ double macroCohensD = matching.stream()
+ .filter(r -> !Double.isNaN(r.cohensD))
+ .mapToDouble(r -> r.cohensD)
+ .average().orElse(Double.NaN);
+ double macroFpr = matching.stream()
+ .mapToDouble(r -> r.fpr)
+ .average().orElse(Double.NaN);
+ double macroTpr = matching.stream()
+ .mapToDouble(r -> r.tpr)
+ .average().orElse(Double.NaN);
+
+ out.printf("%s\t%s\t%d\t%d\t%.3f\t%.3f\t%.3f%n",
+ distortion, param, len, matching.size(),
+ macroCohensD, macroFpr, macroTpr);
+ }
+ }
+
+ double overallD = rows.stream()
+ .filter(r -> !Double.isNaN(r.cohensD))
+ .mapToDouble(r -> r.cohensD)
+ .average().orElse(Double.NaN);
+ double overallFpr = rows.stream()
+ .mapToDouble(r -> r.fpr)
+ .average().orElse(Double.NaN);
+ double overallTpr = rows.stream()
+ .mapToDouble(r -> r.tpr)
+ .average().orElse(Double.NaN);
+ out.println();
+ out.printf("# OVERALL macro_cohens_d=%.3f macro_fpr=%.3f macro_tpr=%.3f%n",
+ overallD, overallFpr, overallTpr);
+
+ System.err.printf("%nOVERALL: macro_cohens_d=%.3f macro_fpr=%.3f macro_tpr=%.3f%n",
+ overallD, overallFpr, overallTpr);
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // Compare eval — pairwise codec arbitration, stratified by string length
+ // -----------------------------------------------------------------------
+
+ /**
+ * For each entry in {@link TrainJunkModel#WRONG_CODEC_PAIRS}, encodes sentences
+ * from the appropriate script's dev file as the source charset, then calls
+ * {@link JunkDetector#compare} with the correct decoding (A) vs the wrong
+ * decoding (B). Reports accuracy (how often A wins) and mean/median delta
+ * at each requested string length.
+ *
+ * Mean delta is the headline metric: larger delta means the model more
+ * confidently picks the correct decoding. At short lengths (5–9 bytes)
+ * delta is expected to be small; at 50 bytes it should be decisive.
+ */
+ private static void writeCompareEval(JunkDetector detector,
+ Path dataDir, String suffix,
+ Path comparePath,
+ int nPerCell, int[] lengths,
+ long seed) throws IOException {
+ try (PrintWriter out = new PrintWriter(
+ Files.newBufferedWriter(comparePath, StandardCharsets.UTF_8))) {
+
+ out.println("source_codec\twrong_codec\tlength"
+ + "\tn_tested\taccuracy\tmean_delta\tmedian_delta\tn_no_diff");
+
+ System.err.printf("%n--- compare() eval ---%n");
+
+ for (String[] pair : TrainJunkModel.WRONG_CODEC_PAIRS) {
+ String sourceCodec = pair[0];
+ String wrongCodec = pair[1];
+
+ Charset srcCharset, wrongCharset;
+ try {
+ srcCharset = Charset.forName(sourceCodec);
+ wrongCharset = Charset.forName(wrongCodec);
+ } catch (UnsupportedCharsetException e) {
+ System.err.printf(" [%s→%s] charset unavailable, skipping%n",
+ sourceCodec, wrongCodec);
+ continue;
+ }
+
+ String script = codecToScript(sourceCodec);
+ Path devFile = dataDir.resolve(script.toLowerCase() + suffix);
+ if (!Files.exists(devFile)) {
+ System.err.printf(" [%s→%s] no dev file for %s, skipping%n",
+ sourceCodec, wrongCodec, script);
+ continue;
+ }
+
+ // Load a large pool; we'll filter down per-length
+ List allSentences = loadSentences(devFile, nPerCell * 50);
+
+ // Pre-filter: keep only sentences that roundtrip through sourceCodec
+ // and produce at least one differing character vs wrongCodec.
+ List candidates = new ArrayList<>(); // {asSource, asWrong}
+ for (String sentence : allSentences) {
+ byte[] bytes = sentence.getBytes(srcCharset);
+ String asSource = new String(bytes, srcCharset);
+ if (!asSource.equals(sentence)) continue; // encoding lost data
+ String asWrong = new String(bytes, wrongCharset);
+ if (asSource.equals(asWrong)) continue; // no differentiating bytes
+ candidates.add(new String[]{asSource, asWrong});
+ }
+
+ if (candidates.isEmpty()) {
+ System.err.printf(" [%s→%s] no qualifying sentences%n",
+ sourceCodec, wrongCodec);
+ continue;
+ }
+
+ System.err.printf(" [%s→%s] %d candidates from %s%n",
+ sourceCodec, wrongCodec, candidates.size(), script);
+
+ for (int targetLen : lengths) {
+ Random rng = new Random(seed);
+ // Shuffle candidates for this length independently
+ List shuffled = new ArrayList<>(candidates);
+ Collections.shuffle(shuffled, rng);
+
+ List deltas = new ArrayList<>();
+ int nCorrect = 0;
+ int nNoDiff = 0;
+
+ for (String[] cand : shuffled) {
+ if (deltas.size() + nNoDiff >= nPerCell * 3 && deltas.size() >= nPerCell) {
+ break;
+ }
+ String asSource = trimToLength(cand[0], targetLen);
+ String asWrong = trimToLength(cand[1], targetLen);
+
+ if (asSource.equals(asWrong)) {
+ nNoDiff++;
+ continue;
+ }
+ if (asSource.isEmpty() || asWrong.isEmpty()) continue;
+
+ TextQualityComparison result = detector.compare(
+ sourceCodec, asSource, wrongCodec, asWrong);
+
+ deltas.add(result.delta());
+ if ("A".equals(result.winner())) nCorrect++;
+ }
+
+ if (deltas.isEmpty()) continue;
+
+ double accuracy = (double) nCorrect / deltas.size();
+ double meanDelta = deltas.stream().mapToDouble(Float::floatValue).average().orElse(0);
+ List sorted = new ArrayList<>(deltas);
+ Collections.sort(sorted);
+ float medianDelta = sorted.get(sorted.size() / 2);
+
+ System.err.printf(" len=%3d n=%3d acc=%.3f mean_delta=%.3f median_delta=%.3f%n",
+ targetLen, deltas.size(), accuracy, meanDelta, medianDelta);
+
+ out.printf("%s\t%s\t%d\t%d\t%.3f\t%.3f\t%.3f\t%d%n",
+ sourceCodec, wrongCodec, targetLen,
+ deltas.size(), accuracy, meanDelta, medianDelta, nNoDiff);
+ }
+ out.flush();
+ }
+ }
+ }
+
+ /**
+ * Returns which dev-file script to use for a given source codec.
+ * CP1251 → CYRILLIC, CP1253 → GREEK, CP1255 → HEBREW, everything else → LATIN.
+ */
+ private static String codecToScript(String codec) {
+ switch (codec.toLowerCase()) {
+ case "windows-1251": return "CYRILLIC";
+ case "windows-1253": return "GREEK";
+ case "windows-1255": return "HEBREW";
+ default: return "LATIN";
+ }
+ }
+
+ /**
+ * Trims a string to approximately {@code targetLen} UTF-8 bytes, aligned to
+ * a codepoint boundary. Used to produce short-string variants for compare() testing.
+ */
+ private static String trimToLength(String s, int targetLen) {
+ byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ if (bytes.length <= targetLen) return s;
+ int end = targetLen;
+ while (end < bytes.length && (bytes[end] & 0xC0) == 0x80) end++;
+ return new String(bytes, 0, end, StandardCharsets.UTF_8);
+ }
+
+ // -----------------------------------------------------------------------
+ // Row (one evaluation cell)
+ // -----------------------------------------------------------------------
+
+ private static final class Row {
+ final String script;
+ final String distortion;
+ final String param;
+ final int length;
+ final int nClean;
+ final int nCorrupt;
+ final double meanCleanZ;
+ final double meanCorruptZ;
+ final double cohensD;
+ final double fpr;
+ final double tpr;
+
+ Row(String script, String distortion, String param, int length,
+ List cleanZ, List corruptZ, float threshold) {
+ this.script = script;
+ this.distortion = distortion;
+ this.param = param;
+ this.length = length;
+ this.nClean = cleanZ.size();
+ this.nCorrupt = corruptZ.size();
+ this.meanCleanZ = mean(cleanZ);
+ this.meanCorruptZ = mean(corruptZ);
+ this.cohensD = computeCohensD(cleanZ, corruptZ);
+ this.fpr = fractionBelow(cleanZ, threshold);
+ this.tpr = fractionBelow(corruptZ, threshold);
+ }
+
+ String toTsv() {
+ return String.format("%s\t%s\t%s\t%d\t%d\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f",
+ script, distortion, param, length,
+ nClean, nCorrupt,
+ meanCleanZ, meanCorruptZ,
+ cohensD, fpr, tpr);
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // Statistics
+ // -----------------------------------------------------------------------
+
+ private static double computeCohensD(List clean, List corrupt) {
+ if (clean.isEmpty() || corrupt.isEmpty()) return Double.NaN;
+ double mc = mean(clean);
+ double mj = mean(corrupt);
+ double vc = variance(clean, mc);
+ double vj = variance(corrupt, mj);
+ double pooledStd = Math.sqrt((vc + vj) / 2.0);
+ if (pooledStd < 1e-9) return Double.NaN;
+ return (mc - mj) / pooledStd;
+ }
+
+ private static double mean(List xs) {
+ return xs.stream().mapToDouble(Float::floatValue).average().orElse(0);
+ }
+
+ private static double variance(List xs, double mu) {
+ return xs.stream().mapToDouble(x -> (x - mu) * (x - mu)).average().orElse(0);
+ }
+
+ private static double fractionBelow(List zs, float threshold) {
+ if (zs.isEmpty()) return Double.NaN;
+ long count = zs.stream().filter(z -> z < threshold).count();
+ return (double) count / zs.size();
+ }
+
+ // -----------------------------------------------------------------------
+ // Scoring helpers
+ // -----------------------------------------------------------------------
+
+ private static List scoreClean(JunkDetector detector, List sentences,
+ int targetLen, int n, Random rng) {
+ List results = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ String s = pickSubstring(sentences, targetLen, rng);
+ TextQualityScore score = detector.score(s);
+ if (!score.isUnknown()) results.add(score.getZScore());
+ }
+ return results;
+ }
+
+ private static List scoreWithInjection(JunkDetector detector,
+ List sentences, int targetLen,
+ double rate, int n, Random rng) {
+ List results = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ String s = pickSubstring(sentences, targetLen, rng);
+ byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ injectRandomBytes(bytes, rate, rng);
+ TextQualityScore score = detector.score(new String(bytes, StandardCharsets.ISO_8859_1));
+ if (!score.isUnknown()) results.add(score.getZScore());
+ }
+ return results;
+ }
+
+ private static List scoreReversed(JunkDetector detector, List sentences,
+ int targetLen, int n, Random rng) {
+ List results = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ String s = reverseCodepoints(pickSubstring(sentences, targetLen, rng));
+ TextQualityScore score = detector.score(s);
+ if (!score.isUnknown()) results.add(score.getZScore());
+ }
+ return results;
+ }
+
+ private static List scoreShuffled(JunkDetector detector, List sentences,
+ int targetLen, int n, Random rng) {
+ List results = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ String s = pickSubstring(sentences, targetLen, rng);
+ byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ shuffleBytes(bytes, rng);
+ TextQualityScore score = detector.score(new String(bytes, StandardCharsets.ISO_8859_1));
+ if (!score.isUnknown()) results.add(score.getZScore());
+ }
+ return results;
+ }
+
+ private static List scoreWrongCodec(JunkDetector detector, List sentences,
+ int targetLen, int n, Random rng) {
+ List results = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ String s = pickSubstring(sentences, targetLen, rng);
+ byte[] garbled = wrongCodecBytes(s.getBytes(StandardCharsets.UTF_8));
+ TextQualityScore score = detector.score(new String(garbled, StandardCharsets.UTF_8));
+ if (!score.isUnknown()) results.add(score.getZScore());
+ }
+ return results;
+ }
+
+ private static List scoreByteSwapped(JunkDetector detector, List sentences,
+ int targetLen, int n, Random rng) {
+ List results = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ String s = pickSubstring(sentences, targetLen, rng);
+ byte[] swapped = swapByteOrder(s.getBytes(StandardCharsets.UTF_8));
+ TextQualityScore score = detector.score(new String(swapped, StandardCharsets.ISO_8859_1));
+ if (!score.isUnknown()) results.add(score.getZScore());
+ }
+ return results;
+ }
+
+ /**
+ * Applies a randomly chosen wrong-codec character remap at {@code rate} to each
+ * sample. Simulates real-world charset misdetection at the character level
+ * (e.g. CP1252-encoded text decoded as CP1255, replacing umlauts with Hebrew letters).
+ */
+ private static List scoreWithRemap(JunkDetector detector, List sentences,
+ int targetLen,
+ List> remapTables,
+ double rate, int n, Random rng) {
+ List results = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ String s = pickSubstring(sentences, targetLen, rng);
+ Map table = remapTables.get(rng.nextInt(remapTables.size()));
+ String corrupted = TrainJunkModel.wrongCodecRemap(s, table, rate, rng);
+ TextQualityScore score = detector.score(corrupted);
+ if (!score.isUnknown()) results.add(score.getZScore());
+ }
+ return results;
+ }
+
+ // -----------------------------------------------------------------------
+ // Distortion primitives
+ // -----------------------------------------------------------------------
+
+ /**
+ * Injects control characters (0x01–0x09) at the given rate.
+ */
+ static void injectRandomBytes(byte[] bytes, double rate, Random rng) {
+ for (int i = 0; i < bytes.length; i++) {
+ if (rng.nextDouble() < rate) {
+ bytes[i] = (byte) (0x01 + rng.nextInt(9));
+ }
+ }
+ }
+
+ /**
+ * Wrong-codec distortion: UTF-8 bytes re-interpreted as ISO-8859-1, then
+ * re-encoded as UTF-8. Produces bogus two-byte sequences for any non-ASCII byte.
+ */
+ static byte[] wrongCodecBytes(byte[] utf8) {
+ String misread = new String(utf8, StandardCharsets.ISO_8859_1);
+ return misread.getBytes(StandardCharsets.UTF_8);
+ }
+
+ /**
+ * Swaps each adjacent pair of bytes — simulates reading a 2-byte encoding
+ * (UTF-16, CP932 two-byte sequences) with wrong byte order.
+ */
+ static byte[] swapByteOrder(byte[] bytes) {
+ byte[] out = bytes.clone();
+ for (int i = 0; i + 1 < out.length; i += 2) {
+ byte tmp = out[i];
+ out[i] = out[i + 1];
+ out[i + 1] = tmp;
+ }
+ return out;
+ }
+
+ static void shuffleBytes(byte[] bytes, Random rng) {
+ for (int i = bytes.length - 1; i > 0; i--) {
+ int j = rng.nextInt(i + 1);
+ byte tmp = bytes[i];
+ bytes[i] = bytes[j];
+ bytes[j] = tmp;
+ }
+ }
+
+ static String reverseCodepoints(String s) {
+ int[] codepoints = s.codePoints().toArray();
+ for (int lo = 0, hi = codepoints.length - 1; lo < hi; lo++, hi--) {
+ int tmp = codepoints[lo];
+ codepoints[lo] = codepoints[hi];
+ codepoints[hi] = tmp;
+ }
+ return new String(codepoints, 0, codepoints.length);
+ }
+
+ // -----------------------------------------------------------------------
+ // Sentence sampling
+ // -----------------------------------------------------------------------
+
+ private static String pickSubstring(List sentences, int targetLen, Random rng) {
+ String s = sentences.get(rng.nextInt(sentences.size()));
+ byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ if (bytes.length <= targetLen) return s;
+ int start = rng.nextInt(bytes.length - targetLen);
+ while (start > 0 && (bytes[start] & 0xC0) == 0x80) start--;
+ int end = Math.min(start + targetLen, bytes.length);
+ while (end < bytes.length && (bytes[end] & 0xC0) == 0x80) end++;
+ return new String(bytes, start, end - start, StandardCharsets.UTF_8);
+ }
+
+ private static List loadSentences(Path devGz, int maxSentences) throws IOException {
+ List result = new ArrayList<>();
+ try (BufferedReader r = new BufferedReader(
+ new InputStreamReader(
+ new GZIPInputStream(Files.newInputStream(devGz)),
+ StandardCharsets.UTF_8))) {
+ String line;
+ while ((line = r.readLine()) != null && result.size() < maxSentences) {
+ String trimmed = line.strip();
+ if (!trimmed.isEmpty()
+ && trimmed.getBytes(StandardCharsets.UTF_8).length >= 5) {
+ result.add(trimmed);
+ }
+ }
+ }
+ return result;
+ }
+}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
new file mode 100644
index 00000000000..fe99f3214e3
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
@@ -0,0 +1,1311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect.tools;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.charset.UnsupportedCharsetException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * Trains the junk detector model from per-script corpus files produced by
+ * {@link BuildJunkTrainingData}.
+ *
+ * For each script group (identified by a {@code {script}.train.gz} file),
+ * four features are trained and then combined by a per-script logistic
+ * regression classifier:
+ *
+ * - Byte-bigram log-probability: 256×256 table of log P(b|a) over
+ * consecutive byte pairs in the UTF-8 encoding.
+ * - Unicode named-block transition log-probability: N×N table of
+ * log P(block_b | block_a), where block ID is determined by
+ * {@link Character.UnicodeBlock#of(int)} — one of the ~327 named Unicode
+ * blocks plus one extra bucket for unassigned codepoints.
+ * - Control-byte fraction: fraction of bytes in control-character
+ * ranges ([0x01–0x08, 0x0B, 0x0C, 0x0E–0x1F, 0x7F]). Stored as
+ * {@code −fraction} so the z-score convention matches the other features
+ * (higher = cleaner).
+ * - Script-transition log-probability: global table of log P(script_b | script_a)
+ * over raw {@link Character.UnicodeScript} values (excluding COMMON, INHERITED, UNKNOWN),
+ * pooled across all training scripts (z4).
+ *
+ *
+ * All four features are calibrated (mu/sigma) on the dev split so their
+ * z-scores are on a common scale. A per-script binary logistic regression
+ * classifier is then fit on (z1, z2, z3, z4) using clean dev windows and corrupted
+ * versions (inject@5%, char-shuffle) as training examples. The learned weights
+ * replace the fixed equal-weight average, allowing the model to automatically
+ * downweight noisy features (e.g. high-variance block transitions for MYANMAR)
+ * and upweight informative ones (e.g. control-byte fraction for inject@0.01).
+ *
+ *
At inference, the final score is the linear combination
+ * {@code w1*z1 + w2*z2 + w3*z3 + w4*z4 + bias}; positive values indicate clean text.
+ * The natural threshold is 0 (probability 0.5); use a negative threshold for
+ * more conservative junk detection.
+ *
+ *
Output format: {@code JUNKDET1} gzipped binary, version 5.
+ * Version 1–4 files can still be loaded by {@code JunkDetector} on the JVM they were trained on.
+ *
+ *
+ * [8 bytes] magic "JUNKDET1" (ASCII)
+ * [1 byte] version = 4
+ * [4 bytes] num_scripts (big-endian int)
+ * [2 bytes] block_N — number of distinct named Unicode blocks + 1 (unassigned)
+ * // Block names section (version 5+): block_N-1 entries for JVM-independence
+ * for i in [0, block_N-1):
+ * [2 bytes] name length (big-endian ushort)
+ * [name bytes] Unicode block name (Character.UnicodeBlock.toString())
+ * // Global script-transition section (version 4+)
+ * [1 byte] num_script_buckets
+ * for each bucket:
+ * [2 bytes] name length (big-endian ushort)
+ * [name bytes] bucket name (UnicodeScript.name() or "OTHER")
+ * [num_script_buckets² × 4 bytes] script-transition log-prob table
+ * [4 bytes] mu4 (float32 big-endian)
+ * [4 bytes] sigma4 (float32 big-endian)
+ * // Per-script data (same as v3 but num_features = 4)
+ * for each script (sorted by name):
+ * [2 bytes] name length (big-endian ushort)
+ * [name bytes] script name (UTF-8)
+ * // Feature 1 — byte bigrams
+ * [4 bytes] mu1 (float32 big-endian)
+ * [4 bytes] sigma1 (float32 big-endian)
+ * [65536×4 bytes] byte-bigram log-prob table (256×256)
+ * // Feature 2 — block transitions
+ * [4 bytes] mu2 (float32 big-endian)
+ * [4 bytes] sigma2 (float32 big-endian)
+ * [block_N²×4 bytes] block-transition log-prob table
+ * // Feature 3 — control-byte fraction
+ * [4 bytes] mu3 (float32 big-endian)
+ * [4 bytes] sigma3 (float32 big-endian)
+ * // Linear classifier weights
+ * [1 byte] num_features (= 4 for v4)
+ * [4 bytes] w1 (float32 big-endian)
+ * [4 bytes] w2 (float32 big-endian)
+ * [4 bytes] w3 (float32 big-endian)
+ * [4 bytes] w4 (float32 big-endian)
+ * [4 bytes] bias (float32 big-endian)
+ *
+ */
+public class TrainJunkModel {
+
+ static final String MAGIC = "JUNKDET1";
+ static final byte VERSION = 5;
+
+ /** Number of clean (and corrupted) windows used to train the per-script classifier. */
+ static final int NUM_CLASSIFIER_SAMPLES = 500;
+
+ /** Fraction of characters replaced with control characters for inject distortion. */
+ static final double CLASSIFIER_INJECT_RATE = 0.05;
+
+ /**
+ * Minimum sigma for the control-byte feature. Because clean dev text
+ * typically has zero control bytes in every sentence, the sample standard
+ * deviation collapses to 0 and would be clamped to 1.0 by the generic
+ * {@link #muSigma} helper — making the feature useless. This floor
+ * ensures a 1% control-byte injection ({@code inject@0.01}) produces
+ * approximately z = −2, providing meaningful signal.
+ */
+ static final float CONTROL_BYTE_MIN_SIGMA = 0.005f;
+
+ /**
+ * Codec pairs used to build wrong-codec remap tables for training.
+ * Each entry is {sourceCodec, wrongCodec}: text encoded in sourceCodec but
+ * decoded as wrongCodec. Pairs within the same script family (e.g. CP1250↔CP1252)
+ * produce wrong-accent distortions that shift characters between Unicode blocks
+ * while staying in LATIN. Cross-script pairs (CP1252↔CP1255) additionally change
+ * the Unicode script, which z4 also detects.
+ */
+ static final String[][] WRONG_CODEC_PAIRS = {
+ {"windows-1252", "windows-1250"}, // Western ↔ Central European (wrong accents)
+ {"windows-1250", "windows-1252"}, // reverse
+ {"windows-1252", "windows-1257"}, // Western ↔ Baltic (wrong accents)
+ {"windows-1257", "windows-1252"}, // reverse
+ {"windows-1252", "windows-1254"}, // Western ↔ Turkish (wrong accents)
+ {"windows-1251", "windows-1252"}, // Cyrillic → Latin (cross-script)
+ {"windows-1252", "windows-1251"}, // Latin → Cyrillic (cross-script)
+ {"windows-1253", "windows-1252"}, // Greek → Latin (cross-script)
+ {"windows-1252", "windows-1253"}, // Latin → Greek (cross-script)
+ {"windows-1255", "windows-1252"}, // Hebrew → Latin (cross-script)
+ {"windows-1252", "windows-1255"}, // Latin → Hebrew (the German vcard case)
+ };
+
+ /**
+ * Target byte-lengths used for calibration sampling, matching the evaluator defaults.
+ */
+ static final int[] CALIB_LENGTHS = {15, 30, 50, 100, 200};
+
+ /**
+ * Number of random byte-window samples drawn from the dev set for calibration.
+ */
+ static final int CALIB_SAMPLES = 5000;
+
+ public static void main(String[] args) throws IOException {
+ Path dataDir = Paths.get(System.getProperty("user.home"),
+ "datasets", "madlad", "junkdetect");
+ Path output = dataDir.resolve("junkdetect.bin");
+
+ for (int i = 0; i < args.length; i++) {
+ switch (args[i]) {
+ case "--data-dir":
+ dataDir = Paths.get(args[++i]);
+ break;
+ case "--output":
+ output = Paths.get(args[++i]);
+ break;
+ default:
+ System.err.println("Unknown argument: " + args[i]);
+ printUsage();
+ System.exit(1);
+ }
+ }
+
+ System.out.println("=== TrainJunkModel (v5) ===");
+ System.out.println(" data-dir: " + dataDir);
+ System.out.println(" output: " + output);
+
+ if (!Files.isDirectory(dataDir)) {
+ System.err.println("ERROR: data-dir not found: " + dataDir);
+ System.exit(1);
+ }
+
+ System.out.print("Building Unicode named-block index... ");
+ long t0 = System.currentTimeMillis();
+ Map blockIndex = buildBlockIndex();
+ int blockN = blockIndex.size() + 1;
+ System.out.printf("%d named blocks → table size %d×%d (%dms)%n",
+ blockIndex.size(), blockN, blockN, System.currentTimeMillis() - t0);
+
+ TreeMap bigramTables = new TreeMap<>();
+ TreeMap bigramCalibrations = new TreeMap<>();
+ TreeMap blockTables = new TreeMap<>();
+ TreeMap blockCalibrations = new TreeMap<>();
+ TreeMap controlCalibrations = new TreeMap<>();
+ TreeMap classifierWeights = new TreeMap<>();
+ TreeMap devFilePaths = new TreeMap<>();
+ List allTrainFiles = new ArrayList<>();
+ List allDevFiles = new ArrayList<>();
+
+ List trainFiles;
+ try (var stream = Files.list(dataDir)) {
+ trainFiles = stream
+ .filter(p -> p.getFileName().toString().endsWith(".train.gz"))
+ .sorted()
+ .toList();
+ }
+
+ if (trainFiles.isEmpty()) {
+ System.err.println("ERROR: no *.train.gz files found in " + dataDir);
+ System.exit(1);
+ }
+
+ // -----------------------------------------------------------------------
+ // Phase 1 — per-script bigram tables, block tables, calibrations
+ // -----------------------------------------------------------------------
+ System.out.println("\n--- Phase 1: per-script tables and calibrations ---");
+ for (Path trainFile : trainFiles) {
+ String filename = trainFile.getFileName().toString();
+ String script = filename.substring(0, filename.length() - ".train.gz".length())
+ .toUpperCase();
+ Path devFile = trainFile.getParent().resolve(
+ filename.replace(".train.gz", ".dev.gz"));
+
+ System.out.printf("%n [%s]%n", script);
+ allTrainFiles.add(trainFile);
+
+ t0 = System.currentTimeMillis();
+ System.out.print(" Training byte-bigram table... ");
+ float[] bigramTable = trainBigramTable(trainFile);
+ System.out.printf("done (%dms)%n", System.currentTimeMillis() - t0);
+
+ t0 = System.currentTimeMillis();
+ System.out.print(" Training named-block table... ");
+ float[] blockTable = trainBlockTable(trainFile, blockIndex, blockN);
+ System.out.printf("done (%dms)%n", System.currentTimeMillis() - t0);
+
+ float[] bigramCal = new float[]{0f, 1f};
+ float[] blockCal = new float[]{0f, 1f};
+ float[] controlCal = new float[]{0f, 1f};
+
+ if (Files.exists(devFile)) {
+ t0 = System.currentTimeMillis();
+ System.out.print(" Calibrating byte bigrams on dev... ");
+ bigramCal = computeBigramCalibration(devFile, bigramTable);
+ System.out.printf("done — mu=%.4f sigma=%.4f (%dms)%n",
+ bigramCal[0], bigramCal[1], System.currentTimeMillis() - t0);
+
+ t0 = System.currentTimeMillis();
+ System.out.print(" Calibrating named blocks on dev... ");
+ blockCal = computeBlockCalibration(devFile, blockTable, blockIndex, blockN);
+ System.out.printf("done — mu=%.4f sigma=%.4f (%dms)%n",
+ blockCal[0], blockCal[1], System.currentTimeMillis() - t0);
+
+ t0 = System.currentTimeMillis();
+ System.out.print(" Calibrating control bytes on dev...");
+ controlCal = computeControlByteCalibration(devFile);
+ System.out.printf("done — mu=%.6f sigma=%.6f (%dms)%n",
+ controlCal[0], controlCal[1], System.currentTimeMillis() - t0);
+
+ devFilePaths.put(script, devFile);
+ allDevFiles.add(devFile);
+ } else {
+ System.out.println(" WARNING: no dev file found, using uncalibrated defaults");
+ }
+
+ bigramTables.put(script, bigramTable);
+ bigramCalibrations.put(script, bigramCal);
+ blockTables.put(script, blockTable);
+ blockCalibrations.put(script, blockCal);
+ controlCalibrations.put(script, controlCal);
+ // Placeholder — set in phase 3
+ classifierWeights.put(script, new float[]{1f / 4, 1f / 4, 1f / 4, 1f / 4, 0f});
+ }
+
+ // -----------------------------------------------------------------------
+ // Phase 2 — global script-transition table
+ // -----------------------------------------------------------------------
+ System.out.println("\n--- Phase 2: global script-transition table ---");
+ List scriptBuckets = buildScriptBuckets();
+ int numScriptBuckets = scriptBuckets.size();
+ Map scriptBucketMap = new LinkedHashMap<>();
+ for (int i = 0; i < numScriptBuckets; i++) {
+ scriptBucketMap.put(scriptBuckets.get(i), i);
+ }
+ System.out.printf(" %d script buckets (including OTHER)%n", numScriptBuckets);
+
+ t0 = System.currentTimeMillis();
+ System.out.print(" Training script-transition table... ");
+ float[] scriptTransTable = trainScriptTransitionTable(allTrainFiles, scriptBucketMap, numScriptBuckets);
+ System.out.printf("done (%dms)%n", System.currentTimeMillis() - t0);
+
+ t0 = System.currentTimeMillis();
+ System.out.print(" Calibrating script transitions... ");
+ float[] scriptTransCal = calibrateScriptTransitions(allDevFiles, scriptTransTable,
+ scriptBucketMap, numScriptBuckets);
+ System.out.printf("done — mu=%.4f sigma=%.4f (%dms)%n",
+ scriptTransCal[0], scriptTransCal[1], System.currentTimeMillis() - t0);
+
+ t0 = System.currentTimeMillis();
+ System.out.print(" Collecting per-script codepoint pools... ");
+ Map> scriptCodepoints = collectScriptCodepoints(allTrainFiles, 200);
+ System.out.printf("done — %d scripts (%dms)%n",
+ scriptCodepoints.size(), System.currentTimeMillis() - t0);
+
+ System.out.print(" Building wrong-codec remap tables... ");
+ List> remapTables = new ArrayList<>();
+ for (String[] pair : WRONG_CODEC_PAIRS) {
+ Map table = buildRemapTable(pair[0], pair[1]);
+ if (!table.isEmpty()) remapTables.add(table);
+ }
+ System.out.printf("%d tables built%n", remapTables.size());
+
+ // -----------------------------------------------------------------------
+ // Phase 3 — per-script linear classifiers (now with z4)
+ // -----------------------------------------------------------------------
+ System.out.println("\n--- Phase 3: per-script linear classifiers (z1,z2,z3,z4) ---");
+ for (String script : bigramTables.keySet()) {
+ Path devFile = devFilePaths.get(script);
+ if (devFile == null) {
+ System.out.printf(" [%s] WARNING: no dev file, keeping equal-weight defaults%n", script);
+ continue;
+ }
+ t0 = System.currentTimeMillis();
+ System.out.printf(" [%s] training classifier... ", script);
+ float[] weights = trainClassifier(devFile,
+ bigramTables.get(script), bigramCalibrations.get(script),
+ blockTables.get(script), blockCalibrations.get(script),
+ controlCalibrations.get(script), blockIndex, blockN,
+ scriptTransTable, scriptTransCal, scriptBucketMap, numScriptBuckets,
+ scriptCodepoints, remapTables);
+ classifierWeights.put(script, weights);
+ System.out.printf("done — w=[%.3f,%.3f,%.3f,%.3f] bias=%.3f (%dms)%n",
+ weights[0], weights[1], weights[2], weights[3], weights[4],
+ System.currentTimeMillis() - t0);
+ }
+
+ System.out.printf("%nWriting model (%d scripts, blockN=%d, scriptBuckets=%d) → %s%n",
+ bigramTables.size(), blockN, numScriptBuckets, output);
+ saveModel(bigramTables, bigramCalibrations,
+ blockTables, blockCalibrations,
+ controlCalibrations, classifierWeights,
+ blockIndex, blockN, scriptBuckets, scriptTransTable, scriptTransCal, output);
+ System.out.printf("Model size: %,d bytes (%.1f MB)%n",
+ Files.size(output), Files.size(output) / 1_000_000.0);
+ System.out.println("Done.");
+ }
+
+ // -----------------------------------------------------------------------
+ // Block index
+ // -----------------------------------------------------------------------
+
+ /**
+ * Builds a stable ordered mapping from {@link Character.UnicodeBlock} to integer index
+ * by scanning all valid Unicode codepoints in order (U+0000 to U+10FFFF) and
+ * recording each block's first occurrence.
+ *
+ * The resulting map has {@code size()} entries (one per named block).
+ * Callers should reserve index {@code size()} as the "unassigned" bucket
+ * (for codepoints where {@code UnicodeBlock.of(cp)} returns null).
+ *
+ * @return immutable ordered map: UnicodeBlock → integer index [0, size)
+ */
+ static Map buildBlockIndex() {
+ LinkedHashMap index = new LinkedHashMap<>();
+ for (int cp = 0; cp <= 0x10FFFF; cp++) {
+ Character.UnicodeBlock b = Character.UnicodeBlock.of(cp);
+ if (b != null) index.putIfAbsent(b, index.size());
+ }
+ return Collections.unmodifiableMap(index);
+ }
+
+ // -----------------------------------------------------------------------
+ // Training
+ // -----------------------------------------------------------------------
+
+ /**
+ * Trains a 256×256 byte-bigram log-probability table from a gzipped sentence file.
+ *
+ * @return float[65536] where index {@code a*256+b} = log P(b|a)
+ */
+ static float[] trainBigramTable(Path trainGz) throws IOException {
+ long[] counts = new long[65536];
+ long totalBigrams = 0;
+ long sentences = 0;
+
+ try (BufferedReader r = openGzipped(trainGz)) {
+ String line;
+ while ((line = r.readLine()) != null) {
+ byte[] bytes = line.getBytes(StandardCharsets.UTF_8);
+ for (int i = 0; i + 1 < bytes.length; i++) {
+ counts[((bytes[i] & 0xFF) << 8) | (bytes[i + 1] & 0xFF)]++;
+ totalBigrams++;
+ }
+ sentences++;
+ }
+ }
+
+ System.out.printf(" %,d sentences, %,d byte bigrams%n", sentences, totalBigrams);
+ return laplaceSmoothLogProb(counts, 256);
+ }
+
+ /**
+ * Trains a {@code blockN×blockN} named-Unicode-block transition log-probability table.
+ *
+ * @param blockIndex ordered mapping from UnicodeBlock to index [0, blockIndex.size())
+ * @param blockN blockIndex.size() + 1 (includes the null bucket)
+ * @return float[blockN*blockN] where index {@code a*blockN+b} = log P(block_b | block_a)
+ */
+ static float[] trainBlockTable(Path trainGz,
+ Map blockIndex,
+ int blockN) throws IOException {
+ long[] counts = new long[blockN * blockN];
+ int nullId = blockN - 1;
+ long totalBigrams = 0;
+ long sentences = 0;
+
+ try (BufferedReader r = openGzipped(trainGz)) {
+ String line;
+ while ((line = r.readLine()) != null) {
+ int prev = -1;
+ for (int i = 0; i < line.length(); ) {
+ int cp = line.codePointAt(i);
+ Character.UnicodeBlock b = Character.UnicodeBlock.of(cp);
+ int blockId = b != null ? blockIndex.getOrDefault(b, nullId) : nullId;
+ if (prev >= 0) {
+ counts[prev * blockN + blockId]++;
+ totalBigrams++;
+ }
+ prev = blockId;
+ i += Character.charCount(cp);
+ }
+ sentences++;
+ }
+ }
+
+ System.out.printf(" %,d sentences, %,d block bigrams%n", sentences, totalBigrams);
+ return laplaceSmoothLogProb(counts, blockN);
+ }
+
+ /**
+ * Applies Laplace (add-1) smoothing per row and converts to log-probabilities.
+ *
+ * @param counts raw bigram counts, length = size*size
+ * @param size number of distinct symbols (256 for byte table, blockN for block table)
+ * @return float[size*size] log-prob table
+ */
+ private static float[] laplaceSmoothLogProb(long[] counts, int size) {
+ float[] table = new float[size * size];
+ for (int a = 0; a < size; a++) {
+ long rowTotal = size; // add-1 pseudocount for each possible next symbol
+ for (int b = 0; b < size; b++) {
+ rowTotal += counts[a * size + b];
+ }
+ for (int b = 0; b < size; b++) {
+ table[a * size + b] =
+ (float) Math.log((counts[a * size + b] + 1.0) / rowTotal);
+ }
+ }
+ return table;
+ }
+
+ // -----------------------------------------------------------------------
+ // Calibration
+ // -----------------------------------------------------------------------
+
+ /**
+ * Loads all sentences from a gzipped file and draws {@code nSamples} random
+ * byte-window substrings of target lengths cycling through {@code lengths}.
+ *
+ * This mirrors the evaluator's {@code pickSubstring}: takes a random
+ * UTF-8-aligned window of {@code targetLen} bytes from a randomly chosen
+ * sentence, or the whole sentence if it is shorter.
+ *
+ * @param nSamples number of windows to sample
+ * @param lengths target byte-lengths to cycle through (round-robin)
+ * @param seed RNG seed for reproducibility
+ */
+ static List sampleSubstrings(Path devGz, int nSamples,
+ int[] lengths, long seed) throws IOException {
+ List sentenceBytes = new ArrayList<>();
+ try (BufferedReader r = openGzipped(devGz)) {
+ String line;
+ while ((line = r.readLine()) != null) {
+ byte[] b = line.getBytes(StandardCharsets.UTF_8);
+ if (b.length >= 2) sentenceBytes.add(b);
+ }
+ }
+ if (sentenceBytes.isEmpty()) return Collections.emptyList();
+
+ Random rng = new Random(seed);
+ List result = new ArrayList<>(nSamples);
+ for (int i = 0; i < nSamples; i++) {
+ byte[] bytes = sentenceBytes.get(rng.nextInt(sentenceBytes.size()));
+ int targetLen = lengths[i % lengths.length];
+
+ if (bytes.length <= targetLen) {
+ result.add(new String(bytes, StandardCharsets.UTF_8));
+ continue;
+ }
+ int start = rng.nextInt(bytes.length - targetLen);
+ while (start > 0 && (bytes[start] & 0xC0) == 0x80) {
+ start--;
+ }
+ int end = Math.min(start + targetLen, bytes.length);
+ while (end < bytes.length && (bytes[end] & 0xC0) == 0x80) {
+ end++;
+ }
+ result.add(new String(bytes, start, end - start, StandardCharsets.UTF_8));
+ }
+ return result;
+ }
+
+ /** @return float[2] = {mu, sigma} of byte-bigram mean log-prob on dev windows */
+ static float[] computeBigramCalibration(Path devGz, float[] bigramTable) throws IOException {
+ List windows = sampleSubstrings(devGz, CALIB_SAMPLES, CALIB_LENGTHS, 42);
+ List scores = new ArrayList<>(windows.size());
+ for (String window : windows) {
+ byte[] bytes = window.getBytes(StandardCharsets.UTF_8);
+ if (bytes.length < 2) continue;
+ double sum = 0;
+ for (int i = 0; i + 1 < bytes.length; i++) {
+ sum += bigramTable[((bytes[i] & 0xFF) << 8) | (bytes[i + 1] & 0xFF)];
+ }
+ scores.add(sum / (bytes.length - 1));
+ }
+ System.out.printf(" %,d dev windows%n", scores.size());
+ return muSigma(scores);
+ }
+
+ /** @return float[2] = {mu, sigma} of block-transition mean log-prob on dev windows */
+ static float[] computeBlockCalibration(Path devGz, float[] blockTable,
+ Map blockIndex,
+ int blockN) throws IOException {
+ List windows = sampleSubstrings(devGz, CALIB_SAMPLES, CALIB_LENGTHS, 43);
+ List scores = new ArrayList<>(windows.size());
+ int nullId = blockN - 1;
+ for (String window : windows) {
+ int[] ids = new int[window.length()];
+ int len = 0;
+ for (int i = 0; i < window.length(); ) {
+ int cp = window.codePointAt(i);
+ Character.UnicodeBlock b = Character.UnicodeBlock.of(cp);
+ ids[len++] = b != null ? blockIndex.getOrDefault(b, nullId) : nullId;
+ i += Character.charCount(cp);
+ }
+ if (len < 2) continue;
+ double sum = 0;
+ for (int i = 0; i + 1 < len; i++) {
+ sum += blockTable[ids[i] * blockN + ids[i + 1]];
+ }
+ scores.add(sum / (len - 1));
+ }
+ System.out.printf(" %,d dev windows%n", scores.size());
+ return muSigma(scores);
+ }
+
+ /** @return float[2] = {mu, sigma} of control-byte fraction on dev windows */
+ static float[] computeControlByteCalibration(Path devGz) throws IOException {
+ List windows = sampleSubstrings(devGz, CALIB_SAMPLES, CALIB_LENGTHS, 44);
+ List scores = new ArrayList<>(windows.size());
+ for (String window : windows) {
+ byte[] bytes = window.getBytes(StandardCharsets.UTF_8);
+ if (bytes.length == 0) continue;
+ long controlCount = 0;
+ for (byte b : bytes) {
+ if (isControlByte(b & 0xFF)) controlCount++;
+ }
+ scores.add(-(double) controlCount / bytes.length);
+ }
+ System.out.printf(" %,d dev windows%n", scores.size());
+ if (scores.isEmpty()) return new float[]{0f, CONTROL_BYTE_MIN_SIGMA};
+ double mu = scores.stream().mapToDouble(Double::doubleValue).average().orElse(0);
+ double variance = scores.stream()
+ .mapToDouble(s -> (s - mu) * (s - mu))
+ .average().orElse(0);
+ double sigma = Math.max(Math.sqrt(variance), CONTROL_BYTE_MIN_SIGMA);
+ return new float[]{(float) mu, (float) sigma};
+ }
+
+ // -----------------------------------------------------------------------
+ // Linear classifier training
+ // -----------------------------------------------------------------------
+
+ /**
+ * Trains a per-script binary logistic regression classifier on (z1, z2, z3, z4).
+ *
+ * Clean examples: {@link #NUM_CLASSIFIER_SAMPLES} random dev windows (seed 100).
+ * Corrupted examples: same count, cycling through four distortions (seed 102):
+ *
+ * - inject@5% control chars
+ * - char-shuffle
+ * - cross-script substitution — replaces ~5% of characters with codepoints from
+ * foreign scripts, simulating charset encoding errors such as German umlauts
+ * becoming Hebrew letters when CP1252 text is decoded as CP1255
+ * - wrong-codec remap — replaces ~5% of characters using a random pre-computed
+ * charset remap table (e.g. CP1252→CP1250 for wrong accents, CP1252→CP1255
+ * for script crossings), simulating real-world charset misdetection
+ *
+ *
+ * @param remapTables list of pre-built wrong-codec remap tables from {@link #buildRemapTable}
+ * @return float[5] = {w1, w2, w3, w4, bias} — classifier weights; positive logit = clean
+ */
+ static float[] trainClassifier(Path devGz,
+ float[] bigramTable, float[] bigramCal,
+ float[] blockTable, float[] blockCal,
+ float[] controlCal,
+ Map blockIndex,
+ int blockN,
+ float[] scriptTransTable, float[] scriptTransCal,
+ Map scriptBucketMap, int numScriptBuckets,
+ Map> scriptCodepoints,
+ List> remapTables)
+ throws IOException {
+ int nEach = NUM_CLASSIFIER_SAMPLES;
+
+ // Clean windows
+ List cleanWindows = sampleSubstrings(devGz, nEach, CALIB_LENGTHS, 100);
+
+ // Corrupted windows: sample base windows (seed 101), then distort
+ // Four-way rotation: inject / shuffle / cross-script / wrong-codec remap
+ List baseWindows = sampleSubstrings(devGz, nEach, CALIB_LENGTHS, 101);
+ Random rng = new Random(102);
+ List corruptedWindows = new ArrayList<>(nEach);
+ for (int i = 0; i < baseWindows.size(); i++) {
+ String w = baseWindows.get(i);
+ switch (i % 4) {
+ case 0:
+ corruptedWindows.add(injectControlChars(w, CLASSIFIER_INJECT_RATE, rng));
+ break;
+ case 1:
+ corruptedWindows.add(shuffleChars(w, rng));
+ break;
+ case 2:
+ corruptedWindows.add(injectCrossScriptChars(w, CLASSIFIER_INJECT_RATE, rng,
+ scriptCodepoints));
+ break;
+ default:
+ if (!remapTables.isEmpty()) {
+ Map table =
+ remapTables.get(rng.nextInt(remapTables.size()));
+ corruptedWindows.add(wrongCodecRemap(w, table, CLASSIFIER_INJECT_RATE, rng));
+ } else {
+ corruptedWindows.add(injectControlChars(w, CLASSIFIER_INJECT_RATE, rng));
+ }
+ break;
+ }
+ }
+
+ // Build (z1, z2, z3, z4) feature matrix
+ List features = new ArrayList<>(cleanWindows.size() + corruptedWindows.size());
+ List labels = new ArrayList<>(cleanWindows.size() + corruptedWindows.size());
+
+ for (String w : cleanWindows) {
+ features.add(extractFeatures(w, bigramTable, bigramCal,
+ blockTable, blockCal, blockN, controlCal, blockIndex,
+ scriptTransTable, scriptTransCal, scriptBucketMap, numScriptBuckets));
+ labels.add(1); // clean
+ }
+ for (String w : corruptedWindows) {
+ features.add(extractFeatures(w, bigramTable, bigramCal,
+ blockTable, blockCal, blockN, controlCal, blockIndex,
+ scriptTransTable, scriptTransCal, scriptBucketMap, numScriptBuckets));
+ labels.add(0); // corrupted
+ }
+
+ float[] weights = fitLogisticRegression(features, labels, 4);
+
+ // Calibrate bias using only short (len=15) windows so that FPR ≤ 2.5%
+ // even at the worst-case (shortest) window length.
+ List shortWindows = sampleSubstrings(devGz, nEach, new int[]{15}, 200);
+ List shortLogits = new ArrayList<>(shortWindows.size());
+ int nFeat = weights.length - 1;
+ for (String w : shortWindows) {
+ float[] x = extractFeatures(w, bigramTable, bigramCal,
+ blockTable, blockCal, blockN, controlCal, blockIndex,
+ scriptTransTable, scriptTransCal, scriptBucketMap, numScriptBuckets);
+ float logit = weights[nFeat];
+ for (int j = 0; j < nFeat; j++) logit += weights[j] * x[j];
+ shortLogits.add(logit);
+ }
+ if (!shortLogits.isEmpty()) {
+ Collections.sort(shortLogits);
+ int pIdx = (int) (0.025 * shortLogits.size());
+ float p025 = shortLogits.get(Math.max(0, pIdx));
+ weights[nFeat] -= p025;
+ }
+
+ return weights;
+ }
+
+ /**
+ * Extracts calibrated z-scores (z1, z2, z3, z4) for a single text window.
+ *
+ * @return float[4] = {z1_bigram, z2_block, z3_control, z4_scriptTrans}
+ */
+ static float[] extractFeatures(String window,
+ float[] bigramTable, float[] bigramCal,
+ float[] blockTable, float[] blockCal,
+ int blockN, float[] controlCal,
+ Map blockIndex,
+ float[] scriptTransTable, float[] scriptTransCal,
+ Map scriptBucketMap, int numScriptBuckets) {
+ byte[] utf8 = window.getBytes(StandardCharsets.UTF_8);
+
+ // z1: byte-bigram mean log-prob
+ float z1 = 0f;
+ if (utf8.length >= 2) {
+ double sum = 0;
+ int count = 0;
+ for (int i = 0; i + 1 < utf8.length; i++) {
+ sum += bigramTable[((utf8[i] & 0xFF) << 8) | (utf8[i + 1] & 0xFF)];
+ count++;
+ }
+ z1 = ((float) (sum / count) - bigramCal[0]) / bigramCal[1];
+ }
+
+ // z2: block-transition mean log-prob
+ float z2 = 0f;
+ if (blockTable != null && window.length() >= 2) {
+ int nullId = blockN - 1;
+ int prev = -1;
+ double sum = 0;
+ int count = 0;
+ for (int i = 0; i < window.length(); ) {
+ int cp = window.codePointAt(i);
+ Character.UnicodeBlock b = Character.UnicodeBlock.of(cp);
+ int blockId = b != null ? blockIndex.getOrDefault(b, nullId) : nullId;
+ if (prev >= 0) {
+ sum += blockTable[prev * blockN + blockId];
+ count++;
+ }
+ prev = blockId;
+ i += Character.charCount(cp);
+ }
+ if (count > 0) {
+ z2 = ((float) (sum / count) - blockCal[0]) / blockCal[1];
+ }
+ }
+
+ // z3: control-byte fraction (stored as −fraction, so higher = cleaner)
+ float z3 = 0f;
+ if (utf8.length > 0 && controlCal != null) {
+ long controlCount = 0;
+ for (byte b : utf8) {
+ if (isControlByte(b & 0xFF)) controlCount++;
+ }
+ float score = -(float) controlCount / utf8.length;
+ z3 = (score - controlCal[0]) / controlCal[1];
+ }
+
+ // z4: script-transition mean log-prob (raw UnicodeScript, no model fallback)
+ float z4 = 0f;
+ if (scriptTransTable != null && scriptTransCal != null) {
+ double raw = rawScriptTransitionLogProb(window, scriptTransTable,
+ scriptBucketMap, numScriptBuckets, numScriptBuckets - 1);
+ if (!Double.isNaN(raw)) {
+ z4 = ((float) raw - scriptTransCal[0]) / scriptTransCal[1];
+ }
+ }
+
+ return new float[]{z1, z2, z3, z4};
+ }
+
+ /**
+ * Replaces a random fraction of characters with Unicode control characters.
+ * Operates at the codepoint level to produce well-formed strings with actual
+ * control bytes in the UTF-8 encoding.
+ *
+ * @param rate fraction of characters to replace [0, 1]
+ */
+ static String injectControlChars(String text, double rate, Random rng) {
+ if (text.isEmpty()) return text;
+ int[] codepoints = text.codePoints().toArray();
+ int[] controlChars = {0x01, 0x02, 0x03, 0x04, 0x07, 0x0B, 0x0C, 0x0E, 0x0F, 0x1A, 0x1B, 0x7F};
+ for (int i = 0; i < codepoints.length; i++) {
+ if (rng.nextDouble() < rate) {
+ codepoints[i] = controlChars[rng.nextInt(controlChars.length)];
+ }
+ }
+ return new String(codepoints, 0, codepoints.length);
+ }
+
+ /**
+ * Randomly permutes all characters in the text (Fisher-Yates shuffle).
+ * Destroys both bigram and block-transition structure while preserving script
+ * distribution, making it a good test of transition-based features.
+ */
+ static String shuffleChars(String text, Random rng) {
+ if (text.isEmpty()) return text;
+ int[] codepoints = text.codePoints().toArray();
+ for (int i = codepoints.length - 1; i > 0; i--) {
+ int j = rng.nextInt(i + 1);
+ int tmp = codepoints[i];
+ codepoints[i] = codepoints[j];
+ codepoints[j] = tmp;
+ }
+ return new String(codepoints, 0, codepoints.length);
+ }
+
+ /**
+ * Fits a binary logistic regression classifier on the given feature matrix.
+ *
+ * Label convention: 1 = clean, 0 = corrupted. At inference, positive
+ * logit → clean text; negative logit → corrupted text.
+ *
+ *
Uses full-batch gradient descent with L2 regularization and a
+ * non-negativity constraint on feature weights (projected gradient descent:
+ * {@code w[j] = max(0, w[j])} after each step). The constraint enforces the
+ * semantic invariant that every feature is calibrated so higher = cleaner;
+ * a negative weight would mean "more unusual transitions → cleaner text", which
+ * is semantically wrong and causes pathological behaviour when collinear features
+ * (e.g. z2 block-transitions and z4 script-transitions) are both present.
+ * The bias term is unconstrained. Converges reliably for {@code numFeatures} ≤ 10
+ * with the default hyperparameters.
+ *
+ * @param features list of feature vectors, each of length {@code numFeatures}
+ * @param labels parallel list of labels (0 or 1)
+ * @param numFeatures number of features
+ * @return float[numFeatures + 1] = {w[0], ..., w[numFeatures-1], bias}
+ */
+ static float[] fitLogisticRegression(List features, List labels,
+ int numFeatures) {
+ int n = features.size();
+ float[] w = new float[numFeatures]; // zero-initialized
+ float bias = 0f;
+
+ if (n == 0) {
+ float[] result = new float[numFeatures + 1];
+ for (int i = 0; i < numFeatures; i++) result[i] = 1f / numFeatures;
+ return result;
+ }
+
+ float lr = 0.05f;
+ float lambda = 0.01f; // L2 regularization
+ int epochs = 500;
+
+ for (int epoch = 0; epoch < epochs; epoch++) {
+ double[] gradW = new double[numFeatures];
+ double gradB = 0;
+
+ for (int i = 0; i < n; i++) {
+ float[] x = features.get(i);
+ int y = labels.get(i);
+
+ double logit = bias;
+ for (int j = 0; j < numFeatures; j++) logit += w[j] * x[j];
+
+ // Numerically stable sigmoid
+ double p;
+ if (logit >= 0) {
+ double e = Math.exp(-logit);
+ p = 1.0 / (1.0 + e);
+ } else {
+ double e = Math.exp(logit);
+ p = e / (1.0 + e);
+ }
+
+ double err = p - y;
+ for (int j = 0; j < numFeatures; j++) gradW[j] += err * x[j];
+ gradB += err;
+ }
+
+ for (int j = 0; j < numFeatures; j++) {
+ w[j] -= lr * (float) (gradW[j] / n + lambda * w[j]);
+ w[j] = Math.max(0f, w[j]); // projected gradient: feature weights are non-negative by design
+ }
+ bias -= lr * (float) (gradB / n);
+ }
+
+ float[] result = new float[numFeatures + 1];
+ for (int j = 0; j < numFeatures; j++) result[j] = w[j];
+ result[numFeatures] = bias;
+ return result;
+ }
+
+ // -----------------------------------------------------------------------
+ // Model serialisation
+ // -----------------------------------------------------------------------
+
+ /**
+ * Writes the trained model (version 4) to a gzipped binary file.
+ *
+ * Format documented in the class Javadoc. All multi-byte integers are
+ * big-endian; floats are IEEE 754 big-endian.
+ *
+ * @param classifierWeights per-script float[5] = {w1, w2, w3, w4, bias}
+ * @param blockN the block table dimension (blockIndex.size() + 1)
+ * @param scriptBuckets ordered list of script bucket names (last = "OTHER")
+ * @param scriptTransTable global script-transition log-prob table
+ * @param scriptTransCal float[2] = {mu, sigma} for script-transition feature
+ */
+ static void saveModel(TreeMap bigramTables,
+ TreeMap bigramCalibrations,
+ TreeMap blockTables,
+ TreeMap blockCalibrations,
+ TreeMap controlCalibrations,
+ TreeMap classifierWeights,
+ Map blockIndex,
+ int blockN,
+ List scriptBuckets,
+ float[] scriptTransTable,
+ float[] scriptTransCal,
+ Path output) throws IOException {
+ try (DataOutputStream dos = new DataOutputStream(
+ new GZIPOutputStream(Files.newOutputStream(output)))) {
+
+ dos.write(MAGIC.getBytes(StandardCharsets.UTF_8));
+ dos.writeByte(VERSION);
+ dos.writeInt(bigramTables.size());
+ dos.writeShort(blockN);
+
+ // Block names section (v5+): write ordered block names for JVM-independence
+ String[] blockNames = new String[blockN - 1];
+ for (Map.Entry e : blockIndex.entrySet()) {
+ blockNames[e.getValue()] = e.getKey().toString();
+ }
+ for (String name : blockNames) {
+ byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8);
+ dos.writeShort(nameBytes.length);
+ dos.write(nameBytes);
+ }
+
+ // Global script-transition section (v4+)
+ int numBuckets = scriptBuckets.size();
+ dos.writeByte(numBuckets);
+ for (String bucketName : scriptBuckets) {
+ byte[] nameBytes = bucketName.getBytes(StandardCharsets.UTF_8);
+ dos.writeShort(nameBytes.length);
+ dos.write(nameBytes);
+ }
+ dos.write(toBytes(scriptTransTable));
+ dos.writeFloat(scriptTransCal[0]); // mu
+ dos.writeFloat(scriptTransCal[1]); // sigma
+
+ for (var entry : bigramTables.entrySet()) {
+ String script = entry.getKey();
+ float[] bigramTable = entry.getValue();
+ float[] bigramCal = bigramCalibrations.getOrDefault(script, new float[]{0f, 1f});
+ float[] blockTable = blockTables.getOrDefault(script, new float[blockN * blockN]);
+ float[] blockCal = blockCalibrations.getOrDefault(script, new float[]{0f, 1f});
+ float[] controlCal = controlCalibrations.getOrDefault(script, new float[]{0f, 1f});
+ float[] weights = classifierWeights.getOrDefault(script,
+ new float[]{1f / 4, 1f / 4, 1f / 4, 1f / 4, 0f});
+
+ byte[] nameBytes = script.getBytes(StandardCharsets.UTF_8);
+ dos.writeShort(nameBytes.length);
+ dos.write(nameBytes);
+
+ dos.writeFloat(bigramCal[0]);
+ dos.writeFloat(bigramCal[1]);
+ dos.write(toBytes(bigramTable));
+
+ dos.writeFloat(blockCal[0]);
+ dos.writeFloat(blockCal[1]);
+ dos.write(toBytes(blockTable));
+
+ dos.writeFloat(controlCal[0]);
+ dos.writeFloat(controlCal[1]);
+
+ int numFeatures = weights.length - 1;
+ dos.writeByte(numFeatures);
+ for (float v : weights) dos.writeFloat(v);
+ }
+ }
+ }
+
+ private static byte[] toBytes(float[] table) {
+ ByteBuffer buf = ByteBuffer.allocate(table.length * 4).order(ByteOrder.BIG_ENDIAN);
+ for (float v : table) buf.putFloat(v);
+ return buf.array();
+ }
+
+ // -----------------------------------------------------------------------
+ // Helpers
+ // -----------------------------------------------------------------------
+
+ /**
+ * Returns true if the byte value is a control character that should not appear
+ * in natural-language UTF-8 text: {@code [0x01–0x08, 0x0B, 0x0C, 0x0E–0x1F, 0x7F]}.
+ *
+ * Excluded: 0x00 (null), 0x09 (tab), 0x0A (newline), 0x0D (carriage return)
+ * — all appear legitimately in text.
+ */
+ static boolean isControlByte(int b) {
+ return (b >= 0x01 && b <= 0x08)
+ || b == 0x0B || b == 0x0C
+ || (b >= 0x0E && b <= 0x1F)
+ || b == 0x7F;
+ }
+
+ private static float[] muSigma(List scores) {
+ if (scores.isEmpty()) return new float[]{0f, 1f};
+ double mu = scores.stream().mapToDouble(Double::doubleValue).average().orElse(0);
+ double variance = scores.stream()
+ .mapToDouble(s -> (s - mu) * (s - mu))
+ .average().orElse(1.0);
+ double sigma = Math.sqrt(variance);
+ if (sigma < 1e-9) sigma = 1.0;
+ return new float[]{(float) mu, (float) sigma};
+ }
+
+ static BufferedReader openGzipped(Path path) throws IOException {
+ return new BufferedReader(
+ new InputStreamReader(
+ new GZIPInputStream(Files.newInputStream(path)),
+ StandardCharsets.UTF_8));
+ }
+
+ /**
+ * Returns an ordered list of all recognized {@link Character.UnicodeScript} names
+ * (excluding COMMON, INHERITED, UNKNOWN pseudo-scripts), sorted alphabetically,
+ * with "OTHER" appended as the final fallback bucket.
+ *
+ * Using raw UnicodeScript names (not the SCRIPT_MODEL_FALLBACK-mapped names)
+ * preserves discrimination power: clean Japanese text has characteristic
+ * KANJI→HIRAGANA→KATAKANA transitions that char-shuffle disrupts, which would
+ * be lost if all three were merged into "HAN".
+ */
+ static List buildScriptBuckets() {
+ List buckets = new ArrayList<>();
+ for (Character.UnicodeScript s : Character.UnicodeScript.values()) {
+ if (s != Character.UnicodeScript.COMMON
+ && s != Character.UnicodeScript.INHERITED
+ && s != Character.UnicodeScript.UNKNOWN) {
+ buckets.add(s.name());
+ }
+ }
+ Collections.sort(buckets);
+ buckets.add("OTHER");
+ return buckets;
+ }
+
+ /**
+ * Trains a global {@code numBuckets×numBuckets} script-transition log-probability
+ * table by pooling all training files. Uses raw {@link Character.UnicodeScript}
+ * values (not the SCRIPT_MODEL_FALLBACK mapping) so that HIRAGANA, KATAKANA, and
+ * HAN remain distinct buckets.
+ *
+ * @return float[numBuckets * numBuckets] where index {@code a*numBuckets+b} = log P(script_b | script_a)
+ */
+ static float[] trainScriptTransitionTable(List trainFiles,
+ Map scriptBucketMap,
+ int numBuckets) throws IOException {
+ long[] counts = new long[numBuckets * numBuckets];
+ int otherBucket = numBuckets - 1;
+ long totalTransitions = 0;
+
+ for (Path trainFile : trainFiles) {
+ try (BufferedReader r = openGzipped(trainFile)) {
+ String line;
+ while ((line = r.readLine()) != null) {
+ int prev = -1;
+ for (int i = 0; i < line.length(); ) {
+ int cp = line.codePointAt(i);
+ i += Character.charCount(cp);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ continue;
+ }
+ int bucket = scriptBucketMap.getOrDefault(s.name(), otherBucket);
+ if (prev >= 0) {
+ counts[prev * numBuckets + bucket]++;
+ totalTransitions++;
+ }
+ prev = bucket;
+ }
+ }
+ }
+ }
+ System.out.printf("%,d script transitions across %d files%n", totalTransitions, trainFiles.size());
+ return laplaceSmoothLogProb(counts, numBuckets);
+ }
+
+ /**
+ * Calibrates the script-transition feature by computing mu and sigma over pooled
+ * dev windows from all scripts.
+ *
+ * @return float[2] = {mu, sigma}
+ */
+ static float[] calibrateScriptTransitions(List devFiles,
+ float[] scriptTransTable,
+ Map scriptBucketMap,
+ int numBuckets) throws IOException {
+ List scores = new ArrayList<>();
+ int otherBucket = numBuckets - 1;
+ for (Path devFile : devFiles) {
+ List windows = sampleSubstrings(devFile, 200, CALIB_LENGTHS, 45);
+ for (String window : windows) {
+ double raw = rawScriptTransitionLogProb(window, scriptTransTable,
+ scriptBucketMap, numBuckets, otherBucket);
+ if (!Double.isNaN(raw)) {
+ scores.add(raw);
+ }
+ }
+ }
+ System.out.printf("%,d dev windows pooled%n", scores.size());
+ return muSigma(scores);
+ }
+
+ /**
+ * Returns the mean script-transition log-probability for a string, or
+ * {@link Double#NaN} if there are fewer than two non-neutral codepoints.
+ * Uses raw {@link Character.UnicodeScript} values (no SCRIPT_MODEL_FALLBACK).
+ */
+ private static double rawScriptTransitionLogProb(String text, float[] table,
+ Map bucketMap,
+ int numBuckets, int otherBucket) {
+ int prev = -1;
+ double sum = 0;
+ int count = 0;
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ continue;
+ }
+ int bucket = bucketMap.getOrDefault(s.name(), otherBucket);
+ if (prev >= 0) {
+ sum += table[prev * numBuckets + bucket];
+ count++;
+ }
+ prev = bucket;
+ }
+ return count > 0 ? sum / count : Double.NaN;
+ }
+
+ /**
+ * Builds a character→character remap table for a (sourceCodec, wrongCodec) pair.
+ * For every byte 0x80–0xFF, if the two codecs decode it to different characters
+ * (and neither produces the replacement character U+FFFD), the source character
+ * maps to the wrong-codec character.
+ *
+ * Returns an empty map if either codec is unavailable on this JVM.
+ */
+ static Map buildRemapTable(String sourceCodec, String wrongCodec) {
+ Charset src, wrong;
+ try {
+ src = Charset.forName(sourceCodec);
+ wrong = Charset.forName(wrongCodec);
+ } catch (UnsupportedCharsetException e) {
+ return Collections.emptyMap();
+ }
+ Map table = new HashMap<>();
+ byte[] singleByte = new byte[1];
+ for (int b = 0x80; b <= 0xFF; b++) {
+ singleByte[0] = (byte) b;
+ String fromSrc = new String(singleByte, src);
+ String fromWrong = new String(singleByte, wrong);
+ if (fromSrc.length() == 1 && fromWrong.length() == 1
+ && fromSrc.charAt(0) != '\uFFFD' && fromWrong.charAt(0) != '\uFFFD'
+ && fromSrc.charAt(0) != fromWrong.charAt(0)) {
+ table.put(fromSrc.charAt(0), fromWrong.charAt(0));
+ }
+ }
+ return table;
+ }
+
+ /**
+ * Replaces characters using a pre-computed wrong-codec remap table, simulating
+ * the effect of encoding text in one charset and decoding it in another.
+ * Only characters present in the remap table are candidates for replacement.
+ *
+ * This produces realistic mojibake: German umlauts becoming Hebrew letters,
+ * Polish characters becoming Western accents, Cyrillic becoming Latin symbols, etc.
+ *
+ * @param remapTable source-char → wrong-char substitution table (from {@link #buildRemapTable})
+ * @param rate fraction of remappable characters to replace [0, 1]
+ */
+ static String wrongCodecRemap(String text, Map remapTable,
+ double rate, Random rng) {
+ if (text.isEmpty() || remapTable.isEmpty()) {
+ return text;
+ }
+ int[] codepoints = text.codePoints().toArray();
+ for (int i = 0; i < codepoints.length; i++) {
+ if (codepoints[i] < 0x10000 && rng.nextDouble() < rate) {
+ Character replacement = remapTable.get((char) codepoints[i]);
+ if (replacement != null) {
+ codepoints[i] = replacement;
+ }
+ }
+ }
+ return new String(codepoints, 0, codepoints.length);
+ }
+
+ /**
+ * Collects a sample of codepoints from each raw {@link Character.UnicodeScript}
+ * found across all training files. Used to build the foreign-script codepoint
+ * pools for the cross-script substitution distortion.
+ *
+ * @param maxPerScript maximum distinct codepoints to collect per script
+ * @return map from raw UnicodeScript name → list of sampled codepoints
+ */
+ static Map> collectScriptCodepoints(List trainFiles,
+ int maxPerScript)
+ throws IOException {
+ Map> collected = new HashMap<>();
+ for (Path trainFile : trainFiles) {
+ try (BufferedReader r = openGzipped(trainFile)) {
+ String line;
+ while ((line = r.readLine()) != null) {
+ for (int i = 0; i < line.length(); ) {
+ int cp = line.codePointAt(i);
+ i += Character.charCount(cp);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s == Character.UnicodeScript.COMMON
+ || s == Character.UnicodeScript.INHERITED
+ || s == Character.UnicodeScript.UNKNOWN) {
+ continue;
+ }
+ Set pool = collected.computeIfAbsent(
+ s.name(), k -> new HashSet<>());
+ if (pool.size() < maxPerScript) {
+ pool.add(cp);
+ }
+ }
+ }
+ }
+ }
+ Map> result = new HashMap<>(collected.size() * 2);
+ for (Map.Entry> e : collected.entrySet()) {
+ result.put(e.getKey(), new ArrayList<>(e.getValue()));
+ }
+ return result;
+ }
+
+ /**
+ * Replaces a random fraction of characters with codepoints drawn from scripts
+ * that do NOT appear in the source text. Simulates real-world charset encoding
+ * errors where accented characters in one script are misread as characters from
+ * a completely different script — e.g., German umlauts (ä, ö, ü) becoming
+ * Hebrew letters when CP1252-encoded text is decoded as CP1255.
+ *
+ * @param rate fraction of characters to replace [0, 1]
+ * @param scriptCodepoints map from raw UnicodeScript name → pool of codepoints
+ */
+ static String injectCrossScriptChars(String text, double rate, Random rng,
+ Map> scriptCodepoints) {
+ if (text.isEmpty() || scriptCodepoints.isEmpty()) {
+ return text;
+ }
+
+ // Identify which scripts appear in the source text
+ Set sourceScripts = new HashSet<>();
+ for (int i = 0; i < text.length(); ) {
+ int cp = text.codePointAt(i);
+ i += Character.charCount(cp);
+ Character.UnicodeScript s = Character.UnicodeScript.of(cp);
+ if (s != Character.UnicodeScript.COMMON
+ && s != Character.UnicodeScript.INHERITED
+ && s != Character.UnicodeScript.UNKNOWN) {
+ sourceScripts.add(s.name());
+ }
+ }
+
+ // Build pool of codepoints from all other scripts
+ List foreignPool = new ArrayList<>();
+ for (Map.Entry> e : scriptCodepoints.entrySet()) {
+ if (!sourceScripts.contains(e.getKey())) {
+ foreignPool.addAll(e.getValue());
+ }
+ }
+ if (foreignPool.isEmpty()) {
+ return text;
+ }
+
+ int[] codepoints = text.codePoints().toArray();
+ for (int i = 0; i < codepoints.length; i++) {
+ if (rng.nextDouble() < rate) {
+ codepoints[i] = foreignPool.get(rng.nextInt(foreignPool.size()));
+ }
+ }
+ return new String(codepoints, 0, codepoints.length);
+ }
+
+ private static void printUsage() {
+ System.err.println("Usage: TrainJunkModel [options]");
+ System.err.println(" --data-dir Directory with {script}.train.gz / .dev.gz files");
+ System.err.println(" (default: ~/datasets/madlad/junkdetect)");
+ System.err.println(" --output Output model file");
+ System.err.println(" (default: {data-dir}/junkdetect.bin)");
+ }
+}
diff --git a/tika-ml/tika-ml-junkdetect/src/main/resources/META-INF/services/org.apache.tika.quality.TextQualityDetector b/tika-ml/tika-ml-junkdetect/src/main/resources/META-INF/services/org.apache.tika.quality.TextQualityDetector
new file mode 100644
index 00000000000..d9d21fa0892
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/src/main/resources/META-INF/services/org.apache.tika.quality.TextQualityDetector
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.tika.ml.junkdetect.JunkDetector
diff --git a/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin b/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin
new file mode 100644
index 00000000000..feb9da112e7
Binary files /dev/null and b/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin differ
diff --git a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorSmokeTest.java b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorSmokeTest.java
new file mode 100644
index 00000000000..88a5a8c16fa
--- /dev/null
+++ b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorSmokeTest.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Random;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.quality.TextQualityComparison;
+import org.apache.tika.quality.TextQualityScore;
+
+/**
+ * Smoke tests verifying the bundled model meets minimum quality thresholds.
+ * Failures indicate the model needs more data or feature extraction is wrong.
+ */
+public class JunkDetectorSmokeTest {
+
+ private static JunkDetector detector;
+
+ @BeforeAll
+ static void loadModel() throws Exception {
+ detector = JunkDetector.loadFromClasspath();
+ }
+
+ /**
+ * Clean English should score higher than random high-byte garbage interpreted
+ * as ISO-8859-1. Simulates binary data mixed into a text extraction.
+ */
+ @Test
+ void cleanVsGarbage() {
+ String clean = "The quick brown fox jumps over the lazy dog. "
+ + "Pack my box with five dozen liquor jugs.";
+
+ byte[] garbageBytes = new byte[80];
+ new Random(42).nextBytes(garbageBytes);
+ for (int i = 0; i < garbageBytes.length; i++) {
+ garbageBytes[i] = (byte) (0x80 | (garbageBytes[i] & 0x7F));
+ }
+ // Decode as ISO-8859-1 so the string contains high-codepoint characters
+ String garbage = new String(garbageBytes, StandardCharsets.ISO_8859_1);
+
+ TextQualityScore cleanScore = detector.score(clean);
+ TextQualityScore garbageScore = detector.score(garbage);
+
+ System.out.println("clean: " + cleanScore);
+ System.out.println("garbage: " + garbageScore);
+
+ assertTrue(cleanScore.getZScore() > garbageScore.getZScore(),
+ "Clean text should score higher than garbage");
+ }
+
+ /**
+ * Forward Arabic should score higher than character-reversed Arabic.
+ * Character (codepoint) reversal produces valid UTF-8 but wrong reading order —
+ * analogous to bidirectional rendering failures or incorrectly stored RTL text.
+ */
+ @Test
+ void forwardVsReversedArabic() {
+ String arabic = "اللغة العربية جميلة وغنية بالمفردات والتعبيرات";
+ String reversed = reverseString(arabic);
+
+ TextQualityScore fwd = detector.score(arabic);
+ TextQualityScore rev = detector.score(reversed);
+
+ System.out.println("arabic forward: " + fwd);
+ System.out.println("arabic reversed: " + rev);
+
+ assertTrue(fwd.getZScore() > rev.getZScore(),
+ "Forward Arabic should score higher than character-reversed Arabic");
+ }
+
+ /**
+ * cp1257 (Baltic) decoding of Lithuanian text should win over cp1252.
+ *
+ * Tests the {@link JunkDetector#compare} API: given raw bytes that were
+ * encoded as cp1257, comparing both decodings should prefer the correct one.
+ * A low delta is expected because the LATIN model is trained across ~322 languages
+ * and Baltic-specific bigrams are diluted.
+ *
+ *
TODO: improve separation with a Baltic sub-model or Baltic-weighted retraining.
+ */
+ @Test
+ void cp1252VsCp1257OnBalticText() throws Exception {
+ String lithuanian = "Lietuvių kalba yra labai graži ir turtinga";
+ byte[] cp1257bytes = lithuanian.getBytes("cp1257");
+
+ String ascp1252 = new String(cp1257bytes, "cp1252");
+ String ascp1257 = new String(cp1257bytes, "cp1257");
+
+ TextQualityComparison result = detector.compare("cp1252", ascp1252, "cp1257", ascp1257);
+
+ System.out.println("Baltic comparison: " + result);
+
+ assertEquals("B", result.winner(),
+ "cp1257 should be identified as the correct encoding for Lithuanian text");
+ // Delta is weak (pooled LATIN model dilutes Baltic-specific bigrams).
+ // Production threshold is delta > 1.0; PoC floor is 0.1.
+ assertTrue(result.delta() > 0.1,
+ "Should have some separation: delta=" + result.delta());
+ }
+
+ /**
+ * cp1251 decoding of Russian text should win over cp1252.
+ *
+ *
This is the canonical Cyrillic mojibake scenario: Windows-1251-encoded Russian
+ * text misinterpreted as Windows-1252 (Western European). The cp1252 decoding
+ * produces Latin symbols interspersed with control characters, while cp1251 produces
+ * proper Cyrillic. The model should strongly prefer cp1251.
+ *
+ *
Note: character-reversal of LTR Cyrillic is NOT a useful test — byte-bigram
+ * statistics are nearly identical forward and backward for LTR scripts. Codec
+ * comparison is the correct test for LTR scripts.
+ */
+ @Test
+ void cp1252VsCp1251OnRussianText() throws Exception {
+ String russian = "Русский язык является одним из восточнославянских языков";
+ byte[] cp1251bytes = russian.getBytes("cp1251");
+
+ String ascp1252 = new String(cp1251bytes, "cp1252");
+ String ascp1251 = new String(cp1251bytes, "cp1251");
+
+ TextQualityComparison result = detector.compare("cp1252", ascp1252, "cp1251", ascp1251);
+
+ System.out.println("Russian Cyrillic comparison: " + result);
+
+ assertEquals("B", result.winner(),
+ "cp1251 should be identified as the correct encoding for Russian text");
+ assertTrue(result.delta() > 1.0,
+ "Cyrillic codec separation should be strong: delta=" + result.delta());
+ }
+
+ /**
+ * Clean Japanese (CJK) should score higher than byte-shuffled Japanese.
+ */
+ @Test
+ void cleanVsShuffledCjk() {
+ String japanese = "日本語は美しい言語であり、世界中で約1億3千万人が話している。";
+ byte[] cleanBytes = japanese.getBytes(StandardCharsets.UTF_8);
+ byte[] shuffledBytes = shuffled(cleanBytes, 42);
+
+ // Shuffled bytes are not valid UTF-8; decode as ISO-8859-1 to get a scoreable string
+ String shuffledText = new String(shuffledBytes, StandardCharsets.ISO_8859_1);
+
+ TextQualityScore cleanScore = detector.score(japanese);
+ TextQualityScore shuffledScore = detector.score(shuffledText);
+
+ System.out.println("Japanese clean: " + cleanScore);
+ System.out.println("Japanese shuffled: " + shuffledScore);
+
+ assertTrue(cleanScore.getZScore() > shuffledScore.getZScore(),
+ "Clean Japanese should score higher than shuffled bytes");
+ }
+
+ /**
+ * Shift-JIS zip entry name (9 bytes) decoded as Shift-JIS should beat the same
+ * bytes decoded as UTF-8 (which produces mojibake with FFFD replacement chars).
+ *
+ *
This is the canonical short-text use case: zip parsers encounter raw filename
+ * bytes with no BOM or language tag. At 9 bytes the z-score signal is weak, but
+ * the corrupted UTF-8 decode contains FFFD sequences (0xEF 0xBF 0xBD) which are
+ * very unlikely in LATIN text, yielding a clearly negative bigram z-score.
+ *
+ *
"テスト.tx" is pure katakana — KATAKANA script maps to the HAN model via
+ * {@link JunkDetector#SCRIPT_MODEL_FALLBACK}.
+ */
+ @Test
+ void shiftJisZipEntryNameVsUtf8() throws Exception {
+ // 9 Shift-JIS bytes: テスト.tx
+ byte[] sjisBytes = "テスト.tx".getBytes("Shift_JIS");
+ assertEquals(9, sjisBytes.length, "fixture sanity: expect exactly 9 Shift-JIS bytes");
+
+ String asShiftJis = new String(sjisBytes, "Shift_JIS"); // "テスト.tx"
+ String asUtf8 = new String(sjisBytes, StandardCharsets.UTF_8); // "?e?X?g.tx" (mojibake)
+
+ TextQualityComparison result = detector.compare("Shift-JIS", asShiftJis, "UTF-8", asUtf8);
+
+ System.out.println("Shift-JIS zip entry: " + result);
+
+ assertEquals("A", result.winner(),
+ "Shift-JIS decode should beat garbled UTF-8 for short Japanese filename");
+ }
+
+ // -----------------------------------------------------------------------
+
+ /**
+ * Reverses the string at codepoint granularity (not char granularity), so
+ * surrogate pairs are kept intact. Produces valid Unicode in reverse reading
+ * order — a realistic distortion for RTL-language tests.
+ */
+ static String reverseString(String s) {
+ int[] codepoints = s.codePoints().toArray();
+ for (int i = 0, j = codepoints.length - 1; i < j; i++, j--) {
+ int tmp = codepoints[i];
+ codepoints[i] = codepoints[j];
+ codepoints[j] = tmp;
+ }
+ return new String(codepoints, 0, codepoints.length);
+ }
+
+ private static byte[] shuffled(byte[] bytes, long seed) {
+ byte[] copy = bytes.clone();
+ Random rng = new Random(seed);
+ for (int i = copy.length - 1; i > 0; i--) {
+ int j = rng.nextInt(i + 1);
+ byte tmp = copy[i];
+ copy[i] = copy[j];
+ copy[j] = tmp;
+ }
+ return copy;
+ }
+}