updates based on copilot, we've hit diminishing returns

Author: tballison

tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java

@@ -353,6 +353,16 @@ public static JunkDetector load(InputStream rawIs) throws IOException {
                 f1TablesByScript.put(script, BigramTables.readFrom(dis));
             }
 
+            requireUsableSigma("scriptTransition", scriptTransitionCalibration);
+            requireUsableSigma("block", blockCalibration);
+            requireUsableSigma("control", controlCalibration);
+            requireUsableSigma("z5", z5Calibration);
+            requireUsableSigma("z6", z6Calibration);
+            requireUsableSigma("z9", z9Calibration);
+            for (Map.Entry<String, float[]> e : calibrations.entrySet()) {
+                requireUsableSigma("z1[" + e.getKey() + "]", e.getValue());
+            }
+
             return new JunkDetector(calibrations,
                     blockTable, blockTableQuant, blockCalibration,
                     controlCalibration, combinerWeights,
@@ -363,6 +373,22 @@ public static JunkDetector load(InputStream rawIs) throws IOException {
         }
     }
 
+    /**
+     * Validates a calibration {@code {mu, sigma}} from the model file: sigma is the
+     * divisor in every z-score, so it must be finite and &gt; 0.  Single enforcement
+     * point for that invariant -- inference divides without re-checking.
+     */
+    static void requireUsableSigma(String name, float[] calibration) throws IOException {
+        boolean ok = calibration != null && calibration.length >= 2
+                && Float.isFinite(calibration[1]) && calibration[1] > 0f;
+        if (!ok) {
+            String sigma = (calibration == null || calibration.length < 2)
+                    ? "absent" : Float.toString(calibration[1]);
+            throw new IOException("Invalid model: " + name
+                    + " calibration sigma must be finite and > 0 but was " + sigma);
+        }
+    }
+
     /** Read {@code size} big-endian int16 values as a short[]. */
     private static short[] readShortTable(DataInputStream dis, int size) throws IOException {
         byte[] raw = dis.readNBytes(size * 2);
@@ -642,7 +668,7 @@ public static final class FeatureComponents {
      */
     public float computeZ5LetterAdjacentToMarkRatio(String text) {
         double raw = TextQualityFeatures.letterAdjacentToMarkRatio(text);
-        if (Double.isNaN(raw) || z5Calibration == null || z5Calibration[1] <= 0) {
+        if (Double.isNaN(raw) || z5Calibration == null) {
             return 0f;
         }
         return ((float) raw - z5Calibration[0]) / z5Calibration[1];
@@ -658,7 +684,7 @@ public float computeZ5LetterAdjacentToMarkRatio(String text) {
      */
     public float computeZ6ReplacementRatio(String text) {
         double raw = TextQualityFeatures.replacementRatio(text);
-        if (Double.isNaN(raw) || z6Calibration == null || z6Calibration[1] <= 0) {
+        if (Double.isNaN(raw) || z6Calibration == null) {
             return 0f;
         }
         // Flip sign: higher replacement = lower quality, so feature is
@@ -676,7 +702,7 @@ public float computeZ6ReplacementRatio(String text) {
      */
     public float computeZ9AlternationRatio(String text) {
         double raw = TextQualityFeatures.scriptAlternationRatio(text);
-        if (Double.isNaN(raw) || z9Calibration == null || z9Calibration[1] <= 0) {
+        if (Double.isNaN(raw) || z9Calibration == null) {
             return 0f;
         }
         // Higher alternation = junkier; (mu - raw) / sigma so clean text → positive z9.

tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java

@@ -16,7 +16,9 @@
  */
 package org.apache.tika.ml.junkdetect;
 
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.BufferedWriter;
@@ -52,6 +54,21 @@
  */
 public class JunkDetectorRoundTripTest {
 
+    @Test
+    void requireUsableSigmaRejectsNonPositiveOrNonFinite() {
+        assertDoesNotThrow(() -> JunkDetector.requireUsableSigma("ok", new float[]{-3f, 0.5f}));
+        for (float badSigma : new float[]{0f, -0.1f, Float.NaN,
+                Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY}) {
+            assertThrows(IOException.class,
+                    () -> JunkDetector.requireUsableSigma("bad", new float[]{0f, badSigma}),
+                    "sigma=" + badSigma + " must be rejected");
+        }
+        assertThrows(IOException.class,
+                () -> JunkDetector.requireUsableSigma("null", null));
+        assertThrows(IOException.class,
+                () -> JunkDetector.requireUsableSigma("short", new float[]{1f}));
+    }
+
     @Test
     void roundTripSeenPairAndUnigramBackoff(@TempDir Path tmp) throws IOException {
         // -----------------------------------------------------------------