TIKA-4745 -- efficiency improvements

Author: tballison

tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java

@@ -40,8 +40,19 @@
 public class EncodingDetectorContext {
 
     private final List<Result> results = new ArrayList<>();
+    private final EncodingProbeCache probeCache = new EncodingProbeCache();
     private String arbitrationInfo;
 
+    /**
+     * Per-detection cache of the raw detection probe, shared across the detectors in
+     * this chain so they don't each re-read the same leading bytes. It lives and dies
+     * with this context (which is removed after detection), so it never leaks into
+     * recursive/attachment parsing.
+     */
+    public EncodingProbeCache getProbeCache() {
+        return probeCache;
+    }
+
     /**
      * Record the ranked results from a child detector.
      *

tika-core/src/main/java/org/apache/tika/detect/EncodingProbeCache.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+/**
+ * Caches the raw encoding-detection probe (the leading bytes read for detection)
+ * so that multiple detectors in a chain do not each re-read and re-tag-strip the
+ * same bytes. For example a statistical detector and a downstream meta detector
+ * that re-reads the bytes for arbitration can share one probe.
+ * <p>
+ * An instance is held by {@link EncodingDetectorContext}, so it inherits that
+ * context's per-detection lifecycle: created fresh per detection and discarded
+ * with the context immediately afterwards. That matters because a
+ * {@link org.apache.tika.parser.ParseContext} flows on into recursive
+ * (attachment/embedded) parsing — a probe must never outlive the single detection
+ * it was read for.
+ * <p>
+ * Not thread-safe: a single detection runs its detectors sequentially on one
+ * thread. The cache is keyed by the probe parameters — {@link #get} returns the
+ * cached probe only when both {@code contentTarget} and {@code rawCap} match what
+ * it was stored with, so a detector that wants a differently-sized probe
+ * transparently reads (and caches) its own.
+ * <p>
+ * The cached array is shared read-only state; callers must not mutate it in place.
+ */
+public class EncodingProbeCache {
+
+    private byte[] probe;
+    private int contentTarget = -1;
+    private int rawCap = -1;
+
+    /**
+     * @return the cached probe if one was stored with the same {@code contentTarget} and
+     * {@code rawCap}; otherwise {@code null}
+     */
+    public byte[] get(int contentTarget, int rawCap) {
+        if (probe != null && this.contentTarget == contentTarget && this.rawCap == rawCap) {
+            return probe;
+        }
+        return null;
+    }
+
+    /**
+     * Stores the probe bytes read with the given parameters.
+     */
+    public void put(byte[] probe, int contentTarget, int rawCap) {
+        this.probe = probe;
+        this.contentTarget = contentTarget;
+        this.rawCap = rawCap;
+    }
+}

tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java

@@ -162,6 +162,14 @@ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
         }
         tis.reset();
 
+        // findCharset only ever matches a meta tag (HTTP_META_PATTERN = "<\s*meta...").
+        // If the probe has no such tag, the full ASCII decode + comment-stripping
+        // regex below can only produce null — skip them. Byte-level, no allocation;
+        // a strict necessary condition for any non-empty result.
+        if (!containsMetaTag(buffer, n)) {
+            return Collections.emptyList();
+        }
+
         String head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n)).toString();
         String headNoComments = head.replaceAll("<!--.*?(-->|$)", " ");
         Charset charset = findCharset(headNoComments);
@@ -175,6 +183,39 @@ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
                 EncodingResult.ResultType.DECLARATIVE));
     }
 
+    /**
+     * Byte-level scan for an opening meta tag, mirroring the {@code <\s*meta} prefix of
+     * {@link #HTTP_META_PATTERN} (ASCII, case-insensitive). Lets {@link #detect} skip the
+     * full ASCII decode + comment-stripping regex on probes that cannot contain a meta
+     * charset declaration. {@code <}, ASCII whitespace and {@code meta} are all ASCII, so a
+     * raw-byte scan is equivalent to scanning the decoded head.
+     */
+    private static boolean containsMetaTag(byte[] buf, int len) {
+        for (int i = 0; i < len; i++) {
+            if (buf[i] != '<') {
+                continue;
+            }
+            int j = i + 1;
+            while (j < len) {
+                int c = buf[j] & 0xFF;
+                if (c == ' ' || c == '\t' || c == '\n' || c == 0x0B || c == '\f'
+                        || c == '\r') {
+                    j++;
+                } else {
+                    break;
+                }
+            }
+            if (j + 4 <= len
+                    && (((buf[j] & 0xFF) | 0x20) == 'm')
+                    && (((buf[j + 1] & 0xFF) | 0x20) == 'e')
+                    && (((buf[j + 2] & 0xFF) | 0x20) == 't')
+                    && (((buf[j + 3] & 0xFF) | 0x20) == 'a')) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     //returns null if no charset was found
     private Charset findCharset(String s) {
 

tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/AdaptiveProbe.java

@@ -55,11 +55,20 @@ public static byte[] read(TikaInputStream tis, int contentTarget, int rawCap)
             throws IOException {
         tis.mark(rawCap);
         try {
-            byte[] buf = new byte[rawCap];
-            byte[] stripDst = new byte[rawCap];
+            // Grow on demand rather than allocating (and zeroing) the full rawCap
+            // (e.g. 512 KB) twice up front: the vast majority of probes are far
+            // smaller. Bytes returned are identical to the eager-allocation version.
+            int cap = Math.min(rawCap, contentTarget);
+            byte[] buf = new byte[cap];
+            byte[] stripDst = new byte[cap];
             int total = 0;
             while (total < rawCap) {
                 int want = Math.min(rawCap - total, contentTarget);
+                if (total + want > buf.length) {
+                    int newCap = Math.min(rawCap, Math.max(buf.length * 2, total + want));
+                    buf = Arrays.copyOf(buf, newCap);
+                    stripDst = Arrays.copyOf(stripDst, newCap);
+                }
                 int n = IOUtils.read(tis, buf, total, want);
                 total += n;
                 HtmlByteStripper.Result r =
@@ -72,7 +81,7 @@ public static byte[] read(TikaInputStream tis, int contentTarget, int rawCap)
             if (total == 0) {
                 return new byte[0];
             }
-            return total == rawCap ? buf : Arrays.copyOf(buf, total);
+            return total == buf.length ? buf : Arrays.copyOf(buf, total);
         } finally {
             tis.reset();
         }

tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java

@@ -29,6 +29,8 @@
 
 import org.apache.tika.config.TikaComponent;
 import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.EncodingDetectorContext;
+import org.apache.tika.detect.EncodingProbeCache;
 import org.apache.tika.detect.EncodingResult;
 import org.apache.tika.detect.HighByteLetterStats;
 import org.apache.tika.io.TikaInputStream;
@@ -208,7 +210,7 @@ private static NaiveBayesBigramEncodingDetector loadFromClasspath() throws IOExc
     @Override
     public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
                                        ParseContext parseContext) throws IOException {
-        byte[] probe = readProbe(tis);
+        byte[] probe = readProbe(tis, parseContext);
         return detect(probe, metadata);
     }
 
@@ -749,7 +751,20 @@ private static boolean shouldTryStrip(String contentType) {
         return lower.contains("html") || lower.contains("xml");
     }
 
-    private static byte[] readProbe(TikaInputStream tis) throws IOException {
-        return AdaptiveProbe.read(tis, PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
+    private static byte[] readProbe(TikaInputStream tis, ParseContext parseContext)
+            throws IOException {
+        EncodingDetectorContext context = parseContext.get(EncodingDetectorContext.class);
+        EncodingProbeCache cache = context == null ? null : context.getProbeCache();
+        if (cache != null) {
+            byte[] cached = cache.get(PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
+            if (cached != null) {
+                return cached;
+            }
+        }
+        byte[] probe = AdaptiveProbe.read(tis, PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
+        if (cache != null) {
+            cache.put(probe, PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
+        }
+        return probe;
     }
 }

tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/NaiveBayesBigramEncodingDetector.java

@@ -309,9 +309,19 @@ public NaiveBayesBigramEncodingDetector(InputStream modelStream) throws IOExcept
                 for (int bg = 0; bg < BIGRAM_SPACE; bg++) {
                     logP8[bg * numClasses + c] = u;
                 }
-                // Overwrite with trained pairs.
+                // Overwrite with trained pairs. Bigram ids are sorted ascending and
+                // stored as varint deltas (LEB128) from the previous id.
+                int bigram = 0;
                 for (int i = 0; i < vocabSize; i++) {
-                    int bigram = dis.readUnsignedShort();
+                    int delta = 0;
+                    int shift = 0;
+                    int b;
+                    do {
+                        b = dis.readUnsignedByte();
+                        delta |= (b & 0x7F) << shift;
+                        shift += 7;
+                    } while ((b & 0x80) != 0);
+                    bigram += delta;
                     byte q = dis.readByte();
                     logP8[bigram * numClasses + c] = q;
                 }

tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/nb-bigram.bin

@@ -244,9 +244,17 @@ public static String preprocess(String rawText) {
      * @return cleaned, NFC-normalized text
      */
     public static String preprocessNoTruncate(String rawText) {
-        // Strip URLs and emails
-        String text = URL_REGEX.matcher(rawText).replaceAll(" ");
-        text = MAIL_REGEX.matcher(text).replaceAll(" ");
+        // Strip URLs and emails. Both regexes scan the entire input on every call;
+        // skip each unless its required marker is present ("://" for URL_REGEX, "@"
+        // for MAIL_REGEX). This is a no-op for the common (markerless) case — the
+        // output is identical — but avoids a full-buffer regex scan + Matcher alloc.
+        String text = rawText;
+        if (text.indexOf("://") >= 0) {
+            text = URL_REGEX.matcher(text).replaceAll(" ");
+        }
+        if (text.indexOf('@') >= 0) {
+            text = MAIL_REGEX.matcher(text).replaceAll(" ");
+        }
 
         // NFC normalize
         if (!Normalizer.isNormalized(text, Normalizer.Form.NFC)) {

tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupFeatureExtractor.java

@@ -427,8 +427,8 @@ public static void main(String[] args) throws IOException {
      *     float32 scale                      (per-class dequant)
      *     byte    unseenQ                    (int8 quantized unseen floor)
      *     int32 vocabSize                    (number of trained pairs)
-     *     for each kept bigram:
-     *       uint16 bigramKey
+     *     bigram keys sorted ascending, each pair stored as:
+     *       varint deltaFromPrevKey         (LEB128; first delta is the key itself)
      *       byte   logP8                     (int8 quantized)
      *
      * <p>Sparse representation: only trained bigram pairs are stored;
@@ -497,11 +497,22 @@ private static void save(Path path, String[] labels,
                 dos.writeByte(unseenQ[c]);
                 float scale = perClassScale[c];
                 dos.writeInt(logProbsPerClass[c].size());
-                for (Map.Entry<Integer, Float> e : logProbsPerClass[c].entrySet()) {
-                    int q = Math.round(e.getValue() / scale);
+                // Bigram ids sorted ascending, stored as varint (LEB128) deltas from
+                // the previous id — most deltas fit in a single byte.
+                int[] keys = logProbsPerClass[c].keySet().stream()
+                        .mapToInt(Integer::intValue).sorted().toArray();
+                int prev = 0;
+                for (int key : keys) {
+                    int q = Math.round(logProbsPerClass[c].get(key) / scale);
                     if (q > 127) q = 127;
                     if (q < -127) q = -127;
-                    dos.writeShort(e.getKey());
+                    int delta = key - prev;
+                    prev = key;
+                    while ((delta & ~0x7F) != 0) {
+                        dos.writeByte((delta & 0x7F) | 0x80);
+                        delta >>>= 7;
+                    }
+                    dos.writeByte(delta);
                     dos.writeByte(q);
                 }
             }

tika-ml/tika-ml-chardetect/src/main/java/org/apache/tika/ml/chardetect/tools/TrainNaiveBayesBigram.java

@@ -36,12 +36,11 @@
  *       script.  Codepoint → dense index is a binary search; index →
  *       codepoint is direct array access.  Typical sizes: ~7K-15K for HAN,
  *       ~200-500 for most other scripts.
- *   <li>{@code bigramKeys} / {@code bigramValues} — parallel arrays
- *       implementing an open-addressed hash table with linear probing.
- *       Each key is a 32-bit value {@code (idxA << 16) | idxB}; key {@code
- *       -1} means "empty slot."  Indices are bounded at 16 bits (65535),
- *       which is comfortably above the largest per-script codepoint count
- *       we observe.
+ *   <li>{@code bigramKeys} / {@code bigramValues} — parallel arrays of the
+ *       occupied entries only, sorted ascending by key for binary-search
+ *       lookup.  Each key is a 32-bit value {@code (idxA << 16) | idxB}.
+ *       Indices are bounded at 16 bits (65535), comfortably above the
+ *       largest per-script codepoint count we observe.
  *   <li>{@code unigramTable} — {@code byte[numCodepoints]}, quantized
  *       unigram log-probabilities indexed by the same codepoint→index map.
  *   <li>{@code bigramQuantMin/Max}, {@code unigramQuantMin/Max} —
@@ -56,10 +55,9 @@
  *       independence sum.
  * </ul>
  *
- * <p>Membership semantics: no Bloom filter.  The empty-slot sentinel is
- * the membership oracle — a pair is "seen" iff binary-search finds both
- * codepoints in the index AND a probe sequence hits a matching key before
- * an empty slot.  Lookups are therefore exact.
+ * <p>Membership semantics: no Bloom filter.  A pair is "seen" iff
+ * binary-search finds both codepoints in the index AND finds the packed
+ * key in {@code bigramKeys}.  Lookups are therefore exact.
  *
  * <p>Fields are package-private so the
  * {@link org.apache.tika.ml.junkdetect.tools.TrainJunkModel} trainer can