TIKA-4745 -- efficiency improvements (#2878)

Author: tballison

tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java

@@ -40,8 +40,19 @@
 public class EncodingDetectorContext {
 
     private final List<Result> results = new ArrayList<>();
+    private final EncodingProbeCache probeCache = new EncodingProbeCache();
     private String arbitrationInfo;
 
+    /**
+     * Per-detection cache of the raw detection probe, shared across the detectors in
+     * this chain so they don't each re-read the same leading bytes. It lives and dies
+     * with this context (which is removed after detection), so it never leaks into
+     * recursive/attachment parsing.
+     */
+    public EncodingProbeCache getProbeCache() {
+        return probeCache;
+    }
+
     /**
      * Record the ranked results from a child detector.
      *

tika-core/src/main/java/org/apache/tika/detect/EncodingProbeCache.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+/**
+ * Caches the raw encoding-detection probe (the leading bytes read for detection)
+ * so that multiple detectors in a chain do not each re-read and re-tag-strip the
+ * same bytes. For example a statistical detector and a downstream meta detector
+ * that re-reads the bytes for arbitration can share one probe.
+ * <p>
+ * An instance is held by {@link EncodingDetectorContext}, so it inherits that
+ * context's per-detection lifecycle: created fresh per detection and discarded
+ * with the context immediately afterwards. That matters because a
+ * {@link org.apache.tika.parser.ParseContext} flows on into recursive
+ * (attachment/embedded) parsing — a probe must never outlive the single detection
+ * it was read for.
+ * <p>
+ * Not thread-safe: a single detection runs its detectors sequentially on one
+ * thread. The cache is keyed by the probe parameters — {@link #get} returns the
+ * cached probe only when both {@code contentTarget} and {@code rawCap} match what
+ * it was stored with, so a detector that wants a differently-sized probe
+ * transparently reads (and caches) its own.
+ * <p>
+ * The cached array is shared read-only state; callers must not mutate it in place.
+ */
+public class EncodingProbeCache {
+
+    private byte[] probe;
+    private int contentTarget = -1;
+    private int rawCap = -1;
+
+    /**
+     * @return the cached probe if one was stored with the same {@code contentTarget} and
+     * {@code rawCap}; otherwise {@code null}
+     */
+    public byte[] get(int contentTarget, int rawCap) {
+        if (probe != null && this.contentTarget == contentTarget && this.rawCap == rawCap) {
+            return probe;
+        }
+        return null;
+    }
+
+    /**
+     * Stores the probe bytes read with the given parameters.
+     */
+    public void put(byte[] probe, int contentTarget, int rawCap) {
+        this.probe = probe;
+        this.contentTarget = contentTarget;
+        this.rawCap = rawCap;
+    }
+}

tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java

@@ -20,7 +20,6 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Serializable;
-import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
@@ -85,6 +84,7 @@ public void setMarkLimit(int markLimit) {
     private static final Pattern FLEXIBLE_CHARSET_ATTR_PATTERN =
             Pattern.compile(("(?is)\\bcharset\\s*=\\s*(?:['\\\"]\\s*)?([-_:\\.a-z0-9]+)"));
     private static final Charset ASCII = Charset.forName("US-ASCII");
+    private static final Pattern HTML_COMMENT_PATTERN = Pattern.compile("<!--.*?(-->|$)");
     /**
      * HTML can include non-iana supported charsets that Java
      * recognizes, e.g. "unicode".  This can lead to incorrect detection/mojibake.
@@ -162,10 +162,20 @@ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
         }
         tis.reset();
 
-        String head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n)).toString();
-        String headNoComments = head.replaceAll("<!--.*?(-->|$)", " ");
+        // findCharset only ever matches a meta tag (HTTP_META_PATTERN = "<\s*meta...").
+        // If the probe has no such tag, the full ASCII decode + comment-stripping
+        // regex below can only produce null — skip them. Byte-level, no allocation;
+        // a strict necessary condition for any non-empty result.
+        if (!containsMetaTag(buffer, n)) {
+            return Collections.emptyList();
+        }
+
+        String head = new String(buffer, 0, n, ASCII);
+        boolean hasComment = head.indexOf("<!--") >= 0;
+        String headNoComments =
+                hasComment ? HTML_COMMENT_PATTERN.matcher(head).replaceAll(" ") : head;
         Charset charset = findCharset(headNoComments);
-        if (charset == null) {
+        if (charset == null && hasComment) {
             charset = findCharset(head);
         }
         if (charset == null) {
@@ -175,6 +185,39 @@ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
                 EncodingResult.ResultType.DECLARATIVE));
     }
 
+    /**
+     * Byte-level scan for an opening meta tag, mirroring the {@code <\s*meta} prefix of
+     * {@link #HTTP_META_PATTERN} (ASCII, case-insensitive). Lets {@link #detect} skip the
+     * full ASCII decode + comment-stripping regex on probes that cannot contain a meta
+     * charset declaration. {@code <}, ASCII whitespace and {@code meta} are all ASCII, so a
+     * raw-byte scan is equivalent to scanning the decoded head.
+     */
+    private static boolean containsMetaTag(byte[] buf, int len) {
+        for (int i = 0; i < len; i++) {
+            if (buf[i] != '<') {
+                continue;
+            }
+            int j = i + 1;
+            while (j < len) {
+                int c = buf[j] & 0xFF;
+                if (c == ' ' || c == '\t' || c == '\n' || c == 0x0B || c == '\f'
+                        || c == '\r') {
+                    j++;
+                } else {
+                    break;
+                }
+            }
+            if (j + 4 <= len
+                    && (((buf[j] & 0xFF) | 0x20) == 'm')
+                    && (((buf[j + 1] & 0xFF) | 0x20) == 'e')
+                    && (((buf[j + 2] & 0xFF) | 0x20) == 't')
+                    && (((buf[j + 3] & 0xFF) | 0x20) == 'a')) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     //returns null if no charset was found
     private Charset findCharset(String s) {
 

…StandardCharsets_unsupported_by_IANA.txt …StandardCharsets_unsupported_by_IANA.txttika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/resources/org/apache/tika/parser/html/StandardCharsets_unsupported_by_IANA.txt renamed to tika-encoding-detectors/tika-encoding-detector-html/src/main/resources/org/apache/tika/parser/html/StandardCharsets_unsupported_by_IANA.txt tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/resources/org/apache/tika/parser/html/StandardCharsets_unsupported_by_IANA.txt renamed to tika-encoding-detectors/tika-encoding-detector-html/src/main/resources/org/apache/tika/parser/html/StandardCharsets_unsupported_by_IANA.txt

@@ -55,11 +55,20 @@ public static byte[] read(TikaInputStream tis, int contentTarget, int rawCap)
             throws IOException {
         tis.mark(rawCap);
         try {
-            byte[] buf = new byte[rawCap];
-            byte[] stripDst = new byte[rawCap];
+            // Grow on demand rather than allocating (and zeroing) the full rawCap
+            // (e.g. 512 KB) twice up front: the vast majority of probes are far
+            // smaller. Bytes returned are identical to the eager-allocation version.
+            int cap = Math.min(rawCap, contentTarget);
+            byte[] buf = new byte[cap];
+            byte[] stripDst = new byte[cap];
             int total = 0;
             while (total < rawCap) {
                 int want = Math.min(rawCap - total, contentTarget);
+                if (total + want > buf.length) {
+                    int newCap = Math.min(rawCap, Math.max(buf.length * 2, total + want));
+                    buf = Arrays.copyOf(buf, newCap);
+                    stripDst = Arrays.copyOf(stripDst, newCap);
+                }
                 int n = IOUtils.read(tis, buf, total, want);
                 total += n;
                 HtmlByteStripper.Result r =
@@ -72,7 +81,7 @@ public static byte[] read(TikaInputStream tis, int contentTarget, int rawCap)
             if (total == 0) {
                 return new byte[0];
             }
-            return total == rawCap ? buf : Arrays.copyOf(buf, total);
+            return total == buf.length ? buf : Arrays.copyOf(buf, total);
         } finally {
             tis.reset();
         }

…arser/html/HtmlEncodingDetectorTest.java …arser/html/HtmlEncodingDetectorTest.javatika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java renamed to tika-encoding-detectors/tika-encoding-detector-html/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java renamed to tika-encoding-detectors/tika-encoding-detector-html/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java

@@ -29,6 +29,8 @@
 
 import org.apache.tika.config.TikaComponent;
 import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.EncodingDetectorContext;
+import org.apache.tika.detect.EncodingProbeCache;
 import org.apache.tika.detect.EncodingResult;
 import org.apache.tika.detect.HighByteLetterStats;
 import org.apache.tika.io.TikaInputStream;
@@ -208,7 +210,7 @@ private static NaiveBayesBigramEncodingDetector loadFromClasspath() throws IOExc
     @Override
     public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
                                        ParseContext parseContext) throws IOException {
-        byte[] probe = readProbe(tis);
+        byte[] probe = readProbe(tis, parseContext);
         return detect(probe, metadata);
     }
 
@@ -749,7 +751,21 @@ private static boolean shouldTryStrip(String contentType) {
         return lower.contains("html") || lower.contains("xml");
     }
 
-    private static byte[] readProbe(TikaInputStream tis) throws IOException {
-        return AdaptiveProbe.read(tis, PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
+    private static byte[] readProbe(TikaInputStream tis, ParseContext parseContext)
+            throws IOException {
+        EncodingDetectorContext context =
+                parseContext == null ? null : parseContext.get(EncodingDetectorContext.class);
+        EncodingProbeCache cache = context == null ? null : context.getProbeCache();
+        if (cache != null) {
+            byte[] cached = cache.get(PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
+            if (cached != null) {
+                return cached;
+            }
+        }
+        byte[] probe = AdaptiveProbe.read(tis, PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
+        if (cache != null) {
+            cache.put(probe, PROBE_CONTENT_TARGET, PROBE_RAW_CAP);
+        }
+        return probe;
     }
 }

…ml/StandardHtmlEncodingDetectorTest.java …ml/StandardHtmlEncodingDetectorTest.javatika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/StandardHtmlEncodingDetectorTest.java renamed to tika-encoding-detectors/tika-encoding-detector-html/src/test/java/org/apache/tika/parser/html/StandardHtmlEncodingDetectorTest.java tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/StandardHtmlEncodingDetectorTest.java renamed to tika-encoding-detectors/tika-encoding-detector-html/src/test/java/org/apache/tika/parser/html/StandardHtmlEncodingDetectorTest.java

@@ -62,12 +62,11 @@ public class NaiveBayesBigramEncodingDetector implements EncodingDetector {
     private static final int BIGRAM_SPACE = 65_536;
 
     /**
-     * Cap probe scanning at 10 KB.  Bigram-based identification
+     * Cap probe scanning at 16 KB.  Bigram-based identification
      * saturates quickly — beyond the first 500-1000 bytes every
      * additional bigram nudges scores by < 0.1 log-likelihood and
-     * doesn't change the argmax.  Reducing the cap from 4 KB to 1 KB
-     * quartes the inner-loop work on long probes at no measurable
-     * accuracy cost.
+     * doesn't change the argmax, so capping the scan bounds the
+     * inner-loop work on long probes at no measurable accuracy cost.
      */
     private static final int MAX_PROBE_BYTES = 16 * 1024;
 
@@ -216,8 +215,8 @@ private static Map<String, Cohort> buildCohortTable() {
     /**
      * Bigram-major int8 logP layout.  Quantized at load time via
      * per-class scale {@code scale[c] = maxAbs(class c's logP column) / 127}.
-     * In-memory footprint: {@code 65_536 × numClasses} bytes ≈ 2 MB for
-     * 32 classes, 4× smaller than float32.  The hot-loop accumulates
+     * In-memory footprint: {@code 65_536 × numClasses} bytes ≈ 2.1 MB for
+     * 34 classes, 4× smaller than float32.  The hot-loop accumulates
      * raw int8 products and applies dequantization once at the end of
      * the probe, CharSoup-style.
      */
@@ -309,9 +308,28 @@ public NaiveBayesBigramEncodingDetector(InputStream modelStream) throws IOExcept
                 for (int bg = 0; bg < BIGRAM_SPACE; bg++) {
                     logP8[bg * numClasses + c] = u;
                 }
-                // Overwrite with trained pairs.
+                // Overwrite with trained pairs. Bigram ids are sorted ascending and
+                // stored as varint deltas (LEB128) from the previous id.
+                int bigram = 0;
                 for (int i = 0; i < vocabSize; i++) {
-                    int bigram = dis.readUnsignedShort();
+                    long delta = 0;
+                    int shift = 0;
+                    int b;
+                    do {
+                        if (shift >= 35) {
+                            throw new IOException(
+                                    "Malformed varint in bigram-id deltas (too long)");
+                        }
+                        b = dis.readUnsignedByte();
+                        delta |= (long) (b & 0x7F) << shift;
+                        shift += 7;
+                    } while ((b & 0x80) != 0);
+                    long next = bigram + delta;
+                    if (next < 0 || next >= BIGRAM_SPACE) {
+                        throw new IOException("Bigram id out of range: " + next
+                                + " (expected [0, " + BIGRAM_SPACE + "))");
+                    }
+                    bigram = (int) next;
                     byte q = dis.readByte();
                     logP8[bigram * numClasses + c] = q;
                 }
@@ -521,6 +539,7 @@ public ScoreResult scoreClassesAndCount(byte[] probe) {
         // diagnostic path.
         double[] score = new double[numClasses];
         double[] contributions = new double[numClasses];
+        double[] bestPerCohort = new double[Cohort.values().length];
         int hashCap = counts.capacity();
         for (int slot = 0; slot < hashCap; slot++) {
             int bigram = counts.keyAt(slot);
@@ -549,6 +568,15 @@ public ScoreResult scoreClassesAndCount(byte[] probe) {
             // cohort, so the cap engages on cross-cohort gaps that a
             // max-vs-overall-runner-up cap missed when multiple classes
             // in top-1's cohort sat close together.
+            //
+            // Single per-class pass computes the contributions, the running
+            // max/topClass, AND the best contribution per cohort; bestCrossCohort
+            // then reduces over the (few) cohorts instead of a second full
+            // per-class pass, and the clip is fused into the accumulate.
+            // Bit-identical to the prior four-pass form: max/cross-cohort are exact
+            // (comparisons over the same value set), and the contribution formula
+            // and score[] accumulation order are unchanged.
+            java.util.Arrays.fill(bestPerCohort, Double.NEGATIVE_INFINITY);
             int topClass = -1;
             double max = Double.NEGATIVE_INFINITY;
             for (int c = 0; c < numClasses; c++) {
@@ -558,25 +586,27 @@ public ScoreResult scoreClassesAndCount(byte[] probe) {
                     max = contrib;
                     topClass = c;
                 }
+                int co = cohorts[c].ordinal();
+                if (contrib > bestPerCohort[co]) {
+                    bestPerCohort[co] = contrib;
+                }
             }
-            Cohort topCohort = cohorts[topClass];
+            int topCohort = cohorts[topClass].ordinal();
             double bestCrossCohort = Double.NEGATIVE_INFINITY;
-            for (int c = 0; c < numClasses; c++) {
-                if (cohorts[c] != topCohort && contributions[c] > bestCrossCohort) {
-                    bestCrossCohort = contributions[c];
+            for (int k = 0; k < bestPerCohort.length; k++) {
+                if (k != topCohort && bestPerCohort[k] > bestCrossCohort) {
+                    bestCrossCohort = bestPerCohort[k];
                 }
             }
             // bestCrossCohort is always finite here: load requires >=2 cohorts.
             double capValue = bestCrossCohort + CAP_PER_BIGRAM_NATS;
-            if (max > capValue) {
-                for (int c = 0; c < numClasses; c++) {
-                    if (contributions[c] > capValue) {
-                        contributions[c] = capValue;
-                    }
-                }
-            }
+            boolean clip = max > capValue;
             for (int c = 0; c < numClasses; c++) {
-                score[c] += contributions[c];
+                double v = contributions[c];
+                if (clip && v > capValue) {
+                    v = capValue;
+                }
+                score[c] += v;
             }
         }
         return new ScoreResult(score, scored, total);