diff --git a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java index 6957601e2ca..426f508c14d 100644 --- a/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java +++ b/tika-core/src/main/java/org/apache/tika/detect/EncodingDetectorContext.java @@ -40,8 +40,19 @@ public class EncodingDetectorContext { private final List results = new ArrayList<>(); + private final EncodingProbeCache probeCache = new EncodingProbeCache(); private String arbitrationInfo; + /** + * Per-detection cache of the raw detection probe, shared across the detectors in + * this chain so they don't each re-read the same leading bytes. It lives and dies + * with this context (which is removed after detection), so it never leaks into + * recursive/attachment parsing. + */ + public EncodingProbeCache getProbeCache() { + return probeCache; + } + /** * Record the ranked results from a child detector. * diff --git a/tika-core/src/main/java/org/apache/tika/detect/EncodingProbeCache.java b/tika-core/src/main/java/org/apache/tika/detect/EncodingProbeCache.java new file mode 100644 index 00000000000..a4d783fb029 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/detect/EncodingProbeCache.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect; + +/** + * Caches the raw encoding-detection probe (the leading bytes read for detection) + * so that multiple detectors in a chain do not each re-read and re-tag-strip the + * same bytes. For example a statistical detector and a downstream meta detector + * that re-reads the bytes for arbitration can share one probe. + *

+ * An instance is held by {@link EncodingDetectorContext}, so it inherits that + * context's per-detection lifecycle: created fresh per detection and discarded + * with the context immediately afterwards. That matters because a + * {@link org.apache.tika.parser.ParseContext} flows on into recursive + * (attachment/embedded) parsing — a probe must never outlive the single detection + * it was read for. + *

+ * Not thread-safe: a single detection runs its detectors sequentially on one + * thread. The cache is keyed by the probe parameters — {@link #get} returns the + * cached probe only when both {@code contentTarget} and {@code rawCap} match what + * it was stored with, so a detector that wants a differently-sized probe + * transparently reads (and caches) its own. + *

+ * The cached array is shared read-only state; callers must not mutate it in place. + */ +public class EncodingProbeCache { + + private byte[] probe; + private int contentTarget = -1; + private int rawCap = -1; + + /** + * @return the cached probe if one was stored with the same {@code contentTarget} and + * {@code rawCap}; otherwise {@code null} + */ + public byte[] get(int contentTarget, int rawCap) { + if (probe != null && this.contentTarget == contentTarget && this.rawCap == rawCap) { + return probe; + } + return null; + } + + /** + * Stores the probe bytes read with the given parameters. + */ + public void put(byte[] probe, int contentTarget, int rawCap) { + this.probe = probe; + this.contentTarget = contentTarget; + this.rawCap = rawCap; + } +} diff --git a/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java index c052b062b60..1d9398ce5e8 100644 --- a/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java +++ b/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Serializable; -import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Collections; @@ -85,6 +84,7 @@ public void setMarkLimit(int markLimit) { private static final Pattern FLEXIBLE_CHARSET_ATTR_PATTERN = Pattern.compile(("(?is)\\bcharset\\s*=\\s*(?:['\\\"]\\s*)?([-_:\\.a-z0-9]+)")); private static final Charset ASCII = Charset.forName("US-ASCII"); + private static final Pattern HTML_COMMENT_PATTERN = Pattern.compile("|$)"); /** * HTML can include non-iana supported charsets that Java * recognizes, e.g. "unicode". This can lead to incorrect detection/mojibake. @@ -162,10 +162,20 @@ public List detect(TikaInputStream tis, Metadata metadata, } tis.reset(); - String head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n)).toString(); - String headNoComments = head.replaceAll("|$)", " "); + // findCharset only ever matches a meta tag (HTTP_META_PATTERN = "<\s*meta..."). + // If the probe has no such tag, the full ASCII decode + comment-stripping + // regex below can only produce null — skip them. Byte-level, no allocation; + // a strict necessary condition for any non-empty result. + if (!containsMetaTag(buffer, n)) { + return Collections.emptyList(); + } + + String head = new String(buffer, 0, n, ASCII); + boolean hasComment = head.indexOf(" + + + tika-ml + org.apache.tika + ${revision} + + 4.0.0 + + tika-ml-junkdetect-tools + Apache Tika ML junk detector — training and evaluation tools + + Build-time training, evaluation, and diagnostic CLIs for the junk detector + (TrainJunkModel, BuildJunkTrainingData, and diagnostics). These are not part + of the runtime detector — they are kept out of the tika-ml-junkdetect runtime + jar and built into a self-contained tools jar via the 'train' profile. + + + + + org.apache.tika + tika-ml-junkdetect + ${revision} + + + org.apache.tika + tika-encoding-detector-mojibuster + ${revision} + + + + + + org.apache.tika + tika-serialization + ${revision} + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.tika.ml.junkdetect.tools + + + + + + + de.thetaphi + forbiddenapis + + true + + + + + + + + + train + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + shade + + true + tools + + + org.apache.tika.ml.junkdetect.tools.TrainJunkModel + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BoundaryBigramAudit.java b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/BoundaryBigramAudit.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BoundaryBigramAudit.java rename to tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/BoundaryBigramAudit.java diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java rename to tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/BuildJunkTrainingData.java diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/DebugScriptRuns.java b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/DebugScriptRuns.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/DebugScriptRuns.java rename to tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/DebugScriptRuns.java diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java rename to tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfig.java diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/LineScriptFractions.java b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/LineScriptFractions.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/LineScriptFractions.java rename to tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/LineScriptFractions.java diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/ScriptCensus.java b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/ScriptCensus.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/ScriptCensus.java rename to tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/ScriptCensus.java diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java similarity index 97% rename from tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java rename to tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java index 63bce5317f2..13cbc20381b 100644 --- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java +++ b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java @@ -46,7 +46,7 @@ * *

z1 (codepoint-bigram log-probability) is trained per script by bucketing * every bigram to its script ({@link JunkDetector#forEachScriptBigram}) and - * building a per-script open-addressing bigram table with unigram backoff. + * building a per-script sorted-occupied bigram table with unigram backoff. * z2 (Unicode block-transition), z3 (control-byte fraction), and z4 * (script-transition) are single global document-level features. All features * are calibrated (mu/sigma) and combined by a single global contrastive @@ -932,7 +932,7 @@ static String byteLevelMojibake(String text, String sourceCs, String wrongCs) { * * @param trainFile the per-script {@code *.train.gz} * @param minBigramCount drop pairs whose count is below this - * @param loadFactor target OA table load factor (e.g. 0.5) + * @param loadFactor unused (retained for signature compatibility) * @param keyIndexBits bit-width per index in the packed key * (each side of the pair must fit) */ @@ -970,9 +970,12 @@ public static BigramTables trainBigramTablesForScript(Path trainFile, /** * Builds the {@link BigramTables} carrier from pre-tallied pair/unigram * counts. Drops pairs below - * {@code minBigramCount}, assigns dense codepoint indices, and packs an - * open-addressing bigram table; unigram log-probs use {@code unigramTotal} + * {@code minBigramCount}, assigns dense codepoint indices, and packs the + * sorted-occupied bigram table; unigram log-probs use {@code unigramTotal} * as the denominator. + * + *

{@code loadFactor} is retained for signature compatibility but unused: + * the sorted-occupied table (binary-search lookup) has no load factor. */ public static BigramTables buildBigramTablesFromCounts( HashMap pairCounts, @@ -1037,12 +1040,8 @@ public static BigramTables buildBigramTablesFromCounts( // Quantize unigram log-probs. QuantizedFloats qUnigram = quantizeFloats(unigramLogP); - // --- Build the open-addressing bigram table. --- - int slots = nextPowerOfTwo((int) Math.max(2, Math.ceil(keptPairs / loadFactor))); - int[] keys = new int[slots]; - java.util.Arrays.fill(keys, BigramTables.EMPTY_KEY); - // Compute log-probs first, quantize once, then write into the table - // alongside its key. + // --- Build the sorted-occupied bigram table (binary-search lookup). --- + // Compute log-probs first, quantize once, then sort by key. float[] keptLogP = new float[keptPairs]; int[] keptKeys = new int[keptPairs]; int writeIdx = 0; @@ -1067,16 +1066,25 @@ public static BigramTables buildBigramTablesFromCounts( } // Quantize all kept log-probs together so they share min/max. QuantizedFloats qBigram = quantizeFloats(keptLogP); - byte[] values = new byte[slots]; + // Sort (key, value) ascending by signed key so the loader can binary-search. + // Pack into a long (key in high 32 bits, value byte in low 8) for one sort. + long[] sortable = new long[keptPairs]; + for (int i = 0; i < keptPairs; i++) { + sortable[i] = (((long) keptKeys[i]) << 32) | (qBigram.bytes[i] & 0xFFL); + } + java.util.Arrays.sort(sortable); + int[] keys = new int[keptPairs]; + byte[] values = new byte[keptPairs]; for (int i = 0; i < keptPairs; i++) { - insertOA(keys, values, keptKeys[i], qBigram.bytes[i]); + keys[i] = (int) (sortable[i] >> 32); + values[i] = (byte) (sortable[i] & 0xFF); } System.out.printf( " pair_counts: distinct=%,d, kept=%,d (>=%d), dropped=%,d " - + "cp_index=%,d slots=%,d (load=%.2f)%n", + + "cp_index=%,d bigram_entries=%,d%n", totalDistinct, keptPairs, minBigramCount, dropped, - cpIndex.length, slots, keptPairs / (double) slots); + cpIndex.length, keptPairs); return new BigramTables(cpIndex, keys, values, qUnigram.bytes, qBigram.min, qBigram.max, @@ -1084,32 +1092,6 @@ public static BigramTables buildBigramTablesFromCounts( unigramFallbackLogP, BACKOFF_ALPHA); } - /** - * Inserts a {@code (packedKey, value)} pair into the open-addressing - * table. The caller is responsible for sizing the table large enough - * to avoid an infinite probe (any load < 1.0 is safe). - */ - private static void insertOA(int[] keys, byte[] values, int packedKey, byte value) { - int mask = keys.length - 1; - int h = JunkDetector.mixIndexKey(packedKey) & mask; - while (keys[h] != BigramTables.EMPTY_KEY) { - if (keys[h] == packedKey) { - // Same key twice — shouldn't happen with our dedup, but be - // defensive and overwrite rather than corrupt. - values[h] = value; - return; - } - h = (h + 1) & mask; - } - keys[h] = packedKey; - values[h] = value; - } - - private static int nextPowerOfTwo(int n) { - if (n < 1) return 1; - int p = Integer.highestOneBit(n - 1) << 1; - return Math.max(1, p); - } // ----------------------------------------------------------------------- // Global contrastive combiner training diff --git a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java similarity index 93% rename from tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java rename to tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java index 07efc64dd85..7433820efea 100644 --- a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java +++ b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java @@ -111,14 +111,12 @@ void roundTripAllSeenPairsScoreHigher(@TempDir Path tmp) throws IOException { // Same shape as the first test but with BOTH (A,B) and (B,A) in the // bigram table. mean log-prob = -1.0, z1 = +4.0, logit = +4.0. int[] cpIndex = new int[]{'A', 'B'}; - int[] keys = new int[4]; - Arrays.fill(keys, BigramTables.EMPTY_KEY); - byte[] values = new byte[4]; float bMin = -10.0f; float bMax = -1.0f; byte b = quantizeOne(-1.0f, bMin, bMax); - insertOA(keys, values, JunkDetector.packBigramKey(0, 1), b); - insertOA(keys, values, JunkDetector.packBigramKey(1, 0), b); + // sorted-occupied: packBigramKey(0,1)=1 < packBigramKey(1,0)=65536 + int[] keys = {JunkDetector.packBigramKey(0, 1), JunkDetector.packBigramKey(1, 0)}; + byte[] values = {b, b}; float uMin = -5.0f; float uMax = -2.0f; @@ -266,17 +264,14 @@ void trainerRoundTripIntegration(@TempDir Path tmp) throws IOException { private static BigramTables buildLatinTablesAB() { int[] cpIndex = new int[]{'A', 'B'}; - // 4 slots ≈ 25% load for 1 pair. Open-addressing with linear probe. - int[] keys = new int[4]; - Arrays.fill(keys, BigramTables.EMPTY_KEY); - byte[] values = new byte[4]; - // Manual quantization with a chosen range so we don't hit the // degenerate single-element case. range=[-10, -1] → -1.0 → byte 255. float bMin = -10.0f; float bMax = -1.0f; byte b = quantizeOne(-1.0f, bMin, bMax); - insertOA(keys, values, JunkDetector.packBigramKey(0, 1), b); + // sorted-occupied table with a single trained pair. + int[] keys = {JunkDetector.packBigramKey(0, 1)}; + byte[] values = {b}; float uMin = -5.0f; float uMax = -2.0f; @@ -358,13 +353,10 @@ void caseFoldedBackoffRescuesAllCapsButNotMixedOrMojibake() { * (uppercase 'A'/'B' are absent from the index, so they must fold). */ private static BigramTables buildLatinTablesLowerAB() { int[] cpIndex = new int[]{'a', 'b'}; - int[] keys = new int[4]; - Arrays.fill(keys, BigramTables.EMPTY_KEY); - byte[] values = new byte[4]; float bMin = -10.0f; float bMax = -1.0f; - insertOA(keys, values, JunkDetector.packBigramKey(0, 1), - quantizeOne(-1.0f, bMin, bMax)); + int[] keys = {JunkDetector.packBigramKey(0, 1)}; + byte[] values = {quantizeOne(-1.0f, bMin, bMax)}; float uMin = -5.0f; float uMax = -2.0f; byte[] unigramBytes = new byte[]{ @@ -403,25 +395,6 @@ private static byte quantizeOne(float v, float min, float max) { return (byte) q; } - /** - * Replica of {@code TrainJunkModel.insertOA} (package-private) for the - * test's hand-constructed tables. Uses the same mix-hash as the - * production code path. - */ - private static void insertOA(int[] keys, byte[] values, int packedKey, byte value) { - int mask = keys.length - 1; - int h = JunkDetector.mixIndexKey(packedKey) & mask; - while (keys[h] != BigramTables.EMPTY_KEY) { - if (keys[h] == packedKey) { - values[h] = value; - return; - } - h = (h + 1) & mask; - } - keys[h] = packedKey; - values[h] = value; - } - /** * Saves a minimal model containing only LATIN, with the block / control / * script-transition features zeroed out and pure-z1 combiner weights diff --git a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationData.java b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationData.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationData.java rename to tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationData.java diff --git a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationDataTest.java b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationDataTest.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationDataTest.java rename to tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/BuildJunkAugmentationDataTest.java diff --git a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfigTest.java b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfigTest.java similarity index 100% rename from tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfigTest.java rename to tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/tools/JunkDetectorTrainingConfigTest.java diff --git a/tika-ml/tika-ml-junkdetect/pom.xml b/tika-ml/tika-ml-junkdetect/pom.xml index fe717998cfe..5027cbe7433 100644 --- a/tika-ml/tika-ml-junkdetect/pom.xml +++ b/tika-ml/tika-ml-junkdetect/pom.xml @@ -61,18 +61,6 @@ - - - org.apache.tika - tika-serialization - ${revision} - test - org.junit.jupiter junit-jupiter-api @@ -108,7 +96,7 @@ - + de.thetaphi forbiddenapis @@ -119,57 +107,6 @@ - - - - train - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - shade - - true - tools - - - org.apache.tika.ml.junkdetect.tools.TrainJunkModel - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - - - 3.0.0-rc1 diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java index 5c7e7382900..05974bd9272 100644 --- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java +++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java @@ -36,12 +36,11 @@ * script. Codepoint → dense index is a binary search; index → * codepoint is direct array access. Typical sizes: ~7K-15K for HAN, * ~200-500 for most other scripts. - *

  • {@code bigramKeys} / {@code bigramValues} — parallel arrays - * implementing an open-addressed hash table with linear probing. - * Each key is a 32-bit value {@code (idxA << 16) | idxB}; key {@code - * -1} means "empty slot." Indices are bounded at 16 bits (65535), - * which is comfortably above the largest per-script codepoint count - * we observe. + *
  • {@code bigramKeys} / {@code bigramValues} — parallel arrays of the + * occupied entries only, sorted ascending by key for binary-search + * lookup. Each key is a 32-bit value {@code (idxA << 16) | idxB}. + * Indices are bounded at 16 bits (65535), comfortably above the + * largest per-script codepoint count we observe. *
  • {@code unigramTable} — {@code byte[numCodepoints]}, quantized * unigram log-probabilities indexed by the same codepoint→index map. *
  • {@code bigramQuantMin/Max}, {@code unigramQuantMin/Max} — @@ -56,14 +55,12 @@ * independence sum. * * - *

    Membership semantics: no Bloom filter. The empty-slot sentinel is - * the membership oracle — a pair is "seen" iff binary-search finds both - * codepoints in the index AND a probe sequence hits a matching key before - * an empty slot. Lookups are therefore exact. + *

    Membership semantics: no Bloom filter. A pair is "seen" iff + * binary-search finds both codepoints in the index AND finds the packed + * key in {@code bigramKeys}. Lookups are therefore exact. * - *

    Fields are package-private so the - * {@link org.apache.tika.ml.junkdetect.tools.TrainJunkModel} trainer can - * construct instances directly without going through accessors. + *

    Instances are built by the trainer ({@code TrainJunkModel}, in the + * tika-ml-junkdetect-tools module) and read back via {@link #readFrom}. */ public final class BigramTables { @@ -124,14 +121,23 @@ public void writeTo(DataOutputStream dos) throws IOException { cpBuf.asIntBuffer().put(codepointIndex); dos.write(cpBuf.array()); - // Bigram open-addressing table (keys + values). + // Bigram table: sorted-occupied keys (ascending) + parallel values. + // Store key[0] raw, then varint (LEB128) deltas from the previous key; + // deltas are small because the keys are sorted and dense. dos.writeInt(bigramKeys.length); dos.writeFloat(bigramQuantMin); dos.writeFloat(bigramQuantMax); - ByteBuffer keyBuf = ByteBuffer.allocate(bigramKeys.length * 4) - .order(ByteOrder.BIG_ENDIAN); - keyBuf.asIntBuffer().put(bigramKeys); - dos.write(keyBuf.array()); + if (bigramKeys.length > 0) { + dos.writeInt(bigramKeys[0]); + for (int i = 1; i < bigramKeys.length; i++) { + long delta = (long) bigramKeys[i] - (long) bigramKeys[i - 1]; + if (delta <= 0) { + throw new IOException("bigramKeys must be strictly ascending " + + "(no duplicates); non-increasing at index " + i); + } + writeVarLong(dos, delta); + } + } dos.write(bigramValues); // Unigram table. @@ -153,9 +159,18 @@ public static BigramTables readFrom(DataInputStream dis) throws IOException { int slots = dis.readInt(); float bMin = dis.readFloat(); float bMax = dis.readFloat(); - byte[] keyBytes = dis.readNBytes(slots * 4); int[] keys = new int[slots]; - ByteBuffer.wrap(keyBytes).order(ByteOrder.BIG_ENDIAN).asIntBuffer().get(keys); + if (slots > 0) { + keys[0] = dis.readInt(); + for (int i = 1; i < slots; i++) { + long next = (long) keys[i - 1] + readVarLong(dis); + if (next <= keys[i - 1] || next > Integer.MAX_VALUE) { + throw new IOException("Corrupt bigram keys: not strictly " + + "ascending / out of range at index " + i); + } + keys[i] = (int) next; + } + } byte[] values = dis.readNBytes(slots); float uMin = dis.readFloat(); @@ -167,11 +182,36 @@ public static BigramTables readFrom(DataInputStream dis) throws IOException { bMin, bMax, uMin, uMax, uFallback, backoffAlpha); } + /** Writes a non-negative long as an unsigned LEB128 varint. */ + private static void writeVarLong(DataOutputStream dos, long v) throws IOException { + while ((v & ~0x7FL) != 0) { + dos.writeByte((int) ((v & 0x7F) | 0x80)); + v >>>= 7; + } + dos.writeByte((int) v); + } + + /** Reads an unsigned LEB128 varint written by {@link #writeVarLong}. */ + private static long readVarLong(DataInputStream dis) throws IOException { + long v = 0; + int shift = 0; + int b; + do { + if (shift >= 64) { + throw new IOException("Malformed varint in bigram key deltas (too long)"); + } + b = dis.readUnsignedByte(); + v |= (long) (b & 0x7F) << shift; + shift += 7; + } while ((b & 0x80) != 0); + return v; + } + /** * Returns a one-line summary for trainer progress output. */ public String statsString() { - return String.format( + return String.format(java.util.Locale.ROOT, " cp_index=%d, bigram_slots=%d (load≈%.2f), " + "bigram_range=[%.3f, %.3f], unigram_range=[%.3f, %.3f]", codepointIndex.length, bigramKeys.length, diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java index 2f117479d04..b63f8787e8e 100644 --- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java +++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java @@ -235,41 +235,41 @@ public static JunkDetector loadFromPath(Path path) throws IOException { * [4 bytes] num_scripts (int BE) * [1 byte] block_scheme_version (must equal * {@link UnicodeBlockRanges#SCHEME_VERSION}) + * // z4 — global script-transition section * [1 byte] num_script_buckets * for each bucket: * [2 bytes] name length (ushort BE) * [name bytes] bucket name (UTF-8) - * [num_script_buckets² × 4 bytes] script-transition log-prob table (F4) - * [4 bytes] mu4 (float32 BE) - * [4 bytes] sigma4 (float32 BE) + * [4 bytes] scriptTrans_quant_min (float32 BE) + * [4 bytes] scriptTrans_quant_max (float32 BE) + * [num_script_buckets² × 2 bytes] script-transition table (z4, int16-quantized) + * [4 bytes] mu4 (z4 calibration, float32 BE) + * [4 bytes] sigma4 + * // z2 — global block-transition section + * [4 bytes] block_quant_min (float32 BE) + * [4 bytes] block_quant_max (float32 BE) + * [block_N² × 2 bytes] block-transition table (z2, int16-quantized) + * [4 bytes] mu2 (z2 calibration) + * [4 bytes] sigma2 + * // global per-feature calibrations, {mu, sigma} float32 pairs + * [8 bytes] z3 calibration (control-byte ratio) + * [8 bytes] z5 calibration (letter-adjacent-to-mark) + * [8 bytes] z6 calibration (replacement-char ratio) + * [8 bytes] z9 calibration (script-alternation) + * // global combiner + * [1 byte] num_features + * [(num_features+1) × 4 bytes] combiner weights w1..wN and bias + * // per-script section * for each script (sorted by name): * [2 bytes] name length * [name bytes] script name (UTF-8) - * [4 bytes] mu1 (F1 calibration, codepoint-bigram mean log-prob) + * [4 bytes] mu1 (z1 calibration, codepoint-bigram mean log-prob) * [4 bytes] sigma1 - * // bigram tables for this script — see {@link BigramTables#writeTo} - * [4 bytes] backoff_alpha (float32 BE) - * [4 bytes] codepoint_count - * [codepoint_count × 4 bytes] codepoint index (sorted, ascending) - * [4 bytes] bigram_slots (power of 2) - * [4 bytes] bigram_quant_min (float32 BE) - * [4 bytes] bigram_quant_max (float32 BE) - * [bigram_slots × 4 bytes] bigram open-addressing keys - * ((idxA<<16)|idxB, or {@link BigramTables#EMPTY_KEY}) - * [bigram_slots bytes] bigram values (8-bit quantized log-probs) - * [4 bytes] unigram_quant_min (float32 BE) - * [4 bytes] unigram_quant_max (float32 BE) - * [4 bytes] unigram_fallback_log_prob (float32 BE; used for - * codepoints not in index) - * [codepoint_count bytes] unigram values (8-bit quantized log-probs) - * // F2/F3/classifier - * [4 bytes] mu2 (F2 calibration) - * [4 bytes] sigma2 - * [block_N² × 4 bytes] block-transition log-prob table (F2) - * [4 bytes] mu3 (F3 calibration) - * [4 bytes] sigma3 - * [1 byte] num_features - * [(num_features+1) × 4 bytes] classifier weights w1..wN and bias + * [variable] bigram + unigram tables — exact layout in + * {@link BigramTables#writeTo}: codepoint index, then the + * sorted-occupied bigram keys (key[0] as int32 BE followed + * by LEB128 varint deltas) and 8-bit quantized bigram and + * unigram log-prob values * */ public static JunkDetector load(InputStream rawIs) throws IOException { @@ -498,6 +498,16 @@ private Agg aggregate(String text) { int[] cps = text.codePoints().toArray(); Map buckets = new HashMap<>(); // script -> {sumLogP, count} + // Left-index memo. forEachScriptBigram emits (^,x),(x,y),(y,$)... so within + // a run each pair's right codepoint b is the next pair's left codepoint a. + // Reuse the previous pair's right-index as this pair's left-index when they + // match (same codepoint AND same script => same table), so each codepoint is + // binary-searched in the script's index once instead of twice. Bit-identical + // to scoring each pair independently; the guard falls back to a fresh search + // whenever the overlap doesn't hold (run boundary, sentinel, script change). + String[] lastScript = {null}; + int[] lastB = {Integer.MIN_VALUE}; + int[] lastBIdx = {-1}; forEachScriptBigram(cps, (script, a, b) -> { if (!calibrations.containsKey(script)) { return; @@ -506,7 +516,13 @@ private Agg aggregate(String text) { if (t == null) { return; } - double lp = computeF1MeanLogP(new int[]{a, b}, t); + int idxA = (a == lastB[0] && script.equals(lastScript[0])) + ? lastBIdx[0] : codepointToIndex(t, a); + int idxB = codepointToIndex(t, b); + lastScript[0] = script; + lastB[0] = b; + lastBIdx[0] = idxB; + double lp = scorePairF1(a, idxA, b, idxB, t); if (Double.isNaN(lp)) { return; } @@ -839,7 +855,7 @@ public static float computeZ4ScriptTransition(String text, for (int i = 0; i < text.length(); ) { int cp = text.codePointAt(i); i += Character.charCount(cp); - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = TextQualityFeatures.scriptOf(cp); if (s == Character.UnicodeScript.COMMON || s == Character.UnicodeScript.INHERITED || s == Character.UnicodeScript.UNKNOWN) { @@ -970,22 +986,6 @@ public static int codepointToIndex(BigramTables tables, int cp) { return java.util.Arrays.binarySearch(tables.codepointIndex, cp); } - /** - * Mixing function used to scatter packed (idxA, idxB) keys across - * the open-addressing table. A simple integer finalizer (splitmix32 - * style) gives good distribution for sequential index values. - * - *

    Public so the trainer's open-addressing insertion routine uses - * the same probe order as inference — drift here would silently - * corrupt every lookup. - */ - public static int mixIndexKey(int packedKey) { - int x = packedKey; - x = (x ^ (x >>> 16)) * 0x7feb352d; - x = (x ^ (x >>> 15)) * 0x846ca68b; - x = x ^ (x >>> 16); - return x; - } /** * Packed bigram key for indices {@code (a, b)} where each index fits in @@ -1085,20 +1085,12 @@ private static double scorePairF1(int cpA, int idxA, int cpB, int idxB, * for {@code (idxA, idxB)}, or {@code -1} if not present (probe hit an * empty slot first). * - *

    Linear probing with the same mix-hash used at training time — - * required for the table to be readable, not just writable. + *

    {@code bigramKeys} is sorted ascending (signed), so this is a binary search. */ static int lookupBigramSlot(BigramTables tables, int idxA, int idxB) { int packedKey = packBigramKey(idxA, idxB); - int[] keys = tables.bigramKeys; - int mask = keys.length - 1; - int h = mixIndexKey(packedKey) & mask; - while (true) { - int k = keys[h]; - if (k == BigramTables.EMPTY_KEY) return -1; - if (k == packedKey) return h; - h = (h + 1) & mask; - } + int slot = java.util.Arrays.binarySearch(tables.bigramKeys, packedKey); + return slot >= 0 ? slot : -1; } private static double unigramLogProb(BigramTables tables, int idx) { @@ -1134,7 +1126,7 @@ private float computeScriptTransitionZ(String text) { for (int i = 0; i < text.length(); ) { int cp = text.codePointAt(i); i += Character.charCount(cp); - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = TextQualityFeatures.scriptOf(cp); if (s == Character.UnicodeScript.COMMON || s == Character.UnicodeScript.INHERITED || s == Character.UnicodeScript.UNKNOWN) { @@ -1170,7 +1162,7 @@ private float computeScriptTransitionZ(String text) { /** COMMON-class predicate: COMMON, INHERITED, UNKNOWN all pool into COMMON. */ static String classKey(int cp) { - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = TextQualityFeatures.scriptOf(cp); if (s == Character.UnicodeScript.COMMON || s == Character.UnicodeScript.INHERITED || s == Character.UnicodeScript.UNKNOWN) { @@ -1381,7 +1373,7 @@ static String detectDominantScript(String text) { Map counts = new HashMap<>(); for (int i = 0; i < text.length(); ) { int cp = text.codePointAt(i); - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = TextQualityFeatures.scriptOf(cp); if (s != Character.UnicodeScript.COMMON && s != Character.UnicodeScript.INHERITED && s != Character.UnicodeScript.UNKNOWN) { diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java index b8cb75de017..0f627744bb8 100644 --- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java +++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java @@ -35,6 +35,7 @@ import org.apache.tika.config.TikaComponent; import org.apache.tika.detect.CharsetSupersets; import org.apache.tika.detect.EncodingDetectorContext; +import org.apache.tika.detect.EncodingProbeCache; import org.apache.tika.detect.EncodingResult; import org.apache.tika.detect.HighByteLetterStats; import org.apache.tika.detect.MetaEncodingDetector; @@ -156,7 +157,7 @@ public List detect(TikaInputStream tis, Metadata metadata, return Collections.emptyList(); } - byte[] bytes = readProbe(tis); + byte[] bytes = readProbe(tis, context); if (bytes == null || bytes.length == 0) { context.setArbitrationInfo("junk-filter-empty-stream"); return Collections.emptyList(); @@ -246,31 +247,20 @@ public List detect(TikaInputStream tis, Metadata metadata, Charset champion = null; double championZ = Double.NEGATIVE_INFINITY; Map scoreByCharset = new LinkedHashMap<>(); - Map diffByCharset = new LinkedHashMap<>(); - // Dedup by text: [0] = whole-text z (the champion + anchor metric, kept - // exactly as before); [1] = script-letter "diff" z (codepoints >= 0x80 - // that are letters/ideographs — the high bytes where the candidate - // decodes actually differ), used ONLY for the family gate below. - Map zByText = new HashMap<>(); + // Whole-text z (the champion + anchor metric), deduped by decoded text. + Map wholeZByText = new HashMap<>(); for (Map.Entry entry : candidates.entrySet()) { String text = entry.getValue(); - float[] zs = zByText.get(text); - if (zs == null) { + Float wholeZ = wholeZByText.get(text); + if (wholeZ == null) { org.apache.tika.quality.TextQualityScore sc = qualityDetector.score(text); - float wholeZ = sc.isUnknown() ? Float.NEGATIVE_INFINITY : sc.getZScore(); - String diff = scriptLetters(text); - float diffZ = Float.NEGATIVE_INFINITY; - if (!diff.isEmpty()) { - org.apache.tika.quality.TextQualityScore d = qualityDetector.score(diff); - diffZ = d.isUnknown() ? Float.NEGATIVE_INFINITY : d.getZScore(); - } - zs = new float[]{wholeZ, diffZ}; - zByText.put(text, zs); + wholeZ = sc.isUnknown() ? Float.NEGATIVE_INFINITY : sc.getZScore(); + wholeZByText.put(text, wholeZ); } - scoreByCharset.put(entry.getKey(), (double) zs[0]); - diffByCharset.put(entry.getKey(), (double) zs[1]); - if (zs[0] > championZ) { - championZ = zs[0]; + double z = wholeZ; + scoreByCharset.put(entry.getKey(), z); + if (z > championZ) { + championZ = z; champion = entry.getKey(); } } @@ -284,34 +274,53 @@ public List detect(TikaInputStream tis, Metadata metadata, // CJK/non-CJK BOUNDARY for COMMON-dominated docs (markup/digits/punct // decode identically and swamp the few discriminating high bytes), // producing false-CJK and real-CJK demotion. The script-letter "diff" z - // reads that boundary cleanly (coherent CJK vs garbage), so use it to - // decide ONLY the family; within a family the whole-text champion stands - // (Latin-vs-Latin etc. untouched — a blanket diff-score regressed there). - // Override only on a clear diff margin. - double bestCjkDiff = Double.NEGATIVE_INFINITY; - double bestNonCjkDiff = Double.NEGATIVE_INFINITY; - for (Map.Entry e : diffByCharset.entrySet()) { - if (isCjkCharset(e.getKey().name())) { - bestCjkDiff = Math.max(bestCjkDiff, e.getValue()); - } else { - bestNonCjkDiff = Math.max(bestNonCjkDiff, e.getValue()); + // (codepoints >= 0x80 that are letters/ideographs — the high bytes where + // candidate decodes actually differ) reads that boundary cleanly, so use + // it to decide ONLY the family; within a family the whole-text champion + // stands (Latin-vs-Latin etc. untouched — a blanket diff-score regressed). + // + // DEMOTE-ONLY and CJK-champion-only: the gate fires only to demote a CJK + // champion to non-CJK (the false-CJK fix). The reverse (promote non-CJK + // -> CJK) is NOT done: measured at 29k, the diff z reliably says "this CJK + // pick is really non-CJK" (OOV improves on every such flip) but UNreliably + // the reverse (the junk model over-rates ideograph mojibake vs sparse + // Latin letters); the promote direction is also unnecessary — genuine CJK + // is html-meta-declared upstream. Because the gate can only act when the + // champion is CJK, the second "diff" score per candidate is needed ONLY + // then — compute it lazily and skip it entirely for the common non-CJK + // champion (halving the score() calls there). + if (isCjkCharset(champion.name())) { + double bestCjkDiff = Double.NEGATIVE_INFINITY; + double bestNonCjkDiff = Double.NEGATIVE_INFINITY; + Map diffZByText = new HashMap<>(); + for (Map.Entry entry : candidates.entrySet()) { + String text = entry.getValue(); + Float diffZ = diffZByText.get(text); + if (diffZ == null) { + String diff = scriptLetters(text); + float dz = Float.NEGATIVE_INFINITY; + if (!diff.isEmpty()) { + org.apache.tika.quality.TextQualityScore d = qualityDetector.score(diff); + dz = d.isUnknown() ? Float.NEGATIVE_INFINITY : d.getZScore(); + } + diffZ = dz; + diffZByText.put(text, diffZ); + } + double dz = diffZ; + if (isCjkCharset(entry.getKey().name())) { + bestCjkDiff = Math.max(bestCjkDiff, dz); + } else { + bestNonCjkDiff = Math.max(bestNonCjkDiff, dz); + } } - } - // DEMOTE-ONLY: fire only to demote a CJK champion to non-CJK when the - // diff z clearly prefers non-CJK (the false-CJK fix). The reverse - // (promote non-CJK -> CJK) is NOT done: measured at 29k, the diff z - // reliably says "this CJK pick is really non-CJK" (OOV improves on every - // such flip) but UNreliably says "this non-CJK pick is really CJK" (the - // junk model over-rates ideograph mojibake vs sparse Latin letters — OOV - // worsened on every promote flip). The promote direction is also - // unnecessary: genuine CJK is html-meta-declared upstream. - if (isCjkCharset(champion.name()) - && bestNonCjkDiff > bestCjkDiff + FAMILY_DIFF_MARGIN) { - Charset reFam = bestInFamily(scoreByCharset, false); - if (reFam != null) { - LOG.trace("junk-filter family gate: {} (CJK) -> {} (non-CJK by diff z)", - champion.name(), reFam.name()); - champion = reFam; + // Override only on a clear diff margin. + if (bestNonCjkDiff > bestCjkDiff + FAMILY_DIFF_MARGIN) { + Charset reFam = bestInFamily(scoreByCharset, false); + if (reFam != null) { + LOG.trace("junk-filter family gate: {} (CJK) -> {} (non-CJK by diff z)", + champion.name(), reFam.name()); + champion = reFam; + } } } @@ -584,9 +593,21 @@ private static boolean allDecodingsIdentical(Map candidates) { return true; } - private byte[] readProbe(TikaInputStream tis) throws IOException { + private byte[] readProbe(TikaInputStream tis, EncodingDetectorContext context) + throws IOException { // readLimit is the tag-stripped content target; cap raw reads at 512 KB. - byte[] probe = AdaptiveProbe.read(tis, readLimit, AdaptiveProbe.DEFAULT_RAW_CAP); + int rawCap = AdaptiveProbe.DEFAULT_RAW_CAP; + EncodingProbeCache cache = context == null ? null : context.getProbeCache(); + if (cache != null) { + byte[] cached = cache.get(readLimit, rawCap); + if (cached != null) { + return cached.length == 0 ? null : cached; + } + } + byte[] probe = AdaptiveProbe.read(tis, readLimit, rawCap); + if (cache != null) { + cache.put(probe, readLimit, rawCap); + } return probe.length == 0 ? null : probe; } diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/TextQualityFeatures.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/TextQualityFeatures.java index 2ae926a927d..b5b7c5aeb28 100644 --- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/TextQualityFeatures.java +++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/TextQualityFeatures.java @@ -43,6 +43,46 @@ public final class TextQualityFeatures { private TextQualityFeatures() { } + // ----------------------------------------------------------------------- + // Memoized Unicode-script lookup + // ----------------------------------------------------------------------- + + /** Cached {@code UnicodeScript.values()} so an ordinal->enum lookup never + * re-allocates the values array. */ + private static final Character.UnicodeScript[] SCRIPT_VALUES = + Character.UnicodeScript.values(); + + /** + * Memoized {@link Character.UnicodeScript#of(int)} for the BMP. Scoring a + * document classifies every codepoint's script ~5 times (z4/z7/z8/z9 plus the + * z1 bigram bucketing), and {@code UnicodeScript.of} is a binary search over + * the script-range table (measured ~12 ns/cp, 10-20x {@code Character.getType}). + * The result is a pure function of the codepoint for a given JVM, so cache it: + * BMP codepoints (>99% of text) become an O(1) array lookup after first + * sight, shared across every call site and every document. Slot 0 means "not + * yet computed"; otherwise {@code ordinal + 1}. The fill is a benign data race + * — every writer stores the same deterministic value and {@code short} writes + * do not tear. + */ + private static final short[] BMP_SCRIPT_CACHE = new short[0x10000]; + + /** + * Script of {@code codePoint}, memoized for the BMP — identical result to + * {@link Character.UnicodeScript#of(int)} (the same singleton enum constant). + */ + static Character.UnicodeScript scriptOf(int codePoint) { + if (codePoint >= 0 && codePoint < 0x10000) { + short v = BMP_SCRIPT_CACHE[codePoint]; + if (v != 0) { + return SCRIPT_VALUES[v - 1]; + } + Character.UnicodeScript s = Character.UnicodeScript.of(codePoint); + BMP_SCRIPT_CACHE[codePoint] = (short) (s.ordinal() + 1); + return s; + } + return Character.UnicodeScript.of(codePoint); + } + // ----------------------------------------------------------------------- // Strip modes // ----------------------------------------------------------------------- @@ -99,7 +139,7 @@ private static boolean shouldStrip(int cp, StripMode mode) { return type == Character.CONTROL || type == Character.FORMAT; } case ALL_COMMON: { - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = scriptOf(cp); return s == Character.UnicodeScript.COMMON || s == Character.UnicodeScript.INHERITED || s == Character.UnicodeScript.UNKNOWN; @@ -398,7 +438,7 @@ public static double scriptDensity(String text) { continue; } total++; - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = scriptOf(cp); if (s != Character.UnicodeScript.COMMON && s != Character.UnicodeScript.INHERITED && s != Character.UnicodeScript.UNKNOWN) { @@ -442,7 +482,7 @@ public static double scriptFragmentation(String text) { for (int i = 0; i < text.length(); ) { int cp = text.codePointAt(i); i += Character.charCount(cp); - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = scriptOf(cp); if (s == Character.UnicodeScript.COMMON || s == Character.UnicodeScript.INHERITED || s == Character.UnicodeScript.UNKNOWN) { @@ -508,7 +548,7 @@ public static double scriptAlternationRatio(String text) { for (int i = 0; i < text.length(); ) { int cp = text.codePointAt(i); i += Character.charCount(cp); - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = scriptOf(cp); if (s == Character.UnicodeScript.COMMON || s == Character.UnicodeScript.INHERITED || s == Character.UnicodeScript.UNKNOWN) { @@ -538,7 +578,7 @@ public static double scriptAlternationRatio(String text) { for (int i = 0; i < text.length(); ) { int cp = text.codePointAt(i); i += Character.charCount(cp); - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = scriptOf(cp); if (s == Character.UnicodeScript.COMMON || s == Character.UnicodeScript.INHERITED || s == Character.UnicodeScript.UNKNOWN) { @@ -616,7 +656,7 @@ static boolean sameScriptCluster(int cpA, int cpB) { } private static String scriptClusterOf(int cp) { - Character.UnicodeScript s = Character.UnicodeScript.of(cp); + Character.UnicodeScript s = scriptOf(cp); switch (s) { case HAN: case HIRAGANA: diff --git a/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin b/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin index ead028cbb37..06179e75e68 100644 Binary files a/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin and b/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml index b558f836a21..ead93cf0835 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/pom.xml @@ -30,10 +30,13 @@ Apache Tika html parser module + org.apache.tika tika-encoding-detector-html ${project.version} + test