TIKA-4754 -- move to bloom filters for common_tokens

tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/tools/CommonTokenOverlapCounter.java

@@ -16,56 +16,84 @@
  */
 package org.apache.tika.eval.app.tools;
 
-import java.io.File;
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 
-import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
-
+/**
+ * Dev tool that reports pairs of languages whose common-token lists overlap by more than 1%.
+ * <p>
+ * The shipped runtime resources are Bloom filters, which cannot be enumerated, so this reads
+ * the raw {@code token<TAB>df<TAB>cf} source lists directly. Point it at a directory of those
+ * files (e.g. {@code tika-eval-core/src/test/resources/common_tokens}).
+ */
 public class CommonTokenOverlapCounter {
 
     public static void main(String[] args) throws Exception {
         Path commonTokensDir = Paths.get(args[0]);
-        CommonTokenOverlapCounter counter = new CommonTokenOverlapCounter();
-        counter.execute(commonTokensDir);
+        new CommonTokenOverlapCounter().execute(commonTokensDir);
     }
 
     private void execute(Path commonTokensDir) throws IOException {
-        List<String> langs = new ArrayList<>();
-        for (File f : commonTokensDir
-                .toFile()
-                .listFiles()) {
-            langs.add(f.getName());
+        Map<String, Set<String>> tokensByLang = new HashMap<>();
+        try (DirectoryStream<Path> ds = Files.newDirectoryStream(commonTokensDir)) {
+            for (Path p : ds) {
+                if (Files.isRegularFile(p)) {
+                    tokensByLang.put(p.getFileName().toString(), loadTokens(p));
+                }
+            }
         }
-        CommonTokenCountManager mgr = new CommonTokenCountManager(commonTokensDir, "");
+        List<String> langs = new ArrayList<>(tokensByLang.keySet());
         for (int i = 0; i < langs.size() - 1; i++) {
             for (int j = i + 1; j < langs.size(); j++) {
-                compare(langs.get(i), langs.get(j), mgr);
+                compare(langs.get(i), langs.get(j), tokensByLang);
+            }
+        }
+    }
+
+    private static Set<String> loadTokens(Path p) throws IOException {
+        Set<String> tokens = new HashSet<>();
+        try (BufferedReader reader = Files.newBufferedReader(p, StandardCharsets.UTF_8)) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                line = line.trim();
+                if (line.isEmpty() || line.startsWith("#")) {
+                    continue;
+                }
+                String token = line.split("\t", 2)[0].trim();
+                if (!token.isEmpty()) {
+                    tokens.add(token);
+                }
             }
         }
+        return tokens;
     }
 
-    private void compare(String langA, String langB, CommonTokenCountManager mgr) {
+    private void compare(String langA, String langB, Map<String, Set<String>> tokensByLang) {
+        Set<String> setA = tokensByLang.get(langA);
+        Set<String> setB = tokensByLang.get(langB);
         int overlap = 0;
-        int denom = 0;
-        Set<String> setA = mgr.getTokens(langA);
-        Set<String> setB = mgr.getTokens(langB);
         for (String a : setA) {
             if (setB.contains(a)) {
                 overlap += 2;
             }
         }
-        denom = setA.size() + setB.size();
+        int denom = setA.size() + setB.size();
         double percent = (double) overlap / (double) denom;
         if (percent > 0.01) {
             System.out.printf(Locale.US, "%s %s %.2f%n", langA, langB, percent);
         }
     }
 
-
 }

tika-eval/tika-eval-core/pom.xml

@@ -55,6 +55,10 @@
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-math3</artifactId>
@@ -99,8 +103,10 @@
         <artifactId>apache-rat-plugin</artifactId>
         <configuration>
           <inputExcludes>
-            <inputExclude>src/main/resources/common_tokens/*</inputExclude>
+            <!-- binary per-language Bloom filters; cannot carry a license header -->
+            <inputExclude>src/main/resources/common_tokens_bloom/*</inputExclude>
             <inputExclude>src/main/resources/*.json</inputExclude>
+            <!-- raw token lists used only to (re)generate the Bloom filters -->
             <inputExclude>src/test/resources/common_tokens/*</inputExclude>
             <inputExclude>src/test/resources/test-dirs/**</inputExclude>
           </inputExcludes>

tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokens.java

@@ -18,14 +18,14 @@
 
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.commons.lang3.tuple.Pair;
 
 import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
 import org.apache.tika.eval.core.tokens.CommonTokenResult;
 import org.apache.tika.eval.core.tokens.LangModel;
+import org.apache.tika.eval.core.tokens.TikaEvalTokenizer;
 import org.apache.tika.eval.core.tokens.TokenCounts;
 import org.apache.tika.language.detect.LanguageResult;
 
@@ -65,7 +65,9 @@ public CommonTokenResult calculate(List<LanguageResult> languages, TokenCounts t
         Pair<String, LangModel> pair =
                 commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
         String actualLangCode = pair.getKey();
-        Set<String> commonTokens = pair.getValue().getTokens();
+        // Membership is via a Bloom filter, so a token not in the common-tokens list may
+        // very occasionally be counted as common (false positive); never the reverse.
+        LangModel commonTokens = pair.getValue();
         int numUniqueCommonTokens = 0;
         int numCommonTokens = 0;
         int numUniqueAlphabeticTokens = 0;
@@ -77,7 +79,10 @@ public CommonTokenResult calculate(List<LanguageResult> languages, TokenCounts t
                 numAlphabeticTokens += count;
                 numUniqueAlphabeticTokens++;
             }
-            if (commonTokens.contains(token)) {
+            // Only test tokens that could actually be in a common-token list. A number,
+            // short token, or HTML term is never in the list, so testing it can only yield a
+            // (Bloom) false positive; skipping them is a no-op for the exact list.
+            if (TikaEvalTokenizer.isCommonTokenCandidate(token) && commonTokens.contains(token)) {
                 numCommonTokens += count;
                 numUniqueCommonTokens++;
             }