diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/tools/CommonTokenOverlapCounter.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/tools/CommonTokenOverlapCounter.java
index ed1b09e3e08..80be9ebb9cc 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/tools/CommonTokenOverlapCounter.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/tools/CommonTokenOverlapCounter.java
@@ -16,56 +16,84 @@
  */
 package org.apache.tika.eval.app.tools;
 
-import java.io.File;
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 
-import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
-
+/**
+ * Dev tool that reports pairs of languages whose common-token lists overlap by more than 1%.
+ * <p>
+ * The shipped runtime resources are Bloom filters, which cannot be enumerated, so this reads
+ * the raw {@code token<TAB>df<TAB>cf} source lists directly. Point it at a directory of those
+ * files (e.g. {@code tika-eval-core/src/test/resources/common_tokens}).
+ */
 public class CommonTokenOverlapCounter {
 
     public static void main(String[] args) throws Exception {
         Path commonTokensDir = Paths.get(args[0]);
-        CommonTokenOverlapCounter counter = new CommonTokenOverlapCounter();
-        counter.execute(commonTokensDir);
+        new CommonTokenOverlapCounter().execute(commonTokensDir);
     }
 
     private void execute(Path commonTokensDir) throws IOException {
-        List<String> langs = new ArrayList<>();
-        for (File f : commonTokensDir
-                .toFile()
-                .listFiles()) {
-            langs.add(f.getName());
+        Map<String, Set<String>> tokensByLang = new HashMap<>();
+        try (DirectoryStream<Path> ds = Files.newDirectoryStream(commonTokensDir)) {
+            for (Path p : ds) {
+                if (Files.isRegularFile(p)) {
+                    tokensByLang.put(p.getFileName().toString(), loadTokens(p));
+                }
+            }
         }
-        CommonTokenCountManager mgr = new CommonTokenCountManager(commonTokensDir, "");
+        List<String> langs = new ArrayList<>(tokensByLang.keySet());
         for (int i = 0; i < langs.size() - 1; i++) {
             for (int j = i + 1; j < langs.size(); j++) {
-                compare(langs.get(i), langs.get(j), mgr);
+                compare(langs.get(i), langs.get(j), tokensByLang);
+            }
+        }
+    }
+
+    private static Set<String> loadTokens(Path p) throws IOException {
+        Set<String> tokens = new HashSet<>();
+        try (BufferedReader reader = Files.newBufferedReader(p, StandardCharsets.UTF_8)) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                line = line.trim();
+                if (line.isEmpty() || line.startsWith("#")) {
+                    continue;
+                }
+                String token = line.split("\t", 2)[0].trim();
+                if (!token.isEmpty()) {
+                    tokens.add(token);
+                }
             }
         }
+        return tokens;
     }
 
-    private void compare(String langA, String langB, CommonTokenCountManager mgr) {
+    private void compare(String langA, String langB, Map<String, Set<String>> tokensByLang) {
+        Set<String> setA = tokensByLang.get(langA);
+        Set<String> setB = tokensByLang.get(langB);
         int overlap = 0;
-        int denom = 0;
-        Set<String> setA = mgr.getTokens(langA);
-        Set<String> setB = mgr.getTokens(langB);
         for (String a : setA) {
             if (setB.contains(a)) {
                 overlap += 2;
             }
         }
-        denom = setA.size() + setB.size();
+        int denom = setA.size() + setB.size();
         double percent = (double) overlap / (double) denom;
         if (percent > 0.01) {
             System.out.printf(Locale.US, "%s %s %.2f%n", langA, langB, percent);
         }
     }
 
-
 }
diff --git a/tika-eval/tika-eval-core/pom.xml b/tika-eval/tika-eval-core/pom.xml
index 5bf5da8a531..8b07e85b3e6 100644
--- a/tika-eval/tika-eval-core/pom.xml
+++ b/tika-eval/tika-eval-core/pom.xml
@@ -55,6 +55,10 @@
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-math3</artifactId>
@@ -99,8 +103,10 @@
         <artifactId>apache-rat-plugin</artifactId>
         <configuration>
           <inputExcludes>
-            <inputExclude>src/main/resources/common_tokens/*</inputExclude>
+            <!-- binary per-language Bloom filters; cannot carry a license header -->
+            <inputExclude>src/main/resources/common_tokens_bloom/*</inputExclude>
             <inputExclude>src/main/resources/*.json</inputExclude>
+            <!-- raw token lists used only to (re)generate the Bloom filters -->
             <inputExclude>src/test/resources/common_tokens/*</inputExclude>
             <inputExclude>src/test/resources/test-dirs/**</inputExclude>
           </inputExcludes>
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokens.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokens.java
index 72aeee1e6e4..fb5b5615b7f 100644
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokens.java
+++ b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokens.java
@@ -18,7 +18,6 @@
 
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.commons.lang3.tuple.Pair;
@@ -26,6 +25,7 @@
 import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
 import org.apache.tika.eval.core.tokens.CommonTokenResult;
 import org.apache.tika.eval.core.tokens.LangModel;
+import org.apache.tika.eval.core.tokens.TikaEvalTokenizer;
 import org.apache.tika.eval.core.tokens.TokenCounts;
 import org.apache.tika.language.detect.LanguageResult;
 
@@ -65,7 +65,9 @@ public CommonTokenResult calculate(List<LanguageResult> languages, TokenCounts t
         Pair<String, LangModel> pair =
                 commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
         String actualLangCode = pair.getKey();
-        Set<String> commonTokens = pair.getValue().getTokens();
+        // Membership is via a Bloom filter, so a token not in the common-tokens list may
+        // very occasionally be counted as common (false positive); never the reverse.
+        LangModel commonTokens = pair.getValue();
         int numUniqueCommonTokens = 0;
         int numCommonTokens = 0;
         int numUniqueAlphabeticTokens = 0;
@@ -77,7 +79,10 @@ public CommonTokenResult calculate(List<LanguageResult> languages, TokenCounts t
                 numAlphabeticTokens += count;
                 numUniqueAlphabeticTokens++;
             }
-            if (commonTokens.contains(token)) {
+            // Only test tokens that could actually be in a common-token list. A number,
+            // short token, or HTML term is never in the list, so testing it can only yield a
+            // (Bloom) false positive; skipping them is a no-op for the exact list.
+            if (TikaEvalTokenizer.isCommonTokenCandidate(token) && commonTokens.contains(token)) {
                 numCommonTokens += count;
                 numUniqueCommonTokens++;
             }
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensBhattacharyya.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensBhattacharyya.java
deleted file mode 100644
index f23c17252ab..00000000000
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensBhattacharyya.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.eval.core.textstats;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.math3.util.FastMath;
-
-import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
-import org.apache.tika.eval.core.tokens.LangModel;
-import org.apache.tika.eval.core.tokens.TokenCounts;
-import org.apache.tika.language.detect.LanguageResult;
-
-public class CommonTokensBhattacharyya implements LanguageAwareTokenCountStats<Double> {
-
-    private final CommonTokenCountManager commonTokenCountManager;
-
-    public CommonTokensBhattacharyya(CommonTokenCountManager mgr) {
-        this.commonTokenCountManager = mgr;
-    }
-
-    @Override
-    public Double calculate(List<LanguageResult> languages, TokenCounts tokenCounts) {
-        Pair<String, LangModel> pair =
-                commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
-        LangModel model = pair.getValue();
-        double sum = 0.0;
-        if (tokenCounts.getTokens().entrySet().size() == 0) {
-            return 0.0;
-        }
-        for (Map.Entry<String, MutableInt> e : tokenCounts.getTokens().entrySet()) {
-            double p = (double) e.getValue().intValue() / (double) tokenCounts.getTotalTokens();
-            if (p == 0.0) { //shouldn't happen, but be defensive
-                continue;
-            }
-            double q = model.getProbability(e.getKey());
-            sum += FastMath.sqrt(p * q);
-        }
-        return sum;
-    }
-}
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensCosine.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensCosine.java
deleted file mode 100644
index d0b27524937..00000000000
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensCosine.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.eval.core.textstats;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.math3.util.FastMath;
-
-import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
-import org.apache.tika.eval.core.tokens.LangModel;
-import org.apache.tika.eval.core.tokens.TokenCounts;
-import org.apache.tika.language.detect.LanguageResult;
-
-public class CommonTokensCosine implements LanguageAwareTokenCountStats<Double> {
-
-    private final CommonTokenCountManager commonTokenCountManager;
-
-    public CommonTokensCosine(CommonTokenCountManager mgr) {
-        this.commonTokenCountManager = mgr;
-    }
-
-    @Override
-    public Double calculate(List<LanguageResult> languages, TokenCounts tokenCounts) {
-        Pair<String, LangModel> pair =
-                commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
-        LangModel model = pair.getValue();
-        double kl = 0.0;
-        if (tokenCounts.getTokens().entrySet().size() == 0) {
-            return 1.0;
-        }
-        double numerator = 0.0;
-        double denominator = 0.0;
-        for (Map.Entry<String, MutableInt> e : tokenCounts.getTokens().entrySet()) {
-            long modelCount = model.getCount(e.getKey());
-            if (modelCount > 0) {
-                numerator += e.getValue().intValue() * modelCount;
-            }
-            double sq = FastMath.pow(e.getValue().intValue(), 2);
-            denominator += sq;
-        }
-        denominator = Math.sqrt(denominator);
-        denominator *= calcNorm(model.getCounts().values());
-        return numerator / denominator;
-    }
-
-    private double calcNorm(Collection<Long> counts) {
-        double d = 0.0;
-        for (long lng : counts) {
-            d += FastMath.pow(lng, 2);
-        }
-        return FastMath.sqrt(d);
-    }
-}
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensHellinger.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensHellinger.java
deleted file mode 100644
index cbbcacc46d4..00000000000
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensHellinger.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.eval.core.textstats;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.math3.util.FastMath;
-
-import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
-import org.apache.tika.eval.core.tokens.LangModel;
-import org.apache.tika.eval.core.tokens.TokenCounts;
-import org.apache.tika.language.detect.LanguageResult;
-
-public class CommonTokensHellinger implements LanguageAwareTokenCountStats<Double> {
-
-    private final CommonTokenCountManager commonTokenCountManager;
-
-    public CommonTokensHellinger(CommonTokenCountManager mgr) {
-        this.commonTokenCountManager = mgr;
-    }
-
-    @Override
-    public Double calculate(List<LanguageResult> languages, TokenCounts tokenCounts) {
-        Pair<String, LangModel> pair =
-                commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
-        LangModel model = pair.getValue();
-        double sum = 0.0;
-        if (tokenCounts.getTokens().entrySet().size() == 0) {
-            return 0.0;
-        }
-        for (Map.Entry<String, MutableInt> e : tokenCounts.getTokens().entrySet()) {
-            double p = (double) e.getValue().intValue() / (double) tokenCounts.getTotalTokens();
-            if (p == 0.0) { //shouldn't happen, but be defensive
-                continue;
-            }
-            double q = model.getProbability(e.getKey());
-            sum += FastMath.pow(FastMath.sqrt(p) - FastMath.sqrt(q), 2.0);
-        }
-        return 1 / FastMath.sqrt(2.0) * Math.sqrt(sum);
-    }
-}
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensKLDNormed.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensKLDNormed.java
deleted file mode 100644
index ec0a99ba7b8..00000000000
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensKLDNormed.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.eval.core.textstats;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.math3.util.FastMath;
-
-import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
-import org.apache.tika.eval.core.tokens.LangModel;
-import org.apache.tika.eval.core.tokens.TokenCounts;
-import org.apache.tika.language.detect.LanguageResult;
-
-public class CommonTokensKLDNormed implements LanguageAwareTokenCountStats<Double> {
-
-    private final CommonTokenCountManager commonTokenCountManager;
-
-    public CommonTokensKLDNormed(CommonTokenCountManager mgr) {
-        this.commonTokenCountManager = mgr;
-    }
-
-    @Override
-    public Double calculate(List<LanguageResult> languages, TokenCounts tokenCounts) {
-        Pair<String, LangModel> pair =
-                commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
-        LangModel model = pair.getValue();
-        double kl = 0.0;
-        if (tokenCounts.getTokens().entrySet().size() == 0) {
-            return 1.0;
-        }
-        double worstCase = 0.0;
-        for (Map.Entry<String, MutableInt> e : tokenCounts.getTokens().entrySet()) {
-            double p = (double) e.getValue().intValue() / (double) tokenCounts.getTotalTokens();
-            if (p == 0.0) { //shouldn't happen, but be defensive
-                continue;
-            }
-            double q = model.getProbability(e.getKey());
-            kl += p * FastMath.log(q / p);
-        }
-        for (int i = 0; i < tokenCounts.getTotalTokens(); i++) {
-            double worstCaseP = 1 / (double) tokenCounts.getTotalTokens();
-            worstCase += worstCaseP * FastMath.log(model.getUnseenProbability() / worstCaseP);
-
-        }
-        return (kl / worstCase);
-    }
-}
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensKLDivergence.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensKLDivergence.java
deleted file mode 100644
index 59d31aeb996..00000000000
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/textstats/CommonTokensKLDivergence.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.eval.core.textstats;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.math3.util.FastMath;
-
-import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
-import org.apache.tika.eval.core.tokens.LangModel;
-import org.apache.tika.eval.core.tokens.TokenCounts;
-import org.apache.tika.language.detect.LanguageResult;
-
-public class CommonTokensKLDivergence implements LanguageAwareTokenCountStats<Double> {
-
-    private final CommonTokenCountManager commonTokenCountManager;
-
-    public CommonTokensKLDivergence(CommonTokenCountManager mgr) {
-        this.commonTokenCountManager = mgr;
-    }
-
-    @Override
-    public Double calculate(List<LanguageResult> languages, TokenCounts tokenCounts) {
-        Pair<String, LangModel> pair =
-                commonTokenCountManager.getLangTokens(languages.get(0).getLanguage());
-        LangModel model = pair.getValue();
-        double kl = 0.0;
-        if (tokenCounts.getTokens().entrySet().size() == 0) {
-            return 1.0;
-        }
-        for (Map.Entry<String, MutableInt> e : tokenCounts.getTokens().entrySet()) {
-            double p = (double) e.getValue().intValue() / (double) tokenCounts.getTotalTokens();
-            if (p == 0.0) { //shouldn't happen, but be defensive
-                continue;
-            }
-            double q = model.getProbability(e.getKey());
-            kl += p * FastMath.log(q / p);
-        }
-        return -1.0 * kl;
-    }
-}
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/CommonTokenCountManager.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/CommonTokenCountManager.java
index e6959c97a7e..9c7a66c328f 100644
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/CommonTokenCountManager.java
+++ b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/CommonTokenCountManager.java
@@ -16,39 +16,36 @@
  */
 package org.apache.tika.eval.core.tokens;
 
-import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
+import org.apache.commons.collections4.bloomfilter.BloomFilter;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Lazily loads the per-language "common tokens" Bloom filters (see {@link CommonTokensBloom})
+ * from the classpath ({@code /common_tokens_bloom/<langCode>}) or from a configured directory.
+ */
 public class CommonTokenCountManager {
+    static final String COMMON_TOKENS_BLOOM_PATH = "/common_tokens_bloom/";
+
     private static final Logger LOG = LoggerFactory.getLogger(CommonTokenCountManager.class);
 
-    private static final Charset COMMON_TOKENS_CHARSET = StandardCharsets.UTF_8;
-    private static final String TERM_FREQS = "#SUM_TERM_FREQS\t";
     private final Path commonTokensDir;
     //if we have no model or if no langid is passed in
     //make this configurable
     private final String defaultLangCode;
     Map<String, LangModel> commonTokenMap = new ConcurrentHashMap<>();
     Set<String> alreadyTriedToLoad = new HashSet<>();
-    private Matcher digitsMatcher = Pattern.compile("(\\d+)").matcher("");
 
     public CommonTokenCountManager() {
         this(null, null);
@@ -75,19 +72,13 @@ public CommonTokenCountManager(Path commonTokensDir, String defaultLangCode) {
         }
     }
 
-    public Set<String> getTokens(String lang) {
-        return Collections.unmodifiableSet(
-                new HashSet(commonTokenMap.get(getActualLangCode(lang)).getTokens()));
-    }
-
     public Set<String> getLangs() {
         return commonTokenMap.keySet();
     }
 
     /**
-     * @param lang
-     * @return pair of actual language code used and a set of common
-     * tokens for that language
+     * @param lang requested language code
+     * @return pair of the actual language code used and the common-tokens model for it
      */
     public Pair<String, LangModel> getLangTokens(String lang) {
         String actualLangCode = getActualLangCode(lang);
@@ -132,60 +123,26 @@ private synchronized void tryToLoad(String langCode) {
 
         try {
             if (p == null || !Files.isRegularFile(p)) {
-                is = this.getClass().getResourceAsStream("/common_tokens/" + langCode);
+                is = this.getClass().getResourceAsStream(COMMON_TOKENS_BLOOM_PATH + langCode);
             } else {
                 is = Files.newInputStream(p);
             }
 
+            alreadyTriedToLoad.add(langCode);
 
             if (is == null) {
-                String path = (p == null) ? "resource on class path: /common_tokens/" + langCode :
+                String path = (p == null) ?
+                        "resource on class path: " + COMMON_TOKENS_BLOOM_PATH + langCode :
                         p.toAbsolutePath().toString();
                 LOG.warn("Couldn't find common tokens file for: '" + langCode + "' tried here: " +
                         path);
-                alreadyTriedToLoad.add(langCode);
                 return;
             }
 
-            LangModel model = null;
-            try (BufferedReader reader = new BufferedReader(
-                    new InputStreamReader(is, COMMON_TOKENS_CHARSET))) {
-                alreadyTriedToLoad.add(langCode);
-                String line = reader.readLine();
-                while (line != null) {
-                    line = line.trim();
-                    if (line.startsWith("#")) {
-                        if (line.startsWith(TERM_FREQS)) {
-                            digitsMatcher.reset(line);
-                            if (digitsMatcher.find()) {
-                                model = new LangModel(Long.parseLong(digitsMatcher.group(1)));
-                            }
-                        }
-                        line = reader.readLine();
-                        continue;
-                    }
-                    //allow language models with, e.g. tab-delimited counts after the term
-                    String[] cols = line.split("\t");
-                    String t = cols[0].trim();
-                    if (t.length() > 0 && cols.length > 2) {
-                        if (model == null) {
-                            throw new IllegalArgumentException(
-                                    "Common tokens file must have included comment line " +
-                                            " with " + TERM_FREQS);
-                        }
-                        //document frequency
-                        String df = cols[1];
-                        //token frequency
-                        long tf = Long.parseLong(cols[2]);
-                        model.add(t, tf);
-                    }
-
-                    line = reader.readLine();
-                }
-            }
-            commonTokenMap.put(langCode, model);
+            BloomFilter filter = CommonTokensBloom.read(is);
+            commonTokenMap.put(langCode, new LangModel(filter));
         } catch (IOException e) {
-            LOG.warn("IOException trying to read: '" + langCode + "'");
+            LOG.warn("IOException trying to read: '" + langCode + "'", e);
         } finally {
             IOUtils.closeQuietly(is);
         }
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/CommonTokensBloom.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/CommonTokensBloom.java
new file mode 100644
index 00000000000..00e219ad86d
--- /dev/null
+++ b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/CommonTokensBloom.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.core.tokens;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.codec.digest.MurmurHash3;
+import org.apache.commons.collections4.bloomfilter.BitMapExtractor;
+import org.apache.commons.collections4.bloomfilter.BloomFilter;
+import org.apache.commons.collections4.bloomfilter.EnhancedDoubleHasher;
+import org.apache.commons.collections4.bloomfilter.Hasher;
+import org.apache.commons.collections4.bloomfilter.Shape;
+import org.apache.commons.collections4.bloomfilter.SimpleBloomFilter;
+
+/**
+ * Shared serialization, sizing and hashing for the per-language "common tokens" Bloom filters.
+ * <p>
+ * The common-token lists are only ever queried for membership ("is this token one of the N
+ * most common tokens in language X?"), so they are shipped as one Bloom filter per language
+ * instead of the raw token lists. This is dramatically smaller (the filter is sized by the
+ * token <em>count</em>, not the token text) at the cost of a small, bounded false-positive
+ * rate: a token that is not common may occasionally be reported as common. There are never
+ * false negatives. For the OOV/common-token ratio this feeds, that bias is negligible.
+ * <p>
+ * Both the runtime loader ({@link CommonTokenCountManager}) and the offline generator use the
+ * methods here so the hashing and on-disk format can never drift apart. The format is:
+ * <pre>
+ *   int  magic   (0x54424631, "TBF1")
+ *   int  k       (number of hash functions)
+ *   int  m       (number of bits)
+ *   int  nLongs  (length of the bit-map array)
+ *   long[nLongs] bit map (big-endian)
+ * </pre>
+ */
+public final class CommonTokensBloom {
+
+    /**
+     * Target false-positive rate for the generated filters. See the class comment for why a
+     * non-zero rate is acceptable here. Changing this requires regenerating the shipped
+     * {@code common_tokens_bloom/} resources.
+     * <p>
+     * 0.0001 (1 in 10,000) keeps the OOV metric close to a literal lookup list: the candidate
+     * filter already removes numeric/short/HTML false positives, and this rate makes a residual
+     * collision on a genuine word ~10x rarer, which matters for repetitive junk docs where a
+     * single dominant token can swing occurrence-weighted OOV. Costs ~2 MB over 0.001.
+     */
+    public static final double DEFAULT_FPP = 0.0001;
+
+    private static final int MAGIC = 0x54424631; // "TBF1"
+
+    private CommonTokensBloom() {
+    }
+
+    /**
+     * @param numTokens number of tokens that will be added to the filter
+     * @return a {@link Shape} sized for {@code numTokens} at {@link #DEFAULT_FPP}
+     */
+    public static Shape shapeFor(int numTokens) {
+        return Shape.fromNP(Math.max(1, numTokens), DEFAULT_FPP);
+    }
+
+    /**
+     * Deterministic hasher for a token, used identically at generation and query time.
+     */
+    public static Hasher hasher(String token) {
+        long[] h = MurmurHash3.hash128x64(token.getBytes(StandardCharsets.UTF_8));
+        return new EnhancedDoubleHasher(h[0], h[1]);
+    }
+
+    /**
+     * @return {@code true} if {@code token} might be in {@code filter} (subject to the
+     * filter's false-positive rate); {@code false} means it is definitely not present.
+     */
+    public static boolean mightContain(BloomFilter filter, String token) {
+        return filter.contains(hasher(token).indices(filter.getShape()));
+    }
+
+    /**
+     * Writes {@code filter} to {@code os} in the format documented on this class. Does not
+     * close the stream.
+     */
+    public static void write(OutputStream os, BloomFilter filter) throws IOException {
+        Shape shape = filter.getShape();
+        long[] bits = filter.asBitMapArray();
+        DataOutputStream d = new DataOutputStream(os);
+        d.writeInt(MAGIC);
+        d.writeInt(shape.getNumberOfHashFunctions());
+        d.writeInt(shape.getNumberOfBits());
+        d.writeInt(bits.length);
+        for (long b : bits) {
+            d.writeLong(b);
+        }
+        d.flush();
+    }
+
+    /**
+     * Reads a filter written by {@link #write}. Does not close the stream.
+     */
+    public static BloomFilter read(InputStream is) throws IOException {
+        DataInputStream d = new DataInputStream(is);
+        int magic = d.readInt();
+        if (magic != MAGIC) {
+            throw new IOException(
+                    "Not a common-tokens Bloom filter (bad magic: 0x" + Integer.toHexString(magic) +
+                            ")");
+        }
+        int k = d.readInt();
+        int m = d.readInt();
+        int nLongs = d.readInt();
+        long[] bits = new long[nLongs];
+        for (int i = 0; i < nLongs; i++) {
+            bits[i] = d.readLong();
+        }
+        SimpleBloomFilter filter = new SimpleBloomFilter(Shape.fromKM(k, m));
+        filter.merge(BitMapExtractor.fromBitMapArray(bits));
+        return filter;
+    }
+}
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/LangModel.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/LangModel.java
index 2c7f889b5be..ef776893c0e 100644
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/LangModel.java
+++ b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/LangModel.java
@@ -16,63 +16,31 @@
  */
 package org.apache.tika.eval.core.tokens;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
+import org.apache.commons.collections4.bloomfilter.BloomFilter;
+
+/**
+ * The set of "common tokens" for a single language, backed by a Bloom filter.
+ * <p>
+ * Membership is approximate: {@link #contains(String)} may occasionally return {@code true}
+ * for a token that is not actually common (a false positive), but never returns {@code false}
+ * for one that is. See {@link CommonTokensBloom} for the rationale and false-positive rate.
+ */
 public class LangModel {
-    public static LangModel EMPTY_MODEL = new LangModel(-1);
 
-    private final long totalTokens;
-    private final double unseenPercentage;
+    /** Model used when a language has no common-tokens resource; nothing is "common". */
+    public static final LangModel EMPTY_MODEL = new LangModel(null);
 
+    private final BloomFilter filter;
 
-    private Map<String, Double> percentages = new HashMap<>();
-    private Map<String, Long> counts = new HashMap<>();
-
-    public LangModel(long totalTokens) {
-        this.totalTokens = totalTokens;
-        this.unseenPercentage = (double) 1 / (double) totalTokens;
+    public LangModel(BloomFilter filter) {
+        this.filter = filter;
     }
 
-
+    /**
+     * @return {@code true} if {@code token} is (probably) one of the common tokens for this
+     * language; {@code false} means it is definitely not.
+     */
     public boolean contains(String token) {
-        return (percentages.containsKey(token));
-    }
-
-    public Set<String> getTokens() {
-        return percentages.keySet();
-    }
-
-    public double getProbability(String token) {
-        Double p = percentages.get(token);
-        if (p != null) {
-            return p;
-        } else {
-            return unseenPercentage;
-        }
-    }
-
-    public void add(String t, long tf) {
-        double p = (double) tf / (double) totalTokens;
-        percentages.put(t, p);
-        counts.put(t, tf);
-    }
-
-    public double getUnseenProbability() {
-        return unseenPercentage;
-    }
-
-    public long getCount(String token) {
-        Long cnt = counts.get(token);
-        if (cnt == null) {
-            return 0;
-        } else {
-            return cnt;
-        }
-    }
-
-    public Map<String, Long> getCounts() {
-        return counts;
+        return filter != null && CommonTokensBloom.mightContain(filter, token);
     }
 }
diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/TikaEvalTokenizer.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/TikaEvalTokenizer.java
index 8cfc07b001e..f84e0fb3381 100644
--- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/TikaEvalTokenizer.java
+++ b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/TikaEvalTokenizer.java
@@ -104,6 +104,43 @@ public enum Mode {
     private TikaEvalTokenizer() {
     }
 
+    /**
+     * Tests whether {@code token} could be a {@link Mode#COMMON_TOKENS} candidate, i.e. whether
+     * it is the kind of token the common-token lists are built from. Use this to gate
+     * common-token membership lookups on tokens produced in {@link Mode#STANDARD} mode (which
+     * also emits numbers and short tokens): a token that could never be in a list should not be
+     * tested against it, since the only possible outcome is a false positive.
+     * <p>
+     * Mirrors {@link #flushWord}: alphabetic/ideographic only (no digits or punctuation),
+     * length 1..{@link #MAX_TOKEN_LENGTH}, at least {@link #MIN_ALPHA_TOKEN_LENGTH} characters
+     * for non-ideographic tokens (CJK tokens are exempt), and not an excluded HTML term.
+     */
+    public static boolean isCommonTokenCandidate(String token) {
+        int len = token.length();
+        if (len == 0 || len > MAX_TOKEN_LENGTH) {
+            return false;
+        }
+        boolean ideographic = false;
+        int i = 0;
+        while (i < len) {
+            int cp = token.codePointAt(i);
+            i += Character.charCount(cp);
+            if (Character.isDigit(cp)) {
+                return false;
+            }
+            if (!Character.isAlphabetic(cp) && !Character.isIdeographic(cp) && cp != '_') {
+                return false;
+            }
+            if (Character.isIdeographic(cp)) {
+                ideographic = true;
+            }
+        }
+        if (!ideographic && len < MIN_ALPHA_TOKEN_LENGTH) {
+            return false;
+        }
+        return !SKIP_SET.contains(token);
+    }
+
     /**
      * Tokenize in {@link Mode#COMMON_TOKENS} mode and return tokens as a list.
      *
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ace b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ace
new file mode 100644
index 00000000000..f6d4bcf7383
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ace differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/afr b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/afr
new file mode 100644
index 00000000000..f758d31b981
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/afr differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/aka b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/aka
new file mode 100644
index 00000000000..56ff504aaae
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/aka differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/alt b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/alt
new file mode 100644
index 00000000000..d34245c7532
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/alt differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/amh b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/amh
new file mode 100644
index 00000000000..dcd9c245535
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/amh differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ami b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ami
new file mode 100644
index 00000000000..5f237937531
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ami differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ara b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ara
new file mode 100644
index 00000000000..5ceb4b4a7e5
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ara differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/arg b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/arg
new file mode 100644
index 00000000000..7b2e7351574
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/arg differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/asm b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/asm
new file mode 100644
index 00000000000..db7dda4594e
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/asm differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ava b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ava
new file mode 100644
index 00000000000..fcaec3f6d20
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ava differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/avk b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/avk
new file mode 100644
index 00000000000..05909084a89
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/avk differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/azb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/azb
new file mode 100644
index 00000000000..9f0085918bc
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/azb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/aze b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/aze
new file mode 100644
index 00000000000..00ce2987098
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/aze differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bak b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bak
new file mode 100644
index 00000000000..0d1a89a05a6
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bak differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ban b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ban
new file mode 100644
index 00000000000..a28bf5a8e2d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ban differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bar b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bar
new file mode 100644
index 00000000000..ea6af2a2e11
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bar differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bcl b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bcl
new file mode 100644
index 00000000000..bbf747a7812
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bcl differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/be-x-old b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/be-x-old
new file mode 100644
index 00000000000..2e5b5b83d81
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/be-x-old differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bel b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bel
new file mode 100644
index 00000000000..40d52692cb4
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bel differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ben b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ben
new file mode 100644
index 00000000000..f8b08d3aa54
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ben differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bjn b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bjn
new file mode 100644
index 00000000000..70927667633
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bjn differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bod b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bod
new file mode 100644
index 00000000000..68e0270e941
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bod differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bre b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bre
new file mode 100644
index 00000000000..12340c0cb1c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bre differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bul b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bul
new file mode 100644
index 00000000000..bb70273b2fd
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bul differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bxr b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bxr
new file mode 100644
index 00000000000..025cc0bd9bc
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/bxr differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cat b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cat
new file mode 100644
index 00000000000..4b78d4139b7
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cat differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cdo-x-rom b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cdo-x-rom
new file mode 100644
index 00000000000..c7968058946
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cdo-x-rom differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ceb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ceb
new file mode 100644
index 00000000000..b1d9ef3c4af
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ceb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ces b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ces
new file mode 100644
index 00000000000..025b3543aa5
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ces differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/che b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/che
new file mode 100644
index 00000000000..cb76d2c65bb
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/che differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/chv b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/chv
new file mode 100644
index 00000000000..bbd6ddbb884
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/chv differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ckb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ckb
new file mode 100644
index 00000000000..8d7f92db420
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ckb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cnh b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cnh
new file mode 100644
index 00000000000..e71478c0251
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cnh differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cor b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cor
new file mode 100644
index 00000000000..2bdd25c95f0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cor differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cos b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cos
new file mode 100644
index 00000000000..a931d8186a7
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cos differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/csb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/csb
new file mode 100644
index 00000000000..cdc85fc2565
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/csb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cym b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cym
new file mode 100644
index 00000000000..e6825b32704
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/cym differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dag b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dag
new file mode 100644
index 00000000000..0de06759f3a
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dag differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dan b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dan
new file mode 100644
index 00000000000..601e3aa46e0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dan differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/deu b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/deu
new file mode 100644
index 00000000000..37a84c25259
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/deu differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/diq b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/diq
new file mode 100644
index 00000000000..96d899f3b3c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/diq differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/div b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/div
new file mode 100644
index 00000000000..0aa82aab83d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/div differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dsb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dsb
new file mode 100644
index 00000000000..c22852adb44
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/dsb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ell b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ell
new file mode 100644
index 00000000000..af0d9a1dcf5
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ell differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/eng b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/eng
new file mode 100644
index 00000000000..54d67e299e5
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/eng differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/epo b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/epo
new file mode 100644
index 00000000000..b019d1ad4c3
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/epo differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/est b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/est
new file mode 100644
index 00000000000..353b3e02384
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/est differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/eus b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/eus
new file mode 100644
index 00000000000..46d725d1736
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/eus differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ewe b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ewe
new file mode 100644
index 00000000000..118a0dc2796
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ewe differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ext b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ext
new file mode 100644
index 00000000000..6217a55c01d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ext differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fao b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fao
new file mode 100644
index 00000000000..d1dcbdd3efa
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fao differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fas b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fas
new file mode 100644
index 00000000000..c795100fbb1
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fas differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fin b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fin
new file mode 100644
index 00000000000..bd38bbca98d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fin differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fra b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fra
new file mode 100644
index 00000000000..21e3770f756
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fra differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/frr b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/frr
new file mode 100644
index 00000000000..61f15f3e7b4
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/frr differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fry b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fry
new file mode 100644
index 00000000000..9e1e84e7e4f
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/fry differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gla b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gla
new file mode 100644
index 00000000000..d838bdd62b6
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gla differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gle b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gle
new file mode 100644
index 00000000000..8d474d5740c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gle differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/glg b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/glg
new file mode 100644
index 00000000000..5cde117461b
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/glg differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/glv b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/glv
new file mode 100644
index 00000000000..2db57abece7
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/glv differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gom b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gom
new file mode 100644
index 00000000000..b92df26b4e5
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gom differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/grn b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/grn
new file mode 100644
index 00000000000..3a252c4db56
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/grn differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gsw b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gsw
new file mode 100644
index 00000000000..5205710334e
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/gsw differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/guj b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/guj
new file mode 100644
index 00000000000..0be7149ce21
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/guj differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hak-x-rom b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hak-x-rom
new file mode 100644
index 00000000000..681a527e67d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hak-x-rom differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hau b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hau
new file mode 100644
index 00000000000..5de9f9bdcea
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hau differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/heb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/heb
new file mode 100644
index 00000000000..fa71b003172
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/heb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hil b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hil
new file mode 100644
index 00000000000..6016bcc0ae6
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hil differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hin b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hin
new file mode 100644
index 00000000000..4e301e6107c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hin differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hrv b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hrv
new file mode 100644
index 00000000000..fd0a3d355d0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hrv differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hsb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hsb
new file mode 100644
index 00000000000..f034ca48fdc
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hsb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hun b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hun
new file mode 100644
index 00000000000..172b8e602b7
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hun differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hye b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hye
new file mode 100644
index 00000000000..44d4ebd33fa
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hye differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hyw b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hyw
new file mode 100644
index 00000000000..4482419567f
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/hyw differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ibo b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ibo
new file mode 100644
index 00000000000..528122b4d44
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ibo differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ido b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ido
new file mode 100644
index 00000000000..b2edddf5c77
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ido differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ile b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ile
new file mode 100644
index 00000000000..64db0c5f115
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ile differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ilo b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ilo
new file mode 100644
index 00000000000..645ddd72527
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ilo differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ina b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ina
new file mode 100644
index 00000000000..d21583dc38e
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ina differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ind b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ind
new file mode 100644
index 00000000000..a042a40adf8
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ind differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/isl b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/isl
new file mode 100644
index 00000000000..6a96bb1ea10
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/isl differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ita b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ita
new file mode 100644
index 00000000000..133d097332c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ita differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jav b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jav
new file mode 100644
index 00000000000..dd8d00d72a0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jav differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jbo b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jbo
new file mode 100644
index 00000000000..6d6aae700cf
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jbo differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jpn b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jpn
new file mode 100644
index 00000000000..7adc32c8caf
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/jpn differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kaa b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kaa
new file mode 100644
index 00000000000..b1f86dc8581
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kaa differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kab b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kab
new file mode 100644
index 00000000000..39572bbfe4d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kab differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kan b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kan
new file mode 100644
index 00000000000..095f8ee11a0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kan differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kat b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kat
new file mode 100644
index 00000000000..bed0c542c75
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kat differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kaz b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kaz
new file mode 100644
index 00000000000..919c29a7267
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kaz differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kha b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kha
new file mode 100644
index 00000000000..18d73153cfa
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kha differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/khm b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/khm
new file mode 100644
index 00000000000..7c096655ad0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/khm differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kin b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kin
new file mode 100644
index 00000000000..96078a432be
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kin differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kir b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kir
new file mode 100644
index 00000000000..bdf57d5ba83
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kir differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kor b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kor
new file mode 100644
index 00000000000..d080e903662
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kor differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kpv b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kpv
new file mode 100644
index 00000000000..1d8027c7c4d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kpv differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ksh b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ksh
new file mode 100644
index 00000000000..8c41b10fc62
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ksh differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kur b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kur
new file mode 100644
index 00000000000..b31cc9b81b5
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/kur differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lao b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lao
new file mode 100644
index 00000000000..2ff090d3b45
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lao differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lat b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lat
new file mode 100644
index 00000000000..47dfdc7c7ef
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lat differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lav b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lav
new file mode 100644
index 00000000000..ab503a9b78f
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lav differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lez b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lez
new file mode 100644
index 00000000000..a43de04d4a1
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lez differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lfn b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lfn
new file mode 100644
index 00000000000..971a30d1679
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lfn differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lim b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lim
new file mode 100644
index 00000000000..75d399ad9fb
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lim differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lit b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lit
new file mode 100644
index 00000000000..3a7ab5dbc02
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lit differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ltz b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ltz
new file mode 100644
index 00000000000..af9863c74e3
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ltz differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lug b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lug
new file mode 100644
index 00000000000..c16eaef8fcc
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lug differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lus b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lus
new file mode 100644
index 00000000000..1c7aee505d8
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/lus differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mal b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mal
new file mode 100644
index 00000000000..3e244b841f4
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mal differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mar b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mar
new file mode 100644
index 00000000000..f86e0f00407
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mar differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mhr b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mhr
new file mode 100644
index 00000000000..3f4b1119fe0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mhr differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/min b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/min
new file mode 100644
index 00000000000..0c7afe1a7bb
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/min differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mkd b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mkd
new file mode 100644
index 00000000000..dcf89c0a50a
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mkd differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mlg b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mlg
new file mode 100644
index 00000000000..62ccca7a57c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mlg differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mlt b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mlt
new file mode 100644
index 00000000000..87f9c0b2242
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mlt differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mon b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mon
new file mode 100644
index 00000000000..f385cab02aa
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mon differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mrj b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mrj
new file mode 100644
index 00000000000..6c77fa0102d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mrj differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/msa b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/msa
new file mode 100644
index 00000000000..dbf4b107793
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/msa differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mwl b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mwl
new file mode 100644
index 00000000000..56cc0ec84db
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mwl differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mya b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mya
new file mode 100644
index 00000000000..1a98483228a
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mya differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/myv b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/myv
new file mode 100644
index 00000000000..1fc2e0c9aec
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/myv differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mzn b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mzn
new file mode 100644
index 00000000000..dbed1501709
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/mzn differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nds b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nds
new file mode 100644
index 00000000000..0bf852ebb5a
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nds differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nep b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nep
new file mode 100644
index 00000000000..d4f89fa552a
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nep differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nld b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nld
new file mode 100644
index 00000000000..bdb23a5b284
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nld differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nno b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nno
new file mode 100644
index 00000000000..177c94474d4
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nno differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nob b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nob
new file mode 100644
index 00000000000..4932c74aeb8
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nob differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nqo b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nqo
new file mode 100644
index 00000000000..2579482e9d8
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nqo differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nso b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nso
new file mode 100644
index 00000000000..c8f47408f62
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nso differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nya b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nya
new file mode 100644
index 00000000000..6da41170d04
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/nya differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/olo b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/olo
new file mode 100644
index 00000000000..ff9c2a39a8c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/olo differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ori b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ori
new file mode 100644
index 00000000000..2d003a600f1
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ori differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/orm b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/orm
new file mode 100644
index 00000000000..e01b343594e
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/orm differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/oss b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/oss
new file mode 100644
index 00000000000..90c5206691d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/oss differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pam b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pam
new file mode 100644
index 00000000000..095250b4d6d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pam differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pan b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pan
new file mode 100644
index 00000000000..9bf7456d04f
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pan differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pap b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pap
new file mode 100644
index 00000000000..9e77ee1117c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pap differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pfl b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pfl
new file mode 100644
index 00000000000..2705cf0c61c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pfl differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pnb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pnb
new file mode 100644
index 00000000000..0d1acf52af6
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pnb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pol b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pol
new file mode 100644
index 00000000000..255fa0da541
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pol differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/por b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/por
new file mode 100644
index 00000000000..07627111216
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/por differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pus b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pus
new file mode 100644
index 00000000000..832e32d5ca5
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/pus differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/roh b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/roh
new file mode 100644
index 00000000000..d6dee8e30b2
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/roh differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ron b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ron
new file mode 100644
index 00000000000..45c122e5870
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ron differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/rue b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/rue
new file mode 100644
index 00000000000..e3c16e6e3d8
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/rue differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/rus b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/rus
new file mode 100644
index 00000000000..23b0fbaa5c3
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/rus differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sah b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sah
new file mode 100644
index 00000000000..092fc6bd247
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sah differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/san b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/san
new file mode 100644
index 00000000000..76ed8251446
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/san differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sat b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sat
new file mode 100644
index 00000000000..be0c15ce3c6
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sat differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sgs b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sgs
new file mode 100644
index 00000000000..338362153a0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sgs differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sin b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sin
new file mode 100644
index 00000000000..5f0712b5e33
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sin differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/skr b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/skr
new file mode 100644
index 00000000000..ae821edb26b
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/skr differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/slk b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/slk
new file mode 100644
index 00000000000..fca5a034869
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/slk differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/slv b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/slv
new file mode 100644
index 00000000000..7aface6a51e
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/slv differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sme b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sme
new file mode 100644
index 00000000000..6fca6b2fa35
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sme differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/smn b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/smn
new file mode 100644
index 00000000000..bfe0d3cc3f0
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/smn differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/smo b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/smo
new file mode 100644
index 00000000000..b33cb2aa87e
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/smo differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sna b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sna
new file mode 100644
index 00000000000..9ba118ff957
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sna differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/snd b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/snd
new file mode 100644
index 00000000000..35d3a2fe013
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/snd differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/som b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/som
new file mode 100644
index 00000000000..06da6a74be5
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/som differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/spa b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/spa
new file mode 100644
index 00000000000..a4a5c67fcd2
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/spa differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sqi b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sqi
new file mode 100644
index 00000000000..9e1558e56dd
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sqi differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/srp b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/srp
new file mode 100644
index 00000000000..c1b98728506
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/srp differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/stq b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/stq
new file mode 100644
index 00000000000..3a1c9a4f6fb
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/stq differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sun b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sun
new file mode 100644
index 00000000000..fe1e51c5896
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/sun differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/swe b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/swe
new file mode 100644
index 00000000000..12e6f27d27d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/swe differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/swh b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/swh
new file mode 100644
index 00000000000..67d4298ce80
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/swh differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/szl b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/szl
new file mode 100644
index 00000000000..88f34d6c100
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/szl differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/szy b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/szy
new file mode 100644
index 00000000000..f45e0c67a97
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/szy differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tam b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tam
new file mode 100644
index 00000000000..421f3cb03cf
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tam differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tat b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tat
new file mode 100644
index 00000000000..6ae87251cac
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tat differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tay b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tay
new file mode 100644
index 00000000000..89cd8c29ec2
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tay differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tel b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tel
new file mode 100644
index 00000000000..cfb8cf2626a
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tel differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tet b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tet
new file mode 100644
index 00000000000..91fbfe267bc
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tet differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tgk b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tgk
new file mode 100644
index 00000000000..4669bb7fc21
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tgk differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tgl b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tgl
new file mode 100644
index 00000000000..2e412ab4f99
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tgl differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tha b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tha
new file mode 100644
index 00000000000..9bddd2d6206
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tha differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tir b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tir
new file mode 100644
index 00000000000..923a6fd63e6
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tir differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/trv b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/trv
new file mode 100644
index 00000000000..80ca795b267
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/trv differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tsn b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tsn
new file mode 100644
index 00000000000..8d426eec398
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tsn differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tso b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tso
new file mode 100644
index 00000000000..460795ff2a6
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tso differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tuk b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tuk
new file mode 100644
index 00000000000..27101e1b9c4
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tuk differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tum b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tum
new file mode 100644
index 00000000000..174b4569be7
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tum differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tur b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tur
new file mode 100644
index 00000000000..81fbcb32692
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tur differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tyv b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tyv
new file mode 100644
index 00000000000..effb5493e22
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/tyv differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/udm b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/udm
new file mode 100644
index 00000000000..e60c40b491c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/udm differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/uig b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/uig
new file mode 100644
index 00000000000..3d2bf847f21
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/uig differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ukr b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ukr
new file mode 100644
index 00000000000..682111630c6
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ukr differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/urd b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/urd
new file mode 100644
index 00000000000..7167cfeae52
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/urd differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/uzb b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/uzb
new file mode 100644
index 00000000000..571055c729d
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/uzb differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vep b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vep
new file mode 100644
index 00000000000..06dedc27621
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vep differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vie b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vie
new file mode 100644
index 00000000000..c26ca22ccee
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vie differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vls b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vls
new file mode 100644
index 00000000000..84d5ffad321
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vls differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vol b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vol
new file mode 100644
index 00000000000..79920db0d98
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vol differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vro b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vro
new file mode 100644
index 00000000000..c5f7ccb2075
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/vro differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/war b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/war
new file mode 100644
index 00000000000..f78ce7416f7
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/war differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/wln b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/wln
new file mode 100644
index 00000000000..a602fd4a296
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/wln differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/xho b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/xho
new file mode 100644
index 00000000000..9248ea47185
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/xho differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/xmf b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/xmf
new file mode 100644
index 00000000000..a9008f814e9
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/xmf differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ydd b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ydd
new file mode 100644
index 00000000000..935f3cd4105
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/ydd differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/yor b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/yor
new file mode 100644
index 00000000000..227d16a1248
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/yor differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/yue b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/yue
new file mode 100644
index 00000000000..3f20af22434
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/yue differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/zho b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/zho
new file mode 100644
index 00000000000..97364b3d43c
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/zho differ
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/zul b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/zul
new file mode 100644
index 00000000000..0cd235afa66
Binary files /dev/null and b/tika-eval/tika-eval-core/src/main/resources/common_tokens_bloom/zul differ
diff --git a/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/langid/LangIdTest.java b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/langid/LangIdTest.java
index caabd10a7f3..447c40f79b5 100644
--- a/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/langid/LangIdTest.java
+++ b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/langid/LangIdTest.java
@@ -19,44 +19,55 @@
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Set;
+import java.util.stream.Stream;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
-import org.apache.tika.eval.core.tokens.CommonTokenCountManager;
-
 public class LangIdTest {
 
 
     @Test
     @Disabled("run manually when updating common tokens or the language model")
     public void testCommonTokensCoverage() throws Exception {
-        //make sure that there is a common tokens file for every
-        //language
-        CommonTokenCountManager commonTokens = new CommonTokenCountManager(null, "eng");
+        //make sure that there is a common tokens (source) list for every language and no extras.
+        //The shipped runtime resources are Bloom filters (not enumerable), so check the raw
+        //token lists under src/test/resources/common_tokens instead.
+        Path commonTokensDir = Paths.get(getClass().getResource("/common_tokens").toURI());
 
         Set<String> supported = LanguageIDWrapper.getSupportedLanguages();
         String[] langs = supported.toArray(new String[0]);
         Arrays.sort(langs);
         for (String lang : langs) {
-            Set<String> tokens = commonTokens.getTokens(lang);
-            if (tokens.size() == 0) {
+            Path f = commonTokensDir.resolve(lang);
+            if (!Files.isRegularFile(f)) {
                 fail(String.format(Locale.US, "missing common tokens for: %s", lang));
-            } else if (tokens.size() < 250) {
-                fail(String.format(Locale.US, "common tokens too small (%s) for: %s",
-                        tokens.size(), lang));
+            }
+            long tokens = countTokens(f);
+            if (tokens < 250) {
+                fail(String.format(Locale.US, "common tokens too small (%d) for: %s", tokens, lang));
             }
         }
-        Path p = Paths.get(getClass().getResource("/common_tokens").toURI());
-        for (File f : p.toFile().listFiles()) {
+        for (File f : commonTokensDir.toFile().listFiles()) {
             if (!supported.contains(f.getName())) {
                 fail("extra common tokens for: " + f.getName());
             }
         }
     }
+
+    private static long countTokens(Path f) throws IOException {
+        try (Stream<String> lines = Files.lines(f, StandardCharsets.UTF_8)) {
+            return lines.map(String::trim)
+                    .filter(l -> !l.isEmpty() && !l.startsWith("#"))
+                    .count();
+        }
+    }
 }
diff --git a/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/tokens/tools/CommonTokensBloomGenerator.java b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/tokens/tools/CommonTokensBloomGenerator.java
new file mode 100644
index 00000000000..c8c21760cdd
--- /dev/null
+++ b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/tokens/tools/CommonTokensBloomGenerator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.core.tokens.tools;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.collections4.bloomfilter.SimpleBloomFilter;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.eval.core.tokens.CommonTokensBloom;
+
+/**
+ * Regenerates the shipped per-language common-token Bloom filters from the raw
+ * {@code token<TAB>df<TAB>cf} source lists.
+ * <p>
+ * This is a developer tool, not a unit test: it is skipped unless
+ * {@code -DgenerateCommonTokensBloom=true} is passed. To regenerate:
+ * <pre>
+ *   ./mvnw -pl tika-eval/tika-eval-core test \
+ *       -Dtest=CommonTokensBloomGenerator -DgenerateCommonTokensBloom=true
+ * </pre>
+ * Inputs/outputs default to the module's source tree and can be overridden with
+ * {@code -DcommonTokensSrc=...} and {@code -DcommonTokensBloomOut=...}.
+ */
+public class CommonTokensBloomGenerator {
+
+    private static final String SRC_DEFAULT = "src/test/resources/common_tokens";
+    private static final String OUT_DEFAULT = "src/main/resources/common_tokens_bloom";
+
+    @Test
+    public void generate() throws Exception {
+        Assumptions.assumeTrue(Boolean.getBoolean("generateCommonTokensBloom"),
+                "set -DgenerateCommonTokensBloom=true to (re)generate the common-token Bloom filters");
+
+        Path srcDir = Paths.get(System.getProperty("commonTokensSrc", SRC_DEFAULT));
+        Path outDir = Paths.get(System.getProperty("commonTokensBloomOut", OUT_DEFAULT));
+        Files.createDirectories(outDir);
+
+        long totalIn = 0;
+        long totalOut = 0;
+        int langs = 0;
+        try (DirectoryStream<Path> ds = Files.newDirectoryStream(srcDir)) {
+            for (Path in : ds) {
+                if (!Files.isRegularFile(in)) {
+                    continue;
+                }
+                Path out = outDir.resolve(in.getFileName().toString());
+                List<String> tokens = readTokens(in);
+                writeFilter(tokens, out);
+                langs++;
+                totalIn += Files.size(in);
+                totalOut += Files.size(out);
+            }
+        }
+        System.out.printf(Locale.ROOT, "Generated %d Bloom filters at fpp=%s: %.1f MB -> %.1f MB%n",
+                langs, CommonTokensBloom.DEFAULT_FPP, totalIn / 1048576.0, totalOut / 1048576.0);
+    }
+
+    private static List<String> readTokens(Path in) throws IOException {
+        List<String> tokens = new ArrayList<>();
+        try (BufferedReader reader = Files.newBufferedReader(in, StandardCharsets.UTF_8)) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                line = line.trim();
+                if (line.isEmpty() || line.startsWith("#")) {
+                    continue;
+                }
+                String token = line.split("\t", 2)[0].trim();
+                if (!token.isEmpty()) {
+                    tokens.add(token);
+                }
+            }
+        }
+        return tokens;
+    }
+
+    private static void writeFilter(List<String> tokens, Path out) throws IOException {
+        SimpleBloomFilter filter = new SimpleBloomFilter(CommonTokensBloom.shapeFor(tokens.size()));
+        for (String token : tokens) {
+            filter.merge(CommonTokensBloom.hasher(token));
+        }
+        try (OutputStream os = Files.newOutputStream(out)) {
+            CommonTokensBloom.write(os, filter);
+        }
+    }
+}
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ace b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ace
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ace
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ace
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/afr b/tika-eval/tika-eval-core/src/test/resources/common_tokens/afr
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/afr
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/afr
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/aka b/tika-eval/tika-eval-core/src/test/resources/common_tokens/aka
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/aka
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/aka
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/alt b/tika-eval/tika-eval-core/src/test/resources/common_tokens/alt
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/alt
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/alt
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/amh b/tika-eval/tika-eval-core/src/test/resources/common_tokens/amh
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/amh
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/amh
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ami b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ami
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ami
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ami
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ara b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ara
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ara
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ara
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/arg b/tika-eval/tika-eval-core/src/test/resources/common_tokens/arg
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/arg
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/arg
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/asm b/tika-eval/tika-eval-core/src/test/resources/common_tokens/asm
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/asm
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/asm
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ava b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ava
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ava
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ava
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/avk b/tika-eval/tika-eval-core/src/test/resources/common_tokens/avk
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/avk
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/avk
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/azb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/azb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/azb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/azb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/aze b/tika-eval/tika-eval-core/src/test/resources/common_tokens/aze
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/aze
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/aze
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bak b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bak
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bak
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bak
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ban b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ban
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ban
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ban
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bar b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bar
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bar
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bar
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bcl b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bcl
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bcl
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bcl
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/be-x-old b/tika-eval/tika-eval-core/src/test/resources/common_tokens/be-x-old
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/be-x-old
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/be-x-old
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bel b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bel
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bel
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bel
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ben b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ben
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ben
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ben
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bjn b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bjn
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bjn
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bjn
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bod b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bod
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bod
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bod
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bre b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bre
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bre
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bre
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bul b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bul
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bul
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bul
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/bxr b/tika-eval/tika-eval-core/src/test/resources/common_tokens/bxr
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/bxr
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/bxr
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/cat b/tika-eval/tika-eval-core/src/test/resources/common_tokens/cat
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/cat
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/cat
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/cdo-x-rom b/tika-eval/tika-eval-core/src/test/resources/common_tokens/cdo-x-rom
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/cdo-x-rom
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/cdo-x-rom
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ceb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ceb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ceb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ceb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ces b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ces
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ces
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ces
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/che b/tika-eval/tika-eval-core/src/test/resources/common_tokens/che
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/che
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/che
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/chv b/tika-eval/tika-eval-core/src/test/resources/common_tokens/chv
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/chv
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/chv
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ckb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ckb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ckb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ckb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/cnh b/tika-eval/tika-eval-core/src/test/resources/common_tokens/cnh
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/cnh
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/cnh
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/cor b/tika-eval/tika-eval-core/src/test/resources/common_tokens/cor
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/cor
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/cor
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/cos b/tika-eval/tika-eval-core/src/test/resources/common_tokens/cos
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/cos
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/cos
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/csb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/csb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/csb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/csb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/cym b/tika-eval/tika-eval-core/src/test/resources/common_tokens/cym
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/cym
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/cym
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/dag b/tika-eval/tika-eval-core/src/test/resources/common_tokens/dag
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/dag
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/dag
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/dan b/tika-eval/tika-eval-core/src/test/resources/common_tokens/dan
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/dan
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/dan
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/deu b/tika-eval/tika-eval-core/src/test/resources/common_tokens/deu
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/deu
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/deu
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/diq b/tika-eval/tika-eval-core/src/test/resources/common_tokens/diq
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/diq
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/diq
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/div b/tika-eval/tika-eval-core/src/test/resources/common_tokens/div
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/div
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/div
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/dsb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/dsb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/dsb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/dsb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ell b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ell
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ell
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ell
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/eng b/tika-eval/tika-eval-core/src/test/resources/common_tokens/eng
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/eng
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/eng
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/epo b/tika-eval/tika-eval-core/src/test/resources/common_tokens/epo
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/epo
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/epo
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/est b/tika-eval/tika-eval-core/src/test/resources/common_tokens/est
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/est
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/est
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/eus b/tika-eval/tika-eval-core/src/test/resources/common_tokens/eus
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/eus
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/eus
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ewe b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ewe
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ewe
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ewe
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ext b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ext
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ext
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ext
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/fao b/tika-eval/tika-eval-core/src/test/resources/common_tokens/fao
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/fao
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/fao
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/fas b/tika-eval/tika-eval-core/src/test/resources/common_tokens/fas
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/fas
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/fas
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/fin b/tika-eval/tika-eval-core/src/test/resources/common_tokens/fin
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/fin
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/fin
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/fra b/tika-eval/tika-eval-core/src/test/resources/common_tokens/fra
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/fra
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/fra
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/frr b/tika-eval/tika-eval-core/src/test/resources/common_tokens/frr
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/frr
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/frr
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/fry b/tika-eval/tika-eval-core/src/test/resources/common_tokens/fry
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/fry
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/fry
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/gla b/tika-eval/tika-eval-core/src/test/resources/common_tokens/gla
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/gla
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/gla
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/gle b/tika-eval/tika-eval-core/src/test/resources/common_tokens/gle
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/gle
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/gle
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/glg b/tika-eval/tika-eval-core/src/test/resources/common_tokens/glg
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/glg
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/glg
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/glv b/tika-eval/tika-eval-core/src/test/resources/common_tokens/glv
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/glv
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/glv
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/gom b/tika-eval/tika-eval-core/src/test/resources/common_tokens/gom
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/gom
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/gom
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/grn b/tika-eval/tika-eval-core/src/test/resources/common_tokens/grn
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/grn
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/grn
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/gsw b/tika-eval/tika-eval-core/src/test/resources/common_tokens/gsw
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/gsw
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/gsw
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/guj b/tika-eval/tika-eval-core/src/test/resources/common_tokens/guj
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/guj
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/guj
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hak-x-rom b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hak-x-rom
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hak-x-rom
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hak-x-rom
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hau b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hau
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hau
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hau
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/heb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/heb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/heb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/heb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hil b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hil
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hil
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hil
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hin b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hin
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hin
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hin
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hrv b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hrv
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hrv
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hrv
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hsb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hsb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hsb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hsb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hun b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hun
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hun
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hun
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hye b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hye
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hye
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hye
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/hyw b/tika-eval/tika-eval-core/src/test/resources/common_tokens/hyw
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/hyw
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/hyw
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ibo b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ibo
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ibo
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ibo
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ido b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ido
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ido
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ido
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ile b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ile
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ile
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ile
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ilo b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ilo
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ilo
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ilo
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ina b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ina
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ina
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ina
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ind b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ind
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ind
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ind
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/isl b/tika-eval/tika-eval-core/src/test/resources/common_tokens/isl
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/isl
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/isl
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ita b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ita
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ita
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ita
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/jav b/tika-eval/tika-eval-core/src/test/resources/common_tokens/jav
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/jav
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/jav
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/jbo b/tika-eval/tika-eval-core/src/test/resources/common_tokens/jbo
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/jbo
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/jbo
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/jpn b/tika-eval/tika-eval-core/src/test/resources/common_tokens/jpn
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/jpn
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/jpn
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kaa b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kaa
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kaa
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kaa
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kab b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kab
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kab
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kab
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kan b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kan
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kan
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kan
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kat b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kat
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kat
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kat
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kaz b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kaz
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kaz
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kaz
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kha b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kha
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kha
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kha
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/khm b/tika-eval/tika-eval-core/src/test/resources/common_tokens/khm
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/khm
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/khm
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kin b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kin
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kin
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kin
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kir b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kir
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kir
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kir
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kor b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kor
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kor
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kor
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kpv b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kpv
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kpv
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kpv
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ksh b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ksh
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ksh
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ksh
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/kur b/tika-eval/tika-eval-core/src/test/resources/common_tokens/kur
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/kur
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/kur
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lao b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lao
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lao
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lao
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lat b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lat
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lat
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lat
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lav b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lav
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lav
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lav
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lez b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lez
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lez
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lez
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lfn b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lfn
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lfn
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lfn
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lim b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lim
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lim
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lim
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lit b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lit
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lit
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lit
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ltz b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ltz
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ltz
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ltz
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lug b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lug
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lug
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lug
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/lus b/tika-eval/tika-eval-core/src/test/resources/common_tokens/lus
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/lus
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/lus
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mal b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mal
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mal
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mal
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mar b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mar
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mar
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mar
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mhr b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mhr
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mhr
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mhr
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/min b/tika-eval/tika-eval-core/src/test/resources/common_tokens/min
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/min
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/min
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mkd b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mkd
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mkd
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mkd
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mlg b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mlg
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mlg
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mlg
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mlt b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mlt
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mlt
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mlt
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mon b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mon
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mon
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mon
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mrj b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mrj
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mrj
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mrj
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/msa b/tika-eval/tika-eval-core/src/test/resources/common_tokens/msa
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/msa
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/msa
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mwl b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mwl
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mwl
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mwl
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mya b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mya
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mya
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mya
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/myv b/tika-eval/tika-eval-core/src/test/resources/common_tokens/myv
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/myv
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/myv
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/mzn b/tika-eval/tika-eval-core/src/test/resources/common_tokens/mzn
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/mzn
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/mzn
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/nds b/tika-eval/tika-eval-core/src/test/resources/common_tokens/nds
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/nds
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/nds
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/nep b/tika-eval/tika-eval-core/src/test/resources/common_tokens/nep
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/nep
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/nep
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/nld b/tika-eval/tika-eval-core/src/test/resources/common_tokens/nld
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/nld
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/nld
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/nno b/tika-eval/tika-eval-core/src/test/resources/common_tokens/nno
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/nno
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/nno
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/nob b/tika-eval/tika-eval-core/src/test/resources/common_tokens/nob
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/nob
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/nob
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/nqo b/tika-eval/tika-eval-core/src/test/resources/common_tokens/nqo
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/nqo
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/nqo
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/nso b/tika-eval/tika-eval-core/src/test/resources/common_tokens/nso
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/nso
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/nso
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/nya b/tika-eval/tika-eval-core/src/test/resources/common_tokens/nya
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/nya
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/nya
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/olo b/tika-eval/tika-eval-core/src/test/resources/common_tokens/olo
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/olo
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/olo
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ori b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ori
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ori
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ori
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/orm b/tika-eval/tika-eval-core/src/test/resources/common_tokens/orm
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/orm
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/orm
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/oss b/tika-eval/tika-eval-core/src/test/resources/common_tokens/oss
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/oss
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/oss
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/pam b/tika-eval/tika-eval-core/src/test/resources/common_tokens/pam
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/pam
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/pam
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/pan b/tika-eval/tika-eval-core/src/test/resources/common_tokens/pan
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/pan
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/pan
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/pap b/tika-eval/tika-eval-core/src/test/resources/common_tokens/pap
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/pap
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/pap
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/pfl b/tika-eval/tika-eval-core/src/test/resources/common_tokens/pfl
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/pfl
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/pfl
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/pnb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/pnb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/pnb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/pnb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/pol b/tika-eval/tika-eval-core/src/test/resources/common_tokens/pol
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/pol
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/pol
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/por b/tika-eval/tika-eval-core/src/test/resources/common_tokens/por
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/por
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/por
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/pus b/tika-eval/tika-eval-core/src/test/resources/common_tokens/pus
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/pus
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/pus
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/roh b/tika-eval/tika-eval-core/src/test/resources/common_tokens/roh
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/roh
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/roh
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ron b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ron
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ron
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ron
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/rue b/tika-eval/tika-eval-core/src/test/resources/common_tokens/rue
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/rue
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/rue
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/rus b/tika-eval/tika-eval-core/src/test/resources/common_tokens/rus
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/rus
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/rus
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/sah b/tika-eval/tika-eval-core/src/test/resources/common_tokens/sah
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/sah
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/sah
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/san b/tika-eval/tika-eval-core/src/test/resources/common_tokens/san
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/san
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/san
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/sat b/tika-eval/tika-eval-core/src/test/resources/common_tokens/sat
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/sat
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/sat
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/sgs b/tika-eval/tika-eval-core/src/test/resources/common_tokens/sgs
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/sgs
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/sgs
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/sin b/tika-eval/tika-eval-core/src/test/resources/common_tokens/sin
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/sin
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/sin
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/skr b/tika-eval/tika-eval-core/src/test/resources/common_tokens/skr
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/skr
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/skr
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/slk b/tika-eval/tika-eval-core/src/test/resources/common_tokens/slk
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/slk
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/slk
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/slv b/tika-eval/tika-eval-core/src/test/resources/common_tokens/slv
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/slv
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/slv
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/sme b/tika-eval/tika-eval-core/src/test/resources/common_tokens/sme
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/sme
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/sme
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/smn b/tika-eval/tika-eval-core/src/test/resources/common_tokens/smn
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/smn
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/smn
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/smo b/tika-eval/tika-eval-core/src/test/resources/common_tokens/smo
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/smo
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/smo
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/sna b/tika-eval/tika-eval-core/src/test/resources/common_tokens/sna
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/sna
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/sna
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/snd b/tika-eval/tika-eval-core/src/test/resources/common_tokens/snd
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/snd
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/snd
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/som b/tika-eval/tika-eval-core/src/test/resources/common_tokens/som
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/som
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/som
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/spa b/tika-eval/tika-eval-core/src/test/resources/common_tokens/spa
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/spa
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/spa
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/sqi b/tika-eval/tika-eval-core/src/test/resources/common_tokens/sqi
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/sqi
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/sqi
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/srp b/tika-eval/tika-eval-core/src/test/resources/common_tokens/srp
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/srp
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/srp
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/stq b/tika-eval/tika-eval-core/src/test/resources/common_tokens/stq
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/stq
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/stq
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/sun b/tika-eval/tika-eval-core/src/test/resources/common_tokens/sun
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/sun
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/sun
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/swe b/tika-eval/tika-eval-core/src/test/resources/common_tokens/swe
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/swe
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/swe
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/swh b/tika-eval/tika-eval-core/src/test/resources/common_tokens/swh
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/swh
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/swh
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/szl b/tika-eval/tika-eval-core/src/test/resources/common_tokens/szl
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/szl
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/szl
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/szy b/tika-eval/tika-eval-core/src/test/resources/common_tokens/szy
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/szy
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/szy
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tam b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tam
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tam
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tam
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tat b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tat
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tat
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tat
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tay b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tay
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tay
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tay
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tel b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tel
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tel
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tel
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tet b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tet
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tet
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tet
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tgk b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tgk
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tgk
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tgk
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tgl b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tgl
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tgl
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tgl
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tha b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tha
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tha
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tha
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tir b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tir
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tir
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tir
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/trv b/tika-eval/tika-eval-core/src/test/resources/common_tokens/trv
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/trv
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/trv
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tsn b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tsn
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tsn
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tsn
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tso b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tso
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tso
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tso
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tuk b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tuk
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tuk
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tuk
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tum b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tum
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tum
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tum
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tur b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tur
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tur
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tur
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/tyv b/tika-eval/tika-eval-core/src/test/resources/common_tokens/tyv
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/tyv
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/tyv
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/udm b/tika-eval/tika-eval-core/src/test/resources/common_tokens/udm
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/udm
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/udm
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/uig b/tika-eval/tika-eval-core/src/test/resources/common_tokens/uig
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/uig
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/uig
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ukr b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ukr
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ukr
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ukr
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/urd b/tika-eval/tika-eval-core/src/test/resources/common_tokens/urd
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/urd
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/urd
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/uzb b/tika-eval/tika-eval-core/src/test/resources/common_tokens/uzb
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/uzb
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/uzb
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/vep b/tika-eval/tika-eval-core/src/test/resources/common_tokens/vep
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/vep
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/vep
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/vie b/tika-eval/tika-eval-core/src/test/resources/common_tokens/vie
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/vie
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/vie
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/vls b/tika-eval/tika-eval-core/src/test/resources/common_tokens/vls
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/vls
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/vls
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/vol b/tika-eval/tika-eval-core/src/test/resources/common_tokens/vol
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/vol
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/vol
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/vro b/tika-eval/tika-eval-core/src/test/resources/common_tokens/vro
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/vro
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/vro
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/war b/tika-eval/tika-eval-core/src/test/resources/common_tokens/war
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/war
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/war
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/wln b/tika-eval/tika-eval-core/src/test/resources/common_tokens/wln
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/wln
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/wln
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/xho b/tika-eval/tika-eval-core/src/test/resources/common_tokens/xho
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/xho
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/xho
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/xmf b/tika-eval/tika-eval-core/src/test/resources/common_tokens/xmf
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/xmf
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/xmf
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/ydd b/tika-eval/tika-eval-core/src/test/resources/common_tokens/ydd
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/ydd
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/ydd
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/yor b/tika-eval/tika-eval-core/src/test/resources/common_tokens/yor
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/yor
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/yor
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/yue b/tika-eval/tika-eval-core/src/test/resources/common_tokens/yue
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/yue
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/yue
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/zho b/tika-eval/tika-eval-core/src/test/resources/common_tokens/zho
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/zho
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/zho
diff --git a/tika-eval/tika-eval-core/src/main/resources/common_tokens/zul b/tika-eval/tika-eval-core/src/test/resources/common_tokens/zul
similarity index 100%
rename from tika-eval/tika-eval-core/src/main/resources/common_tokens/zul
rename to tika-eval/tika-eval-core/src/test/resources/common_tokens/zul