OPENNLP-1261: The Language Detector should not ignore ngram counts.

git push

Author: kottmann

Diff for: opennlp-tools/src/main/java/opennlp/tools/langdetect/DefaultLanguageDetectorContextGenerator.java

@@ -20,8 +20,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 
-import opennlp.tools.ngram.NGramModel;
-import opennlp.tools.util.StringList;
+import opennlp.tools.util.StringUtil;
 import opennlp.tools.util.normalizer.AggregateCharSequenceNormalizer;
 import opennlp.tools.util.normalizer.CharSequenceNormalizer;
 
@@ -58,14 +57,19 @@ public DefaultLanguageDetectorContextGenerator(int minLength, int maxLength,
   public String[] getContext(CharSequence document) {
     Collection<String> context = new ArrayList<>();
 
-    NGramModel model = new NGramModel();
-    model.add(normalizer.normalize(document), minLength, maxLength);
+    CharSequence chars = normalizer.normalize(document);
 
-    for (StringList tokenList : model) {
-      if (tokenList.size() > 0) {
-        context.add(tokenList.getToken(0));
+    for (int lengthIndex = minLength; lengthIndex < maxLength + 1; lengthIndex++) {
+      for (int textIndex = 0;
+           textIndex + lengthIndex - 1 < chars.length(); textIndex++) {
+
+        String gram = StringUtil.toLowerCase(
+            chars.subSequence(textIndex, textIndex + lengthIndex));
+
+        context.add(gram);
       }
     }
+
     return context.toArray(new String[context.size()]);
   }
 }

Diff for: opennlp-tools/src/test/java/opennlp/tools/langdetect/LanguageDetectorCrossValidatorTest.java

@@ -58,7 +58,6 @@ public void missclassified(LanguageSample reference,
     cv.evaluate(sampleStream, 2);
 
     Assert.assertEquals(99, cv.getDocumentCount());
-    Assert.assertEquals(0.98989898989899, cv.getDocumentAccuracy(), 0.01);
+    Assert.assertEquals(1, cv.getDocumentAccuracy(), 0.01);
   }
-
 }