Add lingua for language recognition (as an optional alternative) (#1239)

* add Lingua as language detection library

* fix: improve language detection handling and update configuration comments

Author: lfoppiano

build.gradle

@@ -180,6 +180,7 @@ subprojects {
         implementation "com.google.guava:guava:31.0.1-jre"
         implementation "org.apache.httpcomponents:httpclient:4.5.3"
         implementation "black.ninia:jep:4.2.2"
+        implementation 'com.github.pemistahl:lingua:1.2.2'
 
         implementation "com.fasterxml.jackson.core:jackson-core:2.14.3"
         implementation "com.fasterxml.jackson.core:jackson-databind:2.14.3"
@@ -216,7 +217,7 @@ subprojects {
                 "--add-opens", "java.base/java.lang=ALL-UNNAMED",
                 "--add-opens", "java.base/java.util=ALL-UNNAMED"
         }
-        
+
         systemProperty "java.library.path", "${javaLibraryPath}"
     }
 }
@@ -596,7 +597,7 @@ project(":grobid-trainer") {
         classpath = sourceSets.main.runtimeClasspath
         args 'tei', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0'), getArg('flavor', '')
         def javaLibraryPath = getJavaLibraryPath()
-        
+
         if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
             jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.lang=ALL-UNNAMED"
         } else {

grobid-core/src/main/java/org/grobid/core/utilities/LanguageUtilities.java

@@ -22,7 +22,6 @@ public static LanguageUtilities getInstance() {
 		if (instance == null) {
 		    synchronized (LanguageUtilities.class) {
                 if (instance == null) {
-                    LOGGER.debug("synchronized getNewInstance");
                     instance = new LanguageUtilities();
                 }
             }

grobid-core/src/main/kotlin/org/grobid/core/lang/impl/LinguaLanguageDetector.kt

@@ -0,0 +1,39 @@
+package org.grobid.core.lang.impl
+
+import com.github.pemistahl.lingua.api.LanguageDetectorBuilder
+import org.grobid.core.lang.Language
+import org.grobid.core.lang.LanguageDetector
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+class LinguaLanguageDetector : LanguageDetector {
+    private val detector: com.github.pemistahl.lingua.api.LanguageDetector = LanguageDetectorBuilder
+        .fromAllLanguages()
+        .withLowAccuracyMode()
+        .build()
+
+    override fun detect(text: String?): Language? {
+        if (text.isNullOrBlank()) {
+            return null
+        }
+
+        val languages = detector.computeLanguageConfidenceValues(text = text)
+
+        if (LOGGER.isDebugEnabled) {
+            LOGGER.debug(languages.toString())
+        }
+
+        if (languages.isEmpty()) {
+            return null
+        }
+
+        val l = languages.firstKey()
+        val p = languages[l] ?: 0.0
+
+        return Language(l.isoCode639_1.toString(), p)
+    }
+
+    companion object {
+        private val LOGGER: Logger = LoggerFactory.getLogger(LinguaLanguageDetector::class.java)
+    }
+}

grobid-core/src/main/kotlin/org/grobid/core/lang/impl/LinguaLanguageDetectorFactory.java

@@ -0,0 +1,29 @@
+package org.grobid.core.lang.impl;
+
+import org.grobid.core.lang.LanguageDetector;
+import org.grobid.core.lang.LanguageDetectorFactory;
+
+/**
+ * Implementation of a language detector factory with Lingua language identifier
+ */
+public class LinguaLanguageDetectorFactory implements LanguageDetectorFactory {
+    private static volatile LanguageDetector instance = null;
+
+    private static void init() {
+
+    }
+
+    public LanguageDetector getInstance() {
+        if (instance == null) {
+            synchronized (this) {
+                if(instance == null) {
+                    init();
+                    instance = new LinguaLanguageDetector();
+                }
+            }
+
+        }
+        return instance;
+    }
+
+}

grobid-core/src/test/kotlin/org/grobid/core/lang/impl/LinguaLanguageDetectorTest.kt

@@ -0,0 +1,57 @@
+package org.grobid.core.lang.impl
+
+import org.hamcrest.CoreMatchers.`is`
+import org.hamcrest.CoreMatchers.nullValue
+import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.greaterThan
+import kotlin.test.Test
+
+class LinguaLanguageDetectorTest {
+
+    private val target = LinguaLanguageDetector()
+
+    @Test
+    fun testDetect_englishText_shouldReturnEnglish() {
+        val result = target.detect("This is a simple English sentence for language detection.")
+
+        assertThat(result!!.lang, `is`("en"))
+        assertThat(result.conf, greaterThan(0.0))
+    }
+
+    @Test
+    fun testDetect_frenchText_shouldReturnFrench() {
+        val result = target.detect("Ceci est une phrase en français pour la détection de langue.")
+
+        assertThat(result!!.lang, `is`("fr"))
+        assertThat(result.conf, greaterThan(0.0))
+    }
+
+    @Test
+    fun testDetect_germanText_shouldReturnGerman() {
+        val result = target.detect("Dies ist ein einfacher deutscher Satz zur Spracherkennung.")
+
+        assertThat(result!!.lang, `is`("de"))
+        assertThat(result.conf, greaterThan(0.0))
+    }
+
+    @Test
+    fun testDetect_emptyText_shouldReturnNull() {
+        val result = target.detect("")
+
+        assertThat(result, `is`(nullValue()))
+    }
+
+    @Test
+    fun testDetect_nullText_shouldReturnNull() {
+        val result = target.detect(null)
+
+        assertThat(result, `is`(nullValue()))
+    }
+
+    @Test
+    fun testDetect_blankText_shouldReturnNull() {
+        val result = target.detect("   ")
+
+        assertThat(result, `is`(nullValue()))
+    }
+}

grobid-home/config/grobid-evaluation.yaml

@@ -50,8 +50,9 @@ grobid:
   corsAllowedMethods: "OPTIONS,GET,PUT,POST,DELETE,HEAD"
   corsAllowedHeaders: "X-Requested-With,Content-Type,Accept,Origin"
 
-  # the actual implementation for language recognition to be used
+  # the actual implementation for language recognition to be used (Cybozu or Lingua)
   languageDetectorFactory: "org.grobid.core.lang.impl.CybozuLanguageDetectorFactory"
+  #languageDetectorFactory: "org.grobid.core.lang.impl.LinguaLanguageDetectorFactory"
 
   # the actual implementation for optional sentence segmentation to be used (PragmaticSegmenter or OpenNLP)
   #sentenceDetectorFactory: "org.grobid.core.lang.impl.PragmaticSentenceDetectorFactory"

grobid-home/config/grobid-full.yaml

@@ -50,8 +50,9 @@ grobid:
   corsAllowedMethods: "OPTIONS,GET,PUT,POST,DELETE,HEAD"
   corsAllowedHeaders: "X-Requested-With,Content-Type,Accept,Origin"
 
-  # the actual implementation for language recognition to be used
+  # the actual implementation for language recognition to be used (Cybozu or Lingua)
   languageDetectorFactory: "org.grobid.core.lang.impl.CybozuLanguageDetectorFactory"
+  #languageDetectorFactory: "org.grobid.core.lang.impl.LinguaLanguageDetectorFactory"
 
   # the actual implementation for optional sentence segmentation to be used (PragmaticSegmenter or OpenNLP)
   #sentenceDetectorFactory: "org.grobid.core.lang.impl.PragmaticSentenceDetectorFactory"

grobid-home/config/grobid.yaml

@@ -50,8 +50,9 @@ grobid:
   corsAllowedMethods: "OPTIONS,GET,PUT,POST,DELETE,HEAD"
   corsAllowedHeaders: "X-Requested-With,Content-Type,Accept,Origin"
 
-  # the actual implementation for language recognition to be used
+  # the actual implementation for language recognition to be used (Cybozu or Lingua)
   languageDetectorFactory: "org.grobid.core.lang.impl.CybozuLanguageDetectorFactory"
+  #languageDetectorFactory: "org.grobid.core.lang.impl.LinguaLanguageDetectorFactory"
 
   # the actual implementation for optional sentence segmentation to be used (PragmaticSegmenter or OpenNLP)
   #sentenceDetectorFactory: "org.grobid.core.lang.impl.PragmaticSentenceDetectorFactory"