Merge pull request #14 from word-view/improve-tokenize-kanji-perf

Performance improvements

Author: 64ArthurAraujo

pom.xml

@@ -88,6 +88,11 @@
             <artifactId>gson</artifactId>
             <version>2.11.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlinx</groupId>
+            <artifactId>kotlinx-coroutines-core</artifactId>
+            <version>1.9.0</version>
+        </dependency>
 
     </dependencies>
 

src/main/kotlin/cc/wordview/gengolex/languages/Tokenizer.kt

@@ -5,49 +5,60 @@ import cc.wordview.gengolex.word.DerivatableWord
 import cc.wordview.gengolex.word.Word
 import com.google.gson.Gson
 import com.google.gson.reflect.TypeToken
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.IO
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
 import java.io.File
 import java.util.HashMap
+import kotlin.collections.emptyList
+import kotlin.collections.flatten
 
 interface Tokenizer {
     val dictionary: ArrayList<DerivatableWord>
 
+    companion object {
+        private val gson = Gson()
+        private val typeToken = object : TypeToken<List<DerivatableWord>>() {}.type
+    }
+
     fun tokenize(words: List<String>): ArrayList<Word>
 
     fun initializeDictionary(path: String)
 
     fun initializeDictionary(path: String, lang: String) {
         val dictionaryDir = File("$path/$lang")
 
-        val files = dictionaryDir.listFiles()?.filter { it.isFile && it.name.endsWith(".json") }
-
-        if (files.isNullOrEmpty())
+        val files = dictionaryDir.listFiles()?.filter { it.isFile && it.name.endsWith(".json") } ?:
             throw NoDictionaryException("Unable to find a dictionary for $lang")
 
-        for (file in files) {
-            val content = file.inputStream().readBytes().toString(Charsets.UTF_8)
-
-            if (content.isEmpty()) continue
+        dictionary.clear()
+        // at the moment of this commit the largest dictionary contains 38 words,
+        dictionary.ensureCapacity(50)
 
-            val typeToken = object : TypeToken<List<DerivatableWord>>() {}.type
+        runBlocking(Dispatchers.IO) {
+            val parsedDictionaries = files.map { file ->
+                async {
+                    val content = file.inputStream().use { it.readBytes().toString(Charsets.UTF_8) }
+                    if (content.isEmpty()) emptyList() else gson.fromJson<List<DerivatableWord>>(content, typeToken)
+                }
+            }.awaitAll()
 
-            val parsedDictionary = Gson().fromJson<List<DerivatableWord>>(content, typeToken)
-
-            dictionary.addAll(parsedDictionary)
+            dictionary.addAll(parsedDictionaries.flatten())
         }
     }
 
     fun initializeDictionary(dictionaries: HashMap<String, String>)
 
     fun initializeDictionary(dictionaries: HashMap<String, String>, lang: String) {
-        val portugueseDictionary = dictionaries[lang]
-
-        if (portugueseDictionary.isNullOrEmpty())
-            throw NoDictionaryException("Unable to find a dictionary for $lang")
-
-        val typeToken = object : TypeToken<List<DerivatableWord>>() {}.type
+        val hashmapDictionary = dictionaries[lang] ?: throw NoDictionaryException("Unable to find a dictionary for $lang")
 
-        val parsedDictionary = Gson().fromJson<List<DerivatableWord>>(portugueseDictionary, typeToken)
+        dictionary.clear()
+        // at the moment of this commit the largest dictionary contains 38 words,
+        dictionary.ensureCapacity(50)
 
+        val parsedDictionary = gson.fromJson<List<DerivatableWord>>(hashmapDictionary, typeToken)
         dictionary.addAll(parsedDictionary)
     }
 }

src/main/kotlin/cc/wordview/gengolex/languages/japanese/JapaneseTokenizer.kt

@@ -1,56 +1,48 @@
 package cc.wordview.gengolex.languages.japanese
 
 import cc.wordview.gengolex.languages.Tokenizer
+import cc.wordview.gengolex.languages.japanese.JapaneseKanjiStrategy.*
 import cc.wordview.gengolex.word.DerivatableWord
 import cc.wordview.gengolex.word.Word
 import java.util.HashMap
-import java.util.regex.Pattern
 
 object JapaneseTokenizer : Tokenizer {
     override var dictionary: ArrayList<DerivatableWord> = arrayListOf()
+    private val wordMap: HashMap<String, DerivatableWord> = HashMap()
 
     @Suppress("MemberVisibilityCanBePrivate")
-    var kanjiStrategy = JapaneseKanjiStrategy.PREFER_DERIVATION
-
-    private val kanjiPattern: Pattern = Pattern.compile("[一-龯]")
+    var kanjiStrategy = PREFER_DERIVATION
 
     override fun tokenize(words: List<String>): ArrayList<Word> {
-        val wordsByChars = words.joinToString().replace("、", "").replace("。", "").split("")
-        val joinedWordsString = wordsByChars.joinToString("")
-
-        val wordsFound = ArrayList<Word>()
-        var charsToSkipNext = 0
+        val input = words.joinToString()
+            .replace("、", "")
+            .replace("。", "")
 
-        for (i in wordsByChars.indices) {
-            if (charsToSkipNext > 0) {
-                charsToSkipNext--
-                continue
-            }
+        val chars = input.toCharArray()
 
-            val char = wordsByChars[i]
-
-            val currentWordsString =
-                wordsFound.fold(joinedWordsString) { acc, foundWord -> acc.replace(foundWord.word, "") }
+        val wordsFound = ArrayList<Word>()
+        var i = 0
 
-            tokenizeKanji(char, currentWordsString)?.let {
+        while (i < chars.size) {
+            val char = chars[i].toString()
+            tokenizeKanji(char, input.substring(i))?.let {
                 wordsFound.add(it)
-                charsToSkipNext = it.word.length - 1
-            }
+                i += it.word.length // Skip characters based on the word's length
+            } ?: i++
         }
 
         return wordsFound
     }
 
     private fun tokenizeKanji(char: String, original: String): Word? {
-        if (!kanjiPattern.matcher(char).matches()) return null
+        if (char.isEmpty() || char[0].code !in 0x4E00..0x9FFF) return null
 
-        dictionary.firstOrNull { it.word == char }?.let { kanjiWord ->
+        wordMap[char]?.let { kanjiWord ->
             return when (kanjiStrategy) {
-                // TODO: Properly address this by removing all derivations that is not present in the phrase.
-                JapaneseKanjiStrategy.PREFER_PARENT -> kanjiWord
-                JapaneseKanjiStrategy.PREFER_DERIVATION -> {
+                PREFER_PARENT -> kanjiWord
+                PREFER_DERIVATION -> {
                     @Suppress("UNNECESSARY_SAFE_CALL")
-                    kanjiWord.derivations?.firstOrNull { original.contains(it.word) } ?: kanjiWord
+                    kanjiWord.derivations?.find { original.startsWith(it.word) } ?: kanjiWord
                 }
             }
         }
@@ -60,9 +52,11 @@ object JapaneseTokenizer : Tokenizer {
 
     override fun initializeDictionary(path: String) {
         super.initializeDictionary(path, "kanji")
+        dictionary.forEach { wordMap[it.word] = it }
     }
 
     override fun initializeDictionary(dictionaries: HashMap<String, String>) {
         super.initializeDictionary(dictionaries, "kanji")
+        dictionary.forEach { wordMap[it.word] = it }
     }
 }