11package cc.wordview.gengolex.languages.japanese
22
33import cc.wordview.gengolex.languages.Tokenizer
4+ import cc.wordview.gengolex.languages.japanese.JapaneseKanjiStrategy.*
45import cc.wordview.gengolex.word.DerivatableWord
56import cc.wordview.gengolex.word.Word
67import java.util.HashMap
78import java.util.regex.Pattern
89
910object JapaneseTokenizer : Tokenizer {
1011 override var dictionary: ArrayList <DerivatableWord > = arrayListOf ()
12+ private val wordMap: HashMap <String , DerivatableWord > = HashMap ()
1113
1214 @Suppress(" MemberVisibilityCanBePrivate" )
13- var kanjiStrategy = JapaneseKanjiStrategy . PREFER_DERIVATION
15+ var kanjiStrategy = PREFER_DERIVATION
1416
1517 private val kanjiPattern: Pattern = Pattern .compile(" [一-龯]" )
1618
@@ -38,11 +40,10 @@ object JapaneseTokenizer : Tokenizer {
3840 private fun tokenizeKanji (char : String , original : String ): Word ? {
3941 if (! kanjiPattern.matcher(char).matches()) return null
4042
41- dictionary.firstOrNull { it.word == char } ?.let { kanjiWord ->
43+ wordMap[ char] ?.let { kanjiWord ->
4244 return when (kanjiStrategy) {
43- // TODO: Properly address this by removing all derivations that is not present in the phrase.
44- JapaneseKanjiStrategy .PREFER_PARENT -> kanjiWord
45- JapaneseKanjiStrategy .PREFER_DERIVATION -> {
45+ PREFER_PARENT -> kanjiWord
46+ PREFER_DERIVATION -> {
4647 @Suppress(" UNNECESSARY_SAFE_CALL" )
4748 kanjiWord.derivations?.firstOrNull { original.contains(it.word) } ? : kanjiWord
4849 }
@@ -54,9 +55,11 @@ object JapaneseTokenizer : Tokenizer {
5455
5556 override fun initializeDictionary (path : String ) {
5657 super .initializeDictionary(path, " kanji" )
58+ dictionary.forEach { wordMap[it.word] = it }
5759 }
5860
5961 override fun initializeDictionary (dictionaries : HashMap <String , String >) {
6062 super .initializeDictionary(dictionaries, " kanji" )
63+ dictionary.forEach { wordMap[it.word] = it }
6164 }
6265}
0 commit comments