|
| 1 | +package com.kienhoang.dualsubreplay.data |
| 2 | + |
| 3 | +import com.atilika.kuromoji.dict.CharacterDefinitions |
| 4 | +import com.atilika.kuromoji.dict.ConnectionCosts |
| 5 | +import com.atilika.kuromoji.dict.InsertedDictionary |
| 6 | +import com.atilika.kuromoji.dict.TokenInfoDictionary |
| 7 | +import com.atilika.kuromoji.dict.UnknownDictionary |
| 8 | +import com.atilika.kuromoji.ipadic.Tokenizer |
| 9 | +import com.atilika.kuromoji.trie.DoubleArrayTrie |
| 10 | +import com.atilika.kuromoji.util.ResourceResolver |
| 11 | +import okhttp3.OkHttpClient |
| 12 | +import okhttp3.Request |
| 13 | +import java.io.File |
| 14 | +import java.io.FileNotFoundException |
| 15 | +import java.io.IOException |
| 16 | +import java.io.InputStream |
| 17 | +import java.security.MessageDigest |
| 18 | +import java.util.concurrent.TimeUnit |
| 19 | +import java.util.zip.ZipFile |
| 20 | + |
| 21 | +/** The published Kuromoji IPADIC artifact the app downloads, pinned by size and SHA-256. */ |
| 22 | +internal object JapaneseDictionaryRelease { |
| 23 | + const val FILE_NAME = "kuromoji-ipadic-0.9.0.jar" |
| 24 | + const val SIZE_BYTES = 13_343_016L |
| 25 | + const val SHA256 = "24909fd751c0b439f7af5131b080eb65bc85062f0c4977ca4a71c76abe74e0b6" |
| 26 | + private const val PATH = "com/atilika/kuromoji/kuromoji-ipadic/0.9.0/kuromoji-ipadic-0.9.0.jar" |
| 27 | + |
| 28 | + /** Maven Central, then Google's mirror of it. The checksum makes the host irrelevant to integrity. */ |
| 29 | + val URLS = |
| 30 | + listOf( |
| 31 | + "https://repo1.maven.org/maven2/$PATH", |
| 32 | + "https://maven-central.storage-download.googleapis.com/maven2/$PATH", |
| 33 | + ) |
| 34 | +} |
| 35 | + |
| 36 | +/** |
| 37 | + * Keeps the Japanese dictionary in app storage. The APK carries Kuromoji's code but not its |
| 38 | + * 13 MB dictionary, so only people who watch Japanese download it, once. The file is the |
| 39 | + * unchanged Maven artifact; Kuromoji reads its dictionary entries straight from it. |
| 40 | + */ |
| 41 | +internal class JapaneseDictionaryStore( |
| 42 | + private val directory: File, |
| 43 | + private val expectedSize: Long = JapaneseDictionaryRelease.SIZE_BYTES, |
| 44 | + private val expectedSha256: String = JapaneseDictionaryRelease.SHA256, |
| 45 | + private val open: (url: String) -> InputStream = ::openWithOkHttp, |
| 46 | +) { |
| 47 | + private val file: File get() = File(directory, JapaneseDictionaryRelease.FILE_NAME) |
| 48 | + |
| 49 | + fun isInstalled(): Boolean = file.isFile && file.length() == expectedSize |
| 50 | + |
| 51 | + /** Downloads the dictionary unless it is already installed. Blocking; returns whether it is installed. */ |
| 52 | + fun install(): Boolean { |
| 53 | + if (isInstalled()) return true |
| 54 | + return JapaneseDictionaryRelease.URLS.any { url -> |
| 55 | + try { |
| 56 | + open(url).use(::installFrom) |
| 57 | + } catch (_: Exception) { |
| 58 | + // Offline, blocked or a broken response: try the next host. |
| 59 | + false |
| 60 | + } |
| 61 | + } |
| 62 | + } |
| 63 | + |
| 64 | + /** Saves [input] as the dictionary only if it is exactly the expected file. */ |
| 65 | + internal fun installFrom(input: InputStream): Boolean { |
| 66 | + directory.mkdirs() |
| 67 | + val partial = File(directory, JapaneseDictionaryRelease.FILE_NAME + ".part") |
| 68 | + val digest = MessageDigest.getInstance("SHA-256") |
| 69 | + var total = 0L |
| 70 | + partial.outputStream().use { output -> |
| 71 | + val buffer = ByteArray(BUFFER_BYTES) |
| 72 | + while (total <= expectedSize) { |
| 73 | + val read = input.read(buffer) |
| 74 | + if (read < 0) break |
| 75 | + total += read |
| 76 | + digest.update(buffer, 0, read) |
| 77 | + output.write(buffer, 0, read) |
| 78 | + } |
| 79 | + } |
| 80 | + val sha256 = digest.digest().joinToString("") { "%02x".format(it) } |
| 81 | + if (total != expectedSize || !sha256.equals(expectedSha256, ignoreCase = true) || !partial.renameTo(file)) { |
| 82 | + partial.delete() |
| 83 | + return false |
| 84 | + } |
| 85 | + return true |
| 86 | + } |
| 87 | + |
| 88 | + /** Builds Kuromoji's analyzer from the installed dictionary. Takes about a second and ~50 MB. */ |
| 89 | + fun loadTokenizer(): Tokenizer = ZipFile(file).use { zip -> ZipDictionaryBuilder(zip).build() } |
| 90 | + |
| 91 | + /** |
| 92 | + * Kuromoji 0.9.0's IPADIC builder always reads the dictionary from its own classpath package. |
| 93 | + * This does the same loading steps, but reads each file from the downloaded jar instead. |
| 94 | + */ |
| 95 | + private class ZipDictionaryBuilder( |
| 96 | + private val zip: ZipFile, |
| 97 | + ) : Tokenizer.Builder() { |
| 98 | + override fun loadDictionaries() { |
| 99 | + // IPADIC's default search-mode penalties; normal mode, used here, does not apply them. |
| 100 | + penalties = arrayListOf(2, 3000, 7, 1700) |
| 101 | + val files = |
| 102 | + ResourceResolver { name -> |
| 103 | + val entry = zip.getEntry(DICTIONARY_PACKAGE + name) ?: throw FileNotFoundException(name) |
| 104 | + zip.getInputStream(entry) |
| 105 | + } |
| 106 | + resolver = files |
| 107 | + doubleArrayTrie = DoubleArrayTrie.newInstance(files) |
| 108 | + connectionCosts = ConnectionCosts.newInstance(files) |
| 109 | + tokenInfoDictionary = TokenInfoDictionary.newInstance(files) |
| 110 | + characterDefinitions = CharacterDefinitions.newInstance(files) |
| 111 | + unknownDictionary = UnknownDictionary.newInstance(files, characterDefinitions, totalFeatures) |
| 112 | + insertedDictionary = InsertedDictionary(totalFeatures) |
| 113 | + } |
| 114 | + } |
| 115 | + |
| 116 | + private companion object { |
| 117 | + const val BUFFER_BYTES = 64 * 1024 |
| 118 | + const val DICTIONARY_PACKAGE = "com/atilika/kuromoji/ipadic/" |
| 119 | + |
| 120 | + val client: OkHttpClient by lazy { |
| 121 | + OkHttpClient |
| 122 | + .Builder() |
| 123 | + .connectTimeout(20, TimeUnit.SECONDS) |
| 124 | + .readTimeout(60, TimeUnit.SECONDS) |
| 125 | + .build() |
| 126 | + } |
| 127 | + |
| 128 | + fun openWithOkHttp(url: String): InputStream { |
| 129 | + val response = client.newCall(Request.Builder().url(url).build()).execute() |
| 130 | + if (!response.isSuccessful) { |
| 131 | + response.close() |
| 132 | + throw IOException("The Japanese dictionary download failed (${response.code}).") |
| 133 | + } |
| 134 | + return response.body?.byteStream() ?: throw IOException("The Japanese dictionary download was empty.") |
| 135 | + } |
| 136 | + } |
| 137 | +} |
0 commit comments