Usif

.github/workflows/build_project.yml

@@ -18,9 +18,6 @@ jobs:
       - name: Download Dependencies
         run: ./gradlew dependencies
 
-      - name: Check Dependency Versions
-        run: ./gradlew dependencyUpdates
-
       #- name: Check Kotlin Formatting
       #  run: ./gradlew ktlintCheck --continue
 

build.gradle.kts

@@ -1,55 +1,42 @@
-import org.gradle.jvm.tasks.Jar
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
     `maven-publish`
-    id("org.jetbrains.dokka") version "0.9.17"
-    kotlin("jvm") version "1.3.72"
+    kotlin("jvm") version "1.4.21"
+    id("org.jetbrains.dokka") version "1.4.20"
 }
 
 group = "com.londogard"
 version = "1.0-SNAPSHOT"
-val kluentVersion = "1.61"
-val smileVersion = "2.4.0"
+val kluentVersion: String by project
+val smileVersion: String by project
 
 repositories {
     jcenter()
     google()
     mavenCentral()
-    maven("https://dl.bintray.com/kotlin/kotlin-dev/")
+    maven("https://jitpack.io")
 }
 
 dependencies {
-    implementation(kotlin("stdlib-jdk8"))
-
+    implementation("com.github.londogard:londogard-nlp-toolkit:main-SNAPSHOT")
     implementation("com.github.haifengl:smile-nlp:$smileVersion")
     implementation("com.github.haifengl:smile-core:$smileVersion")
     implementation("com.github.haifengl:smile-kotlin:$smileVersion")
 
     testImplementation("org.amshove.kluent:kluent:$kluentVersion")
-    testImplementation("org.jetbrains.kotlin:kotlin-test:1.3.72")
-    testImplementation("junit:junit:4.12")
+    testImplementation("org.jetbrains.kotlin:kotlin-test:1.4.21")
+    testImplementation(kotlin("test-junit"))
 }
 
-configure<JavaPluginConvention> {
-    sourceCompatibility = JavaVersion.VERSION_1_8
+tasks.test {
+    useJUnit()
 }
+
 tasks.withType<KotlinCompile> {
     kotlinOptions.jvmTarget = "1.8"
 }
 
-tasks.dokka {
-    outputFormat = "html"
-    outputDirectory = "$buildDir/javadoc"
-}
-
-val dokkaJar by tasks.creating(Jar::class) {
-    group = JavaBasePlugin.DOCUMENTATION_GROUP
-    description = "Assembles Kotlin docs with Dokka"
-    classifier = "javadoc"
-    from(tasks.dokka)
-}
-
 publishing {
     repositories {
         maven {

gradle.properties

@@ -1,2 +1,3 @@
 kotlin.code.style=official
-kluent_version=1.59
+kluentVersion=1.64
+smileVersion=2.6.0

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/kotlin/com/londogard/embeddings/Embeddings.kt

@@ -1,5 +1,6 @@
 package com.londogard.embeddings
 
+import com.londogard.embeddings.utils.SimpleDistances
 import java.nio.file.Files
 import java.nio.file.Paths
 import kotlin.math.sqrt
@@ -34,37 +35,17 @@ abstract class Embeddings {
      * @return The Euclidean distance between the vector representations of the words.
      */
     fun euclidean(w1: String, w2: String): Double? = traverseVectors(listOf(w1, w2))?.let { vectors ->
-        if (vectors.size == 2) euclidean(vectors.first(), vectors.last())
+        if (vectors.size == 2) SimpleDistances.euclidean(vectors.first(), vectors.last())
         else null
     }
 
-    /** Compute the Euclidean distance between two vectors.
-     * @param v1 The first vector.
-     * @param v2 The other vector.
-     * @return The Euclidean distance between the two vectors.
-     */
-    fun euclidean(v1: Array<Float>, v2: Array<Float>): Double =
-        (v1 `--` v2).let { vector -> sqrt(vector.dot(vector)) }
-
-    /** Compute the cosine similarity score between two vectors.
-     * 1.0 means equal, 0 = 90* & -1 is when they're opposite
-     * @param v1 The first vector.
-     * @param v2 The other vector.
-     * @return The cosine similarity score of the two vectors.
-     */
-    fun cosine(v1: Array<Float>, v2: Array<Float>): Double {
-        if (v1.size != v2.size) throw ArithmeticException("Vectors must be same size (v1: ${v1.size} != v2: ${v2.size}")
-
-        return v1.dot(v2) / (sqrt(v1.dot(v1)) * sqrt(v2.dot(v2)))
-    }
-
     /** Compute the cosine similarity score between the vector representations of the words.
      * @param w1 The first word.
      * @param w2 The other word.
      * @return The cosine similarity score between the vector representations of the words.
      */
     fun cosine(w1: String, w2: String): Double? = traverseVectors(listOf(w1, w2))?.let { vectors ->
-        if (vectors.size == 2) cosine(vectors.first(), vectors.last())
+        if (vectors.size == 2) SimpleDistances.cosine(vectors.first(), vectors.last())
         else null
     }
 

src/main/kotlin/com/londogard/embeddings/LightWordEmbeddings.kt

@@ -1,5 +1,7 @@
 package com.londogard.embeddings
 
+import com.londogard.embeddings.utils.DownloadHelper
+
 class LightWordEmbeddings(
     override val dimensions: Int = DownloadHelper.dimension,
     override val filename: String = DownloadHelper.embeddingPath,

src/main/kotlin/com/londogard/embeddings/NormAvgSentenceEmbeddings.kt

@@ -1,9 +1,8 @@
 package com.londogard.embeddings
 
-
 class NormAvgSentenceEmbeddings(val embeddings: Embeddings) : SentenceEmbeddings {
     override fun getSentenceEmbedding(tokens: List<String>): Array<Float> = tokens
-            .mapNotNull(embeddings::vector)
-            .sumByColumns()
-            .normalize()
+        .mapNotNull(embeddings::vector)
+        .sumByColumns()
+        .normalize()
 }

src/main/kotlin/com/londogard/embeddings/SentenceEmbeddings.kt

@@ -2,4 +2,6 @@ package com.londogard.embeddings
 
 interface SentenceEmbeddings {
     fun getSentenceEmbedding(tokens: List<String>): Array<Float>
+    fun getSentenceEmbeddings(listOfTokens: List<List<String>>): List<Array<Float>> =
+        listOfTokens.map(this::getSentenceEmbedding)
 }

src/main/kotlin/com/londogard/embeddings/SifSentenceEmbeddings.kt

@@ -1,43 +1,12 @@
-package com.londogard.summarize.embeddings
+package com.londogard.embeddings
 
-import com.londogard.embeddings.*
-import com.londogard.embeddings.mMul
-import com.londogard.embeddings.sumByColumns
 import smile.math.matrix.Matrix
 import smile.nlp.bag
 import smile.nlp.tfidf
 import smile.nlp.vectorize
 import smile.nlp.words
 import smile.projection.PCA
 
-// weight = idf.. embeddings * weight
-// trunctatedSVD =
-/**
-/* calculate principle components */
-public RealMatrix getTruncatedSVD(RealMatrix m, int k) {
-SingularValueDecomposition svd = new SingularValueDecomposition(m);
-
-double[][] truncatedU = new double[svd.getU().getRowDimension()][k];
-double[][] truncatedS = new double[k][k];
-double[][] truncatedVT = new double[k][svd.getVT().getColumnDimension()];
-
-svd.getU().copySubMatrix(0, truncatedU.length - 1, 0, k - 1, truncatedU);
-svd.getS().copySubMatrix(0, k - 1, 0, k - 1, truncatedS);
-svd.getVT().copySubMatrix(0, k - 1, 0, truncatedVT[0].length - 1, truncatedVT);
-
-RealMatrix u = new Array2DRowRealMatrix(truncatedU);
-RealMatrix s = new Array2DRowRealMatrix(truncatedS);
-RealMatrix vt = new Array2DRowRealMatrix(truncatedVT);
-
-return u.multiply(s).multiply(vt);
-}
-
-/* remove principle components */
-private RealMatrix removePrincipleComponents(RealMatrix m, int k) {
-RealMatrix pc = getTruncatedSVD(m, k);
-return m.subtract(m.multiply(pc.transpose()).multiply(pc));
-}
-*/
 class SifSentenceEmbeddings(val embeddings: Embeddings) : SentenceEmbeddings {
     private lateinit var tfidfMap: Map<String, Float>
     private lateinit var pca: PCA
@@ -48,7 +17,8 @@ class SifSentenceEmbeddings(val embeddings: Embeddings) : SentenceEmbeddings {
         val words = corpus.flatMap { bag -> bag.keys }.distinct()
         val bags = corpus.map { vectorize(words.toTypedArray(), it) }
         val vectors = tfidf(bags)
-        val vector = Matrix.of(vectors.toTypedArray()).colSums()
+
+        val vector = Matrix(embeddings.dimensions, vectors.size, vectors.toTypedArray()).colSums()
         val vecMax = vector.max() ?: 1.0
         tfidfMap = vector
             .map { it / vecMax }
@@ -80,7 +50,7 @@ class SifSentenceEmbeddings(val embeddings: Embeddings) : SentenceEmbeddings {
             .normalize()
             .map(Float::toDouble).toDoubleArray()
 
-        val m = Matrix.of(arrayOf(weightedArray))
+        val m = Matrix(weightedArray)
 
         return m
             .sub(m.mul(pca.projection.transpose()).mul(pca.projection)) // TODO perhaps remove rest of PCA?

src/main/kotlin/com/londogard/embeddings/TfIdfSentenceEmbeddings.kt

@@ -5,7 +5,6 @@ import smile.nlp.bag
 import smile.nlp.tfidf
 import smile.nlp.vectorize
 
-
 /**
  * Not yet fully tested
  */
@@ -18,8 +17,8 @@ class TfIdfSentenceEmbeddings(private val embeddings: Embeddings) : SentenceEmbe
         val words = corpus.flatMap { bag -> bag.keys }.distinct()
         val bags = corpus.map { vectorize(words.toTypedArray(), it) }
         val vectors = tfidf(bags)
-        val vector = Matrix.of(vectors.toTypedArray()).colSums()
-        val vecMax = vector.max() ?: 1.0
+        val vector = Matrix(vectors.toTypedArray()).colSums()
+        val vecMax = vector.maxOrNull() ?: 1.0
 
         tfidfMap = vector
             .map { it / vecMax }

src/main/kotlin/com/londogard/embeddings/USifSentenceEmbeddings.kt

@@ -0,0 +1,112 @@
+package com.londogard.embeddings
+
+import com.londogard.embeddings.utils.SimpleDistances
+import com.londogard.embeddings.utils.SimpleWordTokenizer
+import com.londogard.nlp.stopwords.Stopwords
+import com.londogard.nlp.utils.LanguageSupport
+import com.londogard.nlp.wordfreq.WordFrequencies
+import smile.math.matrix.FloatMatrix
+import smile.nlp.dictionary.StopWords
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.Paths
+import kotlin.math.pow
+
+// Implementation based on: https://github.com/kawine/usif/blob/master/usif.py
+class USifSentenceEmbeddings(
+    private val embeddings: Embeddings,
+    private val wordProb: Map<String, Float>,
+    randomWalkLength: Int, // = n, ~11
+    private val numCommonDiscourseVector: Int = 5 // = m, 0 should work. In practise max 5.
+) : SentenceEmbeddings {
+    private val vocabSize = wordProb.size.toFloat()
+    private val threshold = 1 - (1 - 1 / vocabSize).pow(randomWalkLength)
+    private val alpha = wordProb.count { (_, prob) -> prob > threshold } / vocabSize
+    private val Z = vocabSize / 2
+    private val a = (1 - alpha) / (alpha * Z)
+    private val stopWords = setOf("jag", "hur", "som")
+
+    init {
+        if (randomWalkLength < 0) throw IllegalArgumentException("randomWalkLength must be greater than 0 (was: $randomWalkLength)")
+    }
+
+    private fun weight(word: String): Double = a / (0.5 * a + wordProb.getOrDefault(word, 0f))
+
+    override fun getSentenceEmbeddings(listOfTokens: List<List<String>>): List<Array<Float>> {
+        val vectors = listOfTokens.map(this::getSentenceEmbedding)
+
+        return if (numCommonDiscourseVector == 0) vectors
+        else {
+            val svd = FloatMatrix(vectors.map(Array<Float>::toFloatArray).toTypedArray())
+                .svd(true, true)
+            val singularValueSum = svd.s.sumOf { it.pow(2).toDouble() }.toFloat()
+
+            (0 until numCommonDiscourseVector)
+                .fold(vectors) { acc, i ->
+                    val lambdaI = svd.s[i].pow(2) / singularValueSum
+                    val pc = svd.V.row(i).map(Double::toFloat).toTypedArray()
+                    acc.map { element -> (element `--` (project(element, pc)).mMul(lambdaI)) }
+                }
+        }
+    }
+
+    override fun getSentenceEmbedding(tokens: List<String>): Array<Float> =
+        tokens
+            .filter(embeddings.vocabulary::contains)
+            .let { processedTokens ->
+                if (processedTokens.isEmpty()){
+                    println("Empty array")
+                    Array(embeddings.dimensions) { a }
+                }
+                else {
+                    processedTokens
+                        .mapNotNull(embeddings::vector)
+                        .map(Array<Float>::normalize)
+                        .mapIndexed { i, array -> array.mMul(weight(processedTokens[i]).toFloat()) }
+                        .mean()
+                }
+            }
+}
+
+object test {
+    @JvmStatic
+    fun main(args: Array<String>) {
+        val wordProb = WordFrequencies.getAllWordFrequenciesOrNull()
+        val embedd = WordEmbeddings(filename="/home/londogard/summarize-embeddings/cc.sv.50.vec")
+        val usif = USifSentenceEmbeddings(embedd, wordProb, 11, 0)
+
+        val covidFaq = javaClass
+            .getResourceAsStream("/covid_faq.tsv")
+            .bufferedReader().readLines().asSequence()
+            .drop(1)
+            .filterNot(String::isEmpty)
+            .map { line -> line.split('\t') }
+            .filter { line -> line.size == 2 }
+            .map { line -> line[0] to line[1] }
+            .toMap()
+
+        val stopWords = Stopwords.stopwords(LanguageSupport.sv)
+
+        val tokenizer = SimpleWordTokenizer(true)
+        val covidTitleEmbeddings = covidFaq
+            .map { (question, ans) -> question to tokenizer.split("$question $ans".toLowerCase()).filterNot(stopWords::contains) }
+            .map { (question, tokens) -> question to usif.getSentenceEmbedding(tokens) }
+
+        while (true) {
+            print("Question: ")
+            val q = readLine()?.toLowerCase()
+            if (q == null || q == "q") return
+            println(tokenizer.split(q).filterNot(stopWords::contains))
+            val embedding = usif.getSentenceEmbedding(tokenizer.split(q).filterNot(stopWords::contains))
+            val closest = covidTitleEmbeddings
+                .map { (q, e) -> q to SimpleDistances.cosine(embedding, e) }
+                .sortedByDescending { (_, dist) -> dist }
+                .take(20)
+                .map { (question, dist) -> "$dist: $question" }
+
+            println("Closest (5) questions to '$q' :")
+            println(closest.joinToString("\n"))
+            println("=== New Question Please ===")
+        }
+    }
+}

src/main/kotlin/com/londogard/embeddings/WordEmbeddings.kt

@@ -1,6 +1,7 @@
 package com.londogard.embeddings
 
-import java.nio.file.Files
+import com.londogard.embeddings.utils.DownloadHelper
+import com.londogard.embeddings.utils.SimpleDistances
 
 class WordEmbeddings(
     override val dimensions: Int = DownloadHelper.dimension,
@@ -33,7 +34,7 @@ class WordEmbeddings(
 
         return embeddings
             .filterKeys(inputWords::contains)
-            .map { (k, v) -> k to cosine(vector, v) }
+            .map { (k, v) -> k to SimpleDistances.cosine(vector, v) }
             .sortedByDescending { (_, cosineDist) -> cosineDist }
             .take(N)
     }

src/main/kotlin/com/londogard/embeddings/ArrayHelper.kt → src/main/kotlin/com/londogard/embeddings/utils/ArrayHelper.kt

@@ -39,7 +39,13 @@ internal fun Array<Float>.mMul(weight: Float): Array<Float> {
     return this
 }
 
+internal fun project(a: Array<Float>, b: Array<Float>): Array<Float> = b.mMul(a.dot(b).toFloat())
 internal fun Iterable<Array<Float>>.sumByColumns(): Array<Float> = reduce { agg, vector -> agg `++` vector }
+internal fun List<Array<Float>>.mean(): Array<Float> {
+    val first = (this.firstOrNull() ?: emptyArray())
+
+    return Array(first.size) { i -> sumOf { it[i].toDouble() }.toFloat() / size }
+}
 
 internal fun List<List<Double>>.mutableSumByCols(): List<Double> {
     val columnSum = MutableList(this[0].size) { 0.0 }