Merge remote-tracking branch 'origin/master'

Author: breandan

src/commonMain/kotlin/ai/hypergraph/kaliningraph/CommonUtils.kt

@@ -129,6 +129,8 @@ infix fun Collection<Any>.anyAre(that: Any) = any { it isA that }
  * Minimal pure-Kotlin bit set for indices [0..n-1].
  */
 class KBitSet(private val n: Int) {
+  constructor(n: Int, v: Int) : this(n) { set(v) }
+  constructor(n: Int, v: Collection<Int>) : this(n) { v.forEach { set(it) } }
   // Each element of 'data' holds 64 bits, covering up to n bits total.
   private val data = LongArray((n + 63) ushr 6)
 
@@ -154,8 +156,8 @@ class KBitSet(private val n: Int) {
     for (i in data.indices) data[i] = data[i] and other.data[i]
   }
 
-  fun toSet(): Set<Int> {
-    val result = mutableSetOf<Int>()
+  fun toList(): List<Int> {
+    val result = mutableListOf<Int>()
     for (i in 0 until n) if (get(i)) result.add(i)
     return result
   }

src/commonMain/kotlin/ai/hypergraph/kaliningraph/automata/AFSA.kt

@@ -49,9 +49,13 @@ class AFSA(override val Q: TSA, override val init: Set<Σᐩ>, override val fina
     return result
   }
 
-  // Since the FSA is acyclic, we can use a more efficient topological ordering
+  /** See [FSA.intersectPTree] for why this is needed*/
   override val stateLst by lazy {
-    topSort()
+    // Since the FSA is acyclic, we can use a more efficient topsort -
+    // This trick will only work for Levenshtein FSAs (otherwise use topSort())
+    states.groupBy { it.coords().let { (a, b) -> a + b } }.values.flatten()
+
+//    topSort()
 //      .also {
 //      if (it.size != states.size)
 //        throw Exception("Contained ${states.size} but ${it.size} topsorted indices:\n" +
@@ -64,7 +68,7 @@ class AFSA(override val Q: TSA, override val init: Set<Σᐩ>, override val fina
   }
 
   // Assumes stateLst is already in topological order:
-  override val allPairs: Map<Pair<Int, Int>, Set<Int>> by lazy {
+  override val allPairs: List<List<List<Int>?>> by lazy {
     val fwdAdj = Array(numStates) { mutableListOf<Int>() }
     val revAdj = Array(numStates) { mutableListOf<Int>() }
 
@@ -96,34 +100,14 @@ class AFSA(override val Q: TSA, override val init: Set<Σᐩ>, override val fina
     //    i.e. if post[i].get(j) == false => empty set.
     //
     //    We'll reuse a single KBitSet 'tmp' to avoid allocations:
-    val result = mutableMapOf<Pair<Int, Int>, Set<Int>>()
+    val result: List<MutableList<List<Int>?>> = List(states.size) { MutableList(states.size) { null } }
 
-    for (i in 0 until numStates) {
-      for (j in i until numStates) {
-        when {
-          i == j -> {
-            // The trivial path i->i has just i on it (assuming zero-length path is allowed).
-            result[i to i] = emptySet()
-          }
-          !post[i].get(j) -> {
-            // i < j, but j is not actually reachable from i
-            result[i to j] = emptySet()
-            // In a DAG, j->i is definitely unreachable if j > i, so:
-            result[j to i] = emptySet()
-          }
-          else -> {
-            // i < j and j is reachable from i => do the intersection of post[i] & pre[j].
-            val tmp = KBitSet(numStates)
-            tmp.or(post[i])
-            tmp.and(pre[j])
-            result[i to j] = tmp.toSet()
-
-            // j>i => definitely unreachable for j->i in a DAG
-            result[j to i] = emptySet()
-          }
-        }
+    for (i in 0 until numStates) for (j in i + 1 until numStates)
+      when {
+        !post[i].get(j) -> { }
+        // i < j and j is reachable from i => do the intersection of post[i] & pre[j].
+        else -> result[i][j] = KBitSet(numStates).apply { or(post[i]); and(pre[j]) }.toList()
       }
-    }
 
     result
   }

src/commonMain/kotlin/ai/hypergraph/kaliningraph/automata/FSA.kt

@@ -34,13 +34,19 @@ open class FSA constructor(open val Q: TSA, open val init: Set<Σᐩ>, open val
   val states: Set<Σᐩ> by lazy { Q.states() }
   open val stateLst: List<Σᐩ> by lazy { states.toList() }
 
-  fun allIndexedTxs1(cfg: CFG): List<Π3<Int, Σᐩ, Int>> =
-    (cfg.unitProductions * nominalForm.flattenedTriples).filter { (_, σ: Σᐩ, arc) -> (arc.π2)(σ) }
-      .map { (A: Σᐩ, σ: Σᐩ, arc) -> Triple(stateMap[arc.π1]!!, σ, stateMap[arc.π3]!!) }
+  fun allIndexedTxs1(unitProds: Set<Π2A<Σᐩ>>): List<Π3<Int, Σᐩ, Int>> {
+    val triples = mutableListOf<Π3<Int, Σᐩ, Int>>()
+    for ((A, σ) in unitProds) for (arc in nominalForm.flattenedTriples)
+      if (arc.π2(σ)) triples.add(Triple(stateMap[arc.π1]!!, σ, stateMap[arc.π3]!!))
+    return triples
+  }
 
-  fun allIndexedTxs0(cfg: CFG): List<Π3A<Int>> =
-    (cfg.unitProductions * nominalForm.flattenedTriples).filter { (_, σ: Σᐩ, arc) -> (arc.π2)(σ) }
-      .map { (A: Σᐩ, _, arc) -> Triple(stateMap[arc.π1]!!, cfg.bindex[A], stateMap[arc.π3]!!) }
+  fun allIndexedTxs0(unitProds: Set<Π2A<Σᐩ>>, bindex: Bindex<Σᐩ>): List<Π3A<Int>> {
+    val triples = mutableListOf<Π3A<Int>>()
+    for ((A, σ) in unitProds) for (arc in nominalForm.flattenedTriples)
+        if (arc.π2(σ)) triples.add(Triple(stateMap[arc.π1]!!, bindex[A], stateMap[arc.π3]!!))
+    return triples
+  }
 
   val numStates: Int by lazy { states.size }
 
@@ -54,10 +60,14 @@ open class FSA constructor(open val Q: TSA, open val init: Set<Σᐩ>, open val
   }
 
   // TODO: should be a way to compute this on the fly for L-automata (basically a Cartesian grid)
-  open val allPairs: Map<Pair<Int, Int>, Set<Int>> by lazy {
-    graph.allPairs.entries.associate { (a, b) ->
-      Pair(Pair(stateMap[a.first.label]!!, stateMap[a.second.label]!!), b.map { stateMap[it.label]!! }.toSet())
+  open val allPairs: List<List<List<Int>?>> by lazy {
+    val aps: List<MutableList<MutableList<Int>?>> =
+      List(states.size) { MutableList(states.size) { null } }
+    graph.allPairs.entries.forEach { (a, b) ->
+      val temp = b.map { stateMap[it.label]!! }.toMutableList()
+      aps[stateMap[a.first.label]!!][stateMap[a.second.label]!!] = temp
     }
+    aps
   }
 
   val finalIdxs by lazy { final.map { stateMap[it]!! } }
@@ -122,10 +132,11 @@ open class FSA constructor(open val Q: TSA, open val init: Set<Σᐩ>, open val
       val bindex = cfg.bindex
       val width = cfg.nonterminals.size
       val vindex = cfg.vindex
-      val ap: Map<Pair<Int, Int>, Set<Int>> = levFSA.allPairs
+      val ups = cfg.unitProductions
+      val aps: List<List<List<Int>?>> = levFSA.allPairs
       val dp = Array(levFSA.numStates) { Array(levFSA.numStates) { BooleanArray(width) { false } } }
 
-      levFSA.allIndexedTxs0(cfg).forEach { (q0, nt, q1) -> dp[q0][q1][nt] = true }
+      levFSA.allIndexedTxs0(ups, bindex).forEach { (q0, nt, q1) -> dp[q0][q1][nt] = true }
 
       val startIdx = bindex[START_SYMBOL]
 
@@ -134,8 +145,8 @@ open class FSA constructor(open val Q: TSA, open val init: Set<Σᐩ>, open val
         for (iP in 0 until levFSA.numStates - dist) {
           val p = iP
           val q = iP + dist
-          if (p to q !in ap) continue
-          val appq = ap[p to q]!!
+          if (aps[p][q] == null) continue
+          val appq = aps[p][q]!!
           for ((A, indexArray) in vindex.withIndex()) {
             outerloop@for(j in 0..<indexArray.size step 2) {
               val B = indexArray[j]
@@ -174,6 +185,7 @@ open class FSA constructor(open val Q: TSA, open val init: Set<Σᐩ>, open val
       val bimap = cfg.bimap
       val width = cfg.nonterminals.size
       val vindex = cfg.vindex
+      val ups = cfg.unitProductions
 
       val nStates = levFSA.numStates
       val startIdx = bindex[START_SYMBOL]
@@ -182,7 +194,7 @@ open class FSA constructor(open val Q: TSA, open val init: Set<Σᐩ>, open val
       val dp: Array<Array<Array<PTree?>>> = Array(nStates) { Array(nStates) { Array(width) { null } } }
 
       // 2) Initialize terminal productions A -> a
-      val aitx = levFSA.allIndexedTxs1(cfg)
+      val aitx = levFSA.allIndexedTxs1(ups)
       for ((p, σ, q) in aitx) {
         val Aidxs = bimap.TDEPS[σ]!!.map { bindex[it] }
         for (Aidx in Aidxs) {
@@ -195,8 +207,8 @@ open class FSA constructor(open val Q: TSA, open val init: Set<Σᐩ>, open val
       for (dist in 0 until nStates) {
         for (p in 0 until (nStates - dist)) {
           val q = p + dist
-          if (p to q !in levFSA.allPairs) continue
-          val appq = levFSA.allPairs[p to q]!!
+          if (levFSA.allPairs[p][q] == null) continue
+          val appq = levFSA.allPairs[p][q]!!
           for ((Aidx, indexArray) in vindex.withIndex()) {
             val rhsPairs = dp[p][q][Aidx]?.branches?.toMutableList() ?: mutableListOf()
             outerLoop@for (j in 0..<indexArray.size step 2) {

src/commonMain/kotlin/ai/hypergraph/kaliningraph/automata/GRE.kt

@@ -1,47 +1,85 @@
 package ai.hypergraph.kaliningraph.automata
 
+import ai.hypergraph.kaliningraph.KBitSet
 import ai.hypergraph.kaliningraph.parsing.*
 import ai.hypergraph.kaliningraph.tensor.UTMatrix
+import ai.hypergraph.kaliningraph.tokenizeByWhitespace
 import ai.hypergraph.kaliningraph.types.*
 
 // Generalized regular expression: https://planetmath.org/generalizedregularexpression
 // Parsing with derivatives: https://matt.might.net/papers/might2011derivatives.pdf
-sealed class GRE(vararg val args: GRE) {
-  companion object { operator fun invoke(s: Σᐩ) = ONE(s) }
-
+sealed class GRE(open vararg val args: GRE) {
   class EPS: GRE()
-  class ONE(val s: Σᐩ): GRE()
-  class SET(val s: Set<Σᐩ>): GRE()
-  class NEG(val g: GRE): GRE(g)
-  class UNI(val l: GRE, val r: GRE): GRE(l, r)
+  class SET(val s: KBitSet): GRE() { constructor(size: Int): this(KBitSet(size)) }
+  class UNI(override vararg val args: GRE): GRE(*args)
   class CAT(val l: GRE, val r: GRE): GRE(l, r)
-  class INT(val l: GRE, val r: GRE): GRE(l, r)
 
-  infix fun and(a: GRE): GRE = INT(this, a)
+  fun words(terminals: List<Σᐩ>): Sequence<Σᐩ> =
+    enumerate().distinct().map { it.mapNotNull { terminals[it].let { if (it == "ε") null else it } }.joinToString(" ") }
+
+  // F_s(g) = { s | ∂_s(g) != ∅ }
+//  fun GRE.followSet(): KBitSet = when (this) {
+//    is EPS -> KBitSet()
+//    is SET -> s
+//    is UNI -> args.map { it.followSet() }.fold (KBitSet()) { a, b -> a or b }
+//    is CAT -> l.followSet()
+//  }
+
+  fun enumerate(): Sequence<List<Int>> = sequence {
+    when (this@GRE) {
+      is EPS -> emptyList<Int>()
+      is SET -> yieldAll(s.toList().map { listOf(it) })
+      is UNI -> for (a in args) yieldAll(a.enumerate())
+//      yieldAll(args.map { it.enumerate().toSet() }.reduce { a, b -> a + b })
+      is CAT -> for (lhs in l.enumerate()) for (rhs in r.enumerate())
+        if (lhs.isEmpty()) {
+          if (rhs.isEmpty()) yield(emptyList()) else rhs
+        } else {
+          if (rhs.isEmpty()) yield(lhs)
+          else yield(lhs + rhs)
+        }
+    }
+  }
+
+  // ∂_s(g) = { w | s·w ∈ L(g) }
+//  fun dv(s: Σᐩ): GRE? = when (this) {
+//    is EPS -> null // ∂_s(ε) = ∅
+//    is SET -> if (s in this.s) EPS() else NIL
+//    is UNI -> args.reduce { a, b -> a + b }
+//    is CAT -> {
+//      // ∂_s(E1 · E2) = (∂_s(E1)) · E2   ∪   [if E1 nullable => ∂_s(E2)]
+//      val dLeft = l.dv(s) * r
+//      if (l.nullable()) dLeft + r.dv(s) else dLeft
+//    }
+//  }
+
+  // Check whether 'g' accepts the empty string ε.
+  fun nullable(): Boolean = when (this) {
+    is EPS -> true
+    is SET -> false
+    is UNI -> args.any { it.nullable() }
+    is CAT -> l.nullable() && r.nullable()
+  }
+
   operator fun plus(g: GRE): GRE = UNI(this, g)
   operator fun times(g: GRE): GRE = CAT(this, g)
-  operator fun not(): GRE = NEG(this)
 
-  override fun toString(): String = when (this) {
-    is ONE -> s
-    is SET -> "( ${s.joinToString(" ")} )"
-    is NEG -> "! ( $g )"
-    is UNI -> "( $l ∪ $r )"
-    is CAT -> "$l $r"
-    is INT -> "$l ∩ $r"
-    is EPS -> "ε"
-  }
+//  override fun toString() = when (this) {
+//    is EPS -> "ε"
+//    is SET -> if (s.isEmpty()) "∅" else "( ${s.joinToString(" ")} )"
+//    is UNI -> "( ${args.joinToString(" ∪ "){ "$it" }} )"
+//    is CAT -> "$l $r"
+//  }
 }
 
-
-fun CFG.initGREListMat(tokens: List<String>): UTMatrix<List<GRE?>> =
+fun CFG.initGREListMat(tokens: List<Σᐩ>): UTMatrix<List<GRE?>> =
   UTMatrix(
     ts = tokens.map { token ->
       val ptreeList = MutableList<GRE?>(nonterminals.size) { null }
       (if (token != HOLE_MARKER) bimap[listOf(token)] else unitNonterminals)
         .associateWith { nt ->
-          if (token != HOLE_MARKER) GRE.ONE(token)
-          else bimap.UNITS[nt]?.let { GRE.SET(it) }
+          if (token != HOLE_MARKER) GRE.SET(KBitSet(terminals.size, tmMap[token]!!))
+          else bimap.UNITS[nt]?.let { GRE.SET(KBitSet(tmLst.size, it.map { tmMap[it]!! })) }
         }.forEach { (k, v) -> ptreeList[bindex[k]] = v }
       ptreeList
     }.toTypedArray(),
@@ -66,6 +104,5 @@ fun CFG.greJoin(left: List<GRE?>, right: List<GRE?>): List<GRE?> = vindex2.map {
   if (t.isEmpty()) null else t.reduce { acc, int -> if (acc == null) int else if (int == null) acc else acc + int }
 }
 
-fun CFG.startGRE(tokens: List<String>): GRE? =
-  initGREListMat(tokens).seekFixpoint().diagonals.last()[0][bindex[START_SYMBOL]]
-
+fun CFG.startGRE(tokens: List<Σᐩ>): GRE? =
+  initGREListMat(tokens).seekFixpoint().diagonals.last()[0][bindex[START_SYMBOL]]

src/commonMain/kotlin/ai/hypergraph/kaliningraph/cache/LRUCache.kt

@@ -6,7 +6,7 @@ class LRUCache<K, V>(
   private val sizeOf: (key: K, value: V) -> Int = { _, _ -> 1 }
 ) {
   val map: LinkedHashMap<K, V> = LinkedHashMap(0, .75f)
-  private var size: Int = 0
+  var size: Int = 0
 
   fun getOrPut(key: K, value: () -> V): V =
     map[key] ?: value().also { put(key, it) }

src/commonMain/kotlin/ai/hypergraph/kaliningraph/parsing/CFG.kt

@@ -9,6 +9,7 @@ import kotlin.jvm.JvmName
 import kotlin.random.Random
 import kotlin.time.*
 import kotlin.time.Duration.Companion.seconds
+import kotlin.to
 
 typealias Σᐩ = String
 typealias Production = Π2<Σᐩ, List<Σᐩ>>
@@ -60,7 +61,10 @@ val CFG.unicodeMap by cache { terminals.associateBy { Random(it.hashCode()).next
 val CFG.ntLst by cache { (symbols + "ε").toList() }
 val CFG.ntMap by cache { ntLst.mapIndexed { i, s -> s to i }.toMap() }
 
-val CFG.tripleIntProds: Set<Π3A<Int>> by cache { bimap.TRIPL.map { (a, b, c) -> bindex[a] to bindex[b] to bindex[c] }.toSet() }
+val CFG.tmLst by cache { terminals.toList() }
+val CFG.tmMap by cache { tmLst.mapIndexed { i, s -> s to i }.toMap() }
+
+val CFG.tripleIntProds: Set<Π3A<Int>> by cache { bimap.TRIPL.map { (a, b, c) -> Triple(bindex[a], bindex[b], bindex[c]) }.toSet() }
 val CFG.revUnitProds: Map<Σᐩ, List<Int>> by cache { terminals.associate { it to bimap[listOf(it)].map { bindex[it] } } }
 
 // Maps each nonterminal to the set of nonterminal pairs that can generate it,
@@ -272,7 +276,7 @@ class BiMap(val cfg: CFG) {
   }
   val TRIPL: List<Π3A<Σᐩ>> by lazy {
     R2LHS.filter { it.key.size == 2 }
-      .map { it.value.map { v -> v to it.key[0] to it.key[1] } }.flatten()
+      .map { it.value.map { v -> Triple(v, it.key[0], it.key[1]) } }.flatten()
   }
   val X2WZ: Map<Σᐩ, List<Π3A<Σᐩ>>> by lazy {
     TRIPL.groupBy { it.second }.mapValues { it.value }