now I understand

Author: breandan

src/commonMain/kotlin/ai/hypergraph/kaliningraph/CommonUtils.kt

@@ -133,12 +133,8 @@ class KBitSet(val n: Int) {
   constructor(n: Int, v: Collection<Int>) : this(n) { v.forEach { set(it) } }
   // Each element of 'data' holds 64 bits, covering up to n bits total.
   private val data = LongArray((n + 63) ushr 6)
-//  var set = mutableSetOf<Int>()
-//  var modified = false
 
   fun set(index: Int) {
-//    modified = true
-//    set += index
     val word = index ushr 6
     val bit  = index and 63
     data[word] = data[word] or (1L shl bit)

src/commonMain/kotlin/ai/hypergraph/kaliningraph/parsing/SetValiant.kt

@@ -77,25 +77,28 @@ fun CFG.isValid(str: List<Σᐩ>): Bln =
     dp[0][str.size][bindex[START_SYMBOL]]
   }
 
-//fun CFG.isValidAlt(str: List<Σᐩ>): Bln =
-//  if (str.size == 1) checkUnitWord(str.first()).isNotEmpty()
-//  else {
-//    val dp = Array(str.size + 1) { Array(str.size + 1) { KBitSet(nonterminals.size) } }
-//    str.map {
-//      if (it == "_" || tmMap[it] == null) (0..<nonterminals.size).toList()
-//      else tmToVidx[tmMap[it]!!] }.forEachIndexed { i, it -> it.forEach { vidx -> dp[i][i+1].set(vidx) } }
-//
-//    for (dist: Int in 0 until dp.size) {
-//      for (iP: Int in 0 until dp.size - dist) {
-//        val p = iP
-//        val q = iP + dist
-//        val appq = p..q
-//        for (r in appq) for (lt in dp[p][r].set) for (rt in dp[r][q].set)
-//          bimap.R2LHSI[lt][rt].forEach { dp[p][q].set(it) }
-//      }
-//    }
-//    dp[0][str.size][bindex[START_SYMBOL]]
-//  }
+// Differs only by the JOIN\otimes operation.
+// This strategy only wins over child-enumeration under low ambiguity.
+// If the number of child pairs is high, better to just loop over grammar
+fun CFG.isValidAlt(str: List<Σᐩ>): Bln =
+  if (str.size == 1) checkUnitWord(str.first()).isNotEmpty()
+  else {
+    val dp = Array(str.size + 1) { Array(str.size + 1) { KBitSet(nonterminals.size) } }
+    str.map {
+      if (it == "_" || tmMap[it] == null) (0..<nonterminals.size).toList()
+      else tmToVidx[tmMap[it]!!] }.forEachIndexed { i, it -> it.forEach { vidx -> dp[i][i+1].set(vidx) } }
+
+    for (dist: Int in 0 until dp.size) {
+      for (iP: Int in 0 until dp.size - dist) {
+        val p = iP
+        val q = iP + dist
+        val appq = p..q
+        for (r in appq) for (lt in dp[p][r].toList()) for (rt in dp[r][q].toList())
+          bimap.R2LHSI[lt][rt].forEach { dp[p][q].set(it) }
+      }
+    }
+    dp[0][str.size][bindex[START_SYMBOL]]
+  }
 
 fun CFG.corner(str: Σᐩ) =
  solveFixedpoint(str.tokenizeByWhitespace())[0].last().map { it.root }.toSet()
@@ -298,9 +301,18 @@ fun Σᐩ.isNonterminalStubIn(CJL: CJL): Bln = CJL.cfgs.map { isNonterminalStubI
 fun Σᐩ.containsNonterminal(): Bln = Regex("<[^\\s>]*>") in this
 
 // Converts tokens to UT matrix via constructor: σ_i = { A | (A -> w[i]) ∈ P }
-fun CFG.initialMatrix(str: List<Σᐩ>): TreeMatrix =
+fun CFG.initialMatrix(
+  str: List<Σᐩ>,
+  bmp: BiMap = bimap,
+  unitReach: Map<Σᐩ, Set<Σᐩ>> = originalForm.unitReachability
+): TreeMatrix =
   FreeMatrix(makeForestAlgebra(), str.size + 1) { i, j ->
     if (i + 1 != j) emptySet()
+    else if (str[j - 1] == HOLE_MARKER)
+      unitReach.values.flatten().toSet().map { root ->
+        bmp[root].filter { it.size == 1 }.map { it.first() }.filter { it in terminals }
+          .map { Tree(root = root, terminal = it, span = i until (i + 1)) }
+      }.flatten().toSet()
     else bimap[listOf(str[j - 1])].map {
       Tree(root = it, terminal = str[j - 1], span = (j - 1) until j)
     }.toSet()

src/commonTest/kotlin/ai/hypergraph/kaliningraph/parsing/SetValiantTest.kt

@@ -418,7 +418,7 @@ class SetValiantTest {
   fun testUTMRepresentationEquivalence() {
     with(vanillaS2PCFG) {
       println("SIZE: ${nonterminals.size}")
-      val str = "NAME = NAME ( NAME , NUMBER ) . NAME ( ) NEWLINE NAME = NAME . NAME NEWLINE NAME = STRING . NAME ( NAME for NAME in NAME if NAME . NAME ( ) ) NEWLINE NAME ( NAME ) NEWLINE".tokenizeByWhitespace()
+      val str = "NAME = NAME ( NAME , NUMBER ) . NAME ( ) NEWLINE _ _ NAME . NAME NEWLINE NAME = STRING . NAME ( NAME for NAME in NAME if NAME . NAME ( ) ) NEWLINE NAME ( NAME ) NEWLINE".tokenizeByWhitespace()
 //  with("""P -> ( P ) | P P | ε""".parseCFG()) {
 //    val str = "( ( ) ( ) ) ( ) ( ( ( ) ) ( ) ) ( ( ( ) ) ) ( ) ( ) ( ) ( ( ) ( ) ) ( ) ( ( ) ( ) ) ( ) ( ( ) ( ) ) ( )".tokenizeByWhitespace()
       val iter = ceil(log2(str.size.toDouble())).toInt() + 9
@@ -433,13 +433,17 @@ class SetValiantTest {
         )[0, str.size]
       }.also { println("Slow transition: ${it.duration.inWholeMilliseconds}ms") }.value
 
+//      println(slowTransitionFP)
+
       i = 0
       val fastTransitionFP = measureTimedValue {
         initialUTMatrix(str).seekFixpoint(
 //          debug = { println("ITER=${i++}"); it.toFullMatrix().prettyPrint(); println() }
         ).toFullMatrix()[0, str.size]
       }.also { println("Fast transition: ${it.duration.inWholeMilliseconds}ms") }.value
 
+//      println(fastTransitionFP)
+
       measureTimedValue {
         println(isValidAlt(str))
       }.also { println("DP transition: ${it.duration.inWholeMilliseconds}ms") }.value