[Bug] Indexed filters do not take into account previously applied row permission filters. (#1902)

stonesmi · web-flow · commit c5001ff90469 · 2025-11-26T10:49:21.000Z
* #1901 Fix (WiP) * #1901 Fix (WiP) * #1901 Fix (WiP)
diff --git a/plugin/virtualized-table-plugin/src/main/scala/org/finos/vuu/plugin/virtualized/table/VirtualizedTableKeys.scala b/plugin/virtualized-table-plugin/src/main/scala/org/finos/vuu/plugin/virtualized/table/VirtualizedTableKeys.scala
@@ -36,4 +36,7 @@ case class VirtualizedTableKeys(window: MovingWindow[String], dataSize: Int) ext
   }
 
   override def iterator: Iterator[String] = window.iterator
+
+  override def intersect(keys: Iterable[String]): TablePrimaryKeys = ???
+  
 }
diff --git a/vuu/src/main/scala/org/finos/vuu/core/filter/FilterClause.scala b/vuu/src/main/scala/org/finos/vuu/core/filter/FilterClause.scala
@@ -2,7 +2,7 @@ package org.finos.vuu.core.filter
 
 import org.finos.toolbox.collection.array.ImmutableArray
 import org.finos.vuu.core.filter.FilterClause.joinResults
-import org.finos.vuu.core.index._
+import org.finos.vuu.core.index.*
 import org.finos.vuu.core.table.column.{Error, Result}
 import org.finos.vuu.core.table.{RowData, TablePrimaryKeys}
 import org.finos.vuu.feature.inmem.InMemTablePrimaryKeys
@@ -32,9 +32,11 @@ sealed trait RowFilterClause extends FilterClause {
     ))
 
   override def validate(vpColumns: ViewPortColumns): Result[true] = columnExistsInVpColumns(vpColumns)
+
   private def columnExistsInVpColumns(vpColumns: ViewPortColumns): Result[true] =
     if (vpColumns.columnExists(this.columnName)) Result(true)
     else Error(s"Column `$columnName` not found.")
+
 }
 
 case class NotClause(decorated: FilterClause) extends FilterClause {
@@ -92,12 +94,12 @@ case class InClause(columnName: String, values: List[String]) extends RowFilterC
   override def filterAll(rows: RowSource, rowKeys: TablePrimaryKeys, viewPortColumns: ViewPortColumns): TablePrimaryKeys = {
     val column = rows.asTable.columnForName(columnName)
     rows.asTable.indexForColumn(column) match {
-      case Some(ix: StringIndexedField)  => InMemTablePrimaryKeys( ix.find(values) )
-      case Some(ix: IntIndexedField)     => InMemTablePrimaryKeys( ix.find(values.map(s => s.toInt)))
-      case Some(ix: LongIndexedField)    => InMemTablePrimaryKeys( ix.find(values.map(s => s.toLong)))
-      case Some(ix: DoubleIndexedField)  => InMemTablePrimaryKeys( ix.find(values.map(s => s.toDouble)))
-      case Some(ix: BooleanIndexedField) => InMemTablePrimaryKeys( ix.find(values.map(s => s.toBoolean)))
-      case None                          => super.filterAll(rows, rowKeys, viewPortColumns)
+      case Some(ix: StringIndexedField)  => rowKeys.intersect(ix.find(values))
+      case Some(ix: IntIndexedField)     => rowKeys.intersect(ix.find(values.map(s => s.toInt)))
+      case Some(ix: LongIndexedField)    => rowKeys.intersect(ix.find(values.map(s => s.toLong)))
+      case Some(ix: DoubleIndexedField)  => rowKeys.intersect(ix.find(values.map(s => s.toDouble)))
+      case Some(ix: BooleanIndexedField) => rowKeys.intersect(ix.find(values.map(s => s.toBoolean)))
+      case _                          => super.filterAll(rows, rowKeys, viewPortColumns)
     }
   }
 }
@@ -113,10 +115,10 @@ case class GreaterThanClause(columnName: String, value: Double) extends RowFilte
   override def filterAll(rows: RowSource, rowKeys: TablePrimaryKeys, viewPortColumns: ViewPortColumns): TablePrimaryKeys = {
     val column = rows.asTable.columnForName(columnName)
     rows.asTable.indexForColumn(column) match {
-      case Some(ix: DoubleIndexedField) => InMemTablePrimaryKeys(ix.greaterThan(value))
-      case Some(ix: IntIndexedField) => InMemTablePrimaryKeys(ix.greaterThan(value.toInt))
-      case Some(ix: LongIndexedField) => InMemTablePrimaryKeys(ix.greaterThan(value.toLong))
-      case None => super.filterAll(rows, rowKeys, viewPortColumns)
+      case Some(ix: DoubleIndexedField) => rowKeys.intersect(ix.greaterThan(value))
+      case Some(ix: IntIndexedField) => rowKeys.intersect(ix.greaterThan(value.toInt))
+      case Some(ix: LongIndexedField) => rowKeys.intersect(ix.greaterThan(value.toLong))
+      case _ => super.filterAll(rows, rowKeys, viewPortColumns)
     }
   }
 }
@@ -132,10 +134,10 @@ case class LessThanClause(columnName: String, value: Double) extends RowFilterCl
   override def filterAll(rows: RowSource, rowKeys: TablePrimaryKeys, viewPortColumns: ViewPortColumns): TablePrimaryKeys = {
     val column = rows.asTable.columnForName(columnName)
     rows.asTable.indexForColumn(column) match {
-      case Some(ix: DoubleIndexedField) => InMemTablePrimaryKeys(ix.lessThan(value))
-      case Some(ix: IntIndexedField)    => InMemTablePrimaryKeys(ix.lessThan(value.toInt))
-      case Some(ix: LongIndexedField)   => InMemTablePrimaryKeys(ix.lessThan(value.toInt))
-      case None                         => super.filterAll(rows, rowKeys, viewPortColumns)
+      case Some(ix: DoubleIndexedField) => rowKeys.intersect(ix.lessThan(value))
+      case Some(ix: IntIndexedField)    => rowKeys.intersect(ix.lessThan(value.toInt))
+      case Some(ix: LongIndexedField)   => rowKeys.intersect(ix.lessThan(value.toInt))
+      case _                         => super.filterAll(rows, rowKeys, viewPortColumns)
     }
   }
 }
@@ -156,12 +158,14 @@ case class EqualsClause(columnName: String, value: String) extends RowFilterClau
   override def filterAll(rows: RowSource, rowKeys: TablePrimaryKeys, viewPortColumns: ViewPortColumns): TablePrimaryKeys = {
     val column = rows.asTable.columnForName(columnName)
     rows.asTable.indexForColumn(column) match {
-      case Some(ix: StringIndexedField)   => InMemTablePrimaryKeys(ix.find(value))
-      case Some(ix: IntIndexedField)      => InMemTablePrimaryKeys(ix.find(value.toInt))
-      case Some(ix: LongIndexedField)     => InMemTablePrimaryKeys(ix.find(value.toLong))
-      case Some(ix: DoubleIndexedField)   => InMemTablePrimaryKeys(ix.find(value.toDouble))
-      case Some(ix: BooleanIndexedField)  => InMemTablePrimaryKeys(ix.find(value.toBoolean))
-      case None                           => super.filterAll(rows, rowKeys, viewPortColumns)
+      case Some(ix: StringIndexedField)   => rowKeys.intersect(ix.find(value))
+      case Some(ix: IntIndexedField)      => rowKeys.intersect(ix.find(value.toInt))
+      case Some(ix: LongIndexedField)     => rowKeys.intersect(ix.find(value.toLong))
+      case Some(ix: DoubleIndexedField)   => rowKeys.intersect(ix.find(value.toDouble))
+      case Some(ix: BooleanIndexedField)  => rowKeys.intersect(ix.find(value.toBoolean))
+      case _                           => super.filterAll(rows, rowKeys, viewPortColumns)
     }
   }
 }
+
+
diff --git a/vuu/src/main/scala/org/finos/vuu/core/table/TablePrimaryKeys.scala b/vuu/src/main/scala/org/finos/vuu/core/table/TablePrimaryKeys.scala
@@ -7,24 +7,26 @@ object EmptyTablePrimaryKeys extends TablePrimaryKeys{
   override def length: Int = 0
   override def add(key: String): TablePrimaryKeys = this.+(key)
   override def +(key: String): TablePrimaryKeys = InMemTablePrimaryKeys(ImmutableArray.from(Array(key)))
-  override def remove(key: String): TablePrimaryKeys = EmptyTablePrimaryKeys
+  override def remove(key: String): TablePrimaryKeys = EmptyTablePrimaryKeys  
   override def -(key: String): TablePrimaryKeys = EmptyTablePrimaryKeys
   override def sliceTableKeys(from: Int, until: Int): TablePrimaryKeys = EmptyTablePrimaryKeys
   override def iterator: Iterator[String] = Array[String]().iterator
 
   override def get(index: Int): String = null
   override def set(index: Int, key: String): TablePrimaryKeys = this
+  override def intersect(keys: Iterable[String]): TablePrimaryKeys = EmptyTablePrimaryKeys
 }
 
 trait TablePrimaryKeys extends Iterable[String] {
   def length: Int
   def add(key: String): TablePrimaryKeys
   def +(key: String): TablePrimaryKeys
-  def remove(key: String): TablePrimaryKeys
+  def remove(key: String): TablePrimaryKeys  
   def -(key: String): TablePrimaryKeys
   def sliceTableKeys(from: Int, until: Int): TablePrimaryKeys
   def get(index: Int): String
   def set(index: Int, key: String): TablePrimaryKeys
+  def intersect(keys: Iterable[String]): TablePrimaryKeys
 }
 
 
diff --git a/vuu/src/main/scala/org/finos/vuu/feature/inmem/InMemTablePrimaryKeys.scala b/vuu/src/main/scala/org/finos/vuu/feature/inmem/InMemTablePrimaryKeys.scala
@@ -18,4 +18,9 @@ case class InMemTablePrimaryKeys(keys: ImmutableArray[String]) extends TablePrim
   override def -(key: String): TablePrimaryKeys = InMemTablePrimaryKeys(keys - key)
   override def get(index: Int): String = keys.getIndex(index)
   override def set(index: Int, key: String): TablePrimaryKeys = InMemTablePrimaryKeys(keys.set(index, key))
+
+  override def intersect(otherKeys: Iterable[String]): TablePrimaryKeys = {
+    val intersection = keys.filter(otherKeys.toSet.contains).toArray
+    InMemTablePrimaryKeys(ImmutableArray.from(intersection))
+  }
 }
diff --git a/vuu/src/test/scala/org/finos/vuu/viewport/FilterAndSortTest.scala b/vuu/src/test/scala/org/finos/vuu/viewport/FilterAndSortTest.scala
@@ -3,19 +3,23 @@ package org.finos.vuu.viewport
 import org.finos.toolbox.jmx.{MetricsProvider, MetricsProviderImpl}
 import org.finos.toolbox.lifecycle.LifecycleContainer
 import org.finos.toolbox.time.{Clock, DefaultClock}
+import org.finos.vuu.api.{Index, Indices, TableDef}
 import org.finos.vuu.client.messages.RequestId
+import org.finos.vuu.core.auths.VuuUser
 import org.finos.vuu.core.filter.{EqualsClause, LessThanClause, NoFilter}
 import org.finos.vuu.core.sort.{AlphaSort, AntlrBasedFilter, SortDirection, UserDefinedFilterAndSort}
-import org.finos.vuu.core.table.ViewPortColumnCreator
-import org.finos.vuu.net.ClientSessionId
-import org.finos.vuu.provider.MockProvider
+import org.finos.vuu.core.table.{Columns, RowData, TableContainer, ViewPortColumnCreator}
+import org.finos.vuu.feature.inmem.VuuInMemPlugin
+import org.finos.vuu.net.{ClientSessionId, FilterSpec}
+import org.finos.vuu.plugin.DefaultPluginRegistry
+import org.finos.vuu.provider.{JoinTableProviderImpl, MockProvider, ProviderContainer}
 import org.finos.vuu.util.OutboundRowPublishQueue
 import org.finos.vuu.util.table.TableAsserts
+import org.finos.vuu.util.table.TableAsserts.assertVpEq
 import org.scalatest.GivenWhenThen
 import org.scalatest.featurespec.AnyFeatureSpec
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.prop.Tables.Table
-import org.finos.vuu.core.auths.VuuUser
 
 import java.time.{LocalDateTime, ZoneId}
 
@@ -242,9 +246,8 @@ class FilterAndSortTest extends AnyFeatureSpec with Matchers with ViewPortSetup
       val updates3 = combineQs(viewport).filter( vp => vp.vpUpdate == RowUpdateType)
 
       updates3.size should be (1)
-      updates3(0).vp.size should equal(1)
-      //make sure we clean up the mappings
-      updates3(0).vp.getRowKeyMappingSize_ForTest should equal(1)
+      updates3.head.vp.size should equal(1)
+      updates3.head.vp.getRowKeyMappingSize_ForTest should equal(1)
 
       assertVpEq(updates3){
         Table(
@@ -474,6 +477,52 @@ class FilterAndSortTest extends AnyFeatureSpec with Matchers with ViewPortSetup
 
     }
 
+    Scenario("test index hit does not override row permission filter") {
+
+      given lifecycle: LifecycleContainer = new LifecycleContainer
+
+      val pricesDef = TableDef(name = "prices",
+        keyField = "ric",
+        columns = Columns.fromNames("ric:String", "bid:Double", "ask:Double"),
+        indices = Indices.apply(Index.apply("ric")),
+        joinFields = "ric")
+        .withPermissions((vp, tc) => (row: RowData) => {
+          val ric = row.get("ric").asInstanceOf[String]
+          ric != "VOD.L"
+        })
+
+      val joinProvider = JoinTableProviderImpl()
+      val tableContainer = new TableContainer(joinProvider)
+      val prices = tableContainer.createTable(pricesDef)
+      val pricesProvider = new MockProvider(prices)
+      val providerContainer = new ProviderContainer(joinProvider)
+      val pluginRegistry = new DefaultPluginRegistry()
+      pluginRegistry.registerPlugin(new VuuInMemPlugin)
+      val viewPortContainer = new ViewPortContainer(tableContainer, providerContainer, pluginRegistry)
+
+      pricesProvider.tick("VOD.L", Map("ric" -> "VOD.L", "bid" -> 220.0, "ask" -> 222.0))
+      pricesProvider.tick("BT.L", Map("ric" -> "BT.L", "bid" -> 500.0, "ask" -> 501.0))
+
+      val queue = new OutboundRowPublishQueue()
+
+      val columns = ViewPortColumnCreator.create(prices, prices.getTableDef.columns.map(_.name).toList)
+
+      val viewport = viewPortContainer.create(RequestId.oneNew(), VuuUser("B"),
+        ClientSessionId("A", "C"), queue, prices, ViewPortRange(0, 20), columns,
+        filterSpec = FilterSpec("ric = \"VOD.L\" or ric = \"BT.L\""))
+
+      viewPortContainer.runOnce()
+
+      val updates = combineQs(viewport)
+
+      assertVpEq(updates) {
+        Table(
+          ("ric", "bid", "ask"),
+          ("BT.L", 500.0, 501.0)
+        )
+      }
+
+    }
 
   }
 

Original file line number	Diff line number	Diff line change
`@@ -36,4 +36,7 @@ case class VirtualizedTableKeys(window: MovingWindow[String], dataSize: Int) ext`
`36`	`36`	`}`
`37`	`37`
`38`	`38`	`override def iterator: Iterator[String] = window.iterator`
	`39`	`+`
	`40`	`+ override def intersect(keys: Iterable[String]): TablePrimaryKeys = ???`
	`41`	`+`
`39`	`42`	`}`