diff --git a/plugin/virtualized-table-plugin/src/main/scala/org/finos/vuu/plugin/virtualized/table/VirtualizedTableKeys.scala b/plugin/virtualized-table-plugin/src/main/scala/org/finos/vuu/plugin/virtualized/table/VirtualizedTableKeys.scala index f872b9f24..c9069de6b 100644 --- a/plugin/virtualized-table-plugin/src/main/scala/org/finos/vuu/plugin/virtualized/table/VirtualizedTableKeys.scala +++ b/plugin/virtualized-table-plugin/src/main/scala/org/finos/vuu/plugin/virtualized/table/VirtualizedTableKeys.scala @@ -36,7 +36,5 @@ case class VirtualizedTableKeys(window: MovingWindow[String], dataSize: Int) ext } override def iterator: Iterator[String] = window.iterator - - override def intersect(keys: Iterable[String]): TablePrimaryKeys = ??? } diff --git a/toolbox/src/main/scala/org/finos/toolbox/collection/set/ImmutableArraySet.scala b/toolbox/src/main/scala/org/finos/toolbox/collection/set/ImmutableArraySet.scala index e4a017bbb..adb3f8d30 100644 --- a/toolbox/src/main/scala/org/finos/toolbox/collection/set/ImmutableArraySet.scala +++ b/toolbox/src/main/scala/org/finos/toolbox/collection/set/ImmutableArraySet.scala @@ -38,7 +38,7 @@ object ImmutableArraySet { } } - def from[T <: Object](set: Set[T])(implicit c: ClassTag[T]): ImmutableArraySet[T] = { + def from[T <: Object](set: scala.collection.Set[T])(implicit c: ClassTag[T]): ImmutableArraySet[T] = { if (set.isEmpty) { empty() } else { diff --git a/toolbox/src/main/scala/org/finos/toolbox/collection/set/VectorImmutableArraySet.scala b/toolbox/src/main/scala/org/finos/toolbox/collection/set/VectorImmutableArraySet.scala index b08362e1f..c432cbe6d 100644 --- a/toolbox/src/main/scala/org/finos/toolbox/collection/set/VectorImmutableArraySet.scala +++ b/toolbox/src/main/scala/org/finos/toolbox/collection/set/VectorImmutableArraySet.scala @@ -30,8 +30,8 @@ object VectorImmutableArraySet { VectorImmutableArraySet(builder.result(), seen.toSet) } - def from[T <: Object : ClassTag](set: Set[T]): ImmutableArraySet[T] = { - VectorImmutableArraySet(set.toVector, set) + def from[T <: Object : ClassTag](set: scala.collection.Set[T]): ImmutableArraySet[T] = { + VectorImmutableArraySet(set.toVector, set.toSet) } def empty[T <: Object : ClassTag](): ImmutableArraySet[T] = { diff --git a/vuu/src/main/scala/org/finos/vuu/core/filter/FilterClause.scala b/vuu/src/main/scala/org/finos/vuu/core/filter/FilterClause.scala index 0f8e78a3e..b4e0c9464 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/filter/FilterClause.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/filter/FilterClause.scala @@ -1,6 +1,7 @@ package org.finos.vuu.core.filter -import org.finos.toolbox.collection.array.ImmutableArray +import org.finos.toolbox.collection.array.{ImmutableArray, VectorImmutableArray} +import org.finos.toolbox.collection.set.ImmutableArraySet import org.finos.vuu.core.filter.FilterClause.joinResults import org.finos.vuu.core.index.* import org.finos.vuu.core.table.column.{Error, Result} @@ -47,14 +48,28 @@ sealed trait RowFilterClause extends FilterClause { else Error(s"Column `$columnName` not found.") protected def hitIndex[T](primaryKeys: TablePrimaryKeys, value: T, - indexLookup: T => ImmutableArray[String], firstInChain: Boolean): TablePrimaryKeys = { + indexLookup: T => ImmutableArraySet[String], firstInChain: Boolean): TablePrimaryKeys = { val results = indexLookup.apply(value) if (results.isEmpty) { EmptyTablePrimaryKeys } else if (firstInChain) { - InMemTablePrimaryKeys(results) + InMemTablePrimaryKeys(results.toImmutableArray) } else { - primaryKeys.intersect(results) + + val keyLength = primaryKeys.length + val builder = Vector.newBuilder[String] + builder.sizeHint(results.length) + + var i = 0 + while (i < keyLength) { + val key = primaryKeys.get(i) + if (results.contains(key)) { + builder += key + } + i += 1 + } + + InMemTablePrimaryKeys(VectorImmutableArray.from(builder.result())) } } diff --git a/vuu/src/main/scala/org/finos/vuu/core/filter/Filters.scala b/vuu/src/main/scala/org/finos/vuu/core/filter/Filters.scala index 8a3d8c870..ff7316da8 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/filter/Filters.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/filter/Filters.scala @@ -1,5 +1,6 @@ package org.finos.vuu.core.filter +import com.typesafe.scalalogging.StrictLogging import org.finos.vuu.core.table.{EmptyTablePrimaryKeys, TablePrimaryKeys} import org.finos.vuu.viewport.{RowSource, ViewPortColumns} @@ -7,6 +8,25 @@ trait Filter { def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, firstInChain: Boolean): TablePrimaryKeys } +case class CompoundFilter(filters: Filter*) extends Filter with StrictLogging { + + override def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, firstInChain: Boolean): TablePrimaryKeys = { + logger.trace(s"Starting filter with ${primaryKeys.length} rows") + if (primaryKeys.isEmpty || filters.isEmpty) return primaryKeys + + var currentKeys = primaryKeys + val filterIterator = filters.iterator + + while (filterIterator.hasNext && currentKeys.nonEmpty) { + val filter = filterIterator.next() + val stillFirst = firstInChain && (currentKeys eq primaryKeys) + currentKeys = filter.doFilter(source, currentKeys, stillFirst) + } + + currentKeys + } +} + trait ViewPortFilter { def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, vpColumns:ViewPortColumns, firstInChain: Boolean): TablePrimaryKeys } @@ -21,23 +41,22 @@ object FilterOutEverythingFilter extends ViewPortFilter { vpColumns: ViewPortColumns, firstInChain: Boolean): TablePrimaryKeys = EmptyTablePrimaryKeys } -case class CompoundFilter(filters: ViewPortFilter*) extends ViewPortFilter { +case class CompoundViewPortFilter(filters: ViewPortFilter*) extends ViewPortFilter with StrictLogging { override def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, vpColumns: ViewPortColumns, firstInChain: Boolean): TablePrimaryKeys = { - if (primaryKeys.isEmpty) { - primaryKeys - } else { - filters.foldLeft(primaryKeys) { - (remainingKeys, filter) => { - if (remainingKeys.isEmpty) { - remainingKeys - } else { - val stillFirstInChain = firstInChain && remainingKeys == primaryKeys - filter.doFilter(source, remainingKeys, vpColumns, stillFirstInChain) - } - } - } + logger.trace(s"Starting filter with ${primaryKeys.length} rows") + if (primaryKeys.isEmpty || filters.isEmpty) return primaryKeys + + var currentKeys = primaryKeys + val filterIterator = filters.iterator + + while (filterIterator.hasNext && currentKeys.nonEmpty) { + val filter = filterIterator.next() + val stillFirst = firstInChain && (currentKeys eq primaryKeys) + currentKeys = filter.doFilter(source, currentKeys, vpColumns, stillFirst) } + + currentKeys } } \ No newline at end of file diff --git a/vuu/src/main/scala/org/finos/vuu/core/filter/type/AntlrBasedFilter.scala b/vuu/src/main/scala/org/finos/vuu/core/filter/type/AntlrBasedFilter.scala index ffb2c349b..00533e9f5 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/filter/type/AntlrBasedFilter.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/filter/type/AntlrBasedFilter.scala @@ -1,12 +1,12 @@ package org.finos.vuu.core.filter.`type` -import com.typesafe.scalalogging.LazyLogging +import com.typesafe.scalalogging.StrictLogging import org.finos.vuu.core.filter.{FilterClause, ViewPortFilter} import org.finos.vuu.core.table.column.{Error, Success} import org.finos.vuu.core.table.{EmptyTablePrimaryKeys, TablePrimaryKeys} import org.finos.vuu.viewport.{RowSource, ViewPortColumns} -case class AntlrBasedFilter(clause: FilterClause) extends ViewPortFilter with LazyLogging { +case class AntlrBasedFilter(clause: FilterClause) extends ViewPortFilter with StrictLogging { override def doFilter(source: RowSource, rowKeys: TablePrimaryKeys, vpColumns: ViewPortColumns, firstInChain: Boolean): TablePrimaryKeys = { logger.trace(s"Starting filter with ${rowKeys.length}") diff --git a/vuu/src/main/scala/org/finos/vuu/core/filter/type/BaseFilter.scala b/vuu/src/main/scala/org/finos/vuu/core/filter/type/BaseFilter.scala index b43fb3964..eeb074c2b 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/filter/type/BaseFilter.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/filter/type/BaseFilter.scala @@ -1,6 +1,6 @@ package org.finos.vuu.core.filter.`type` -import org.finos.vuu.core.filter.ViewPortFilter +import org.finos.vuu.core.filter.{CompoundFilter, ViewPortFilter} import org.finos.vuu.core.table.TablePrimaryKeys import org.finos.vuu.core.table.datatype.EpochTimestamp import org.finos.vuu.viewport.{RowSource, ViewPortColumns} @@ -11,7 +11,7 @@ object BaseFilter { def apply(permissionFilter: PermissionFilter, frozenTime: Option[EpochTimestamp]): BaseFilter = { frozenTime match { - case Some(value) => PermissionAndFrozenTimeFilter(permissionFilter, FrozenTimeFilter(value)) + case Some(value) => PermissionAndFrozenTimeFilter(permissionFilter, value) case None => OnlyPermissionFilter(permissionFilter) } } @@ -27,20 +27,13 @@ private case class OnlyPermissionFilter(permissionFilter: PermissionFilter) exte } -private case class PermissionAndFrozenTimeFilter(permissionFilter: PermissionFilter, frozenTimeFilter: FrozenTimeFilter) extends BaseFilter { - - override def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, vpColumns: ViewPortColumns, firstInChain: Boolean): TablePrimaryKeys = { - if (primaryKeys.isEmpty) { - primaryKeys - } else { - val remainingKeys = permissionFilter.doFilter(source, primaryKeys, firstInChain) - if (remainingKeys.isEmpty) { - remainingKeys - } else { - val stillFirstInChain = firstInChain && remainingKeys == primaryKeys - frozenTimeFilter.doFilter(source, remainingKeys, stillFirstInChain) - } - } +private case class PermissionAndFrozenTimeFilter(permissionFilter: PermissionFilter, frozenTime: EpochTimestamp) extends BaseFilter { + + private val internalFilter = CompoundFilter(permissionFilter, FrozenTimeFilter(frozenTime)) + + override def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, + vpColumns: ViewPortColumns, firstInChain: Boolean): TablePrimaryKeys = { + internalFilter.doFilter(source, primaryKeys, firstInChain) } } diff --git a/vuu/src/main/scala/org/finos/vuu/core/filter/type/FrozenTimeFilter.scala b/vuu/src/main/scala/org/finos/vuu/core/filter/type/FrozenTimeFilter.scala index 4019f2a04..65121c8c0 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/filter/type/FrozenTimeFilter.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/filter/type/FrozenTimeFilter.scala @@ -1,9 +1,9 @@ package org.finos.vuu.core.filter.`type` import com.typesafe.scalalogging.LazyLogging -import org.finos.toolbox.collection.array.ImmutableArray +import org.finos.toolbox.collection.array.{ImmutableArray, VectorImmutableArray} import org.finos.vuu.core.filter.Filter -import org.finos.vuu.core.index.{EpochTimestampIndexedField, LongIndexedField} +import org.finos.vuu.core.index.EpochTimestampIndexedField import org.finos.vuu.core.table.datatype.EpochTimestamp import org.finos.vuu.core.table.{DefaultColumn, EmptyTablePrimaryKeys, TablePrimaryKeys} import org.finos.vuu.feature.inmem.InMemTablePrimaryKeys @@ -13,9 +13,10 @@ case class FrozenTimeFilter(frozenTime: EpochTimestamp) extends Filter with Lazy override def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, firstInChain: Boolean): TablePrimaryKeys = { logger.trace(s"Starting filter with ${primaryKeys.length}") + if (primaryKeys.isEmpty) return primaryKeys val column = source.asTable.columnForName(DefaultColumn.CreatedTime.name) - if (column == null || primaryKeys.isEmpty) { + if (column == null) { EmptyTablePrimaryKeys } else { source.asTable.indexForColumn(column) match { @@ -32,18 +33,41 @@ case class FrozenTimeFilter(frozenTime: EpochTimestamp) extends Filter with Lazy if (results.isEmpty) { EmptyTablePrimaryKeys } else if (firstInChain) { - InMemTablePrimaryKeys(results) + InMemTablePrimaryKeys(results.toImmutableArray) } else { - primaryKeys.intersect(results) + val keyLength = primaryKeys.length + val builder = Vector.newBuilder[String] + builder.sizeHint(Math.min(keyLength, results.length)) + + var i = 0 + while (i < keyLength) { + val key = primaryKeys.get(i) + if (results.contains(key)) { + builder += key + } + i += 1 + } + + InMemTablePrimaryKeys(VectorImmutableArray.from(builder.result())) } } - private def filterAll(source: RowSource, rowKeys: TablePrimaryKeys): TablePrimaryKeys = { - val filtered = rowKeys.filter(key => { - val vuuCreatedTimestamp = source.pullRow(key).get(DefaultColumn.CreatedTime.name) - vuuCreatedTimestamp != null && vuuCreatedTimestamp.asInstanceOf[EpochTimestamp] < frozenTime - }) - - InMemTablePrimaryKeys(ImmutableArray.from[String](filtered.toArray)) + private def filterAll(source: RowSource, primaryKeys: TablePrimaryKeys): TablePrimaryKeys = { + val length = primaryKeys.length + val builder = Vector.newBuilder[String] + builder.sizeHint(length) + + var i = 0 + while (i < length) { + val key = primaryKeys.get(i) + val row = source.pullRow(key) + val value = row.get(DefaultColumn.CreatedTime.name) + if (value != null && value.asInstanceOf[EpochTimestamp] < frozenTime) { + builder += key + } + i += 1 + } + + InMemTablePrimaryKeys(ImmutableArray.from[String](builder.result())) } } diff --git a/vuu/src/main/scala/org/finos/vuu/core/filter/type/PermissionFilter.scala b/vuu/src/main/scala/org/finos/vuu/core/filter/type/PermissionFilter.scala index 7999e382b..4f5df047f 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/filter/type/PermissionFilter.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/filter/type/PermissionFilter.scala @@ -1,18 +1,15 @@ package org.finos.vuu.core.filter.`type` -import com.typesafe.scalalogging.LazyLogging -import org.finos.toolbox.collection.array.ImmutableArray +import com.typesafe.scalalogging.StrictLogging +import org.finos.toolbox.collection.array.{ImmutableArray, VectorImmutableArray} +import org.finos.vuu.core.filter.Filter import org.finos.vuu.core.index.{BooleanIndexedField, CharIndexedField, DoubleIndexedField, EpochTimestampIndexedField, IndexedField, IntIndexedField, LongIndexedField, StringIndexedField} import org.finos.vuu.core.table.datatype.EpochTimestamp import org.finos.vuu.core.table.{Column, DataType, EmptyTablePrimaryKeys, RowData, TablePrimaryKeys} import org.finos.vuu.feature.inmem.InMemTablePrimaryKeys import org.finos.vuu.viewport.RowSource -trait PermissionFilter { - - def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, firstInChain: Boolean): TablePrimaryKeys - -} +trait PermissionFilter extends Filter { } object PermissionFilter { @@ -36,68 +33,82 @@ object DenyAllPermissionFilter extends PermissionFilter { } -private case class PermissionFilterChain(filters: Iterable[PermissionFilter]) extends PermissionFilter with LazyLogging { +private case class PermissionFilterChain(filters: Iterable[PermissionFilter]) extends PermissionFilter with StrictLogging { override def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, firstInChain: Boolean): TablePrimaryKeys = { logger.trace(s"Starting filter with ${primaryKeys.length} rows") + if (primaryKeys.isEmpty || filters.isEmpty) return primaryKeys - if (primaryKeys.isEmpty) { - primaryKeys - } else { - filters.foldLeft(primaryKeys) { - (remainingKeys, filter) => { - if (remainingKeys.isEmpty) { - remainingKeys - } else { - val stillFirstInChain = firstInChain && remainingKeys == primaryKeys - filter.doFilter(source, remainingKeys, stillFirstInChain) - } - } - } + var remainingKeys = primaryKeys + val filterIterator = filters.iterator + + while (filterIterator.hasNext && remainingKeys.nonEmpty) { + val filter = filterIterator.next() + val stillFirst = firstInChain && (remainingKeys eq primaryKeys) + remainingKeys = filter.doFilter(source, remainingKeys, stillFirst) } + + remainingKeys } } -private case class RowPermissionFilter(rowPredicate: RowData => Boolean) extends PermissionFilter with LazyLogging { +private case class RowPermissionFilter(rowPredicate: RowData => Boolean) extends PermissionFilter with StrictLogging { override def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, firstInChain: Boolean): TablePrimaryKeys = { logger.trace(s"Starting filter with ${primaryKeys.length} rows") + if (primaryKeys.isEmpty) return primaryKeys - if (primaryKeys.isEmpty) { - primaryKeys - } else { - val filtered = primaryKeys.filter(key => rowPredicate.apply(source.pullRow(key))) - InMemTablePrimaryKeys(ImmutableArray.from[String](filtered.toArray)) + val length = primaryKeys.length + val builder = Vector.newBuilder[String] + builder.sizeHint(length) + + var i = 0 + while (i < length) { + val key = primaryKeys.get(i) + if (rowPredicate.apply(source.pullRow(key))) { + builder += key + } + i += 1 } + + InMemTablePrimaryKeys(ImmutableArray.from[String](builder.result())) } } -private case class ContainsPermissionFilter(columnName: String, allowedValues: Set[String]) extends PermissionFilter with LazyLogging { +private case class ContainsPermissionFilter(columnName: String, allowedValues: Set[String]) extends PermissionFilter with StrictLogging { + + private lazy val longValues = allowedValues.map(f => f.toLong) + private lazy val intValues = allowedValues.map(f => f.toInt) + private lazy val doubleValues = allowedValues.map(f => f.toDouble) + private lazy val booleanValues = allowedValues.map(f => f.toBoolean) + private lazy val epochTimestampValues = allowedValues.map(f => EpochTimestamp(f.toLong)) + private lazy val charValues = allowedValues.map(f => f.charAt(0)) override def doFilter(source: RowSource, primaryKeys: TablePrimaryKeys, firstInChain: Boolean): TablePrimaryKeys = { logger.trace(s"Starting filter with ${primaryKeys.length} rows") + if (primaryKeys.isEmpty) return primaryKeys val column = source.asTable.columnForName(columnName) - if (column == null || primaryKeys.isEmpty || allowedValues.isEmpty) { + if (column == null || allowedValues.isEmpty) { EmptyTablePrimaryKeys } else { source.asTable.indexForColumn(column) match { case Some(index: StringIndexedField) => hitIndex(primaryKeys, index, allowedValues, firstInChain) case Some(index: LongIndexedField) => - hitIndex(primaryKeys, index, allowedValues.map(f => f.toLong), firstInChain) + hitIndex(primaryKeys, index, longValues, firstInChain) case Some(index: IntIndexedField) => - hitIndex(primaryKeys, index, allowedValues.map(f => f.toInt), firstInChain) + hitIndex(primaryKeys, index, intValues, firstInChain) case Some(index: DoubleIndexedField) => - hitIndex(primaryKeys, index, allowedValues.map(f => f.toDouble), firstInChain) + hitIndex(primaryKeys, index, doubleValues, firstInChain) case Some(index: BooleanIndexedField) => - hitIndex(primaryKeys, index, allowedValues.map(f => f.toBoolean), firstInChain) + hitIndex(primaryKeys, index, booleanValues, firstInChain) case Some(index: EpochTimestampIndexedField) => - hitIndex(primaryKeys, index, allowedValues.map(f => EpochTimestamp(f.toLong)), firstInChain) + hitIndex(primaryKeys, index, epochTimestampValues, firstInChain) case Some(index: CharIndexedField) => - hitIndex(primaryKeys, index, allowedValues.map(f => f.charAt(0)), firstInChain) + hitIndex(primaryKeys, index, charValues, firstInChain) case _ => logger.trace(s"Falling back to row filtering for $column as no Index found") filterByRow(source, primaryKeys, firstInChain, column) @@ -107,46 +118,69 @@ private case class ContainsPermissionFilter(columnName: String, allowedValues: S private def hitIndex[T](primaryKeys: TablePrimaryKeys, indexedField: IndexedField[T], values: Set[T], firstInChain: Boolean): TablePrimaryKeys = { - val results = indexedField.find(values.toList) + val results = indexedField.find(values) if (results.isEmpty) { EmptyTablePrimaryKeys } else if (firstInChain) { - InMemTablePrimaryKeys(results) + InMemTablePrimaryKeys(results.toImmutableArray) } else { - primaryKeys.intersect(results) + val keyLength = primaryKeys.length + val builder = Vector.newBuilder[String] + builder.sizeHint(Math.min(keyLength, results.length)) + + var i = 0 + while (i < keyLength) { + val key = primaryKeys.get(i) + if (results.contains(key)) { + builder += key + } + i += 1 + } + + InMemTablePrimaryKeys(VectorImmutableArray.from(builder.result())) } } private def filterByRow(source: RowSource, primaryKeys: TablePrimaryKeys, firstInChain: Boolean, column: Column): TablePrimaryKeys = { - val rowPermissionFilter = column.dataType match { + column.dataType match { case DataType.StringDataType => - buildFilter(column, allowedValues) + applyFilter(source, primaryKeys, column, allowedValues) case DataType.LongDataType => - buildFilter(column, allowedValues.map(f => f.toLong)) + applyFilter(source, primaryKeys, column, longValues) case DataType.IntegerDataType => - buildFilter(column, allowedValues.map(f => f.toInt)) + applyFilter(source, primaryKeys, column, intValues) case DataType.DoubleDataType => - buildFilter(column, allowedValues.map(f => f.toDouble)) + applyFilter(source, primaryKeys, column, doubleValues) case DataType.BooleanDataType => - buildFilter(column, allowedValues.map(f => f.toBoolean)) + applyFilter(source, primaryKeys, column, booleanValues) case DataType.EpochTimestampType => - buildFilter(column, allowedValues.map(f => EpochTimestamp(f.toLong))) + applyFilter(source, primaryKeys, column, epochTimestampValues) case DataType.CharDataType => - buildFilter(column, allowedValues.map(f => f.charAt(0))) + applyFilter(source, primaryKeys, column, charValues) case _ => logger.warn(s"Unable to permission filter $column") - DenyAllPermissionFilter + EmptyTablePrimaryKeys } - rowPermissionFilter.doFilter(source, primaryKeys, firstInChain) } - private def buildFilter[T](column: Column, items: Set[T]): RowPermissionFilter = { - val predicate: RowData => Boolean = r => { - val value = r.get(column) - value != null && items.contains(value.asInstanceOf[T]) + private def applyFilter[T](source: RowSource, primaryKeys: TablePrimaryKeys, column: Column, items: Set[T]): TablePrimaryKeys = { + val length = primaryKeys.length + val builder = Vector.newBuilder[String] + builder.sizeHint(length) + + var i = 0 + while (i < length) { + val key = primaryKeys.get(i) + val row = source.pullRow(key) + val value = row.get(column) + if (value != null && items.contains(value.asInstanceOf[T])) { + builder += key + } + i += 1 } - RowPermissionFilter(predicate) + + InMemTablePrimaryKeys(ImmutableArray.from[String](builder.result())) } - + } diff --git a/vuu/src/main/scala/org/finos/vuu/core/filter/type/VisualLinkedFilter.scala b/vuu/src/main/scala/org/finos/vuu/core/filter/type/VisualLinkedFilter.scala index 71e60f55d..5f8b80550 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/filter/type/VisualLinkedFilter.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/filter/type/VisualLinkedFilter.scala @@ -59,7 +59,7 @@ case class VisualLinkedFilter(viewPortVisualLink: ViewPortVisualLink) extends Vi } private def filterIndexByValues[TYPE](index: IndexedField[TYPE], parentSelected: List[TYPE]): TablePrimaryKeys = { - InMemTablePrimaryKeys(index.find(parentSelected)) + InMemTablePrimaryKeys(index.find(parentSelected).toImmutableArray) } private def doFilterByBruteForce(parentDataValues: Set[Any], childColumn: Column, source: RowSource, primaryKeys: TablePrimaryKeys): TablePrimaryKeys = { diff --git a/vuu/src/main/scala/org/finos/vuu/core/index/IndexedField.scala b/vuu/src/main/scala/org/finos/vuu/core/index/IndexedField.scala index 755f78bc0..7be95299b 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/index/IndexedField.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/index/IndexedField.scala @@ -1,40 +1,35 @@ package org.finos.vuu.core.index import com.typesafe.scalalogging.StrictLogging -import org.finos.toolbox.collection.array.{ImmutableArray, VectorImmutableArray} import org.finos.toolbox.collection.set.ImmutableArraySet import org.finos.vuu.core.table.Column import org.finos.vuu.core.table.datatype.EpochTimestamp import java.util.concurrent.{ConcurrentHashMap, ConcurrentNavigableMap, ConcurrentSkipListMap} -import scala.collection.mutable +import scala.jdk.CollectionConverters.IteratorHasAsScala trait IndexedField[TYPE] { - protected val empty: ImmutableArray[String] = ImmutableArray.empty + protected val empty: ImmutableArraySet[String] = ImmutableArraySet.empty def insert(indexedValue: TYPE, rowKey: String): Unit def replace(oldIndexedValue: TYPE, newIndexedValue: TYPE, rowKey: String): Unit - + def remove(indexedValue: TYPE, rowKey: String): Unit def column: Column - def lessThan(bound: TYPE): ImmutableArray[String] + def lessThan(bound: TYPE): ImmutableArraySet[String] - def greaterThan(bound: TYPE): ImmutableArray[String] + def greaterThan(bound: TYPE): ImmutableArraySet[String] - def find(indexedValue: TYPE): ImmutableArray[String] + def find(indexedValue: TYPE): ImmutableArraySet[String] - def find(indexedValues: List[TYPE]): ImmutableArray[String] = { - if (indexedValues.isEmpty) { - empty - } else if (indexedValues.length == 1) { - find(indexedValues.head) - } else { - ImmutableArray.from(indexedValues.iterator.flatMap(f => find(f))) - } + def find(indexedValues: Iterable[TYPE]): ImmutableArraySet[String] = { + val it = indexedValues.iterator + if (!it.hasNext) empty + else ImmutableArraySet.from(it.flatMap(find)) } } @@ -94,21 +89,21 @@ class HashMapIndexedStringField(val column: Column) extends StringIndexedField w } }) } - - override def find(indexKey: String): ImmutableArray[String] = { + + override def find(indexKey: String): ImmutableArraySet[String] = { logger.debug(s"Hit Index: ${column.name} for key $indexKey") val result = indexMap.get(indexKey) - if (result != null) result.toImmutableArray else empty + if (result != null) result else empty } - override def find(indexedValues: List[String]): ImmutableArray[String] = super.find(indexedValues) + override def find(indexedValues: Iterable[String]): ImmutableArraySet[String] = super.find(indexedValues) - override def lessThan(bound: String): ImmutableArray[String] = { + override def lessThan(bound: String): ImmutableArraySet[String] = { logger.warn("Less than is not supported for Strings") empty } - override def greaterThan(bound: String): ImmutableArray[String] = { + override def greaterThan(bound: String): ImmutableArraySet[String] = { logger.warn("Greater than is not supported for Strings") empty } @@ -145,7 +140,7 @@ class SkipListIndexedField[TYPE](val column: Column) extends IndexedField[TYPE] } }) } - + override def insert(indexKey: TYPE, rowKey: String): Unit = { logger.trace(s"Inserting value $rowKey into ${column.name} index") skipList.compute(indexKey, (_, value) => { @@ -156,50 +151,41 @@ class SkipListIndexedField[TYPE](val column: Column) extends IndexedField[TYPE] }) } - override def find(indexKey: TYPE): ImmutableArray[String] = { + override def find(indexKey: TYPE): ImmutableArraySet[String] = { logger.debug(s"Hit Index: ${column.name} for key $indexKey") val result = skipList.get(indexKey) - if (result != null) result.toImmutableArray else empty + if (result != null) result else empty } - def lessThan(bound: TYPE): ImmutableArray[String] = { + def lessThan(bound: TYPE): ImmutableArraySet[String] = { logger.debug(s"Hit Index (LT): ${column.name}") collect(skipList.headMap(bound, false)) } - def lessThanOrEqual(bound: TYPE): ImmutableArray[String] = { + def lessThanOrEqual(bound: TYPE): ImmutableArraySet[String] = { logger.debug(s"Hit Index (LTE): ${column.name}") collect(skipList.headMap(bound, true)) } - def greaterThan(bound: TYPE): ImmutableArray[String] = { + def greaterThan(bound: TYPE): ImmutableArraySet[String] = { logger.debug(s"Hit Index (GT): ${column.name}") collect(skipList.tailMap(bound, false)) } - def greaterThanOrEqual(bound: TYPE): ImmutableArray[String] = { + def greaterThanOrEqual(bound: TYPE): ImmutableArraySet[String] = { logger.debug(s"Hit Index (GTE): ${column.name}") collect(skipList.tailMap(bound, true)) } - override def find(indexedValues: List[TYPE]): ImmutableArray[String] = super.find(indexedValues) + override def find(indexedValues: Iterable[TYPE]): ImmutableArraySet[String] = super.find(indexedValues) - private def collect(results: ConcurrentNavigableMap[TYPE, ImmutableArraySet[String]]): ImmutableArray[String] = { - if (results.isEmpty) { - empty - } else if (results.size() == 1) { - results.firstEntry().getValue.toImmutableArray - } else { - val uniqueValues = mutable.HashSet.empty[String] - val iterator = results.values().iterator() - while (iterator.hasNext) { - val set = iterator.next() - uniqueValues.addAll(set.iterator) - } - ImmutableArray.from(uniqueValues) - } + private def collect(results: ConcurrentNavigableMap[TYPE, ImmutableArraySet[String]]): ImmutableArraySet[String] = { + if (results.isEmpty) return empty + + if (results.size() == 1) results.firstEntry().getValue + else ImmutableArraySet.from(results.values().iterator().asScala.flatMap(_.iterator)) } - + } class SkipListIndexedDoubleField(column: Column) extends SkipListIndexedField[Double](column) with DoubleIndexedField {} diff --git a/vuu/src/main/scala/org/finos/vuu/core/sort/FilterAndSort.scala b/vuu/src/main/scala/org/finos/vuu/core/sort/FilterAndSort.scala index 4d24b0437..5f2a5bb7e 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/sort/FilterAndSort.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/sort/FilterAndSort.scala @@ -2,7 +2,7 @@ package org.finos.vuu.core.sort import com.typesafe.scalalogging.StrictLogging import org.finos.vuu.core.filter.`type`.BaseFilter -import org.finos.vuu.core.filter.{CompoundFilter, ViewPortFilter} +import org.finos.vuu.core.filter.{CompoundViewPortFilter, ViewPortFilter} import org.finos.vuu.core.table.{EmptyTablePrimaryKeys, TablePrimaryKeys} import org.finos.vuu.viewport.{RowSource, ViewPortColumns} @@ -23,12 +23,12 @@ case class UserDefinedFilterAndSort(filter: ViewPortFilter, sort: Sort) extends baseFilter: BaseFilter): TablePrimaryKeys = { logger.trace(s"Starting filter and sort with ${primaryKeys.length} rows") if (primaryKeys.length == 0) { - return EmptyTablePrimaryKeys + return primaryKeys } try { - val realizedFilter = CompoundFilter(baseFilter, filter) + val realizedFilter = CompoundViewPortFilter(baseFilter, filter) logger.trace("Filtering...") val filteredKeys = realizedFilter.doFilter(source, primaryKeys, vpColumns, firstInChain = true) logger.trace("Filtered. Sorting...") diff --git a/vuu/src/main/scala/org/finos/vuu/core/table/TablePrimaryKeys.scala b/vuu/src/main/scala/org/finos/vuu/core/table/TablePrimaryKeys.scala index 22ffab017..bb3f11153 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/table/TablePrimaryKeys.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/table/TablePrimaryKeys.scala @@ -6,7 +6,7 @@ import org.finos.vuu.feature.inmem.InMemTablePrimaryKeys object EmptyTablePrimaryKeys extends TablePrimaryKeys{ override def length: Int = 0 override def add(key: String): TablePrimaryKeys = this.+(key) - override def +(key: String): TablePrimaryKeys = InMemTablePrimaryKeys(ImmutableArray.from(Array(key))) + override def +(key: String): TablePrimaryKeys = InMemTablePrimaryKeys(ImmutableArray.of(key)) override def remove(key: String): TablePrimaryKeys = EmptyTablePrimaryKeys override def -(key: String): TablePrimaryKeys = EmptyTablePrimaryKeys override def sliceTableKeys(from: Int, until: Int): TablePrimaryKeys = EmptyTablePrimaryKeys @@ -14,7 +14,7 @@ object EmptyTablePrimaryKeys extends TablePrimaryKeys{ override def get(index: Int): String = null override def set(index: Int, key: String): TablePrimaryKeys = this - override def intersect(keys: Iterable[String]): TablePrimaryKeys = EmptyTablePrimaryKeys + } trait TablePrimaryKeys extends Iterable[String] { @@ -26,7 +26,6 @@ trait TablePrimaryKeys extends Iterable[String] { def sliceTableKeys(from: Int, until: Int): TablePrimaryKeys def get(index: Int): String def set(index: Int, key: String): TablePrimaryKeys - def intersect(keys: Iterable[String]): TablePrimaryKeys } diff --git a/vuu/src/main/scala/org/finos/vuu/feature/inmem/InMemTablePrimaryKeys.scala b/vuu/src/main/scala/org/finos/vuu/feature/inmem/InMemTablePrimaryKeys.scala index 3efd06a01..445bbc9e0 100644 --- a/vuu/src/main/scala/org/finos/vuu/feature/inmem/InMemTablePrimaryKeys.scala +++ b/vuu/src/main/scala/org/finos/vuu/feature/inmem/InMemTablePrimaryKeys.scala @@ -19,12 +19,4 @@ case class InMemTablePrimaryKeys(keys: ImmutableArray[String]) extends TablePrim override def get(index: Int): String = keys.apply(index) override def set(index: Int, key: String): TablePrimaryKeys = InMemTablePrimaryKeys(keys.set(index, key)) - override def intersect(otherKeys: Iterable[String]): TablePrimaryKeys = { - if (otherKeys.isEmpty) { - EmptyTablePrimaryKeys - } else { - val intersection = keys.filter(otherKeys.toSet.contains) - InMemTablePrimaryKeys(ImmutableArray.from(intersection)) - } - } } diff --git a/vuu/src/main/scala/org/finos/vuu/viewport/ViewPort.scala b/vuu/src/main/scala/org/finos/vuu/viewport/ViewPort.scala index 6b9fd1298..7d180eb0e 100644 --- a/vuu/src/main/scala/org/finos/vuu/viewport/ViewPort.scala +++ b/vuu/src/main/scala/org/finos/vuu/viewport/ViewPort.scala @@ -5,7 +5,7 @@ import org.finos.toolbox.collection.array.ImmutableArray import org.finos.toolbox.time.Clock import org.finos.vuu.api.ViewPortDef import org.finos.vuu.core.auths.VuuUser -import org.finos.vuu.core.filter.CompoundFilter +import org.finos.vuu.core.filter.{CompoundFilter, CompoundViewPortFilter} import org.finos.vuu.core.filter.`type`.{PermissionFilter, VisualLinkedFilter} import org.finos.vuu.core.sort.* import org.finos.vuu.core.table.datatype.EpochTimestamp @@ -370,7 +370,7 @@ class ViewPortImpl(val id: String, override def sortSpec: SortSpec = structuralFields.get().sortSpec - def sendUpdatesOnChange(currentRange: ViewPortRange): Unit = { + private def sendUpdatesOnChange(currentRange: ViewPortRange): Unit = { //val currentKeys = keys.toArray @@ -621,7 +621,7 @@ class ViewPortImpl(val id: String, val fields = this.structuralFields.get() val newFilterSort = fields.filtAndSort match { case udfs: UserDefinedFilterAndSort => - UserDefinedFilterAndSort(CompoundFilter(VisualLinkedFilter(link), udfs.filter), udfs.sort) + UserDefinedFilterAndSort(CompoundViewPortFilter(VisualLinkedFilter(link), udfs.filter), udfs.sort) case x => x } @@ -639,7 +639,7 @@ class ViewPortImpl(val id: String, case udfs: UserDefinedFilterAndSort => udfs.filter match { //remove the visual linked filter from the filterandsort - case CompoundFilter(first, second) => + case CompoundViewPortFilter(first, second) => UserDefinedFilterAndSort(second, udfs.sort) } } diff --git a/vuu/src/main/scala/org/finos/vuu/viewport/ViewPortContainer.scala b/vuu/src/main/scala/org/finos/vuu/viewport/ViewPortContainer.scala index 79bba19de..dd05b571b 100644 --- a/vuu/src/main/scala/org/finos/vuu/viewport/ViewPortContainer.scala +++ b/vuu/src/main/scala/org/finos/vuu/viewport/ViewPortContainer.scala @@ -12,7 +12,7 @@ import org.finos.vuu.api.{Link, ViewPortDef} import org.finos.vuu.client.messages.ViewPortId import org.finos.vuu.core.auths.VuuUser import org.finos.vuu.core.filter.`type`.{AllowAllPermissionFilter, AntlrBasedFilter, BaseFilter, VisualLinkedFilter} -import org.finos.vuu.core.filter.{CompoundFilter, FilterOutEverythingFilter, FilterSpecParser, NoFilter, ViewPortFilter} +import org.finos.vuu.core.filter.{CompoundViewPortFilter, FilterOutEverythingFilter, FilterSpecParser, NoFilter, ViewPortFilter} import org.finos.vuu.core.sort.* import org.finos.vuu.core.table.{DataTable, SessionTable, TableContainer} import org.finos.vuu.core.tree.TreeSessionTableImpl @@ -421,7 +421,7 @@ class ViewPortContainer(val tableContainer: TableContainer, val providerContaine val filtAndSort = viewPort.getVisualLink match { case Some(visualLink) => - UserDefinedFilterAndSort(CompoundFilter(VisualLinkedFilter(visualLink), aFilter), aSort) + UserDefinedFilterAndSort(CompoundViewPortFilter(VisualLinkedFilter(visualLink), aFilter), aSort) case None => UserDefinedFilterAndSort(aFilter, aSort) } @@ -585,7 +585,6 @@ class ViewPortContainer(val tableContainer: TableContainer, val providerContaine def deselectRow(clientSession: ClientSessionId, vpId: String, rowKey: String, preserveExistingSelection: Boolean): ViewPort = { logger.trace(s"[VP] Deselecting row with key $rowKey in viewport $vpId in session ${clientSession.sessionId}. Preserve: $preserveExistingSelection") - logger.trace(s"[VP] Deselecting row with key $rowKey in viewport $vpId in session ${clientSession.sessionId}") val viewPort = getViewportInSession(vpId, clientSession) viewPort.deselectRow(rowKey, preserveExistingSelection) logger.debug(s"[VP] Deselected row with key $rowKey in viewport $vpId in session ${clientSession.sessionId}. Preserve: $preserveExistingSelection") @@ -593,7 +592,6 @@ class ViewPortContainer(val tableContainer: TableContainer, val providerContaine } def selectRowRange(clientSession: ClientSessionId, vpId: String, fromRowKey: String, toRowKey: String, preserveExistingSelection: Boolean): ViewPort = { - logger.trace(s"[VP] Selecting row range from key $fromRowKey to key $toRowKey in viewport $vpId in session ${clientSession.sessionId}") logger.trace(s"[VP] Selecting row range from key $fromRowKey to key $toRowKey in viewport $vpId in session ${clientSession.sessionId}. Preserve: $preserveExistingSelection") val viewPort = getViewportInSession(vpId, clientSession) viewPort.selectRowRange(fromRowKey, toRowKey, preserveExistingSelection) diff --git a/vuu/src/main/scala/org/finos/vuu/viewport/tree/TreeBuilder.scala b/vuu/src/main/scala/org/finos/vuu/viewport/tree/TreeBuilder.scala index 95fb86e6e..71c67c130 100644 --- a/vuu/src/main/scala/org/finos/vuu/viewport/tree/TreeBuilder.scala +++ b/vuu/src/main/scala/org/finos/vuu/viewport/tree/TreeBuilder.scala @@ -4,7 +4,7 @@ import com.typesafe.scalalogging.StrictLogging import org.finos.toolbox.logging.LogAtFrequency import org.finos.toolbox.time.Clock import org.finos.vuu.core.filter.`type`.{AntlrBasedFilter, BaseFilter, PermissionFilter} -import org.finos.vuu.core.filter.{CompoundFilter, FilterOutEverythingFilter, FilterSpecParser, NoFilter} +import org.finos.vuu.core.filter.{CompoundFilter, CompoundViewPortFilter, FilterOutEverythingFilter, FilterSpecParser, NoFilter} import org.finos.vuu.core.sort.Sort import org.finos.vuu.core.table.datatype.EpochTimestamp import org.finos.vuu.core.table.{Column, EmptyRowData, RowData, RowWithData, TablePrimaryKeys} @@ -65,7 +65,7 @@ class TreeBuilderImpl(val table: TreeSessionTableImpl, val groupBy: GroupBy, val logger.debug("Applying filter") val baseFilter = BaseFilter(permissionFilter, frozenTime) - val realizedFilter = CompoundFilter(baseFilter, theFilter) + val realizedFilter = CompoundViewPortFilter(baseFilter, theFilter) realizedFilter.doFilter(table.sourceTable, table.sourceTable.primaryKeys, vpColumns, firstInChain = true) } diff --git a/vuu/src/test/scala/org/finos/vuu/core/filter/FiltersTest.scala b/vuu/src/test/scala/org/finos/vuu/core/filter/FiltersTest.scala index 46ab23c12..2f05e7f51 100644 --- a/vuu/src/test/scala/org/finos/vuu/core/filter/FiltersTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/core/filter/FiltersTest.scala @@ -28,7 +28,7 @@ class FiltersTest extends AnyFeatureSpec with Matchers { result shouldEqual EmptyTablePrimaryKeys } - Scenario("Compound filter applies filters in succession") { + Scenario("Compound viewport filter applies filters in succession") { val table = setupTable() @@ -52,14 +52,14 @@ class FiltersTest extends AnyFeatureSpec with Matchers { } } - val result = CompoundFilter(Filter1(), Filter2()).doFilter(table, table.primaryKeys, ViewPortColumns(table.columns().toList), true) + val result = CompoundViewPortFilter(Filter1(), Filter2()).doFilter(table, table.primaryKeys, ViewPortColumns(table.columns().toList), true) result.size shouldEqual 5 result.toSet shouldEqual Set("LDN-0001", "LDN-0002", "LDN-0008", "NYC-0002", "NYC-0010") //Should get same result when applying in reverse - val result2 = CompoundFilter(Filter2(), Filter1()).doFilter(table, table.primaryKeys, ViewPortColumns(table.columns().toList), true) + val result2 = CompoundViewPortFilter(Filter2(), Filter1()).doFilter(table, table.primaryKeys, ViewPortColumns(table.columns().toList), true) result2.size shouldEqual result.size result2.toSet shouldEqual result.toSet diff --git a/vuu/src/test/scala/org/finos/vuu/core/index/InMemColumnIndicesTest.scala b/vuu/src/test/scala/org/finos/vuu/core/index/InMemColumnIndicesTest.scala index 5bcedbe96..d40c4b189 100644 --- a/vuu/src/test/scala/org/finos/vuu/core/index/InMemColumnIndicesTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/core/index/InMemColumnIndicesTest.scala @@ -92,7 +92,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.insert(createRow(rowKey, column, "A")) val result = indexField.find("A") result.length shouldEqual 1 - result.apply(0) shouldEqual rowKey + result.contains(rowKey) shouldBe true // 2. Update to new value indices.update(createRow(rowKey, column, "A"), createRow(rowKey, column, "B")) @@ -100,7 +100,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { result2.length shouldEqual 0 val result3 = indexField.find("B") result3.length shouldEqual 1 - result3.apply(0) shouldEqual rowKey + result3.contains(rowKey) shouldBe true // 3. Update to null indices.update(createRow(rowKey, column, "B"), createRow(rowKey, column, null)) @@ -111,7 +111,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.update(createRow(rowKey, column, null), createRow(rowKey, column, "C")) val result5 = indexField.find("C") result5.length shouldEqual 1 - result5.apply(0) shouldEqual rowKey + result5.contains(rowKey) shouldBe true // 5. Remove indices.remove(createRow(rowKey, column, "C")) @@ -128,7 +128,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.insert(createRow(rowKey, column, 1)) val result = indexField.find(1) result.length shouldEqual 1 - result.apply(0) shouldEqual rowKey + result.contains(rowKey) shouldBe true // 2. Update to new value indices.update(createRow(rowKey, column, 1), createRow(rowKey, column, 2)) @@ -136,7 +136,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { result2.length shouldEqual 0 val result3 = indexField.find(2) result3.length shouldEqual 1 - result3.apply(0) shouldEqual rowKey + result3.contains(rowKey) shouldBe true // 3. Update to null indices.update(createRow(rowKey, column, 2), createRow(rowKey, column, null)) @@ -147,7 +147,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.update(createRow(rowKey, column, null), createRow(rowKey, column, 3)) val result5 = indexField.find(3) result5.length shouldEqual 1 - result5.apply(0) shouldEqual rowKey + result5.contains(rowKey) shouldBe true // 5. Remove indices.remove(createRow(rowKey, column, 3)) @@ -164,7 +164,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.insert(createRow(rowKey, column, 1L)) val result = indexField.find(1L) result.length shouldEqual 1 - result.apply(0) shouldEqual rowKey + result.contains(rowKey) shouldBe true // 2. Update to new value indices.update(createRow(rowKey, column, 1L), createRow(rowKey, column, 2L)) @@ -172,7 +172,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { result2.length shouldEqual 0 val result3 = indexField.find(2L) result3.length shouldEqual 1 - result3.apply(0) shouldEqual rowKey + result3.contains(rowKey) shouldBe true // 3. Update to null indices.update(createRow(rowKey, column, 2L), createRow(rowKey, column, null)) @@ -183,7 +183,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.update(createRow(rowKey, column, null), createRow(rowKey, column, 3L)) val result5 = indexField.find(3L) result5.length shouldEqual 1 - result5.apply(0) shouldEqual rowKey + result5.contains(rowKey) shouldBe true // 5. Remove indices.remove(createRow(rowKey, column, 3L)) @@ -200,7 +200,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.insert(createRow(rowKey, column, 1.0)) val result = indexField.find(1.0) result.length shouldEqual 1 - result.apply(0) shouldEqual rowKey + result.contains(rowKey) shouldBe true // 2. Update to new value indices.update(createRow(rowKey, column, 1.0), createRow(rowKey, column, 2.0)) @@ -208,7 +208,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { result2.length shouldEqual 0 val result3 = indexField.find(2.0) result3.length shouldEqual 1 - result3.apply(0) shouldEqual rowKey + result3.contains(rowKey) shouldBe true // 3. Update to null indices.update(createRow(rowKey, column, 2.0), createRow(rowKey, column, null)) @@ -219,7 +219,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.update(createRow(rowKey, column, null), createRow(rowKey, column, 3.0)) val result5 = indexField.find(3.0) result5.length shouldEqual 1 - result5.apply(0) shouldEqual rowKey + result5.contains(rowKey) shouldBe true // 5. Remove indices.remove(createRow(rowKey, column, 3.0)) @@ -236,7 +236,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.insert(createRow(rowKey, column, true)) val result = indexField.find(true) result.length shouldEqual 1 - result.apply(0) shouldEqual rowKey + result.contains(rowKey) shouldBe true // 2. Update to new value indices.update(createRow(rowKey, column, true), createRow(rowKey, column, false)) @@ -244,7 +244,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { result2.length shouldEqual 0 val result3 = indexField.find(false) result3.length shouldEqual 1 - result3.apply(0) shouldEqual rowKey + result3.contains(rowKey) shouldBe true // 3. Update to null indices.update(createRow(rowKey, column, false), createRow(rowKey, column, null)) @@ -255,7 +255,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.update(createRow(rowKey, column, null), createRow(rowKey, column, true)) val result5 = indexField.find(true) result5.length shouldEqual 1 - result5.apply(0) shouldEqual rowKey + result5.contains(rowKey) shouldBe true // 5. Remove indices.remove(createRow(rowKey, column, true)) @@ -272,7 +272,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.insert(createRow(rowKey, column, 'A')) val result = indexField.find('A') result.length shouldEqual 1 - result.apply(0) shouldEqual rowKey + result.contains(rowKey) shouldBe true // 2. Update to new value indices.update(createRow(rowKey, column, 'A'), createRow(rowKey, column, 'B')) @@ -280,7 +280,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { result2.length shouldEqual 0 val result3 = indexField.find('B') result3.length shouldEqual 1 - result3.apply(0) shouldEqual rowKey + result3.contains(rowKey) shouldBe true // 3. Update to null indices.update(createRow(rowKey, column, 'B'), createRow(rowKey, column, null)) @@ -291,7 +291,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.update(createRow(rowKey, column, null), createRow(rowKey, column, 'C')) val result5 = indexField.find('C') result5.length shouldEqual 1 - result5.apply(0) shouldEqual rowKey + result5.contains(rowKey) shouldBe true // 5. Remove indices.remove(createRow(rowKey, column, 'C')) @@ -308,7 +308,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.insert(createRow(rowKey, column, EpochTimestamp(1L))) val result = indexField.find(EpochTimestamp(1L)) result.length shouldEqual 1 - result.apply(0) shouldEqual rowKey + result.contains(rowKey) shouldBe true // 2. Update to new value indices.update(createRow(rowKey, column, EpochTimestamp(1L)), createRow(rowKey, column, EpochTimestamp(2L))) @@ -316,7 +316,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { result2.length shouldEqual 0 val result3 = indexField.find(EpochTimestamp(2L)) result3.length shouldEqual 1 - result3.apply(0) shouldEqual rowKey + result3.contains(rowKey) shouldBe true // 3. Update to null indices.update(createRow(rowKey, column, EpochTimestamp(2L)), createRow(rowKey, column, null)) @@ -327,7 +327,7 @@ class InMemColumnIndicesTest extends AnyFeatureSpec with Matchers { indices.update(createRow(rowKey, column, null), createRow(rowKey, column, EpochTimestamp(3L))) val result5 = indexField.find(EpochTimestamp(3L)) result5.length shouldEqual 1 - result5.apply(0) shouldEqual rowKey + result5.contains(rowKey) shouldBe true // 5. Remove indices.remove(createRow(rowKey, column, EpochTimestamp(3L))) diff --git a/vuu/src/test/scala/org/finos/vuu/core/index/IndexedFieldTest.scala b/vuu/src/test/scala/org/finos/vuu/core/index/IndexedFieldTest.scala index faa1a13e3..6bde44eb4 100644 --- a/vuu/src/test/scala/org/finos/vuu/core/index/IndexedFieldTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/core/index/IndexedFieldTest.scala @@ -23,8 +23,8 @@ class IndexedFieldTest extends AnyFeatureSpec with Matchers with StrictLogging { }) - val rowKeys = index.find(3) - + val rowKeys = index.find(3).toList + rowKeys(0) shouldEqual ("1300") rowKeys(1) shouldEqual ("1301") rowKeys(2) shouldEqual ("1302") @@ -33,7 +33,7 @@ class IndexedFieldTest extends AnyFeatureSpec with Matchers with StrictLogging { index.remove(3, "1302") - val rowKeys2 = index.find(3) + val rowKeys2 = index.find(3).toList rowKeys2.length shouldEqual (9) rowKeys2.indexOf("1302") shouldEqual(-1) @@ -56,30 +56,28 @@ class IndexedFieldTest extends AnyFeatureSpec with Matchers with StrictLogging { }) val results = index.find(key1) - - results.toList shouldEqual List("AaBB0", "AaBB1", "AaBB2", "AaBB3", "AaBB4", "AaBB5") + results.length shouldEqual 6 + results.toSet shouldEqual Set("AaBB0", "AaBB1", "AaBB2", "AaBB3", "AaBB4", "AaBB5") index.remove(key1, "AaBB5") val results2 = index.find(key1) - - results2.toList shouldEqual List("AaBB0", "AaBB1", "AaBB2", "AaBB3", "AaBB4") + results2.length shouldEqual 5 + results2.toSet shouldEqual Set("AaBB0", "AaBB1", "AaBB2", "AaBB3", "AaBB4") val results3 = index.find("lolcats") - results3.length shouldEqual 0 val results4 = index.find(List(key1, key2, "lolcats")) - - results4.toList shouldEqual List("AaBB0", "AaBB1", "AaBB2", "AaBB3", "AaBB4", "BBAa0", "BBAa1", "BBAa2", "BBAa3", "BBAa4", "BBAa5") + results4.length shouldEqual 11 + results4.toSet shouldEqual Set("AaBB0", "AaBB1", "AaBB2", "AaBB3", "AaBB4", "BBAa0", "BBAa1", "BBAa2", "BBAa3", "BBAa4", "BBAa5") val results5 = index.find(List()) - results5.isEmpty shouldBe true val results6 = index.find(List(key2)) - - results6.toList shouldEqual List("BBAa0", "BBAa1", "BBAa2", "BBAa3", "BBAa4", "BBAa5") + results6.length shouldEqual 6 + results6.toSet shouldEqual Set("BBAa0", "BBAa1", "BBAa2", "BBAa3", "BBAa4", "BBAa5") } @@ -126,21 +124,22 @@ class IndexedFieldTest extends AnyFeatureSpec with Matchers with StrictLogging { val values = index.find(EpochTimestamp(1)) - values.toList shouldEqual List("10", "11", "12", "13", "14", "15") + values.length shouldEqual 6 + values.toSet shouldEqual Set("10", "11", "12", "13", "14", "15") index.remove(EpochTimestamp(1), "15") val values2 = index.find(EpochTimestamp(1)) - values2.toList shouldEqual List("10", "11", "12", "13", "14") + values2.length shouldEqual 5 + values2.toSet shouldEqual Set("10", "11", "12", "13", "14") val values3 = index.find(EpochTimestamp(-1)) - - values3.toList shouldEqual List() + values3.length shouldEqual 0 val values4 = index.find(List(EpochTimestamp(2), EpochTimestamp(3))) - - values4.toList shouldEqual List("20", "21", "22", "23", "24", "25", "30", "31", "32", "33", "34", "35") + values4.length shouldEqual 12 + values4.toSet shouldEqual Set("20", "21", "22", "23", "24", "25", "30", "31", "32", "33", "34", "35") } Scenario("Check ordered operations on EpochTimestamp index") {