diff --git a/benchmark/src/main/java/org/finos/vuu/benchmark/BenchmarkHelper.java b/benchmark/src/main/java/org/finos/vuu/benchmark/BenchmarkHelper.java index d4a0c3fa4..d2f3f404c 100644 --- a/benchmark/src/main/java/org/finos/vuu/benchmark/BenchmarkHelper.java +++ b/benchmark/src/main/java/org/finos/vuu/benchmark/BenchmarkHelper.java @@ -77,10 +77,10 @@ public void addTableData(DataTable dataTable, int offset, int size) { rowBuilder.setKey(ric); rowBuilder.setString(ricColumn, ric); rowBuilder.setString(exchangeColumn, "exchange-" + i); - rowBuilder.setDouble(bidColumn, 101); - rowBuilder.setDouble(askColumn, 100); - rowBuilder.setDouble(lastColumn, 105); - rowBuilder.setDouble(closeColumn, 106); + rowBuilder.setDouble(bidColumn, i + 1.0); + rowBuilder.setDouble(askColumn, i + 2.0); + rowBuilder.setDouble(lastColumn, i + 3.0); + rowBuilder.setDouble(closeColumn, i + 4.0); dataTable.processUpdate(rowBuilder.build()); } } diff --git a/benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmark.java b/benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmark.java index ec44b1beb..8b9c44fd9 100644 --- a/benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmark.java +++ b/benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmark.java @@ -1,9 +1,9 @@ package org.finos.vuu.benchmark.sort; import org.finos.vuu.benchmark.BenchmarkHelper; -import org.finos.vuu.core.sort.GenericSort2; import org.finos.vuu.core.sort.Sort; import org.finos.vuu.core.sort.SortDirection; +import org.finos.vuu.core.table.Column; import org.finos.vuu.core.table.DataTable; import org.finos.vuu.core.table.ViewPortColumnCreator; import org.finos.vuu.net.SortDef; @@ -12,25 +12,42 @@ import java.util.List; +import static java.util.Arrays.stream; import static org.finos.vuu.util.ScalaCollectionConverter.toScala; public class SortBenchmark { private final DataTable inMemDataTable; private final ViewPortColumns viewPortColumns; - private final Sort sort; + private final Sort singleSort; + private final Sort multiSort; public SortBenchmark(BenchmarkHelper benchmarkHelper, int size) { inMemDataTable = benchmarkHelper.buildTable(); - viewPortColumns = ViewPortColumnCreator.create(inMemDataTable, toScala(List.of("exchange"))); - sort = new GenericSort2( - SortSpec.apply(toScala(List.of(SortDef.apply("exchange", SortDirection.ASCENDING().external())))), - toScala(List.of(inMemDataTable.getTableDef().columnForName("exchange")))); + viewPortColumns = ViewPortColumnCreator.create(inMemDataTable, + toScala(stream(inMemDataTable.getTableDef().getColumns()).map(Column::name).toList())); + var closeColumn = inMemDataTable.getTableDef().columnForName("close"); + var exchangeColumn = inMemDataTable.getTableDef().columnForName("exchange"); + singleSort = Sort.apply( + SortSpec.apply(toScala(List.of( + SortDef.apply(exchangeColumn.name(), SortDirection.ASCENDING().external()) + ))), + toScala(List.of(exchangeColumn))); + multiSort = Sort.apply( + SortSpec.apply(toScala(List.of( + SortDef.apply(exchangeColumn.name(), SortDirection.DESCENDING().external()), + SortDef.apply(closeColumn.name(), SortDirection.ASCENDING().external()) + ))), + toScala(List.of(exchangeColumn, closeColumn))); benchmarkHelper.addTableData(inMemDataTable, size); } - void sortLargeTable() { - sort.doSort(inMemDataTable, inMemDataTable.primaryKeys(), viewPortColumns); + void sortLargeTableSingleColumn() { + singleSort.doSort(inMemDataTable, inMemDataTable.primaryKeys(), viewPortColumns); + } + + void sortLargeTableMultiColumn() { + multiSort.doSort(inMemDataTable, inMemDataTable.primaryKeys(), viewPortColumns); } } diff --git a/benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmarkRunner.java b/benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmarkRunner.java index ebf924715..47fa7c9e2 100644 --- a/benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmarkRunner.java +++ b/benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmarkRunner.java @@ -14,7 +14,6 @@ import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; -import java.io.IOException; import java.util.concurrent.TimeUnit; @State(Scope.Benchmark) @@ -23,7 +22,7 @@ public class SortBenchmarkRunner { private final BenchmarkHelper benchmarkHelper = new BenchmarkHelper(); private SortBenchmark benchmark; - @Param({ "10000", "100000", "500000", "1000000" }) + @Param({ "10000", "100000", "1000000" }) public int tableSize; @Setup(Level.Trial) @@ -37,8 +36,18 @@ public void setup() { @Measurement(iterations = 5) @Fork(1) @BenchmarkMode(Mode.SampleTime) - public void sortLargeTable() throws IOException { - benchmark.sortLargeTable(); + public void sortLargeTableSingle() { + benchmark.sortLargeTableSingleColumn(); + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MILLISECONDS) + @Warmup(iterations = 5) + @Measurement(iterations = 5) + @Fork(1) + @BenchmarkMode(Mode.SampleTime) + public void sortLargeTableMulti() { + benchmark.sortLargeTableMultiColumn(); } } diff --git a/toolbox/src/main/scala/org/finos/toolbox/collection/array/VectorImmutableArray.scala b/toolbox/src/main/scala/org/finos/toolbox/collection/array/VectorImmutableArray.scala index ed2e55817..98cae5da0 100644 --- a/toolbox/src/main/scala/org/finos/toolbox/collection/array/VectorImmutableArray.scala +++ b/toolbox/src/main/scala/org/finos/toolbox/collection/array/VectorImmutableArray.scala @@ -95,13 +95,10 @@ private class VectorImmutableArrayImpl[T <: Object : ClassTag](private val data: doRemove(logicalIndex) } - override def iterator: Iterator[T] = { - val iterator = new Iterator[Int] { - private val it = activeIndices.getIntIterator - def hasNext: Boolean = it.hasNext - def next(): Int = it.next() - } - iterator.map(data(_)) + override def iterator: Iterator[T] = new Iterator[T] { + private val it = activeIndices.getIntIterator + override def hasNext: Boolean = it.hasNext + override def next(): T = data(it.next()) } override def foreach[U](f: T => U): Unit = { diff --git a/vuu/src/main/scala/org/finos/vuu/core/sort/SortCompares.scala b/vuu/src/main/scala/org/finos/vuu/core/sort/SortCompares.scala deleted file mode 100644 index 65d06b215..000000000 --- a/vuu/src/main/scala/org/finos/vuu/core/sort/SortCompares.scala +++ /dev/null @@ -1,108 +0,0 @@ -package org.finos.vuu.core.sort - -import com.typesafe.scalalogging.StrictLogging -import org.finos.vuu.core.sort.SortDirection.Ascending -import org.finos.vuu.core.table.datatype.{EpochTimestamp, ScaledDecimal2, ScaledDecimal4, ScaledDecimal6, ScaledDecimal8} -import org.finos.vuu.core.table.{Column, DataType, RowData} - -import java.util.function.ToIntBiFunction -import scala.annotation.tailrec - -object SortCompares extends StrictLogging { - - @tailrec - def compare(o1: RowData, o2: RowData, columns: List[Column], sortDirections: List[SortDirection], columnIndex: Int): Int = { - - val activeColumn = columns(columnIndex) - val isAscending = sortDirections(columnIndex) == Ascending - - val compareValue = activeColumn.dataType match { - case DataType.StringDataType => compareString(o1, o2, activeColumn, isAscending) - case DataType.LongDataType => compareLong(o1, o2, activeColumn, isAscending) - case DataType.IntegerDataType => compareInt(o1, o2, activeColumn, isAscending) - case DataType.DoubleDataType => compareDouble(o1, o2, activeColumn, isAscending) - case DataType.BooleanDataType => compareBoolean(o1, o2, activeColumn, isAscending) - case DataType.CharDataType => compareChar(o1, o2, activeColumn, isAscending) - case DataType.EpochTimestampType => compareEpochTimestamp(o1, o2, activeColumn, isAscending) - case DataType.ScaledDecimal2Type => compareScaledDecimal2(o1, o2, activeColumn, isAscending) - case DataType.ScaledDecimal4Type => compareScaledDecimal4(o1, o2, activeColumn, isAscending) - case DataType.ScaledDecimal6Type => compareScaledDecimal6(o1, o2, activeColumn, isAscending) - case DataType.ScaledDecimal8Type => compareScaledDecimal8(o1, o2, activeColumn, isAscending) - case _ => - logger.warn(s"Unable to sort datatype ${activeColumn.dataType}") - 0 - } - - if (compareValue != 0 || columnIndex == (columns.length - 1)) { - compareValue - } else { - compare(o1, o2, columns, sortDirections, columnIndex + 1) - } - } - - def compareChar(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[java.lang.Character](o1, o2, column, isAscending) - } - - def compareDouble(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[java.lang.Double](o1, o2, column, isAscending) - } - - def compareInt(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[java.lang.Integer](o1, o2, column, isAscending) - } - - def compareLong(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[java.lang.Long](o1, o2, column, isAscending) - } - - def compareBoolean(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[java.lang.Boolean](o1, o2, column, isAscending) - } - - def compareString(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareReferenceType[String](o1, o2, column, isAscending, (v1: String, v2: String) => v1.compareToIgnoreCase(v2)) - } - - def compareEpochTimestamp(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[EpochTimestamp](o1, o2, column, isAscending) - } - - def compareScaledDecimal2(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[ScaledDecimal2](o1, o2, column, isAscending) - } - - def compareScaledDecimal4(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[ScaledDecimal4](o1, o2, column, isAscending) - } - - def compareScaledDecimal6(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[ScaledDecimal6](o1, o2, column, isAscending) - } - - def compareScaledDecimal8(o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareComparable[ScaledDecimal8](o1, o2, column, isAscending) - } - - private def compareComparable[T <: AnyRef with Comparable[T]](o1: RowData, o2: RowData, column: Column, isAscending: Boolean): Int = { - compareReferenceType(o1, o2, column, isAscending, (c1: T, c2: T) => c1.compareTo(c2)) - } - - private def compareReferenceType[T <: AnyRef](o1: RowData, o2: RowData, column: Column, isAscending: Boolean, compareFunction: ToIntBiFunction[T,T]): Int = { - val c1 = o1.get(column).asInstanceOf[T] - val c2 = o2.get(column).asInstanceOf[T] - if (c1 eq c2) { //Short circuit for reference equality - 0 - } else if (c1 == null) { - if (isAscending) 1 else -1 - } else if (c2 == null) { - if (isAscending) -1 else 1 - } else if (isAscending) { - compareFunction.applyAsInt(c1, c2) - } else { - compareFunction.applyAsInt(c2, c1) - } - } - -} - diff --git a/vuu/src/main/scala/org/finos/vuu/core/sort/SortProjection.scala b/vuu/src/main/scala/org/finos/vuu/core/sort/SortProjection.scala new file mode 100644 index 000000000..9289da467 --- /dev/null +++ b/vuu/src/main/scala/org/finos/vuu/core/sort/SortProjection.scala @@ -0,0 +1,117 @@ +package org.finos.vuu.core.sort + +import com.typesafe.scalalogging.StrictLogging +import org.finos.vuu.core.sort.SortDirection.Ascending +import org.finos.vuu.core.table.{Column, DataType} + +trait SortProjectionComparator extends java.util.Comparator[Array[AnyRef]] + +object SortProjectionComparator extends StrictLogging { + + def apply(columns: Array[Column], sortDirections: Array[SortDirection]): SortProjectionComparator = { + val comparators = columns.indices.map { i => + val col = columns(i) + val dir = sortDirections(i) + buildColumnComparator(col, i + 1, dir == Ascending) + }.toArray + + comparators.length match { + case 1 => SingleColumnComparatorImpl(comparators.head) + case _ => MultiColumnComparatorImpl(comparators) + } + } + + private def buildColumnComparator(column: Column, index: Int, isAscending: Boolean): ColumnSort = { + column.dataType match { + case DataType.StringDataType => + if (isAscending) StringColumnSortAsc(index) else StringColumnSortDesc(index) + case DataType.LongDataType | DataType.IntegerDataType | DataType.DoubleDataType | + DataType.BooleanDataType | DataType.CharDataType | DataType.EpochTimestampType | + DataType.ScaledDecimal2Type | DataType.ScaledDecimal4Type | + DataType.ScaledDecimal6Type | DataType.ScaledDecimal8Type => + if (isAscending) ComparableColumnSortAsc(index) else ComparableColumnSortDesc(index) + case _ => + logger.warn(s"Unable to sort datatype ${column.dataType}") + NoColumnSort + } + } + +} + +case class SingleColumnComparatorImpl(columnSort: ColumnSort) extends SortProjectionComparator { + + override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = { + columnSort.compare(o1, o2) + } + +} + +case class MultiColumnComparatorImpl(comparators: Array[ColumnSort]) extends SortProjectionComparator { + + override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = { + var i = 0 + val len = comparators.length + while (i < len) { + val res = comparators(i).compare(o1, o2) + if (res != 0) return res + i += 1 + } + 0 + } +} + +sealed trait ColumnSort { + def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int +} + +object NoColumnSort extends ColumnSort { + override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = 0 +} + +case class StringColumnSortAsc(index: Int) extends ColumnSort { + override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = { + val v1 = o1(index).asInstanceOf[String] + val v2 = o2(index).asInstanceOf[String] + + if (v1 eq v2) 0 + else if (v1 == null) 1 + else if (v2 == null) -1 + else v1.compareToIgnoreCase(v2) + } +} + +case class StringColumnSortDesc(index: Int) extends ColumnSort { + override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = { + val v1 = o1(index).asInstanceOf[String] + val v2 = o2(index).asInstanceOf[String] + + if (v1 eq v2) 0 + else if (v1 == null) -1 + else if (v2 == null) 1 + else v2.compareToIgnoreCase(v1) + } +} + +case class ComparableColumnSortAsc(index: Int) extends ColumnSort { + override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = { + val v1 = o1(index).asInstanceOf[Comparable[AnyRef]] + val v2 = o2(index).asInstanceOf[Comparable[AnyRef]] + + if (v1 eq v2) 0 + else if (v1 == null) 1 + else if (v2 == null) -1 + else v1.compareTo(v2) + } +} + +case class ComparableColumnSortDesc(index: Int) extends ColumnSort { + override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = { + val v1 = o1(index).asInstanceOf[Comparable[AnyRef]] + val v2 = o2(index).asInstanceOf[Comparable[AnyRef]] + + if (v1 eq v2) 0 + else if (v1 == null) -1 + else if (v2 == null) 1 + else v2.compareTo(v1) + } +} \ No newline at end of file diff --git a/vuu/src/main/scala/org/finos/vuu/core/sort/Sorts.scala b/vuu/src/main/scala/org/finos/vuu/core/sort/Sorts.scala index 5fe35c901..057b57c2b 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/sort/Sorts.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/sort/Sorts.scala @@ -2,20 +2,23 @@ package org.finos.vuu.core.sort import com.typesafe.scalalogging.StrictLogging import org.finos.toolbox.collection.array.ImmutableArray -import org.finos.toolbox.time.TimeIt.timeIt -import org.finos.vuu.core.table.{Column, RowData, RowWithData, TablePrimaryKeys} +import org.finos.vuu.core.table.{Column, RowWithData, TablePrimaryKeys} import org.finos.vuu.feature.inmem.InMemTablePrimaryKeys import org.finos.vuu.net.SortSpec import org.finos.vuu.viewport.{RowSource, ViewPortColumns} import java.util +import java.util.Comparator trait Sort { def doSort(source: RowSource, primaryKeys: TablePrimaryKeys, vpColumns: ViewPortColumns): TablePrimaryKeys } object Sort { - def apply(spec: SortSpec, columns: List[Column]): Sort = GenericSort2(spec, columns) + def apply(spec: SortSpec, columns: List[Column]): Sort = { + val sortDirections = spec.sortDefs.map(sd => SortDirection.fromExternal(sd.sortType)) + GenericSort2(columns.toArray, sortDirections.toArray) + } } object NoSort extends Sort { @@ -24,44 +27,34 @@ object NoSort extends Sort { } } -private case class GenericSort2(spec: SortSpec, columns: List[Column]) extends Sort with StrictLogging { +private case class GenericSort2(columns: Array[Column], sortDirections: Array[SortDirection]) extends Sort with StrictLogging { - private val sortDirections = spec.sortDefs.map(sd => SortDirection.fromExternal(sd.sortType)) - private val comparator = new java.util.Comparator[RowData] { - override def compare(o1: RowData, o2: RowData): Int = - SortCompares.compare(o1, o2, columns, sortDirections, 0) - } + private val comparator = SortProjectionComparator(columns, sortDirections) + private val columnNames = columns.map(_.name) + private val columnsLength = columns.length + private val projectionLength = columnsLength + 1 override def doSort(source: RowSource, primaryKeys: TablePrimaryKeys, vpColumns: ViewPortColumns): TablePrimaryKeys = { //This has been repeatedly benchmarked using JMH. If you touch this, do a before and after run of SortBenchmark - logger.trace("Starting map") - - val (millisToArray, snapshotAndCount) = timeIt { - createSnapshot(source, primaryKeys, vpColumns) - } - - logger.trace("Starting sort") + logger.trace("Creating projections") + val sortProjections = new Array[Array[AnyRef]](primaryKeys.length) + val rowCount = addProjections(sortProjections, source, primaryKeys, vpColumns) - val (millisSort, _ ) = timeIt { - util.Arrays.sort(snapshotAndCount._1, 0, snapshotAndCount._2, comparator) - } - - logger.trace("Starting build imm arr") + logger.trace("Performing sort") + util.Arrays.sort(sortProjections, 0, rowCount, comparator) - val (millisImmArray, immutableArray) = timeIt { - createKeyArray(snapshotAndCount._1, snapshotAndCount._2) - } + logger.trace("Creating primary keys") + val sortedKeys = createKeyArray(sortProjections, rowCount) - logger.debug(s"[SORT]: Table Size: ${primaryKeys.length} DataToArray: ${millisToArray}ms, Sort: ${millisSort}ms, ImmutArr: ${millisImmArray}ms") - - InMemTablePrimaryKeys(immutableArray) + logger.trace("Finished") + InMemTablePrimaryKeys(sortedKeys) } - private def createSnapshot(source: RowSource, primaryKeys: TablePrimaryKeys, vpColumns: ViewPortColumns): (Array[RowWithData], Int) = { + private def addProjections(sortProjections: Array[Array[AnyRef]], source: RowSource, + primaryKeys: TablePrimaryKeys, vpColumns: ViewPortColumns): Int = { val length = primaryKeys.length - val rowDataArray = new Array[RowWithData](length) var index = 0 var count = 0 @@ -69,23 +62,35 @@ private case class GenericSort2(spec: SortSpec, columns: List[Column]) extends S val key = primaryKeys.get(index) source.pullRow(key, vpColumns) match { case r: RowWithData => - rowDataArray(count) = r + val projection = new Array[AnyRef](projectionLength) + projection(0) = r.key + var columnIndex = 0 + while (columnIndex < columnsLength) { + val columnValue = r.data.getOrElse(columnNames(columnIndex), null).asInstanceOf[AnyRef] + projection(columnIndex + 1) = columnValue + columnIndex += 1 + } + sortProjections(count) = projection count += 1 case _ => } index += 1 } - (rowDataArray, count) + count } - private def createKeyArray(snapshot: Array[RowWithData], length: Int): ImmutableArray[String] = { - val keys = new Array[String](length) - var i = 0 - while (i < length) { - keys(i) = snapshot(i).key - i += 1 + private def createKeyArray(snapshot: Array[Array[AnyRef]], rowCount: Int): ImmutableArray[String] = { + val iterator = new Iterator[String] { + private var i = 0 + override def hasNext: Boolean = i < rowCount + override def next(): String = { + val key = snapshot(i)(0).asInstanceOf[String] + i += 1 + key + } + override def knownSize: Int = rowCount } - ImmutableArray.from(keys) + ImmutableArray.from(iterator) } } diff --git a/vuu/src/test/scala/org/finos/vuu/core/sort/SortComparesTest.scala b/vuu/src/test/scala/org/finos/vuu/core/sort/SortComparesTest.scala deleted file mode 100644 index 80ede73d1..000000000 --- a/vuu/src/test/scala/org/finos/vuu/core/sort/SortComparesTest.scala +++ /dev/null @@ -1,383 +0,0 @@ -package org.finos.vuu.core.sort - -import org.finos.vuu.core.table.datatype.{EpochTimestamp, ScaledDecimal2, ScaledDecimal4, ScaledDecimal6, ScaledDecimal8} -import org.finos.vuu.core.table.{Column, RowData, RowWithData, SimpleColumn} -import org.scalatest.BeforeAndAfterEach -import org.scalatest.featurespec.AnyFeatureSpec -import org.scalatest.matchers.should.Matchers - -import java.util.Comparator - -class SortComparesTest extends AnyFeatureSpec with Matchers with BeforeAndAfterEach { - - override def beforeEach(): Unit = TestComparator.clear() - - Feature("compareString") { - val rowData1 = RowWithData("id-2", Map("stringField" -> "XXX")) - val rowData2 = RowWithData("id-3", Map("stringField" -> "YYY")) - val rowData3 = RowWithData("id-1", Map("stringField" -> "ZZZ")) - val rowData4 = RowWithData("id-4", Map()) - val rowData5 = RowWithData("id-5", Map()) - - val ascending = List(rowData1, rowData2, rowData3) - val unordered = List(rowData1, rowData3, rowData2) - val col = column("stringField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareString(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareString(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can support nulls and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareString(o1, o2, col, isAscending = true)) - val data = List(rowData4, rowData3, rowData5) - val sortedData = List(rowData3, rowData4, rowData5) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - Feature("compareInt") { - val rowData1 = RowWithData("id-2", Map("intField" -> -10)) - val rowData2 = RowWithData("id-3", Map("intField" -> 0)) - val rowData3 = RowWithData("id-1", Map("intField" -> 7)) - val rowData4 = RowWithData("id-4", Map("intField" -> 10)) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData1, rowData3, rowData4, rowData2) - val col = column("intField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareInt(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareInt(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareInt(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - Feature("compareLong") { - val rowData1 = RowWithData("id-2", Map("longField" -> -10L)) - val rowData2 = RowWithData("id-3", Map("longField" -> 0L)) - val rowData3 = RowWithData("id-1", Map("longField" -> 7L)) - val rowData4 = RowWithData("id-4", Map("longField" -> 10L)) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData3, rowData1, rowData4, rowData2) - val col = column("longField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareLong(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareLong(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareLong(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - Feature("compareDouble") { - val rowData1 = RowWithData("id-2", Map("doubleField" -> -10.9)) - val rowData2 = RowWithData("id-3", Map("doubleField" -> 0.0)) - val rowData3 = RowWithData("id-1", Map("doubleField" -> 5.7)) - val rowData4 = RowWithData("id-4", Map("doubleField" -> 5.71)) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData1, rowData3, rowData4, rowData2) - val col = column("doubleField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareDouble(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareDouble(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareDouble(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - Feature("compareChar") { - val rowData1 = RowWithData("id-2", Map("charField" -> '5')) - val rowData2 = RowWithData("id-3", Map("charField" -> 'A')) - val rowData3 = RowWithData("id-1", Map("charField" -> 'Z')) - val rowData4 = RowWithData("id-4", Map("charField" -> 'a')) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData4, rowData1, rowData3, rowData2) - val col = column("charField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareChar(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareChar(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareChar(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - - } - - Feature("compareBoolean") { - val rowData1 = RowWithData("id-1", Map("boolField" -> true)) - val rowData2 = RowWithData("id-2", Map("boolField" -> false)) - val rowData3 = RowWithData("id-3", Map()) - val rowData4 = RowWithData("id-4", Map()) - - val ascending = List(rowData2, rowData1) - val unordered = List(rowData1, rowData2) - val col = column("boolField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareBoolean(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareBoolean(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareBoolean(o1, o2, col, isAscending = true)) - val data = List(rowData2, rowData1, rowData3, rowData4) - val sortedData = List(rowData2, rowData1, rowData3, rowData4) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - - } - - Feature("compareEpochTimeStamp") { - val rowData1 = RowWithData("id-2", Map("epochField" -> EpochTimestamp(-10L))) - val rowData2 = RowWithData("id-3", Map("epochField" -> EpochTimestamp(0L))) - val rowData3 = RowWithData("id-1", Map("epochField" -> EpochTimestamp(7L))) - val rowData4 = RowWithData("id-4", Map("epochField" -> EpochTimestamp(10L))) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData3, rowData1, rowData4, rowData2) - val col = column("epochField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareEpochTimestamp(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareEpochTimestamp(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareEpochTimestamp(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - Feature("compareScaledDecimal2") { - val rowData1 = RowWithData("id-2", Map("scaledDecimalField" -> ScaledDecimal2(-10L))) - val rowData2 = RowWithData("id-3", Map("scaledDecimalField" -> ScaledDecimal2(0L))) - val rowData3 = RowWithData("id-1", Map("scaledDecimalField" -> ScaledDecimal2(7L))) - val rowData4 = RowWithData("id-4", Map("scaledDecimalField" -> ScaledDecimal2(10L))) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData3, rowData1, rowData4, rowData2) - val col = column("scaledDecimalField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal2(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal2(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal2(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - Feature("compareScaledDecimal4") { - val rowData1 = RowWithData("id-2", Map("scaledDecimalField" -> ScaledDecimal4(-10L))) - val rowData2 = RowWithData("id-3", Map("scaledDecimalField" -> ScaledDecimal4(0L))) - val rowData3 = RowWithData("id-1", Map("scaledDecimalField" -> ScaledDecimal4(7L))) - val rowData4 = RowWithData("id-4", Map("scaledDecimalField" -> ScaledDecimal4(10L))) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData3, rowData1, rowData4, rowData2) - val col = column("scaledDecimalField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal4(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal4(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal4(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - Feature("compareScaledDecimal6") { - val rowData1 = RowWithData("id-2", Map("scaledDecimalField" -> ScaledDecimal6(-10L))) - val rowData2 = RowWithData("id-3", Map("scaledDecimalField" -> ScaledDecimal6(0L))) - val rowData3 = RowWithData("id-1", Map("scaledDecimalField" -> ScaledDecimal6(7L))) - val rowData4 = RowWithData("id-4", Map("scaledDecimalField" -> ScaledDecimal6(10L))) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData3, rowData1, rowData4, rowData2) - val col = column("scaledDecimalField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal6(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal6(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal6(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - Feature("compareScaledDecimal8") { - val rowData1 = RowWithData("id-2", Map("scaledDecimalField" -> ScaledDecimal8(-10L))) - val rowData2 = RowWithData("id-3", Map("scaledDecimalField" -> ScaledDecimal8(0L))) - val rowData3 = RowWithData("id-1", Map("scaledDecimalField" -> ScaledDecimal8(7L))) - val rowData4 = RowWithData("id-4", Map("scaledDecimalField" -> ScaledDecimal8(10L))) - val rowData5 = RowWithData("id-5", Map()) - val rowData6 = RowWithData("id-6", Map()) - - val ascending = List(rowData1, rowData2, rowData3, rowData4) - val unordered = List(rowData3, rowData1, rowData4, rowData2) - val col = column("scaledDecimalField") - - Scenario("can support `A` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal8(o1, o2, col, isAscending = true)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending - } - - Scenario("can support `D` sort direction") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal8(o1, o2, col, isAscending = false)) - - unordered.sorted(TestComparator.compare) shouldEqual ascending.reverse - } - - Scenario("can sort null value and they go last in ascending order") { - TestComparator.register((o1, o2) => SortCompares.compareScaledDecimal8(o1, o2, col, isAscending = true)) - val data = List(rowData5, rowData3, rowData6) - val sortedData = List(rowData3, rowData5, rowData6) - data.sorted(TestComparator.compare) shouldEqual sortedData - } - } - - private def column(name: String): Column = SimpleColumn(name, -1, classOf[Any]) - - private object TestComparator extends Comparator[RowData] { - private type RowComparator = (RowData, RowData) => Int - private val dummyTestComparator: RowComparator = (_, _) => 0 - private var testComparator: RowComparator = dummyTestComparator - - override def compare(o1: RowData, o2: RowData): Int = testComparator(o1, o2) - - def register(testComparator: RowComparator): Unit = { - this.testComparator = testComparator - } - - def clear(): Unit = { - this.testComparator = dummyTestComparator - } - } -} diff --git a/vuu/src/test/scala/org/finos/vuu/core/sort/SortProjectionTest.scala b/vuu/src/test/scala/org/finos/vuu/core/sort/SortProjectionTest.scala new file mode 100644 index 000000000..5332b05ac --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/core/sort/SortProjectionTest.scala @@ -0,0 +1,89 @@ +package org.finos.vuu.core.sort + +import org.finos.vuu.core.sort.SortDirection.{Ascending, Descending} +import org.finos.vuu.core.table.{Column, DataType, SimpleColumn} +import org.scalatest.GivenWhenThen +import org.scalatest.featurespec.AnyFeatureSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.prop.TableDrivenPropertyChecks + +class SortProjectionTest extends AnyFeatureSpec with Matchers with GivenWhenThen with TableDrivenPropertyChecks { + + Feature("SortProjectionComparator") { + + def createColumn(name: String, dataType: Class[?]): Column = SimpleColumn(name, 0 , dataType) + + Scenario("Verify sorting and null handling for all supported types and directions") { + + val testCases = Table( + ("Label", "DataType", "Direction", "Val1", "Val2", "Expected (Sign)"), + //Strings + ("String Asc: A vs B", DataType.StringDataType, Ascending, "A", "B", -1), + ("String Asc: B vs A", DataType.StringDataType, Ascending, "B", "A", 1), + ("String Desc: A vs B", DataType.StringDataType, Descending, "A", "B", 1), + ("String Desc: B vs A", DataType.StringDataType, Descending, "B", "A", -1), + //Strings: Null Handling + ("String Asc: Null vs Val", DataType.StringDataType, Ascending, null, "A", 1), + ("String Asc: Val vs Null", DataType.StringDataType, Ascending, "A", null, -1), + ("String Desc: Null vs Val", DataType.StringDataType, Descending, null, "A", -1), + ("String Desc: Val vs Null", DataType.StringDataType, Descending, "A", null, 1), + //Comparable + ("Int Asc: 1 vs 10", DataType.IntegerDataType, Ascending, 1.asInstanceOf[AnyRef], 10.asInstanceOf[AnyRef], -1), + ("Int Desc: 1 vs 10", DataType.IntegerDataType, Descending, 1.asInstanceOf[AnyRef], 10.asInstanceOf[AnyRef], 1), + //Comparable: Null Handling + ("Comparable Asc: Null vs 100", DataType.IntegerDataType, Ascending, null, 100.asInstanceOf[AnyRef], 1), + ("Comparable Desc: Null vs 100", DataType.IntegerDataType, Descending, null, 100.asInstanceOf[AnyRef], -1) + ) + + forAll(testCases) { (label, dataType, direction, val1, val2, expectedSign) => + Given(s"a comparator for $label") + val column = createColumn("testCol", dataType) + val comparator = SortProjectionComparator(Array(column), Array(direction)) + + When("two rows are compared (offset by 1 for row key)") + val row1 = Array[AnyRef]("key1", val1) + val row2 = Array[AnyRef]("key2", val2) + val result = comparator.compare(row1, row2) + + Then(s"the result should have sign $expectedSign") + if (expectedSign == 0) result shouldBe 0 + else if (expectedSign > 0) result should be > 0 + else result should be < 0 + } + } + + Scenario("Multi-column sort: should fall back to second column if first is equal") { + Given("a multi-column comparator (Col1 Asc, Col2 Desc)") + val col1 = createColumn("col1", DataType.StringDataType) + val col2 = createColumn("col2", DataType.IntegerDataType) + + val comparator = SortProjectionComparator( + Array(col1, col2), + Array(Ascending, Descending) + ) + + When("comparing rows where Col1 is identical") + // Projection structure: [Key, Col1, Col2] + val row1 = Array[AnyRef]("k1", "Apple", 10.asInstanceOf[AnyRef]) + val row2 = Array[AnyRef]("k2", "Apple", 20.asInstanceOf[AnyRef]) + + val result = comparator.compare(row1, row2) + + Then("it should sort by Col2 in Descending order (20 > 10, so row2 < row1)") + result should be > 0 // Row1(10) vs Row2(20) in Desc: 10 is "greater" than 20 + } + + Scenario("Identity check: same object references should return 0") { + Given("a comparator and a single row") + val col = createColumn("col", DataType.StringDataType) + val comparator = SortProjectionComparator(Array(col), Array(Ascending)) + val row = Array[AnyRef]("key", "SomeValue") + + When("comparing a row with itself") + val result = comparator.compare(row, row) + + Then("the result should be 0 immediately (eq check)") + result shouldBe 0 + } + } +} \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/core/sort/SortSpecParserTest.scala b/vuu/src/test/scala/org/finos/vuu/core/sort/SortSpecParserTest.scala index 959d12b1c..0d0578a78 100644 --- a/vuu/src/test/scala/org/finos/vuu/core/sort/SortSpecParserTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/core/sort/SortSpecParserTest.scala @@ -44,7 +44,7 @@ class SortSpecParserTest extends AnyFeatureSpec with Matchers { sort.isInstanceOf[NoSort.type] shouldBe true } - Scenario("Spec with a mix of columns") { + Scenario("Spec with a mix of valid and invalid columns and directions") { val validColumn = table.columns().head val spec = SortSpec( @@ -56,17 +56,14 @@ class SortSpecParserTest extends AnyFeatureSpec with Matchers { ) val sort = SortSpecParser.parse(spec, viewPortColumns) - sort.isInstanceOf[GenericSort2] shouldBe true val sortImpl = sort.asInstanceOf[GenericSort2] sortImpl.columns.length shouldEqual 1 sortImpl.columns.head shouldEqual validColumn - sortImpl.spec.sortDefs.length shouldEqual 1 - val sortDef = sortImpl.spec.sortDefs.head - sortDef.column shouldEqual validColumn.name - sortDef.sortType shouldEqual SortDirection.Ascending.external + sortImpl.sortDirections.length shouldEqual 1 + sortImpl.sortDirections.head shouldEqual SortDirection.Ascending } }