diff --git a/plugin/ignite-plugin/pom.xml b/plugin/ignite-plugin/pom.xml index 427277ace2..c740c4c657 100644 --- a/plugin/ignite-plugin/pom.xml +++ b/plugin/ignite-plugin/pom.xml @@ -57,6 +57,12 @@ 0.9.65-SNAPSHOT + + org.finos.vuu.plugin + virtualized-table-plugin + 0.9.65-SNAPSHOT + + org.scala-lang scala-library diff --git a/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala b/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala new file mode 100644 index 0000000000..3403443168 --- /dev/null +++ b/plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/SchemaMapperFunctionalTest.scala @@ -0,0 +1,83 @@ +package org.finos.vuu.feature.ignite + +import org.finos.vuu.core.sort.SortDirection +import org.finos.vuu.core.table.{ColumnValueProvider, Columns} +import org.finos.vuu.net.FilterSpec +import org.finos.vuu.plugin.virtualized.api.VirtualizedSessionTableDef +import org.finos.vuu.provider.VirtualizedProvider +import org.finos.vuu.test.{FakeDataSource, FakeInMemoryTable} +import org.finos.vuu.util.schema.{ExternalEntitySchema, SchemaMapperBuilder, SchemaMapperFunctionalTestBase, SchemaTestData} +import org.finos.vuu.viewport.ViewPort + +class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { + + Feature("Filter data in virtualised table using schema mapper") { + Scenario("When table columns and entity fields has same type") { + + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema + val tableDef = VirtualizedSessionTableDef( + name = "MyExampleVirtualTable", + keyField = "id", + columns = Columns.fromExternalSchema(externalEntitySchema) + ) + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .build() + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + + //simulate using typeahead + givenColumnQueryReturns("unique", "clientId", Array("5","6")) + + val dataProvider = new TestVirtualProvider(fakeDataSource) + val columnValueProvider = dataProvider.asInstanceOf[ColumnValueProvider] + columnValueProvider.getUniqueValues("clientId") + + //todo assert on the result returned for typeahead + + //simulate using user entered filter and sort to the data query + val filterSpec = FilterSpec("orderId > 1 and ric starts \"ABC\"") + val sortSpec = Map("price" -> SortDirection.Ascending) + + val filterAndSortSpecToSql = FilterAndSortSpecToSql(schemaMapper) + filterAndSortSpecToSql.sortToSql(sortSpec) + filterAndSortSpecToSql.filterToSql(filterSpec) + + //todo assert that correct sql query is created - should use real ignite or assert on expected sql query? + + //todo test once query is returned it can be mapped appropriate to table rows & assert on what exist in table + givenQueryReturns("filtered", List( + List("testId1", 5, 10.5), + List("testId2", 6, 11.5), + List("testId3", 5, 12.5), + )) + fakeDataSource.getAsListOfValues("filtered") + } + + Scenario("When table columns and entity fields has different type"){} + } +} + + +class TestVirtualProvider(fakeDataSource:FakeDataSource[SchemaTestData]) extends VirtualizedProvider { + override def runOnce(viewPort: ViewPort): Unit = ??? + + override def getUniqueValues(columnName: String): Array[String] = getColumnQueryResult("unique", columnName) + + override def getUniqueValuesStartingWith(columnName: String, starts: String): Array[String] = ??? + + override def subscribe(key: String): Unit = ??? + + override def doStart(): Unit = ??? + + override def doStop(): Unit = ??? + + override def doInitialize(): Unit = ??? + + override def doDestroy(): Unit = ??? + + override val lifecycleId: String = "SchemaMapperFunctionalTest" + + private def getColumnQueryResult(queryName: String, columnName:String): Array[String] = { + fakeDataSource.getColumnValues(queryName, columnName) + .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) + } +} diff --git a/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java b/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java new file mode 100644 index 0000000000..194f9f8ea8 --- /dev/null +++ b/vuu/src/main/java/org/finos/vuu/util/ScalaCollectionConverter.java @@ -0,0 +1,30 @@ +package org.finos.vuu.util; + +import scala.jdk.CollectionConverters; + +import java.util.List; +import java.util.Map; + +public class ScalaCollectionConverter { + + public static scala.collection.immutable.Map toScala(Map m) { + return scala.collection.immutable.Map.from(scala.jdk.CollectionConverters.MapHasAsScala(m).asScala()); + } + + public static scala.collection.Iterable toScala(Iterable l) { + return CollectionConverters.IterableHasAsScala(l).asScala(); + } + + public static scala.collection.immutable.List toScala(List l) { + return CollectionConverters.IterableHasAsScala(l).asScala().toList(); + } + + public static scala.collection.immutable.Seq toScalaSeq(List l) { + return CollectionConverters.IterableHasAsScala(l).asScala().toSeq(); + } + + public static List toJava(scala.collection.immutable.List l) { + return CollectionConverters.SeqHasAsJava(l).asJava(); + } +} + diff --git a/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala new file mode 100644 index 0000000000..8ffc61f881 --- /dev/null +++ b/vuu/src/main/scala/org/finos/vuu/api/ColumnBuilder.scala @@ -0,0 +1,44 @@ +package org.finos.vuu.api + +import org.finos.vuu.core.table.{Column, Columns} + +import scala.collection.mutable + +class ColumnBuilder { + + val columns = new mutable.ArrayBuilder.ofRef[String]() + + def addString(columnName: String): ColumnBuilder = { + columns += (columnName + ":String") + this + } + + def addDouble(columnName: String): ColumnBuilder = { + columns += (columnName + ":Double") + this + } + + def addInt(columnName: String): ColumnBuilder = { + columns += (columnName + ":Int") + this + } + + def addLong(columnName: String): ColumnBuilder = { + columns += (columnName + ":Long") + this + } + + def addBoolean(columnName: String): ColumnBuilder = { + columns += (columnName + ":Boolean") + this + } + + def addChar(columnName: String): ColumnBuilder = { + columns += (columnName + ":Char") + this + } + + def build(): Array[Column] = Columns.fromNames(columns.result()) +} + + diff --git a/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala b/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala index d5a0ca9e3b..7671fb4304 100644 --- a/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala +++ b/vuu/src/main/scala/org/finos/vuu/core/table/Column.scala @@ -3,6 +3,7 @@ package org.finos.vuu.core.table import com.typesafe.scalalogging.StrictLogging import org.finos.vuu.api.TableDef import org.finos.vuu.core.table.column.CalculatedColumnClause +import org.finos.vuu.util.schema.ExternalEntitySchema import org.finos.vuu.util.types.{DefaultTypeConverters, TypeConverterContainerBuilder} import scala.util.Try @@ -86,8 +87,6 @@ object Columns { table.columns.filter(c => aliased.contains(c.name)) map (c => new AliasedJoinColumn(aliased(c.name), c.index, c.dataType, table, c).asInstanceOf[Column]) } - //def calculated(name: String, definition: String): Array[Column] = ??? - def allFromExcept(table: TableDef, excludeColumns: String*): Array[Column] = { val excluded = excludeColumns.map(s => s -> 1).toMap @@ -95,6 +94,14 @@ object Columns { table.columns.filterNot(c => excluded.contains(c.name)).map(c => new JoinColumn(c.name, c.index, c.dataType, table, c)) } + /** + * Create columns that use same name, type, order as the external entity fields + * */ + def fromExternalSchema(externalSchema: ExternalEntitySchema): Array[Column] = { + externalSchema.fields.map(field => SimpleColumn(field.name, field.index, field.dataType)) + .toArray + } + } trait Column { diff --git a/vuu/src/test/java/org/finos/vuu/util/ScalaList.java b/vuu/src/test/java/org/finos/vuu/util/ScalaList.java new file mode 100644 index 0000000000..11bf3b172a --- /dev/null +++ b/vuu/src/test/java/org/finos/vuu/util/ScalaList.java @@ -0,0 +1,11 @@ +package org.finos.vuu.util; + +import java.util.Arrays; + +import static org.finos.vuu.util.ScalaCollectionConverter.toScala; + +public class ScalaList { + public static scala.collection.immutable.List of(T... args) { + return toScala(Arrays.asList(args)); + } +} diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java new file mode 100644 index 0000000000..d5a32ff09f --- /dev/null +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaJavaTestData.java @@ -0,0 +1,7 @@ +package org.finos.vuu.util; + +public class SchemaJavaTestData { + public String Id; + public int ClientId; + public double NotionalValue; +} diff --git a/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java b/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java new file mode 100644 index 0000000000..0afc516948 --- /dev/null +++ b/vuu/src/test/java/org/finos/vuu/util/SchemaMapperJavaFunctionalTest.java @@ -0,0 +1,179 @@ +package org.finos.vuu.util; + + +import org.finos.toolbox.time.Clock; +import org.finos.toolbox.time.TestFriendlyClock; +import org.finos.vuu.api.ColumnBuilder; +import org.finos.vuu.api.TableDef; +import org.finos.vuu.core.table.Columns; +import org.finos.vuu.core.table.RowWithData; +import org.finos.vuu.test.FakeDataSource; +import org.finos.vuu.test.FakeInMemoryTable; +import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder; +import org.finos.vuu.util.schema.SchemaMapper; +import org.finos.vuu.util.schema.SchemaMapperBuilder; +import org.finos.vuu.util.types.TypeConverter; +import org.finos.vuu.util.types.TypeConverterContainerBuilder; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import scala.jdk.javaapi.OptionConverters; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import static org.finos.vuu.util.ScalaCollectionConverter.*; +import static org.junit.Assert.assertEquals; + + +@RunWith(Enclosed.class) +public class SchemaMapperJavaFunctionalTest { + + private static String queryName = "myQuery"; + private static final FakeDataSource dataSource = new FakeDataSource<>(); + private static final Clock clock = new TestFriendlyClock(10001L); + + @Before + public void setUp() { + queryName += java.util.UUID.randomUUID().toString(); // unique query name for each test run + } + + public static class UpdateInMemoryTable { + @Test + public void when_table_columns_and_entity_fields_match_exactly() throws Exception { + + var externalEntitySchema = ExternalEntitySchemaBuilder.apply() + .withEntity(SchemaJavaTestData.class) + .withIndex("ID_INDEX", toScala(List.of("Id"))) + .build(); + var tableDef = TableDef.apply( + "MyJavaExampleTable", + "Id", + Columns.fromExternalSchema(externalEntitySchema), + toScalaSeq(List.of()) + ); + var schemaMapper = SchemaMapperBuilder.apply(externalEntitySchema, tableDef.columns()) + .build(); + var table = new FakeInMemoryTable("SchemaMapJavaTest", tableDef); + dataSource.setUpResultAsListOfValues( + queryName, + ScalaList.of(ScalaList.of("testId1", 5, 10.5)) + ); + + getDataAndUpdateTable(queryName, schemaMapper, table); + + var existingRows = toJava(table.pullAllRows()); + assertEquals(existingRows.size(), 1); + assertEquals(existingRows.get(0).get("Id"), "testId1"); + assertEquals(existingRows.get(0).get("ClientId"), 5); + assertEquals(existingRows.get(0).get("NotionalValue"), 10.5); + } + + @Test + public void when_table_columns_and_entity_fields_does_not_match_exactly() throws Exception { + + var externalEntitySchema = ExternalEntitySchemaBuilder.apply() + .withEntity(SchemaJavaTestData.class) + .withIndex("ID_INDEX", toScala(List.of("Id"))) + .build(); + var tableDef = TableDef.apply( + "MyJavaExampleTable", + "Id", + new ColumnBuilder() + .addDouble("SomeOtherName") + .addString("Id") + .addInt("ClientId") + .build(), + toScalaSeq(List.of()) + ); + var schemaMapper = SchemaMapperBuilder.apply(externalEntitySchema, tableDef.columns()) + .withFieldsMap( + toScala(Map.of("Id", "Id", + "ClientId", "ClientId", + "NotionalValue", "SomeOtherName" + )) + ) + .build(); + var table = new FakeInMemoryTable("SchemaMapJavaTest", tableDef); + dataSource.setUpResultAsListOfValues( + queryName, + ScalaList.of(ScalaList.of("testId1", 5, 10.5)) + ); + + getDataAndUpdateTable(queryName, schemaMapper, table); + + var existingRows = toJava(table.pullAllRows()); + assertEquals(existingRows.size(), 1); + assertEquals(existingRows.get(0).get("Id"), "testId1"); + assertEquals(existingRows.get(0).get("ClientId"), 5); + assertEquals(existingRows.get(0).get("SomeOtherName"), 10.5); + + } + + @Test + public void when_table_columns_and_entity_fields_has_different_types() throws Exception { + + var externalEntitySchema = ExternalEntitySchemaBuilder.apply() + .withField("Id", Integer.class) + .withField("decimalValue", BigDecimal.class) + .withIndex("ID_INDEX", toScala(List.of("Id"))) + .build(); + var tableDef = TableDef.apply( + "MyJavaExampleTable", + "Id", + new ColumnBuilder() + .addString("Id") + .addDouble("doubleValue") + .build(), + toScalaSeq(List.of()) + ); + var typeConverterContainer = TypeConverterContainerBuilder.apply() + .withConverter(TypeConverter.apply(BigDecimal.class, Double.class, BigDecimal::doubleValue)) + .withConverter(TypeConverter.apply(Double.class, BigDecimal.class, v -> new BigDecimal(v.toString()))) + .build(); + var schemaMapper = SchemaMapperBuilder.apply(externalEntitySchema, tableDef.columns()) + .withFieldsMap( + toScala(Map.of("Id", "Id", + "decimalValue","doubleValue" + )) + ) + .withTypeConverters(typeConverterContainer) + .build(); + var table = new FakeInMemoryTable("SchemaMapJavaTest", tableDef); + dataSource.setUpResultAsListOfValues( + queryName, + ScalaList.of(ScalaList.of(10, new BigDecimal("1.0001"))) + ); + + getDataAndUpdateTable(queryName, schemaMapper, table); + + var existingRows = toJava(table.pullAllRows()); + assertEquals(existingRows.size(), 1); + assertEquals(existingRows.get(0).get("Id"), "10"); + assertEquals(existingRows.get(0).get("doubleValue"), 1.0001d); + } + } + + private static void getDataAndUpdateTable(String queryName, SchemaMapper schemaMapper, FakeInMemoryTable table) throws Exception { + //todo should use fake java store which is more likely usecase and avoid all the type conversions? + var keyFieldName = table.getTableDef().keyField(); + getQueryResult(queryName).stream() + .map(rowValues -> mapToRow(schemaMapper, rowValues, keyFieldName)) + .forEach(row -> table.processUpdate(row.key(), row, clock.now())); + } + + private static RowWithData mapToRow(SchemaMapper schemaMapper, List valueList, String keyFieldName) { + var rowMap = schemaMapper.toInternalRowMap(toScala(valueList)); + var keyOptional = OptionConverters.toJava(rowMap.get(keyFieldName)); + var key = keyOptional.orElseThrow(); + return new RowWithData(key.toString(), rowMap); + } + + private static List> getQueryResult(String queryName) throws Exception { + return OptionConverters.toJava(dataSource.getAsListOfValues(queryName)) + .map(listOfLists -> toJava(listOfLists.map(ScalaCollectionConverter::toJava).toList())) + .orElseThrow(() -> new Exception("Query does not exist in store. make sure it is setup")); + } +} diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala new file mode 100644 index 0000000000..ae462b9aad --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeDataSource.scala @@ -0,0 +1,35 @@ +package org.finos.vuu.test +class FakeDataSource[T]{ + + private val queryToEntityResultMap = scala.collection.mutable.HashMap.empty[String, List[T]] + private val queryToValuesResultMap = scala.collection.mutable.HashMap.empty[String, List[List[Any]]] + private val queryToColumnResultMap = scala.collection.mutable.HashMap.empty[String, Array[String]] + + def setUpResultAsObjects(queryName:String, queryResult:List[T]): Unit = { + queryToEntityResultMap += (queryName -> queryResult) + } + + def getAsObjects(queryName: String): Option[List[T]] = { + queryToEntityResultMap.get(queryName) + } + + def setUpResultAsListOfValues(queryName: String, resultValues: List[List[Any]]): Unit = { + queryToValuesResultMap += (queryName -> resultValues) + } + + def getAsListOfValues(queryName: String): Option[List[List[Any]]] = { + queryToValuesResultMap.get(queryName) + } + + def setUpResultGivenColumn(queryName:String, columnName:String, queryResult:Array[String]): Unit = { + queryToColumnResultMap += (getQueryColumnName(queryName, columnName) -> queryResult) + } + + def getColumnValues(queryName: String, columnName:String): Option[Array[String]] = { + queryToColumnResultMap.get(getQueryColumnName(queryName, columnName)) + } + + private def getQueryColumnName(queryName: String, columnName: String) = { + queryName + ":" + columnName + } +} \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala new file mode 100644 index 0000000000..5df637338e --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/test/FakeInMemoryTable.scala @@ -0,0 +1,62 @@ +package org.finos.vuu.test + +import org.finos.vuu.api.TableDef +import org.finos.vuu.core.index.IndexedField +import org.finos.vuu.core.table.{Column, ColumnValueProvider, DataTable, KeyObserver, RowData, RowKeyUpdate, RowWithData, TableData, TablePrimaryKeys} +import org.finos.vuu.viewport.{RowProcessor, ViewPortColumns} + +class FakeInMemoryTable(val instanceName: String, val tableDef: TableDef) extends DataTable { + + private val rowMap = scala.collection.mutable.HashMap.empty[String, RowWithData] + + override def name: String = instanceName + override def getTableDef: TableDef = tableDef + + override def processUpdate(rowKey: String, rowUpdate: RowWithData, timeStamp: Long): Unit = + rowMap += (rowKey -> rowUpdate) + + override def pullRow(key: String): RowData = + rowMap.getOrElse(key, throw new Exception(s"Could not find row data for key $key in table $name")) + + def pullAllRows() : List[RowWithData] = rowMap.values.toList + + override protected def createDataTableData(): TableData = ??? + + override def updateCounter: Long = ??? + + override def incrementUpdateCounter(): Unit = ??? + + override def indexForColumn(column: Column): Option[IndexedField[_]] = ??? + + override def getColumnValueProvider: ColumnValueProvider = ??? + + override def processDelete(rowKey: String): Unit = ??? + + override def notifyListeners(rowKey: String, isDelete: Boolean): Unit = ??? + + override def linkableName: String = ??? + + override def readRow(key: String, columns: List[String], processor: RowProcessor): Unit = ??? + + override def primaryKeys: TablePrimaryKeys = ??? + + override def pullRow(key: String, columns: ViewPortColumns): RowData = ??? + + override def pullRowFiltered(key: String, columns: ViewPortColumns): RowData = ??? + + override def pullRowAsArray(key: String, columns: ViewPortColumns): Array[Any] = ??? + + override def getObserversByKey(): Map[String, Array[KeyObserver[RowKeyUpdate]]] = ??? + + override def addKeyObserver(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def removeKeyObserver(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def getObserversByKey(key: String): List[KeyObserver[RowKeyUpdate]] = ??? + + override def isKeyObserved(key: String): Boolean = ??? + + override def isKeyObservedBy(key: String, observer: KeyObserver[RowKeyUpdate]): Boolean = ??? + + override def removeAllObservers(): Unit = ??? +} \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala new file mode 100644 index 0000000000..024766c44a --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTest.scala @@ -0,0 +1,168 @@ +package org.finos.vuu.util.schema + +import org.finos.vuu.api.{ColumnBuilder, TableDef} +import org.finos.vuu.core.table.Columns +import org.finos.vuu.test.FakeInMemoryTable +import org.finos.vuu.util.types.{TypeConverter, TypeConverterContainerBuilder} + +class SchemaMapperFunctionalTest extends SchemaMapperFunctionalTestBase { + + Feature("Update in memory table using schema mapper") { + Scenario("When table columns and entity fields match exactly") { + + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "id", + columns = Columns.fromExternalSchema(externalEntitySchema) + ) + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .build() + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + givenQueryReturns(queryName, List(List("testId1", 5, 10.5))) + + getDataAndUpdateTable(table, schemaMapper, queryName) + + val existingTableRows = table.pullAllRows() + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("id") == "testId1") + assert(existingTableRows.head.get("clientId") == 5) + assert(existingTableRows.head.get("notionalValue") == 10.5) + } + + Scenario("When table has fewer columns than fields on external entity") { + + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "id", + columns = new ColumnBuilder() + .addString("id") + .addDouble("notionalValue") + .build() + ) + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .withFieldsMap( + Map( + "id" -> "id", + "notionalValue" -> "notionalValue", + ) + ) + .build() + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + givenQueryReturns(queryName, List(List("testId1", 5, 10.5))) + + getDataAndUpdateTable(table, schemaMapper, queryName) + + val existingTableRows = table.pullAllRows() + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("id") == "testId1") + assert(existingTableRows.head.get("notionalValue") == 10.5) + } + + Scenario("When table has columns with different name than fields on external entity") { + + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "firstColumn", + columns = new ColumnBuilder() + .addString("firstColumn") + .addInt("secondColumn") + .addDouble("thirdColumn") + .build() + ) + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .build() + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + givenQueryReturns(queryName, List(List("testId1", 5, 10.5))) + + getDataAndUpdateTable(table, schemaMapper, queryName) + + val existingTableRows = table.pullAllRows() + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("firstColumn") == "testId1") + assert(existingTableRows.head.get("secondColumn") == 5) + assert(existingTableRows.head.get("thirdColumn") == 10.5) + } + + Scenario("When table has columns are in different order from fields on external entity") { + + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "id", + columns = new ColumnBuilder() + .addDouble("notionalValue") + .addString("id") + .addInt("clientId") + .build() + ) + val externalEntitySchema: ExternalEntitySchema = createExternalEntitySchema + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .withFieldsMap( + Map( + "notionalValue" -> "notionalValue", + "id" -> "id", + "clientId" -> "clientId", + ) + ) + .build() + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + givenQueryReturns(queryName, List(List("testId1", 5, 10.5))) + + getDataAndUpdateTable(table, schemaMapper, queryName) + + val existingTableRows = table.pullAllRows() + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("id") == "testId1") + assert(existingTableRows.head.get("clientId") == 5) + assert(existingTableRows.head.get("notionalValue") == 10.5) + } + + Scenario("When table has columns with different type from fields on external entity") { + + val externalEntitySchema: ExternalEntitySchema = ExternalEntitySchemaBuilder() + .withField("id", classOf[Int]) + .withField("decimalValue", classOf[BigDecimal]) + .withIndex("ID_INDEX", List("id")) + .build() + + val tableDef = TableDef( + name = "MyExampleTable", + keyField = "id", + columns = new ColumnBuilder() + .addString("id") + .addDouble("doubleValue") + .build() + ) + val typeConverterContainer = TypeConverterContainerBuilder() + .withConverter(TypeConverter[BigDecimal, Double](classOf[BigDecimal], classOf[Double], _.doubleValue)) + .withConverter(TypeConverter[Double, BigDecimal](classOf[Double], classOf[BigDecimal], v => BigDecimal(v.toString))) + .build() + val schemaMapper = SchemaMapperBuilder(externalEntitySchema, tableDef.columns) + .withFieldsMap( + Map( + "id" -> "id", + "decimalValue" -> "doubleValue", + ) + ) + .withTypeConverters(typeConverterContainer) + .build() + val table = new FakeInMemoryTable("SchemaMapTest", tableDef) + givenQueryReturns(queryName, List(List(10, BigDecimal("1.0001")))) + + getDataAndUpdateTable(table, schemaMapper, queryName) + + val existingTableRows = table.pullAllRows() + assert(existingTableRows.size == 1) + assert(existingTableRows.head.get("id") == "10") + assert(existingTableRows.head.get("doubleValue") == 1.0001d) + + } + + Scenario("When query result has less number of fields than table columns return useful error") {} + Scenario("When column and field order does not match but no fields map defined on schema return useful error") {} + Scenario("When getting data from source fails return useful error") {} + Scenario("When getting casting data from source to table column type fails return useful error") {} + } +} \ No newline at end of file diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala new file mode 100644 index 0000000000..b6611595e0 --- /dev/null +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperFunctionalTestBase.scala @@ -0,0 +1,53 @@ +package org.finos.vuu.util.schema + +import org.finos.toolbox.time.TestFriendlyClock +import org.finos.vuu.core.table.{DataTable, RowWithData} +import org.finos.vuu.test.FakeDataSource +import org.scalatest.BeforeAndAfterEach +import org.scalatest.featurespec.AnyFeatureSpec + +class SchemaMapperFunctionalTestBase extends AnyFeatureSpec with BeforeAndAfterEach { + + protected val fakeDataSource: FakeDataSource[SchemaTestData] = new FakeDataSource[SchemaTestData] + protected var queryName: String = "myQuery" + private val clock = new TestFriendlyClock(10001L) + + override def beforeEach(): Unit = { + queryName += java.util.UUID.randomUUID.toString // unique query name for each test run + } + + protected def getDataAndUpdateTable(table: DataTable, schemaMapper: SchemaMapper, queryName: String): Unit = { + val keyFieldName = table.getTableDef.keyField + + getQueryResult(queryName) + .map(rowValues => mapToRow(schemaMapper, keyFieldName, rowValues)) + .foreach(row => table.processUpdate(row.key, row, clock.now())) + } + + private def mapToRow(schemaMapper: SchemaMapper, keyFieldName: String, rowValues: List[Any]) = { + val rowMap = schemaMapper.toInternalRowMap(rowValues) + val keyValue = rowMap(keyFieldName).toString + RowWithData(keyValue, rowMap) + } + + protected def givenQueryReturns(queryName: String, results: List[List[Any]]): Unit = { + fakeDataSource.setUpResultAsListOfValues(queryName, results) + } + + private def getQueryResult(queryName: String): List[List[Any]] = { + fakeDataSource.getAsListOfValues(queryName) + .getOrElse(throw new Exception("query does not exist in store. make sure it is setup")) + } + + protected def givenColumnQueryReturns(queryName: String, columnName:String, results: Array[String]): Unit = { + fakeDataSource.setUpResultGivenColumn(queryName, columnName, results) + } + + protected def createExternalEntitySchema: ExternalEntitySchema = + ExternalEntitySchemaBuilder() + .withEntity(classOf[SchemaTestData]) + .withIndex("ID_INDEX", List("id")) + .build() +} + +case class SchemaTestData(id: String, clientId: Int, notionalValue: Double) {} \ No newline at end of file