Skip to content

Commit b5fa0bc

Browse files
kok-foongkok-foong
authored
Merge pull request #1443 from cambridge-cares/dev-modify-existing-timeseries
dev-modify-existing-timeseries
2 parents 126ef51 + 41162fb commit b5fa0bc

File tree

8 files changed

+238
-8
lines changed

8 files changed

+238
-8
lines changed

‎JPS_BASE_LIB/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
<!-- Please refer to the Versioning page on TheWorldAvatar wiki for
1212
details on how version numbers should be selected -->
13-
<version>1.45.0</version>
13+
<version>1.46.0</version>
1414

1515
<!-- Project Properties -->
1616
<properties>

‎JPS_BASE_LIB/src/main/java/uk/ac/cam/cares/jps/base/timeseries/TimeSeriesClient.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1341,6 +1341,41 @@ public void bulkInitTimeSeries(List<List<String>> dataIRIs, List<List<Class<?>>>
13411341
}
13421342
}
13431343

1344+
/**
1345+
* add columns to an existing timeseries, srid is only used when one of the new
1346+
* columns has the geometry type, can be null
1347+
*
1348+
* @param timeSeriesIri
1349+
* @param dataIri
1350+
* @param classes
1351+
* @param srid
1352+
* @param conn
1353+
*/
1354+
public void addColumnsToExistingTimeSeries(String timeSeriesIri, List<String> dataIri, List<Class<?>> classes,
1355+
Integer srid, Connection conn) {
1356+
if (rdbClient.checkAnyDataHasTimeSeries(dataIri, conn) != null) {
1357+
throw new JPSRuntimeException(
1358+
exceptionPrefix + "one or more provided data IRI contains an existing time series");
1359+
}
1360+
1361+
if (!rdbClient.timeSeriesExists(timeSeriesIri, conn)) {
1362+
throw new JPSRuntimeException(
1363+
exceptionPrefix + "provided time series does not exist");
1364+
}
1365+
1366+
rdfClient.addDataToExistingTimeSeries(dataIri, timeSeriesIri);
1367+
rdbClient.addColumnsToExistingTimeSeries(dataIri, classes, timeSeriesIri, srid, conn);
1368+
}
1369+
1370+
public void addColumnsToExistingTimeSeries(String timeSeriesIri, List<String> dataIri, List<Class<?>> classes,
1371+
Integer srid) {
1372+
try (Connection conn = rdbClient.getConnection()) {
1373+
addColumnsToExistingTimeSeries(timeSeriesIri, dataIri, classes, srid, conn);
1374+
} catch (SQLException e) {
1375+
throw new JPSRuntimeException(exceptionPrefix + CONNECTION_ERROR, e);
1376+
}
1377+
}
1378+
13441379
public String getRdbUrl() {
13451380
return rdbClient.getRdbURL();
13461381
}

‎JPS_BASE_LIB/src/main/java/uk/ac/cam/cares/jps/base/timeseries/TimeSeriesRDBClient.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1696,6 +1696,79 @@ public void deleteAll() {
16961696
}
16971697
}
16981698

1699+
/**
1700+
* Adds columns to an existing time series table, set srid to null if no
1701+
* geometry columns are required
1702+
*/
1703+
@Override
1704+
public void addColumnsToExistingTimeSeries(List<String> dataIRIs, List<Class<?>> dataClasses, String tsIri,
1705+
Integer srid, Connection conn) {
1706+
DSLContext context = DSL.using(conn);
1707+
1708+
// there is already a check on whether time series exists before this method is
1709+
// called
1710+
String tsTableName = context.select(TABLENAME_COLUMN).from(getDSLTable(DB_TABLE_NAME))
1711+
.where(TS_IRI_COLUMN.eq(tsIri)).fetch(TABLENAME_COLUMN).get(0);
1712+
1713+
List<String> existingColumns = context.select(COLUMNNAME_COLUMN).from(getDSLTable(DB_TABLE_NAME))
1714+
.where(TS_IRI_COLUMN.eq(tsIri)).fetch(COLUMNNAME_COLUMN);
1715+
1716+
int largestNum = 0;
1717+
for (String column : existingColumns) {
1718+
// extract number
1719+
int number = Integer.parseInt(column.split("column")[1]);
1720+
if (number > largestNum) {
1721+
largestNum = number;
1722+
}
1723+
}
1724+
1725+
// Assign column name for each dataIRI; name for time column is fixed
1726+
Map<String, String> dataColumnNames = new HashMap<>();
1727+
largestNum++;
1728+
for (String s : dataIRIs) {
1729+
dataColumnNames.put(s, "column" + largestNum);
1730+
largestNum++;
1731+
}
1732+
1733+
// Add corresponding entries in central lookup table
1734+
populateCentralTable(tsTableName, dataIRIs, dataColumnNames, tsIri, conn);
1735+
1736+
List<String> additionalGeomColumns = new ArrayList<>();
1737+
List<Class<?>> classForAdditionalGeomColumns = new ArrayList<>();
1738+
1739+
List<Query> allSteps = new ArrayList<>();
1740+
for (int i = 0; i < dataIRIs.size(); i++) {
1741+
if (Geometry.class.isAssignableFrom(dataClasses.get(i))) {
1742+
// these columns will be added with their respective restrictions
1743+
additionalGeomColumns.add(dataColumnNames.get(dataIRIs.get(i)));
1744+
classForAdditionalGeomColumns.add(dataClasses.get(i));
1745+
} else {
1746+
allSteps.add(context.alterTable(getDSLTable(tsTableName)).add(dataColumnNames.get(dataIRIs.get(i)),
1747+
DefaultDataType.getDataType(DIALECT, dataClasses.get(i))));
1748+
}
1749+
}
1750+
1751+
context.batch(allSteps).execute();
1752+
1753+
// add remaining geometry columns with restrictions
1754+
try {
1755+
if (!additionalGeomColumns.isEmpty()) {
1756+
addGeometryColumns(tsTableName, additionalGeomColumns, classForAdditionalGeomColumns, srid, conn);
1757+
}
1758+
} catch (SQLException e) {
1759+
String errmsg = "Failed to add geometry columns";
1760+
LOGGER.error(errmsg);
1761+
LOGGER.error(e.getMessage());
1762+
throw new JPSRuntimeException(errmsg, e);
1763+
}
1764+
}
1765+
1766+
@Override
1767+
public boolean timeSeriesExists(String tsIRI, Connection conn) {
1768+
DSLContext context = DSL.using(conn);
1769+
return context.fetchExists(selectFrom(getDSLTable(DB_TABLE_NAME)).where(TS_IRI_COLUMN.eq(tsIRI)));
1770+
}
1771+
16991772
@Override
17001773
public Connection getConnection() throws SQLException {
17011774
try {

‎JPS_BASE_LIB/src/main/java/uk/ac/cam/cares/jps/base/timeseries/TimeSeriesRDBClientInterface.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ void initTimeSeriesTable(List<String> dataIRI, List<Class<?>> dataClass, String
7878
* @param dataClasses
7979
* @param tsIRIs
8080
* @param srid
81-
* @param conn connection to the RDB
81+
* @param conn connection to the RDB
8282
* @return
8383
*/
8484
List<Integer> bulkInitTimeSeriesTable(List<List<String>> dataIRIs, List<List<Class<?>>> dataClasses,
@@ -378,11 +378,25 @@ List<Integer> bulkInitTimeSeriesTable(List<List<String>> dataIRIs, List<List<Cla
378378
* Check if all given data IRI is attached to a time series in kb
379379
*
380380
* @param dataIRIs data IRIs provided as list of string
381-
* @param conn connection to the RDB
381+
* @param conn connection to the RDB
382382
* @return the first dataIRI that exists and is attached to a time series, null
383383
* otherwise
384384
*/
385385
String checkAnyDataHasTimeSeries(List<String> dataIRIs, Connection conn);
386386

387+
/**
388+
* Add columns to an existing time series
389+
*
390+
* @param dataIRIs list of data IRI
391+
* @param dataClasses corresponding class of data
392+
* @param tsIri IRI of time series
393+
* @param srid srid for geometry types
394+
* @param conn SQL connection to RDB
395+
*/
396+
void addColumnsToExistingTimeSeries(List<String> dataIRIs, List<Class<?>> dataClasses, String tsIri, Integer srid,
397+
Connection conn);
398+
399+
boolean timeSeriesExists(String tsIRI, Connection conn);
400+
387401
Connection getConnection() throws SQLException;
388402
}

‎JPS_BASE_LIB/src/main/java/uk/ac/cam/cares/jps/base/timeseries/TimeSeriesRDBClientWithReducedTables.java‎

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ public List<Integer> bulkInitTimeSeriesTable(List<List<String>> dataIRIs, List<L
282282

283283
List<Map<String, String>> dataColumnNamesMaps = new ArrayList<>();
284284

285-
TimeSeriesDatabaseMetadata timeSeriesDatabaseMetadata = new TimeSeriesDatabaseMetadata();
285+
TimeSeriesDatabaseMetadata timeSeriesDatabaseMetadata = getTimeSeriesDatabaseMetadata(conn, tsTableName);
286286

287287
for (int i = 0; i < dataIRIs.size(); i++) {
288288
try {
@@ -1683,6 +1683,58 @@ public void deleteAll() {
16831683
}
16841684
}
16851685

1686+
@Override
1687+
public void addColumnsToExistingTimeSeries(List<String> dataIRIs, List<Class<?>> dataClasses, String tsIri,
1688+
Integer srid, Connection conn) {
1689+
DSLContext context = DSL.using(conn);
1690+
String tsTableName = getTableWithMatchingTimeColumn(conn);
1691+
1692+
// Assign column name for each dataIRI; name for time column is fixed
1693+
Map<String, String> dataColumnNames = new HashMap<>();
1694+
1695+
List<String> existingColumns = context.select(COLUMNNAME_COLUMN).from(getDSLTable(DB_TABLE_NAME))
1696+
.where(TS_IRI_COLUMN.eq(tsIri)).fetch(COLUMNNAME_COLUMN);
1697+
1698+
if (tsTableName != null) {
1699+
TimeSeriesDatabaseMetadata timeSeriesDatabaseMetadata = getTimeSeriesDatabaseMetadata(conn,
1700+
tsTableName, existingColumns);
1701+
List<Boolean> hasMatchingColumn = timeSeriesDatabaseMetadata.hasMatchingColumn(dataClasses, srid);
1702+
1703+
for (int i = 0; i < hasMatchingColumn.size(); i++) {
1704+
if (Boolean.TRUE.equals(hasMatchingColumn.get(i))) {
1705+
// use existing column
1706+
String existingSuitableColumn = timeSeriesDatabaseMetadata
1707+
.getExistingSuitableColumn(dataClasses.get(i), srid);
1708+
dataColumnNames.put(dataIRIs.get(i), existingSuitableColumn);
1709+
} else {
1710+
// add new columns
1711+
int columnIndex = getNumberOfDataColumns(tsTableName, conn) + 1;
1712+
String columnName = PREFIX_COLUMN + columnIndex;
1713+
try {
1714+
addColumn(tsTableName, dataClasses.get(i), columnName, srid, conn);
1715+
} catch (SQLException e) {
1716+
String errmsg = "Failed to add column for " + dataClasses.get(i).getSimpleName();
1717+
LOGGER.error(errmsg);
1718+
throw new JPSRuntimeException(errmsg);
1719+
}
1720+
1721+
dataColumnNames.put(dataIRIs.get(i), columnName);
1722+
}
1723+
}
1724+
} else {
1725+
String errmsg = "Probably the wrong time class is initialised for TimeSeriesClient";
1726+
throw new JPSRuntimeException(exceptionPrefix + errmsg);
1727+
}
1728+
1729+
populateCentralTable(tsTableName, dataIRIs, dataColumnNames, tsIri, conn);
1730+
}
1731+
1732+
@Override
1733+
public boolean timeSeriesExists(String tsIRI, Connection conn) {
1734+
DSLContext context = DSL.using(conn);
1735+
return context.fetchExists(selectFrom(getDSLTable(DB_TABLE_NAME)).where(TS_IRI_COLUMN.eq(tsIRI)));
1736+
}
1737+
16861738
@Override
16871739
public Connection getConnection() throws SQLException {
16881740
try {
@@ -1715,15 +1767,21 @@ private Table<Record> getDSLTable(String tableName) {
17151767
*
17161768
* @param conn
17171769
* @param tableName
1770+
* @param columnsToExclude // used when adding new columns
17181771
* @return
17191772
*/
1720-
private TimeSeriesDatabaseMetadata getTimeSeriesDatabaseMetadata(Connection conn, String tableName) {
1773+
private TimeSeriesDatabaseMetadata getTimeSeriesDatabaseMetadata(Connection conn, String tableName,
1774+
List<String> columnsToExclude) {
17211775
TimeSeriesDatabaseMetadata timeSeriesDatabaseMetadata = new TimeSeriesDatabaseMetadata();
1722-
addDataTypes(conn, timeSeriesDatabaseMetadata, tableName);
1776+
addDataTypes(conn, timeSeriesDatabaseMetadata, tableName, columnsToExclude);
17231777
addSpecificGeometryClass(conn, timeSeriesDatabaseMetadata, tableName);
17241778
return timeSeriesDatabaseMetadata;
17251779
}
17261780

1781+
private TimeSeriesDatabaseMetadata getTimeSeriesDatabaseMetadata(Connection conn, String tableName) {
1782+
return getTimeSeriesDatabaseMetadata(conn, tableName, new ArrayList<>());
1783+
}
1784+
17271785
/**
17281786
* obtains data from the information_schema.columns table on the initialised
17291787
* time series columns
@@ -1733,7 +1791,7 @@ private TimeSeriesDatabaseMetadata getTimeSeriesDatabaseMetadata(Connection conn
17331791
* @param tableName
17341792
*/
17351793
private void addDataTypes(Connection conn, TimeSeriesDatabaseMetadata timeSeriesDatabaseMetadata,
1736-
String tableName) {
1794+
String tableName, List<String> columnsToExclude) {
17371795
DSLContext context = DSL.using(conn, DIALECT);
17381796
Field<String> dataTypeColumn = DSL.field("data_type", String.class);
17391797
Field<String> udtNameColumn = DSL.field("udt_name", String.class);
@@ -1749,7 +1807,7 @@ private void addDataTypes(Connection conn, TimeSeriesDatabaseMetadata timeSeries
17491807
.select(dataTypeColumn, udtNameColumn, COLUMNNAME_COLUMN).from(COLUMNS_TABLE)
17501808
.where(condition).fetch();
17511809

1752-
queryResult.forEach(rec -> {
1810+
queryResult.stream().filter(rec -> !columnsToExclude.contains(rec.get(COLUMNNAME_COLUMN))).forEach(rec -> {
17531811
String dataType = rec.get(dataTypeColumn);
17541812
String udtName = rec.get(udtNameColumn);
17551813
String columnName = rec.get(COLUMNNAME_COLUMN);

‎JPS_BASE_LIB/src/main/java/uk/ac/cam/cares/jps/base/timeseries/TimeSeriesSparql.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -972,6 +972,20 @@ boolean hasAnyExistingTimeSeries(List<String> dataIriList) {
972972
} else {
973973
return anyExist;
974974
}
975+
}
976+
977+
/**
978+
*
979+
* @param dataIriList
980+
* @param tsIri
981+
*/
982+
void addDataToExistingTimeSeries(List<String> dataIriList, String tsIri) {
983+
ModifyQuery modify = Queries.MODIFY();
984+
985+
dataIriList.forEach(dataIri -> modify.insert(iri(dataIri).has(hasTimeSeries, iri(tsIri))));
975986

987+
modify.prefix(PREFIX_ONTOLOGY);
988+
989+
kbClient.executeUpdate(modify.getQueryString());
976990
}
977991
}

‎JPS_BASE_LIB/src/test/java/uk/ac/cam/cares/jps/base/timeseries/TimeSeriesClientIntegrationTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -664,4 +664,23 @@ void testBulkInitTimeSeries() throws SQLException {
664664
Assertions.assertEquals(dataIRI_2.size(), ts2.getDataIRIs().size());
665665
}
666666
}
667+
668+
@Test
669+
void testAddNewColumns() throws SQLException {
670+
try (Connection conn = rdbStoreClient.getConnection()) {
671+
tsClient.initTimeSeries(dataIRI_1, dataClass_1, timeUnit, conn);
672+
673+
// if there are no errors it can be assumed it is initialised correctly
674+
tsClient.getTimeSeries(dataIRI_1, conn);
675+
676+
TimeSeriesSparql timeSeriesSparql = new TimeSeriesSparql(kbClient);
677+
String tsIri = timeSeriesSparql.getTimeSeries(dataIRI_1.get(0));
678+
679+
tsClient.addColumnsToExistingTimeSeries(tsIri, dataIRI_2, dataClass_2, null, conn);
680+
681+
List<String> combinedList = new ArrayList<>(dataIRI_1);
682+
combinedList.addAll(dataIRI_2);
683+
tsClient.getTimeSeries(combinedList, conn);
684+
}
685+
}
667686
}

‎JPS_BASE_LIB/src/test/java/uk/ac/cam/cares/jps/base/timeseries/TimeSeriesClientIntegrationWithoutConnTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,4 +422,21 @@ void testBulkInitTimeSeries() {
422422
TimeSeries<Instant> ts2 = tsClient.getTimeSeries(dataIRI_2);
423423
Assertions.assertEquals(dataIRI_2.size(), ts2.getDataIRIs().size());
424424
}
425+
426+
@Test
427+
void testAddNewColumns() {
428+
tsClient.initTimeSeries(dataIRI_1, dataClass_1, timeUnit);
429+
430+
// if there are no errors it can be assumed it is initialised correctly
431+
tsClient.getTimeSeries(dataIRI_1);
432+
433+
TimeSeriesSparql timeSeriesSparql = new TimeSeriesSparql(kbClient);
434+
String tsIri = timeSeriesSparql.getTimeSeries(dataIRI_1.get(0));
435+
436+
tsClient.addColumnsToExistingTimeSeries(tsIri, dataIRI_2, dataClass_2, null);
437+
438+
List<String> combinedList = new ArrayList<>(dataIRI_1);
439+
combinedList.addAll(dataIRI_2);
440+
tsClient.getTimeSeries(combinedList);
441+
}
425442
}

0 commit comments

Comments
 (0)