diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/ExportToFiles.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/ExportToFiles.java index 979106f79e..fd48de126b 100644 --- a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/ExportToFiles.java +++ b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/ExportToFiles.java @@ -19,14 +19,35 @@ package com.here.xyz.jobs.steps.compiler; +import static com.here.xyz.events.ContextAwareEvent.SpaceContext.DEFAULT; +import static com.here.xyz.events.ContextAwareEvent.SpaceContext.EXTENSION; + +import com.here.xyz.util.geo.GeoTools; +import com.here.xyz.util.service.BaseHttpServerVerticle.ValidationException; +import com.here.xyz.util.web.HubWebClient; +import com.here.xyz.util.web.XyzWebClient.WebClientException; +import com.here.xyz.events.ContextAwareEvent.SpaceContext; +import com.here.xyz.events.PropertiesQuery; import com.here.xyz.jobs.Job; import com.here.xyz.jobs.datasets.DatasetDescription.Space; import com.here.xyz.jobs.datasets.Files; import com.here.xyz.jobs.datasets.files.GeoJson; +import com.here.xyz.jobs.datasets.filters.SpatialFilter; import com.here.xyz.jobs.steps.CompilationStepGraph; +import com.here.xyz.jobs.steps.Config; +import com.here.xyz.jobs.steps.JobCompiler.CompilationError; import com.here.xyz.jobs.steps.impl.transport.ExportSpaceToFiles; +import com.here.xyz.psql.query.Spatial; +import com.here.xyz.responses.StatisticsResponse; + + import java.util.Map; +import javax.xml.crypto.dsig.TransformException; + +import org.geotools.api.referencing.FactoryException; +import org.locationtech.jts.geom.Geometry; + public class ExportToFiles implements JobCompilationInterceptor { @Override public boolean chooseMe(Job job) { @@ -50,4 +71,63 @@ public static ExportSpaceToFiles compile(String jobId, Space source) { .withVersionRef(source.getVersionRef()) .withOutputMetadata(Map.of(source.getClass().getSimpleName().toLowerCase(), source.getId())); } + + private static HubWebClient hubWebClient() { + return HubWebClient.getInstance(Config.instance.HUB_ENDPOINT); + } + + public static boolean canExportFromDb(Space source) throws ValidationException { + + String spaceId = source.getId(); + SpaceContext sourceContext; + StatisticsResponse sourceStatistics; + + try { + sourceStatistics = hubWebClient().loadSpaceStatistics(spaceId, null); + sourceContext = hubWebClient().loadSpace(spaceId).getExtension() != null ? EXTENSION : DEFAULT; + sourceStatistics = hubWebClient().loadSpaceStatistics(spaceId, sourceContext); + } catch (WebClientException e) { + throw new CompilationError("Error resolving resource information: " + e.getMessage(), e); + } + + boolean hasFilters = source.getFilters() != null + && ( source.getFilters().getPropertyFilter() != null + || source.getFilters().getSpatialFilter() != null ); + + long maxAllowedFeatureCount = 100_000l; + + if(!hasFilters) // less then 100k features ok to export from DB + return sourceStatistics.getCount().getValue() <= maxAllowedFeatureCount; + + SpatialFilter spatialFilter = source.getFilters().getSpatialFilter(); + + if( spatialFilter != null && spatialFilter != null ) + { + try { spatialFilter.validateSpatialFilter(); } + catch (ValidationException e) + { throw e; } + + Geometry jtsGeometry = spatialFilter.getGeometry().getJTSGeometry(); + + if(jtsGeometry != null && !jtsGeometry.isValid()) + throw new ValidationException("Invalid geometry in spatialFilter!"); + + try { + long MAX_ALLOWED_SPATALFILTER_AREA_IN_SQUARE_KM = 1000l; //tbd: 1000km2 + Geometry bufferedGeo = GeoTools.applyBufferInMetersToGeometry(jtsGeometry, spatialFilter.getRadius()); + int areaInSquareKilometersFromGeometry = (int) GeoTools.getAreaInSquareKilometersFromGeometry(bufferedGeo); + + if (areaInSquareKilometersFromGeometry <= MAX_ALLOWED_SPATALFILTER_AREA_IN_SQUARE_KM) + return true; + } catch (FactoryException | TransformException | org.geotools.api.referencing.operation.TransformException e) { + throw new ValidationException("Error calculating area of spatialFilter: " + e.getMessage(), e); + } + } + +//TODO: check if propertyFilter is not null and if so "calculate" the size of the result set and decide if it is ok to export from DB +// PropertiesQuery propertyFilter = source.getFilters().getPropertyFilter(); + + return false; + } + } diff --git a/xyz-jobs/xyz-job-service/src/test/java/com/here/xyz/jobs/steps/compiler/ExportToFilesTest.java b/xyz-jobs/xyz-job-service/src/test/java/com/here/xyz/jobs/steps/compiler/ExportToFilesTest.java index 451050507b..8c9926fdfb 100644 --- a/xyz-jobs/xyz-job-service/src/test/java/com/here/xyz/jobs/steps/compiler/ExportToFilesTest.java +++ b/xyz-jobs/xyz-job-service/src/test/java/com/here/xyz/jobs/steps/compiler/ExportToFilesTest.java @@ -8,12 +8,22 @@ import com.here.xyz.jobs.datasets.FileOutputSettings; import com.here.xyz.jobs.datasets.Files; import com.here.xyz.jobs.datasets.files.GeoJson; +import com.here.xyz.jobs.datasets.filters.Filters; +import com.here.xyz.jobs.datasets.filters.SpatialFilter; import com.here.xyz.jobs.steps.CompilationStepGraph; import com.here.xyz.jobs.steps.impl.transport.ExportSpaceToFiles; +import com.here.xyz.models.geojson.coordinates.LinearRingCoordinates; +import com.here.xyz.models.geojson.coordinates.PolygonCoordinates; +import com.here.xyz.models.geojson.coordinates.Position; +import com.here.xyz.models.geojson.exceptions.InvalidGeometryException; +import com.here.xyz.models.geojson.implementation.Polygon; import com.here.xyz.models.hub.Ref; import com.here.xyz.models.hub.Space; import com.here.xyz.models.hub.Tag; +import com.here.xyz.psql.query.Spatial; import com.here.xyz.util.service.BaseHttpServerVerticle; +import com.here.xyz.util.service.BaseHttpServerVerticle.ValidationException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -71,4 +81,35 @@ private Job buildExportJobWithVersionRef(Ref versionRef) { .withSource(new DatasetDescription.Space<>().withId(SPACE_ID).withVersionRef(versionRef)) .withTarget(new Files<>().withOutputSettings(new FileOutputSettings().withFormat(new GeoJson().withEntityPerLine(Feature)))); } + + + @Test + public void exportFromDb() { + + Polygon spatialSearchGeom; + float xmin = 7.0f, ymin = 50.0f, xmax = 10.1f, ymax = 60.1f; + LinearRingCoordinates lrc = new LinearRingCoordinates(); + lrc.add(new Position(xmin, ymin)); + lrc.add(new Position(xmax, ymin)); + lrc.add(new Position(xmax, ymax)); + lrc.add(new Position(xmin, ymax)); + lrc.add(new Position(xmin, ymin)); + PolygonCoordinates pc = new PolygonCoordinates(); + pc.add(lrc); + spatialSearchGeom = new Polygon().withCoordinates(pc); + + try { + SpatialFilter spatialFilter = new SpatialFilter().withGeometry(spatialSearchGeom).withRadius(3000); + Filters filters = new Filters().withSpatialFilter(spatialFilter); + + DatasetDescription.Space source = new DatasetDescription.Space<>().withId(SPACE_ID).withFilters(filters); + + boolean result = ExportToFiles.canExportFromDb(source); + Assertions.assertFalse(result); + + } catch (ValidationException | InvalidGeometryException e) { + Assertions.fail("Error in exportFromDb: " + e.getMessage()); + } + + } } diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/ExportSpaceToFiles.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/ExportSpaceToFiles.java index d881f53fbb..4362cd3aa0 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/ExportSpaceToFiles.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/ExportSpaceToFiles.java @@ -59,6 +59,8 @@ import com.here.xyz.util.db.SQLQuery; import com.here.xyz.util.geo.GeoTools; import com.here.xyz.util.service.BaseHttpServerVerticle.ValidationException; +import com.here.xyz.util.web.XyzWebClient.WebClientException; + import java.sql.SQLException; import java.util.List; import java.util.Objects; @@ -317,10 +319,9 @@ public boolean validate() throws ValidationException { if(restrictExtendOfSpatialFilter) { try { Geometry bufferedGeo = GeoTools.applyBufferInMetersToGeometry(jtsGeometry, spatialFilter.getRadius()); - int areaInSquareKilometersFromGeometry = (int) GeoTools.getAreaInSquareKilometersFromGeometry(bufferedGeo); - if (GeoTools.getAreaInSquareKilometersFromGeometry(bufferedGeo) > MAX_ALLOWED_SPATALFILTER_AREA_IN_SQUARE_KM) { - throw new ValidationException("Invalid SpatialFilter! Provided area of filter geometry is to large! [" - + areaInSquareKilometersFromGeometry + " km² > " + MAX_ALLOWED_SPATALFILTER_AREA_IN_SQUARE_KM + " km²]"); + double areaInSquareKilometersFromGeometry = GeoTools.getAreaInSquareKilometersFromGeometry(bufferedGeo); + if ( areaInSquareKilometersFromGeometry > MAX_ALLOWED_SPATALFILTER_AREA_IN_SQUARE_KM) { + throw new ValidationException( String.format("Invalid SpatialFilter! Provided area of filter geometry is to large! [%.2f km² > %d km²]",areaInSquareKilometersFromGeometry,MAX_ALLOWED_SPATALFILTER_AREA_IN_SQUARE_KM)); } } catch (FactoryException | org.geotools.api.referencing.operation.TransformException | TransformException | NullPointerException e) {