diff --git a/Cargo.lock b/Cargo.lock index e26294f24..5ccf5e06e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,7 +54,7 @@ dependencies = [ "actix-web", "bitflags 2.9.0", "bytes", - "derive_more 0.99.19", + "derive_more 0.99.20", "futures-core", "http-range", "log", @@ -95,7 +95,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand 0.9.0", + "rand 0.9.1", "sha1", "smallvec", "tokio", @@ -123,7 +123,7 @@ dependencies = [ "actix-multipart-derive", "actix-utils", "actix-web", - "derive_more 0.99.19", + "derive_more 0.99.20", "futures-core", "futures-util", "httparse", @@ -361,7 +361,7 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "version_check", "zerocopy 0.7.35", @@ -1029,9 +1029,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1152,9 +1152,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.19" +version = "1.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a" dependencies = [ "jobserver", "libc", @@ -1246,9 +1246,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.36" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04" +checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" dependencies = [ "clap_builder", "clap_derive", @@ -1256,9 +1256,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.36" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5" +checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" dependencies = [ "anstream", "anstyle", @@ -1345,7 +1345,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "tiny-keccak", ] @@ -1658,9 +1658,9 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" [[package]] name = "der" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", "pem-rfc7468", @@ -1721,9 +1721,9 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.19" +version = "0.99.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" dependencies = [ "convert_case 0.4.0", "proc-macro2", @@ -2248,7 +2248,7 @@ source = "git+https://github.com/lelongg/geo-rand.git?branch=dependabot%2Fcargo% dependencies = [ "geo", "num-traits", - "rand 0.9.0", + "rand 0.9.1", ] [[package]] @@ -2383,7 +2383,7 @@ dependencies = [ "pin-project", "postgres-protocol", "postgres-types", - "rand 0.9.0", + "rand 0.9.1", "rayon", "rustc-hash", "serde", @@ -2456,7 +2456,7 @@ dependencies = [ "proj-sys", "prost 0.12.6", "pwhash", - "rand 0.9.0", + "rand 0.9.1", "rayon", "reqwest", "serde", @@ -2514,9 +2514,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", @@ -3520,9 +3520,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +checksum = "c9627da5196e5d8ed0b0495e61e518847578da83483c37288316d9b2e03a7f72" [[package]] name = "libredox" @@ -3593,12 +3593,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "lockfree-object-pool" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" - [[package]] name = "log" version = "0.4.27" @@ -4029,7 +4023,7 @@ checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ "base64 0.22.1", "chrono", - "getrandom 0.2.15", + "getrandom 0.2.16", "http 1.3.1", "rand 0.8.5", "reqwest", @@ -4215,7 +4209,7 @@ dependencies = [ "glob", "opentelemetry", "percent-encoding", - "rand 0.9.0", + "rand 0.9.1", "serde_json", "thiserror 2.0.12", "tokio", @@ -4623,7 +4617,7 @@ dependencies = [ "hmac 0.12.1", "md-5 0.10.6", "memchr", - "rand 0.9.0", + "rand 0.9.1", "sha2 0.10.8", "stringprep", ] @@ -4980,13 +4974,13 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +checksum = "bcbafbbdbb0f638fe3f35f3c56739f77a8a1d070cb25603226c83339b391472b" dependencies = [ "bytes", "getrandom 0.3.2", - "rand 0.9.0", + "rand 0.9.1", "ring", "rustc-hash", "rustls 0.23.26", @@ -5041,13 +5035,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.24", ] [[package]] @@ -5076,7 +5069,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "serde", ] @@ -5301,7 +5294,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -5540,9 +5533,9 @@ dependencies = [ [[package]] name = "scc" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea091f6cac2595aa38993f04f4ee692ed43757035c36e67c180b6828356385b1" +checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4" dependencies = [ "sdd", ] @@ -5823,9 +5816,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -6362,7 +6355,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.0", + "rand 0.9.1", "socket2", "tokio", "tokio-util", @@ -6403,9 +6396,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -7446,9 +7439,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" +checksum = "6cb8234a863ea0e8cd7284fcdd4f145233eb00fee02bbdd9861aec44e6477bc5" dependencies = [ "memchr", ] @@ -7686,15 +7679,13 @@ dependencies = [ [[package]] name = "zopfli" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" dependencies = [ "bumpalo", "crc32fast", - "lockfree-object-pool", "log", - "once_cell", "simd-adler32", ] diff --git a/datatypes/src/error.rs b/datatypes/src/error.rs index c49f84bf2..935a0fefc 100644 --- a/datatypes/src/error.rs +++ b/datatypes/src/error.rs @@ -345,6 +345,8 @@ pub enum Error { DuplicateBandInQueryBandSelection, QueryBandSelectionMustNotBeEmpty, + TilingGeoTransformOriginCoordinateMismatch, + TilingGeoTransformResolutionMissmatch, #[snafu(display("Invalid number of suffixes, expected {} found {}", expected, found))] InvalidNumberOfSuffixes { expected: usize, @@ -363,6 +365,11 @@ pub enum Error { expected: usize, found: usize, }, + NoIntersectionWithTargetProjection { + srs_in: SpatialReference, + srs_out: SpatialReference, + bounds: BoundingBox2D, + }, } impl From for Error { diff --git a/datatypes/src/operations/reproject.rs b/datatypes/src/operations/reproject.rs index 533e43436..0b08c475c 100644 --- a/datatypes/src/operations/reproject.rs +++ b/datatypes/src/operations/reproject.rs @@ -1,10 +1,14 @@ use crate::{ - error::{self}, + error, primitives::{ - AxisAlignedRectangle, Coordinate2D, Line, MultiLineString, MultiLineStringAccess, - MultiLineStringRef, MultiPoint, MultiPointAccess, MultiPointRef, MultiPolygon, - MultiPolygonAccess, MultiPolygonRef, QueryAttributeSelection, QueryRectangle, - SpatialBounded, SpatialResolution, + AxisAlignedRectangle, BoundingBox2D, Coordinate2D, Line, MultiLineString, + MultiLineStringAccess, MultiLineStringRef, MultiPoint, MultiPointAccess, MultiPointRef, + MultiPolygon, MultiPolygonAccess, MultiPolygonRef, SpatialBounded, SpatialQueryRectangle, + SpatialResolution, + }, + raster::{ + BoundedGrid, GeoTransform, GridBoundingBox, GridBounds, GridIdx, GridIdx2D, GridShape, + GridSize, SamplePoints, SpatialGridDefinition, }, spatial_reference::SpatialReference, util::Result, @@ -352,6 +356,17 @@ fn diag_distance(ul_coord: Coordinate2D, lr_coord: Coordinate2D) -> f64 { (proj_ul_lr_vector.x * proj_ul_lr_vector.x + proj_ul_lr_vector.y * proj_ul_lr_vector.y).sqrt() } +pub fn suggest_pixel_size_like_gdal_helper( + bbox: B, + spatial_resolution: SpatialResolution, + source_srs: SpatialReference, + target: SpatialReference, +) -> Result { + let projector = CoordinateProjector::from_known_srs(source_srs, target)?; + + suggest_pixel_size_like_gdal(bbox, spatial_resolution, &projector) +} + /// This method calculates a suggested pixel size for the translation of a raster into a different projection. /// The source raster is described using a `BoundingBox2D` and a pixel size as `SpatialResolution`. /// A suggested pixel size is calculated using the approach used by GDAL: @@ -375,6 +390,146 @@ pub fn suggest_pixel_size_like_gdal Result { + let projector = CoordinateProjector::from_known_srs(source_srs, target_srs)?; + + suggest_output_spatial_grid_like_gdal(spatial_grid, &projector) +} + +pub fn reproject_spatial_grid_bounds( + spatial_grid: &SpatialGridDefinition, + projector: &P, +) -> Result> { + // First, try to reproject the bounds: + let full_bounds: std::result::Result = spatial_grid + .spatial_partition() + .as_bbox() + .reproject(projector); + + if let Ok(projected_bounds) = full_bounds { + let res = A::from_min_max( + projected_bounds.lower_left(), + projected_bounds.upper_right(), + ); + return Some(res).transpose(); + } + + // Second, create a grid of coordinates project that and use the valid bounds. + + let sample_bounds = SpatialGridDefinition::new( + spatial_grid.geo_transform, + GridBoundingBox::new_unchecked( + spatial_grid.grid_bounds.min_index(), + spatial_grid.grid_bounds.max_index() + GridIdx2D::new_y_x(1, 1), + ), + ); + // let coord_grid = spatial_grid.generate_coord_grid_upper_left_edge(); + // use a "Haus vom Nikolaus" strategy to find the bound. + let mut coord_grid_sample = sample_bounds.sample_outline(2); + coord_grid_sample.append(&mut sample_bounds.sample_diagonals(2)); + coord_grid_sample.append(&mut sample_bounds.sample_cross(2)); + + let proj_outline_coordinates: Vec = + project_coordinates_fail_tolerant(&coord_grid_sample, projector) + .into_iter() + .flatten() + .collect(); + + if proj_outline_coordinates.is_empty() { + return Ok(None); + } + + let out = MultiPoint::new(proj_outline_coordinates)?.spatial_bounds(); + + Some(A::from_min_max(out.lower_left(), out.upper_right())).transpose() +} + +pub fn suggest_output_spatial_grid_like_gdal( + spatial_grid: &SpatialGridDefinition, + projector: &P, +) -> Result { + const ROUND_UP_SIZE: bool = false; + + let in_x_pixels = spatial_grid.grid_bounds().axis_size_x(); + let in_y_pixels = spatial_grid.grid_bounds().axis_size_y(); + + let proj_bbox_option: Option = + reproject_spatial_grid_bounds(spatial_grid, projector)?; + + let Some(proj_bbox) = proj_bbox_option else { + return Err(error::Error::NoIntersectionWithTargetProjection { + srs_in: projector.source_srs(), + srs_out: projector.target_srs(), + bounds: spatial_grid.spatial_partition().as_bbox(), + }); + }; + + let out_x_distance = proj_bbox.size_x(); + let out_y_distance = proj_bbox.size_y(); + + let out_diagonal_dist = + (out_x_distance * out_x_distance + out_y_distance * out_y_distance).sqrt(); + + let pixel_size = + out_diagonal_dist / ((in_x_pixels * in_x_pixels + in_y_pixels * in_y_pixels) as f64).sqrt(); + + let x_pixels_with_frac = out_x_distance / pixel_size; + let y_pixels_with_frac = out_y_distance / pixel_size; + + let (x_pixels, y_pixels) = if ROUND_UP_SIZE { + const EPS_FROM_GDAL: f64 = 1e-5; + ( + (x_pixels_with_frac - EPS_FROM_GDAL).ceil() as usize, + (y_pixels_with_frac - EPS_FROM_GDAL).ceil() as usize, + ) + } else { + ( + (x_pixels_with_frac + 0.5) as usize, + (y_pixels_with_frac + 0.5) as usize, + ) + }; + + // TODO: gdal does some magic to fit to the bounds which might change the pixel size again. + // let x_pixel_size = out_x_distance / x_pixels as f64; + // let y_pixel_size = out_y_distance / y_pixels as f64; + let x_pixel_size = pixel_size; + let y_pixel_size = pixel_size; + + let geo_transform = GeoTransform::new(proj_bbox.upper_left(), x_pixel_size, -y_pixel_size); + let grid_bounds = GridShape::new_2d(y_pixels, x_pixels).bounding_box(); + let out_spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); + + // if the input grid is anchored at the upper left idx then we don't have to move the origin of the geo transform + if spatial_grid.grid_bounds.min_index() == GridIdx([0, 0]) { + return Ok(SpatialGridDefinition::new(geo_transform, grid_bounds)); + } + + let proj_origin = spatial_grid + .geo_transform() + .origin_coordinate() + .reproject(projector)?; + + let out_spatial_grid_moved_origin = + out_spatial_grid.with_moved_origin_to_nearest_grid_edge(proj_origin); + + Ok(out_spatial_grid_moved_origin.replace_origin(proj_origin)) +} + +pub fn suggest_pixel_size_from_diag_cross_helper( + bbox: B, + spatial_resolution: SpatialResolution, + source_srs: SpatialReference, + target: SpatialReference, +) -> Result { + let projector = CoordinateProjector::from_known_srs(source_srs, target)?; + + suggest_pixel_size_from_diag_cross(bbox, spatial_resolution, &projector) +} + /// This approach uses the GDAL way to suggest the pixel size. However, we check both diagonals and take the smaller one. /// This method fails if the bbox cannot be projected pub fn suggest_pixel_size_from_diag_cross( @@ -387,7 +542,7 @@ pub fn suggest_pixel_size_from_diag_cross = projected_diag_distance(bbox.lower_left(), bbox.upper_right(), projector); let min_dist_r = match (proj_ul_lr_distance, proj_ll_ur_distance) { @@ -456,25 +611,15 @@ pub fn project_coordinates_fail_tolerant( /// this method performs the transformation of a query rectangle in `target` projection /// to a new query rectangle with coordinates in the `source` projection -pub fn reproject_query( - query: QueryRectangle, +pub fn reproject_spatial_query( + query: SpatialQueryRectangle, source: SpatialReference, target: SpatialReference, -) -> Result>> { - let (Some(s_bbox), Some(p_bbox)) = - reproject_and_unify_bbox(query.spatial_bounds, target, source)? - else { - return Ok(None); - }; +) -> Result>> { + let proj_to_from = CoordinateProjector::from_known_srs(target, source)?; + let target_bbox_clipped = query.spatial_bounds.reproject_clipped(&proj_to_from)?; - let p_spatial_resolution = - suggest_pixel_size_from_diag_cross_projected(s_bbox, p_bbox, query.spatial_resolution)?; - Ok(Some(QueryRectangle { - spatial_bounds: p_bbox, - spatial_resolution: p_spatial_resolution, - time_interval: query.time_interval, - attributes: query.attributes, - })) + Ok(target_bbox_clipped.map(|b| SpatialQueryRectangle::new(b))) } /// Reproject a bounding box to the `target` projection and return the input and output bounding box @@ -498,6 +643,26 @@ pub fn reproject_and_unify_bbox( } } +/// Reproject the area of use of the `source` projection to the `target` projection and back. Return the back projected bounds and the area of use in the `target` projection. +pub fn reproject_and_unify_proj_bounds( + source: SpatialReference, + target: SpatialReference, +) -> Result<(Option, Option)> { + let proj_from_to = CoordinateProjector::from_known_srs(source, target)?; + let proj_to_from = CoordinateProjector::from_known_srs(target, source)?; + + let target_bbox_clipped = source + .area_of_use_projected::()? + .reproject_clipped(&proj_from_to)?; // TODO: can we intersect areas of use first? + + if let Some(target_b) = target_bbox_clipped { + let source_bbox_clipped = target_b.reproject(&proj_to_from)?; + Ok((Some(source_bbox_clipped), target_bbox_clipped)) + } else { + Ok((None, None)) + } +} + #[cfg(test)] mod tests { diff --git a/datatypes/src/primitives/db_types.rs b/datatypes/src/primitives/db_types.rs index 027f2e34b..e4e098224 100644 --- a/datatypes/src/primitives/db_types.rs +++ b/datatypes/src/primitives/db_types.rs @@ -9,7 +9,7 @@ use crate::{ util::NotNanF64, }; use postgres_types::{FromSql, ToSql}; -use std::collections::HashMap; +use std::collections::BTreeMap; #[derive(Debug, ToSql, FromSql)] #[postgres(name = "Measurement")] @@ -77,7 +77,7 @@ impl TryFrom for Measurement { continuous: None, classification: Some(classification), } => { - let mut classes = HashMap::with_capacity(classification.classes.len()); + let mut classes = BTreeMap::new(); for SmallintTextKeyValue { key, value } in classification.classes { classes.insert( u8::try_from(key).map_err(|_| Error::UnexpectedInvalidDbTypeConversion)?, diff --git a/datatypes/src/primitives/measurement.rs b/datatypes/src/primitives/measurement.rs index 0067b066a..f4ec78051 100644 --- a/datatypes/src/primitives/measurement.rs +++ b/datatypes/src/primitives/measurement.rs @@ -1,6 +1,6 @@ use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use std::fmt; use std::str::FromStr; @@ -17,12 +17,20 @@ impl Measurement { Self::Continuous(ContinuousMeasurement { measurement, unit }) } - pub fn classification(measurement: String, classes: HashMap) -> Self { + pub fn classification(measurement: String, classes: BTreeMap) -> Self { Self::Classification(ClassificationMeasurement { measurement, classes, }) } + + pub fn is_classification(&self) -> bool { + matches!(self, Self::Classification(_)) + } + + pub fn is_continuous(&self) -> bool { + matches!(self, Self::Continuous(_)) + } } impl Default for Measurement { @@ -44,7 +52,7 @@ pub struct ContinuousMeasurement { )] pub struct ClassificationMeasurement { pub measurement: String, - pub classes: HashMap, + pub classes: BTreeMap, } /// A type that is solely for serde's serializability. @@ -73,13 +81,14 @@ impl TryFrom for ClassificationMeasuremen type Error = ::Err; fn try_from(measurement: SerializableClassificationMeasurement) -> Result { - let mut classes = HashMap::with_capacity(measurement.classes.len()); - for (k, v) in measurement.classes { - classes.insert(k.parse::()?, v); - } + let classes: Result, _> = measurement + .classes + .into_iter() + .map(|(k, v)| (k.parse::().map(|x| (x, v)))) + .collect(); Ok(Self { measurement: measurement.measurement, - classes, + classes: classes?, }) } } @@ -96,12 +105,12 @@ impl fmt::Display for Measurement { /// # Examples /// ```rust /// use geoengine_datatypes::primitives::Measurement; - /// use std::collections::HashMap; + /// use std::collections::BTreeMap; /// /// assert_eq!(format!("{}", Measurement::Unitless), ""); /// assert_eq!(format!("{}", Measurement::continuous("foo".into(), Some("bar".into()))), "foo in bar"); /// assert_eq!(format!("{}", Measurement::continuous("foo".into(), None)), "foo"); - /// assert_eq!(format!("{}", Measurement::classification("foobar".into(), HashMap::new())), "foobar"); + /// assert_eq!(format!("{}", Measurement::classification("foobar".into(), BTreeMap::new())), "foobar"); /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -143,7 +152,7 @@ mod tests { fn classification_serialization() { let measurement = Measurement::classification( "foo".into(), - HashMap::from([(1_u8, "bar".to_string()), (2, "baz".to_string())]), + BTreeMap::from([(1_u8, "bar".to_string()), (2, "baz".to_string())]), ); let serialized = serde_json::to_string(&measurement).unwrap(); assert_eq!( diff --git a/datatypes/src/primitives/mod.rs b/datatypes/src/primitives/mod.rs index f89da27d0..3aea17d86 100755 --- a/datatypes/src/primitives/mod.rs +++ b/datatypes/src/primitives/mod.rs @@ -40,7 +40,9 @@ pub use multi_polygon::{MultiPolygon, MultiPolygonAccess, MultiPolygonRef}; pub use no_geometry::NoGeometry; pub use query_rectangle::{ BandSelection, ColumnSelection, PlotQueryRectangle, PlotSeriesSelection, - QueryAttributeSelection, QueryRectangle, RasterQueryRectangle, VectorQueryRectangle, + PlotSpatialQueryRectangle, QueryAttributeSelection, QueryRectangle, RasterQueryRectangle, + RasterSpatialQueryRectangle, SpatialGridQueryRectangle, SpatialQueryRectangle, + VectorQueryRectangle, VectorSpatialQueryRectangle, }; pub use spatial_partition::{ AxisAlignedRectangle, SpatialPartition2D, SpatialPartitioned, partitions_extent, diff --git a/datatypes/src/primitives/query_rectangle.rs b/datatypes/src/primitives/query_rectangle.rs index f29c49fb2..e81bad5ba 100644 --- a/datatypes/src/primitives/query_rectangle.rs +++ b/datatypes/src/primitives/query_rectangle.rs @@ -1,7 +1,8 @@ use super::{ - AxisAlignedRectangle, BoundingBox2D, SpatialPartition2D, SpatialPartitioned, SpatialResolution, + AxisAlignedRectangle, BoundingBox2D, SpatialBounded, SpatialPartition2D, SpatialPartitioned, TimeInterval, }; +use crate::raster::{GeoTransform, GridBoundingBox2D}; use crate::{ error::{DuplicateBandInQueryBandSelection, QueryBandSelectionMustNotBeEmpty}, util::Result, @@ -12,16 +13,177 @@ use snafu::ensure; /// A spatio-temporal rectangle with a specified resolution and the selected bands #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct QueryRectangle< - SpatialBounds: AxisAlignedRectangle, - AttributeSelection: QueryAttributeSelection, -> { - pub spatial_bounds: SpatialBounds, +pub struct QueryRectangle { + pub spatial_query: SpatialQuery, pub time_interval: TimeInterval, - pub spatial_resolution: SpatialResolution, pub attributes: AttributeSelection, } +impl QueryRectangle { + pub fn spatial_query(&self) -> SpatialQuery { + self.spatial_query + } + + pub fn temporal_query(&self) -> TimeInterval { + self.time_interval + } + + pub fn spatial_query_mut(&mut self) -> &mut SpatialQuery { + &mut self.spatial_query + } + + pub fn temporal_query_mut(&mut self) -> &mut TimeInterval { + &mut self.time_interval + } + + pub fn attributes(&self) -> &A { + &self.attributes + } + + pub fn attributes_mut(&mut self) -> &mut A { + &mut self.attributes + } +} + +pub type VectorQueryRectangle = + QueryRectangle, ColumnSelection>; +pub type RasterQueryRectangle = QueryRectangle; +pub type PlotQueryRectangle = + QueryRectangle, PlotSeriesSelection>; + +// Implementation for VectorQueryRectangle and PlotQueryRectangle +impl QueryRectangle, A> +where + S: SpatialBounded, + A: QueryAttributeSelection, +{ + pub fn new( + spatial_bounds: SpatialQueryRectangle, + time_interval: TimeInterval, + attributes: A, + ) -> Self { + Self { + spatial_query: spatial_bounds, + time_interval, + attributes, + } + } + + /// Creates a new `QueryRectangle` from a `BoundingBox2D`, and a `TimeInterval` + pub fn with_bounds(spatial_bounds: S, time_interval: TimeInterval, attributes: A) -> Self { + Self { + spatial_query: SpatialQueryRectangle { spatial_bounds }, + time_interval, + attributes, + } + } + + /// Creates a new `QueryRectangle` with bounds and time from a `RasterQueryRectangle` and supplied attributes. + pub fn from_raster_query_and_geo_transform_replace_attributes( + raster_query: &RasterQueryRectangle, + geo_transform: GeoTransform, + attributes: A, + ) -> QueryRectangle, A> { + let bounds = + geo_transform.grid_to_spatial_bounds(&raster_query.spatial_query.grid_bounds()); + let bounding_box = bounds.as_bbox(); + + QueryRectangle::with_bounds(bounding_box, raster_query.time_interval, attributes) + } +} + +impl RasterQueryRectangle { + /// Creates a new `QueryRectangle` that describes the requested grid. + /// The spatial query is derived from a vector query rectangle and a `GeoTransform`. + /// The temporal query is defined by a `TimeInterval`. + /// NOTE: If the distance between the upper left of the spatial partition and the origin coordinate is not at a multiple of the spatial resolution, the grid bounds will be shifted. + pub fn with_spatial_query_and_geo_transform( + vector_query: &QueryRectangle, + geo_transform: GeoTransform, + attributes: BandSelection, + ) -> Self { + Self::new( + SpatialGridQueryRectangle::with_bounding_box_and_geo_transform( + vector_query.spatial_query.spatial_bounds(), + geo_transform, + ), + vector_query.time_interval, + attributes, + ) + } + + pub fn new_with_grid_bounds( + grid_bounds: GridBoundingBox2D, + time_interval: TimeInterval, + attributes: BandSelection, + ) -> Self { + Self::new( + SpatialGridQueryRectangle::new(grid_bounds), + time_interval, + attributes, + ) + } + + /// Creates a new `QueryRectangle` with a spatial grid query defined by a `SpatialGridQueryRectangle` and a temporal query defined by a `TimeInterval`. + pub fn new( + spatial_bounds: SpatialGridQueryRectangle, + time_interval: TimeInterval, + attributes: BandSelection, + ) -> Self { + Self { + spatial_query: spatial_bounds, + time_interval, + attributes, + } + } + + pub fn from_qrect_and_geo_transform( + query: &QueryRectangle, A>, + bands: BandSelection, + geo_transform: GeoTransform, + ) -> Self { + Self::new( + SpatialGridQueryRectangle::with_bounding_box_and_geo_transform( + query.spatial_query.spatial_bounds, + geo_transform, + ), + query.time_interval, + bands, + ) + } + + #[must_use] + pub fn select_bands(&self, bands: BandSelection) -> Self { + Self { + spatial_query: self.spatial_query, + time_interval: self.time_interval, + attributes: bands, + } + } +} + +pub type RasterSpatialQueryRectangle = SpatialGridQueryRectangle; +pub type VectorSpatialQueryRectangle = SpatialQueryRectangle; +pub type PlotSpatialQueryRectangle = SpatialQueryRectangle; + +/// A spatial rectangle with a specified resolution +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct SpatialQueryRectangle { + pub spatial_bounds: SpatialBounds, +} + +impl SpatialQueryRectangle { + pub fn new(spatial_bounds: S) -> Self { + Self { spatial_bounds } + } +} + +impl SpatialBounded for SpatialQueryRectangle { + fn spatial_bounds(&self) -> BoundingBox2D { + self.spatial_bounds + } +} + pub trait QueryAttributeSelection: Clone + Send + Sync {} #[derive(Clone, Debug, PartialEq, Serialize)] @@ -67,6 +229,10 @@ impl BandSelection { pub fn as_vec(&self) -> Vec { self.0.clone() } + + pub fn is_single(&self) -> bool { + self.count() == 1 + } } impl From for BandSelection { @@ -115,56 +281,15 @@ impl PlotSeriesSelection { impl QueryAttributeSelection for PlotSeriesSelection {} -pub type VectorQueryRectangle = QueryRectangle; -pub type RasterQueryRectangle = QueryRectangle; -pub type PlotQueryRectangle = QueryRectangle; - -impl RasterQueryRectangle { - pub fn from_qrect_and_bands( - query: &QueryRectangle, - bands: BandSelection, - ) -> Self - where - A: QueryAttributeSelection, - QueryRectangle: SpatialPartitioned, - { - Self { - spatial_bounds: query.spatial_partition(), - time_interval: query.time_interval, - spatial_resolution: query.spatial_resolution, - attributes: bands, - } - } - - #[must_use] - pub fn select_bands(&self, bands: BandSelection) -> Self { - Self { - spatial_bounds: self.spatial_bounds, - time_interval: self.time_interval, - spatial_resolution: self.spatial_resolution, - attributes: bands, - } - } -} - -impl SpatialPartitioned for QueryRectangle { - fn spatial_partition(&self) -> SpatialPartition2D { - SpatialPartition2D::with_bbox_and_resolution(self.spatial_bounds, self.spatial_resolution) - } -} - -impl SpatialPartitioned for QueryRectangle { - fn spatial_partition(&self) -> SpatialPartition2D { - SpatialPartition2D::with_bbox_and_resolution(self.spatial_bounds, self.spatial_resolution) - } -} - -impl SpatialPartitioned for QueryRectangle { +impl SpatialPartitioned for SpatialQueryRectangle { fn spatial_partition(&self) -> SpatialPartition2D { self.spatial_bounds } } +/* +impl From> for QueryRectangle { + fn from(value: QueryRectangle) -> Self { impl From> for QueryRectangle { @@ -177,28 +302,53 @@ impl From> } } } +*/ -impl From> - for QueryRectangle -{ - fn from(value: QueryRectangle) -> Self { - Self { - spatial_bounds: value.spatial_bounds, - time_interval: value.time_interval, - spatial_resolution: value.spatial_resolution, - attributes: value.attributes.into(), - } - } +/// A spatio-temporal grid query with a geotransform and a size in pixels. +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridQueryRectangle { + grid_bounds: GridBoundingBox2D, } -impl From for PlotSeriesSelection { - fn from(_: ColumnSelection) -> Self { - Self {} +impl SpatialGridQueryRectangle { + /// Creates a new `SpatialGridQueryRectangle` from a geo transform and a grid bounds. + pub fn new(grid_bounds: GridBoundingBox2D) -> Self { + Self { grid_bounds } } -} -impl From for ColumnSelection { - fn from(_: PlotSeriesSelection) -> Self { - Self {} + pub fn grid_bounds(&self) -> GridBoundingBox2D { + self.grid_bounds + } + + /// Creates a new `SpatialGridQueryRectangle` from a spatial partition and a geo transform. + pub fn with_partition_and_geo_transform( + spatial_partition: SpatialPartition2D, + geo_transform: GeoTransform, + ) -> Self { + let grid_bounds = geo_transform.spatial_to_grid_bounds(&spatial_partition); + + Self::new(grid_bounds) + } + + /// Creates a new `SpatialGridQueryRectangle` from a spatial bounding box and a geo transform. + pub fn with_bounding_box_and_geo_transform( + spatial_bounds: BoundingBox2D, + geo_transform: GeoTransform, + ) -> Self { + let grid_bounds = + geo_transform.bounding_box_2d_to_intersecting_grid_bounds(&spatial_bounds); + + Self::new(grid_bounds) + } + + pub fn with_vector_query_geo_transform( + vector_spatial_query: VectorSpatialQueryRectangle, + geo_transform: GeoTransform, + ) -> Self { + let pixel_bounds = geo_transform + .bounding_box_2d_to_intersecting_grid_bounds(&vector_spatial_query.spatial_bounds()); + + Self::new(pixel_bounds) } } diff --git a/datatypes/src/primitives/spatial_partition.rs b/datatypes/src/primitives/spatial_partition.rs index fd323c709..4e76b48c4 100644 --- a/datatypes/src/primitives/spatial_partition.rs +++ b/datatypes/src/primitives/spatial_partition.rs @@ -306,22 +306,10 @@ impl From<&SpatialPartition2D> for geo::Rect { } /// Compute the extent of all input partitions. If one partition is None, the output will also be None -pub fn partitions_extent>>( - mut bboxes: I, +pub fn partitions_extent>( + bboxes: I, ) -> Option { - let Some(Some(mut extent)) = bboxes.next() else { - return None; - }; - - for bbox in bboxes { - if let Some(bbox) = bbox { - extent.extend(&bbox); - } else { - return None; - } - } - - Some(extent) + bboxes.reduce(|s, other| s.extended(&other)) } #[cfg(test)] @@ -495,37 +483,16 @@ mod tests { #[test] fn extent() { - assert_eq!(partitions_extent([None].into_iter()), None); assert_eq!( partitions_extent( [ - Some(SpatialPartition2D::new((-50., 50.).into(), (50., -50.).into()).unwrap()), - Some(SpatialPartition2D::new((0., 70.).into(), (70., 0.).into()).unwrap()) + SpatialPartition2D::new((-50., 50.).into(), (50., -50.).into()).unwrap(), + SpatialPartition2D::new((0., 70.).into(), (70., 0.).into()).unwrap() ] .into_iter() ), Some(SpatialPartition2D::new((-50., 70.).into(), (70., -50.).into()).unwrap()) ); - assert_eq!( - partitions_extent( - [ - Some(SpatialPartition2D::new((-50., 50.).into(), (50., -50.).into()).unwrap()), - None - ] - .into_iter() - ), - None - ); - assert_eq!( - partitions_extent( - [ - None, - Some(SpatialPartition2D::new((-50., 50.).into(), (50., -50.).into()).unwrap()) - ] - .into_iter() - ), - None - ); } #[test] diff --git a/datatypes/src/primitives/spatial_resolution.rs b/datatypes/src/primitives/spatial_resolution.rs index b892ec046..15d2f2ce1 100644 --- a/datatypes/src/primitives/spatial_resolution.rs +++ b/datatypes/src/primitives/spatial_resolution.rs @@ -2,6 +2,7 @@ use std::{convert::TryFrom, ops::Add, ops::Div, ops::Mul, ops::Sub}; use crate::primitives::error; use crate::util::Result; +use float_cmp::{ApproxEq, F64Margin, approx_eq}; use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -90,6 +91,15 @@ impl Div for SpatialResolution { } } +impl ApproxEq for SpatialResolution { + type Margin = F64Margin; + + fn approx_eq>(self, other: Self, margin: M) -> bool { + let m = margin.into(); + approx_eq!(f64, self.x, other.x, m) && approx_eq!(f64, self.y, other.y, m) + } +} + #[cfg(test)] mod test { use super::*; diff --git a/datatypes/src/primitives/time_interval.rs b/datatypes/src/primitives/time_interval.rs index 8021c0e78..e918f2288 100755 --- a/datatypes/src/primitives/time_interval.rs +++ b/datatypes/src/primitives/time_interval.rs @@ -88,6 +88,23 @@ impl TimeInterval { } ); + ensure!( + start_instant <= end_instant, + error::TimeIntervalEndBeforeStart { + start: start_instant, + end: end_instant + } + ); + ensure!( + start_instant >= TimeInstance::MIN && end_instant <= TimeInstance::MAX, + error::TimeIntervalOutOfBounds { + start: start_instant, + end: end_instant, + min: TimeInstance::MIN, + max: TimeInstance::MAX, + } + ); + Ok(Self { start: start_instant, end: end_instant, @@ -246,10 +263,10 @@ impl TimeInterval { i2: *other, } ); - Ok(Self { - start: TimeInstance::min(self.start, other.start), - end: TimeInstance::max(self.end, other.end), - }) + Self::new( + TimeInstance::min(self.start, other.start), + TimeInstance::max(self.end, other.end), + ) } pub fn start(&self) -> TimeInstance { diff --git a/datatypes/src/raster/db_types.rs b/datatypes/src/raster/db_types.rs new file mode 100644 index 000000000..449cc06f2 --- /dev/null +++ b/datatypes/src/raster/db_types.rs @@ -0,0 +1,39 @@ +use postgres_types::{FromSql, ToSql}; + +use crate::delegate_from_to_sql; + +use super::GridBoundingBox2D; + +#[derive(Debug, PartialEq, ToSql, FromSql)] +#[postgres(name = "GridBoundingBox2D")] +pub struct GridBoundingBox2DDbType { + y_min: i64, + y_max: i64, + x_min: i64, + x_max: i64, +} + +impl From<&GridBoundingBox2D> for GridBoundingBox2DDbType { + fn from(value: &GridBoundingBox2D) -> Self { + Self { + y_min: value.y_min() as i64, + y_max: value.y_max() as i64, + x_min: value.x_min() as i64, + x_max: value.x_max() as i64, + } + } +} + +impl From for GridBoundingBox2D { + fn from(value: GridBoundingBox2DDbType) -> Self { + GridBoundingBox2D::new_min_max( + value.y_min as isize, + value.y_max as isize, + value.x_min as isize, + value.x_max as isize, + ) + .expect("conversion must be correct") + } +} + +delegate_from_to_sql!(GridBoundingBox2D, GridBoundingBox2DDbType); diff --git a/datatypes/src/raster/empty_grid.rs b/datatypes/src/raster/empty_grid.rs index e9842ee61..dd3d43c31 100644 --- a/datatypes/src/raster/empty_grid.rs +++ b/datatypes/src/raster/empty_grid.rs @@ -42,7 +42,7 @@ where impl GridSize for EmptyGrid where - D: GridSize + GridSpaceToLinearSpace, + D: GridSpaceToLinearSpace, { type ShapeArray = D::ShapeArray; @@ -85,23 +85,30 @@ where } } -impl ChangeGridBounds for EmptyGrid +impl ChangeGridBounds for EmptyGrid where - I: AsRef<[isize]> + Clone, - D: GridBounds + Clone, - T: Copy, - GridBoundingBox: GridSize, + D: GridBounds + GridSize, + I: AsRef<[isize]> + Into> + Clone, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, GridIdx: Add> + From, + T: Copy, { - type Output = EmptyGrid, T>; + type BoundedOutput = EmptyGrid, T>; + type UnboundedOutput = EmptyGrid, T>; - fn shift_by_offset(self, offset: GridIdx) -> Self::Output { + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput { EmptyGrid::new(self.shift_bounding_box(offset)) } - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { Ok(EmptyGrid::new(bounds)) } + + fn unbounded(self) -> Self::UnboundedOutput { + EmptyGrid::new(self.grid_shape()) + } } impl ByteSize for EmptyGrid {} diff --git a/datatypes/src/raster/geo_transform.rs b/datatypes/src/raster/geo_transform.rs index beba9a1dc..165ae2039 100644 --- a/datatypes/src/raster/geo_transform.rs +++ b/datatypes/src/raster/geo_transform.rs @@ -1,11 +1,14 @@ +use super::{GeoTransformAccess, GridBoundingBox2D, GridBounds, GridIdx, GridIdx2D}; use crate::{ - primitives::{AxisAlignedRectangle, Coordinate2D, SpatialPartition2D, SpatialResolution}, + primitives::{ + AxisAlignedRectangle, BoundingBox2D, Coordinate2D, SpatialPartition2D, SpatialResolution, + }, util::test::TestDefault, }; +use float_cmp::{ApproxEq, F64Margin, approx_eq}; +use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Deserializer, Serialize, de}; -use super::{GridBoundingBox2D, GridIdx, GridIdx2D}; - /// This is a typedef for the `GDAL GeoTransform`. It represents an affine transformation matrix. pub type GdalGeoTransform = [f64; 6]; @@ -13,7 +16,7 @@ pub type GdalGeoTransform = [f64; 6]; /// In Geo Engine x pixel size is always postive and y pixel size is always negative. For raster tiles /// the origin is always the upper left corner. In the global grid for the `TilingStrategy` the origin /// is always located at (0, 0). -#[derive(Copy, Clone, PartialEq, Debug, Serialize, Deserialize)] +#[derive(Copy, Clone, PartialEq, Debug, Serialize, Deserialize, ToSql, FromSql)] #[serde(rename_all = "camelCase")] pub struct GeoTransform { pub origin_coordinate: Coordinate2D, @@ -34,9 +37,8 @@ impl GeoTransform { /// #[inline] pub fn new(origin_coordinate: Coordinate2D, x_pixel_size: f64, y_pixel_size: f64) -> Self { - debug_assert!(x_pixel_size > 0.0); - debug_assert!(y_pixel_size < 0.0); - + debug_assert!(x_pixel_size != 0.0); + debug_assert!(y_pixel_size != 0.0); Self { origin_coordinate, x_pixel_size, @@ -60,9 +62,8 @@ impl GeoTransform { origin_coordinate_y: f64, y_pixel_size: f64, ) -> Self { - debug_assert!(x_pixel_size > 0.0); - debug_assert!(y_pixel_size < 0.0); - + debug_assert!(x_pixel_size != 0.0); + debug_assert!(y_pixel_size != 0.0); Self { origin_coordinate: (origin_coordinate_x, origin_coordinate_y).into(), x_pixel_size, @@ -78,6 +79,10 @@ impl GeoTransform { self.y_pixel_size } + pub fn y_axis_is_neg(&self) -> bool { + self.y_pixel_size < 0.0 + } + /// Transforms a grid coordinate (row, column) ~ (y, x) into a SRS coordinate (x,y) /// The resulting coordinate is the upper left coordinate of the pixel /// See GDAL documentation for more details (including the two ignored parameters): @@ -114,6 +119,7 @@ impl GeoTransform { } /// Transforms an SRS coordinate (x,y) into a grid coordinate (row, column) ~ (y, x) + /// This method selects the grid index with the center nearest to the coordinate. /// /// # Examples /// @@ -148,6 +154,7 @@ impl GeoTransform { } pub fn spatial_resolution(&self) -> SpatialResolution { + // TODO: should honor negative y size SpatialResolution { x: self.x_pixel_size.abs(), y: self.y_pixel_size.abs(), @@ -156,13 +163,7 @@ impl GeoTransform { /// compute the index of the upper left pixel that is contained in the `partition` pub fn upper_left_pixel_idx(&self, partition: &SpatialPartition2D) -> GridIdx2D { - // choose the epsilon relative to the pixel size - const EPSILON: f64 = 0.000_001; - let epsilon: Coordinate2D = - (self.x_pixel_size() * EPSILON, self.y_pixel_size() * EPSILON).into(); - - let upper_left_coordinate = partition.upper_left() + epsilon; - + let upper_left_coordinate = partition.upper_left(); self.coordinate_to_grid_idx_2d(upper_left_coordinate) } @@ -190,13 +191,25 @@ impl GeoTransform { self.coordinate_to_grid_idx_2d(lower_right) } + /// Transform a `BoundingBox2D` into a `GridBoundingBox2D`. + #[inline] + pub fn bounding_box_2d_to_intersecting_grid_bounds( + &self, + bounding_box: &BoundingBox2D, + ) -> GridBoundingBox2D { + let upper_left = self.coordinate_to_grid_idx_2d(bounding_box.upper_left()); + let lower_right = self.coordinate_to_grid_idx_2d(bounding_box.lower_right()); + + GridBoundingBox2D::new_unchecked(upper_left, lower_right) + } + /// Transform a `SpatialPartition2D` into a `GridBoundingBox` #[inline] pub fn spatial_to_grid_bounds( &self, spatial_partition: &SpatialPartition2D, ) -> GridBoundingBox2D { - let GridIdx([ul_y, ul_x]) = self.upper_left_pixel_idx(spatial_partition); + let GridIdx([ul_y, ul_x]) = self.coordinate_to_grid_idx_2d(spatial_partition.upper_left()); let GridIdx([lr_y, lr_x]) = self.lower_right_pixel_idx(spatial_partition); // this is the pixel inside the spatial partition debug_assert!(ul_x <= lr_x); @@ -222,6 +235,66 @@ impl GeoTransform { Ok(unchecked) } + + pub fn grid_to_spatial_bounds>( + &self, + grid_bounds: &S, + ) -> SpatialPartition2D { + let ul = self.grid_idx_to_pixel_upper_left_coordinate_2d(grid_bounds.min_index()); + let lr = self.grid_idx_to_pixel_upper_left_coordinate_2d(grid_bounds.max_index() + 1); + + SpatialPartition2D::new_unchecked(ul, lr) + } + + pub fn origin_coordinate(&self) -> Coordinate2D { + self.origin_coordinate + } + + #[must_use] + pub fn shift_by_pixel_offset(&self, offset: GridIdx2D) -> Self { + GeoTransform { + origin_coordinate: self.grid_idx_to_pixel_upper_left_coordinate_2d(offset), + x_pixel_size: self.x_pixel_size, + y_pixel_size: self.y_pixel_size, + } + } + + pub fn nearest_pixel_edge(&self, coordinate: Coordinate2D) -> GridIdx2D { + self.coordinate_to_grid_idx_2d( + coordinate + Coordinate2D::new(self.x_pixel_size * 0.5, self.y_pixel_size * 0.5), // by adding a half pixel, we can find flips between edges + ) + } + + pub fn nearest_pixel_edge_coordinate(&self, coordinate: Coordinate2D) -> Coordinate2D { + self.grid_idx_to_pixel_upper_left_coordinate_2d(self.nearest_pixel_edge(coordinate)) + } + + pub fn distance_to_nearest_pixel_edge(&self, coordinate: Coordinate2D) -> Coordinate2D { + let pixel_edge = self.nearest_pixel_edge_coordinate(coordinate); + let dist = coordinate - pixel_edge; + debug_assert!(dist.x.abs() <= self.x_pixel_size.abs() * 0.5); + debug_assert!(dist.y.abs() <= self.y_pixel_size.abs() * 0.5); + dist + } + + pub fn is_valid_pixel_edge(&self, coordinate: Coordinate2D) -> bool { + // TODO: maybe use fraction of pixel size as M? + approx_eq!( + Coordinate2D, + self.distance_to_nearest_pixel_edge(coordinate), + Coordinate2D::new(0., 0.) + ) + } + + pub fn is_compatible_grid(&self, other: GeoTransform) -> bool { + self.is_valid_pixel_edge(other.origin_coordinate) + && approx_eq!(f64, self.x_pixel_size(), other.x_pixel_size()) + && approx_eq!(f64, self.y_pixel_size(), other.y_pixel_size()) + } + + pub fn is_compatible_grid_generic(&self, g: &G) -> bool { + self.is_compatible_grid(g.geo_transform()) + } } impl TestDefault for GeoTransform { @@ -256,6 +329,18 @@ impl From for GdalGeoTransform { } } +impl ApproxEq for GeoTransform { + type Margin = F64Margin; + + fn approx_eq>(self, other: Self, margin: M) -> bool { + let m: F64Margin = margin.into(); + + self.origin_coordinate.approx_eq(other.origin_coordinate, m) + && self.x_pixel_size.approx_eq(other.x_pixel_size, m) + && self.y_pixel_size.approx_eq(other.y_pixel_size, m) + } +} + #[cfg(test)] mod tests { use super::*; @@ -518,4 +603,32 @@ mod tests { assert!(test.is_err()); } + + #[test] + fn shift_by_pixel_offset() { + let geo_transform = GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0); + let shifted = geo_transform.shift_by_pixel_offset([1, 1].into()); + assert_eq!(shifted.origin_coordinate, (1.0, -1.0).into()); + } + + #[test] + fn coordinate_to_nearest_grid_center_idx_2d() { + let geo_transform = GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.); + + let coord = Coordinate2D::new(0.1, -0.1); + let center_coord_idx = geo_transform.coordinate_to_grid_idx_2d(coord); + assert_eq!(center_coord_idx, [0, 0].into()); + + let coord = Coordinate2D::new(0.5, -0.5); + let center_coord_idx = geo_transform.coordinate_to_grid_idx_2d(coord); + assert_eq!(center_coord_idx, [0, 0].into()); + + let coord = Coordinate2D::new(0.9, -0.9); + let center_coord_idx = geo_transform.coordinate_to_grid_idx_2d(coord); + assert_eq!(center_coord_idx, [0, 0].into()); + + let coord = Coordinate2D::new(1.0, -1.0); + let center_coord_idx = geo_transform.coordinate_to_grid_idx_2d(coord); + assert_eq!(center_coord_idx, [1, 1].into()); + } } diff --git a/datatypes/src/raster/grid.rs b/datatypes/src/raster/grid.rs index 5538917ca..1be6a38dd 100644 --- a/datatypes/src/raster/grid.rs +++ b/datatypes/src/raster/grid.rs @@ -45,6 +45,48 @@ pub type GridShape1D = GridShape<[usize; 1]>; pub type GridShape2D = GridShape<[usize; 2]>; pub type GridShape3D = GridShape<[usize; 3]>; +impl GridShape1D { + pub fn new_1d(x_size: usize) -> Self { + Self::new([x_size]) + } + + pub fn x(self) -> usize { + self.shape_array[0] + } +} + +impl GridShape2D { + pub fn new_2d(y_size: usize, x_size: usize) -> Self { + Self::new([y_size, x_size]) + } + + pub fn x(&self) -> usize { + self.shape_array[1] + } + + pub fn y(&self) -> usize { + self.shape_array[0] + } +} + +impl GridShape3D { + pub fn new_3d(z_size: usize, y_size: usize, x_size: usize) -> Self { + Self::new([z_size, y_size, x_size]) + } + + pub fn x(&self) -> usize { + self.shape_array[2] + } + + pub fn y(&self) -> usize { + self.shape_array[1] + } + + pub fn z(&self) -> usize { + self.shape_array[0] + } +} + impl From<[usize; 1]> for GridShape1D { fn from(shape: [usize; 1]) -> Self { GridShape1D { shape_array: shape } @@ -484,26 +526,36 @@ where } } -impl ChangeGridBounds for Grid +impl ChangeGridBounds for Grid where - I: AsRef<[isize]> + Clone, - D: GridBounds + Clone, - T: Clone, - GridBoundingBox: GridSize, + D: GridBounds + GridSize, + I: AsRef<[isize]> + Into> + Clone, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, GridIdx: Add> + From, + T: Copy, { - type Output = Grid, T>; + type BoundedOutput = Grid, T>; + type UnboundedOutput = Grid, T>; - fn shift_by_offset(self, offset: GridIdx) -> Self::Output { + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput { Grid { shape: self.shift_bounding_box(offset), data: self.data, } } - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { Grid::new(bounds, self.data) } + + fn unbounded(self) -> Self::UnboundedOutput { + Grid { + shape: self.grid_shape(), + data: self.data, + } + } } impl ByteSize for Grid diff --git a/datatypes/src/raster/grid_bounds.rs b/datatypes/src/raster/grid_bounds.rs index 83389f85f..e46dbf2ad 100644 --- a/datatypes/src/raster/grid_bounds.rs +++ b/datatypes/src/raster/grid_bounds.rs @@ -1,13 +1,14 @@ -use snafu::ensure; - -use crate::{error, util::Result}; - use super::{ BoundedGrid, GridBounds, GridContains, GridIdx, GridIntersection, GridShape, GridShapeAccess, GridSize, GridSpaceToLinearSpace, }; +use crate::{error, util::Result}; +use serde::{Deserialize, Serialize}; +use snafu::ensure; +use std::ops::Add; -#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone)] +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize)] +/// A bounding box for a grid where the min and max values are inclusive pub struct GridBoundingBox where A: AsRef<[isize]>, @@ -49,6 +50,70 @@ where } } +impl GridBoundingBox1D { + #[inline] + pub fn x_min(&self) -> isize { + let [x_min] = self.min; + x_min + } + + #[inline] + pub fn x_max(&self) -> isize { + let [x_max] = self.max; + x_max + } + + #[inline] + pub fn x_bounds(&self) -> [isize; 2] { + [self.x_min(), self.x_max()] + } + + #[inline] + pub fn new_min_max(x_min: isize, x_max: isize) -> Result { + Self::new([x_min], [x_max]) + } +} + +impl GridBoundingBox2D { + #[inline] + pub fn x_min(&self) -> isize { + let [_y_min, x_min] = self.min; + x_min + } + + #[inline] + pub fn x_max(&self) -> isize { + let [_y_max, x_max] = self.max; + x_max + } + + #[inline] + pub fn x_bounds(&self) -> [isize; 2] { + [self.x_min(), self.x_max()] + } + + #[inline] + pub fn y_min(&self) -> isize { + let [y_min, _x_min] = self.min; + y_min + } + + #[inline] + pub fn y_max(&self) -> isize { + let [y_max, _x_max] = self.max; + y_max + } + + #[inline] + pub fn y_bounds(&self) -> [isize; 2] { + [self.y_min(), self.y_max()] + } + + pub fn new_min_max(y_min: isize, y_max: isize, x_min: isize, x_max: isize) -> Result { + Self::new([y_min, x_min], [y_max, x_max]) + } +} + impl GridSize for GridBoundingBox<[isize; 1]> { type ShapeArray = [usize; 1]; @@ -326,6 +391,92 @@ where } } +impl GridContains for GridBoundingBox2D { + fn contains(&self, other: &GridBoundingBox2D) -> bool { + let [self_y_min, self_x_min] = self.min; + let [other_y_min, other_x_min] = other.min; + + let [self_y_max, self_x_max] = self.max; + let [other_y_max, other_x_max] = other.max; + + self_y_min <= other_y_min + && self_x_min <= other_x_min + && self_y_max >= other_y_max + && self_x_max >= other_x_max + } +} + +pub trait GridBoundingBoxExt: GridBounds { + fn extend(&mut self, other: &Self); + + #[must_use] + fn extended(&self, other: &Self) -> Self + where + Self: Sized + Clone, + { + let mut extended = self.clone(); + extended.extend(other); + extended + } + + fn shift_by_offset( + &self, + offset: GridIdx, + ) -> GridBoundingBox + where + GridIdx: Add> + Clone, + { + GridBoundingBox::new_unchecked(self.min_index() + offset.clone(), self.max_index() + offset) + } +} + +impl GridBoundingBoxExt for GridBoundingBox1D { + fn extend(&mut self, other: &Self) { + let [self_x_min] = self.min; + let [other_x_min] = other.min; + + let [self_x_max] = self.max; + let [other_x_max] = other.max; + + self.min = [self_x_min.min(other_x_min)]; + self.max = [self_x_max.max(other_x_max)]; + } +} + +impl GridBoundingBoxExt for GridBoundingBox2D { + fn extend(&mut self, other: &Self) { + let [self_y_min, self_x_min] = self.min; + let [other_y_min, other_x_min] = other.min; + + let [self_y_max, self_x_max] = self.max; + let [other_y_max, other_x_max] = other.max; + + self.min = [self_y_min.min(other_y_min), self_x_min.min(other_x_min)]; + self.max = [self_y_max.max(other_y_max), self_x_max.max(other_x_max)]; + } +} + +impl GridBoundingBoxExt for GridBoundingBox3D { + fn extend(&mut self, other: &Self) { + let [self_z_min, self_y_min, self_x_min] = self.min; + let [other_z_min, other_y_min, other_x_min] = other.min; + + let [self_z_max, self_y_max, self_x_max] = self.max; + let [other_z_max, other_y_max, other_x_max] = other.max; + + self.min = [ + self_z_min.min(other_z_min), + self_y_min.min(other_y_min), + self_x_min.min(other_x_min), + ]; + self.max = [ + self_z_max.max(other_z_max), + self_y_max.max(other_y_max), + self_x_max.max(other_x_max), + ]; + } +} + #[cfg(test)] mod tests { use super::*; @@ -461,4 +612,36 @@ mod tests { assert_eq!(l2, 1 * 42 * 42 + 1 * 42 + 1); assert_eq!(a.grid_idx_unchecked(l2), [2, 2, 2].into()); } + + #[test] + fn grid_bounding_box_2d_contains() { + let a = GridBoundingBox::new([1, 1], [42, 42]).unwrap(); + let b = GridBoundingBox::new([2, 2], [41, 41]).unwrap(); + assert!(a.contains(&b)); + assert!(!b.contains(&a)); + } + + #[test] + fn extend_1d() { + let mut a = GridBoundingBox::new([1], [42]).unwrap(); + let b = GridBoundingBox::new([2], [69]).unwrap(); + a.extend(&b); + assert_eq!(a, GridBoundingBox::new([1], [69]).unwrap()); + } + + #[test] + fn extend_2d() { + let mut a = GridBoundingBox::new([1, 2], [42, 69]).unwrap(); + let b = GridBoundingBox::new([2, 1], [69, 42]).unwrap(); + a.extend(&b); + assert_eq!(a, GridBoundingBox::new([1, 1], [69, 69]).unwrap()); + } + + #[test] + fn extend_3d() { + let mut a = GridBoundingBox::new([1, 3, 2], [42, 69, 666]).unwrap(); + let b = GridBoundingBox::new([3, 2, 1], [69, 666, 42]).unwrap(); + a.extend(&b); + assert_eq!(a, GridBoundingBox::new([1, 2, 1], [69, 666, 666]).unwrap()); + } } diff --git a/datatypes/src/raster/grid_index.rs b/datatypes/src/raster/grid_index.rs index 05fe9d125..fc5b1da27 100644 --- a/datatypes/src/raster/grid_index.rs +++ b/datatypes/src/raster/grid_index.rs @@ -1,8 +1,10 @@ -use std::ops::{Add, Div, Mul, Rem, Sub}; +use std::ops::{Add, Div, Mul, Neg, Rem, Sub}; use num_traits::{One, Zero}; use serde::{Deserialize, Serialize}; +use super::GridShape2D; + /// /// The grid index struct. This is a wrapper for arrays with added methods and traits, e.g. Add, Sub... /// @@ -228,6 +230,19 @@ where } } +impl Mul for GridIdx2D { + type Output = Self; + + fn mul(self, rhs: GridShape2D) -> Self::Output { + let GridIdx([a, b]) = self; + let GridShape2D { + shape_array: [shape_a, shape_b], + } = rhs; + + GridIdx([a * shape_a as isize, b * shape_b as isize]) + } +} + impl Mul for GridIdx3D where I: Into, @@ -318,3 +333,74 @@ where GridIdx([a % a_other, b % b_other, c % c_other]) } } + +impl GridIdx1D { + pub fn x(self) -> isize { + let [a] = self.0; + a + } + + pub fn to_2d(self) -> GridIdx2D { + let [a] = self.0; + GridIdx([a, 0]) + } + + pub fn to_3d(self) -> GridIdx3D { + let [a] = self.0; + GridIdx([a, 0, 0]) + } + + pub fn new_x(x: isize) -> Self { + GridIdx([x]) + } +} + +impl GridIdx2D { + pub fn x(&self) -> isize { + let [_, x] = self.0; + x + } + + pub fn y(&self) -> isize { + let [y, _] = self.0; + y + } + + pub fn to_3d(self) -> GridIdx3D { + let [a, b] = self.0; + GridIdx([a, b, 0]) + } + + pub fn new_y_x(y: isize, x: isize) -> Self { + GridIdx([y, x]) + } +} + +impl GridIdx3D { + pub fn x(&self) -> isize { + let [_, _, x] = self.0; + x + } + + pub fn y(&self) -> isize { + let [_, y, _] = self.0; + y + } + + pub fn z(&self) -> isize { + let [z, _, _] = self.0; + z + } + + pub fn new_z_y_x(z: isize, y: isize, x: isize) -> Self { + GridIdx([z, y, x]) + } +} + +impl Neg for GridIdx2D { + type Output = Self; + + fn neg(self) -> Self { + GridIdx::new_y_x(-self.y(), -self.x()) + } +} diff --git a/datatypes/src/raster/grid_or_empty.rs b/datatypes/src/raster/grid_or_empty.rs index 5a22943f3..f4c02623e 100644 --- a/datatypes/src/raster/grid_or_empty.rs +++ b/datatypes/src/raster/grid_or_empty.rs @@ -148,7 +148,7 @@ where impl GridBounds for GridOrEmpty where - D: GridBounds + GridSpaceToLinearSpace, + D: GridBounds, T: Clone, I: AsRef<[isize]> + Into>, { @@ -213,29 +213,39 @@ where } } -impl ChangeGridBounds for GridOrEmpty +impl ChangeGridBounds for GridOrEmpty where - I: AsRef<[isize]> + Clone, - D: GridBounds + Clone + GridSpaceToLinearSpace, - T: Copy, - GridBoundingBox: GridSize, + D: GridBounds + GridSize, + I: AsRef<[isize]> + Into> + Clone, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, GridIdx: Add> + From, + T: Copy, { - type Output = GridOrEmpty, T>; + type BoundedOutput = GridOrEmpty, T>; + type UnboundedOutput = GridOrEmpty, T>; - fn shift_by_offset(self, offset: GridIdx) -> Self::Output { + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput { match self { GridOrEmpty::Grid(g) => GridOrEmpty::Grid(g.shift_by_offset(offset)), GridOrEmpty::Empty(n) => GridOrEmpty::Empty(n.shift_by_offset(offset)), } } - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { match self { GridOrEmpty::Grid(g) => g.set_grid_bounds(bounds).map(Into::into), GridOrEmpty::Empty(n) => n.set_grid_bounds(bounds).map(Into::into), } } + + fn unbounded(self) -> Self::UnboundedOutput { + match self { + GridOrEmpty::Grid(g) => GridOrEmpty::Grid(g.unbounded()), + GridOrEmpty::Empty(n) => GridOrEmpty::Empty(n.unbounded()), + } + } } #[cfg(test)] @@ -247,7 +257,7 @@ mod tests { #[test] fn grid_bounds_2d_empty_grid() { let dim: GridShape2D = [3, 2].into(); - let raster2d: GridOrEmpty2D = EmptyGrid::new(dim).into(); // FIXME: find out why type is needed + let raster2d: GridOrEmpty2D = EmptyGrid::new(dim).into(); assert_eq!(raster2d.min_index(), GridIdx([0, 0])); assert_eq!(raster2d.max_index(), GridIdx([2, 1])); diff --git a/datatypes/src/raster/grid_spatial.rs b/datatypes/src/raster/grid_spatial.rs new file mode 100644 index 000000000..7ef7678ae --- /dev/null +++ b/datatypes/src/raster/grid_spatial.rs @@ -0,0 +1,501 @@ +use super::{ + FromIndexFn, GeoTransform, GeoTransformAccess, Grid, GridBoundingBox2D, GridBoundingBoxExt, + GridBounds, GridIdx, GridIdx2D, GridIntersection, TilingSpecification, TilingStrategy, +}; +use crate::{ + operations::reproject::{ + CoordinateProjection, Reproject, ReprojectClipped, suggest_output_spatial_grid_like_gdal, + }, + primitives::{ + AxisAlignedRectangle, Coordinate2D, SpatialPartition2D, SpatialPartitioned, + SpatialResolution, + }, + util::Result, +}; +use float_cmp::approx_eq; +use postgres_types::{FromSql, ToSql}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, ToSql, FromSql, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridDefinition { + pub geo_transform: GeoTransform, + pub grid_bounds: GridBoundingBox2D, +} + +impl SpatialGridDefinition { + pub fn new(geo_transform: GeoTransform, grid_bounds: GridBoundingBox2D) -> Self { + Self { + geo_transform, + grid_bounds, + } + } + + pub fn new_generic>( + geo_transform: GeoTransform, + grid_bounds: G, + ) -> Self { + let grid_bounds: GridBoundingBox2D = grid_bounds.into(); + Self::new(geo_transform, grid_bounds) + } + + pub fn grid_bounds(&self) -> GridBoundingBox2D { + self.grid_bounds + } + + pub fn geo_transform(&self) -> GeoTransform { + self.geo_transform + } + + pub fn spatial_partition(&self) -> SpatialPartition2D { + self.geo_transform.grid_to_spatial_bounds(&self.grid_bounds) + } + + #[must_use] + /// Moves the origin and bounds using a pixel offset. The spatial location stays the same! + pub fn shift_bounds_relative_by_pixel_offset(&self, offset: GridIdx2D) -> Self { + let grid_bounds = self.grid_bounds.shift_by_offset(offset); + let geo_transform = self.geo_transform.shift_by_pixel_offset(-offset); + Self::new(geo_transform, grid_bounds) + } + + /// Moves the origin to another pixel edge. The spatial location stays the same! + /// Check if you can use `shift_bounds_relative_by_pixel_offset`! + pub fn with_moved_origin_exact_grid(&self, new_origin: Coordinate2D) -> Option { + if approx_eq!( + Coordinate2D, + self.geo_transform + .distance_to_nearest_pixel_edge(new_origin), + Coordinate2D::new(0., 0.) + ) { + Some(self.with_moved_origin_to_nearest_grid_edge(new_origin)) + } else { + None + } + } + + /// This method moves the origin to the coordinate of the grid edge nearest to the supplied new origin reference + #[must_use] + pub fn with_moved_origin_to_nearest_grid_edge( + &self, + new_origin_referece: Coordinate2D, + ) -> Self { + let nearest_to_target = self.geo_transform.nearest_pixel_edge(new_origin_referece); + self.shift_bounds_relative_by_pixel_offset(-nearest_to_target) + } + + /// This method moves the origin to the coordinate of the grid edge nearest to the supplied new origin reference + pub fn with_moved_origin_to_nearest_grid_edge_with_distance( + &self, + new_origin_referece: Coordinate2D, + ) -> (Self, Coordinate2D) { + let distance = self + .geo_transform + .distance_to_nearest_pixel_edge(new_origin_referece); + let new_self = self.with_moved_origin_to_nearest_grid_edge(new_origin_referece); + (new_self, distance) + } + + /// Creates a new spatial grid with the self shape and pixel size but new origin. + #[must_use] + pub fn replace_origin(&self, new_origin: Coordinate2D) -> Self { + Self { + geo_transform: GeoTransform::new( + new_origin, + self.geo_transform.x_pixel_size(), + self.geo_transform.y_pixel_size(), + ), + grid_bounds: self.grid_bounds, + } + } + + /// Merges two spatial grids + /// If the second grid is not compatible with selfit returns None + /// If the second grid has a different `GeoTransform` it is transformed to the `GroTransform` of self + pub fn merge(&self, other: &Self) -> Option { + if !self.is_compatible_grid_generic(other) { + return None; + } + + let other_shift = + other.with_moved_origin_exact_grid(self.geo_transform.origin_coordinate)?; + + let merged_bounds = self.grid_bounds().extended(&other_shift.grid_bounds()); + + Some(Self::new(self.geo_transform, merged_bounds)) + } + + pub fn is_compatible_grid_generic(&self, g: &G) -> bool { + self.geo_transform().is_compatible_grid(g.geo_transform()) + } + + /// Computes the intersection of self and other + /// IF other is incompatible with self, None is returned. + /// IF other has a different `GeoTransform` then self it is transformed to to the `GeoTransform` of self. + pub fn intersection(&self, other: &SpatialGridDefinition) -> Option { + if !self.is_compatible_grid_generic(other) { + return None; + } + + let (other_shift, dist) = other.with_moved_origin_to_nearest_grid_edge_with_distance( + self.geo_transform.origin_coordinate, + ); + if dist.x.abs() > self.geo_transform().x_pixel_size().abs() * 0.00001 // TODO: maybe use exact_grid and another epsilon? + || dist.y.abs() > self.geo_transform().y_pixel_size().abs() * 0.00001 + { + return None; + } + + let intersection_bounds = self + .grid_bounds() + .intersection(&(other_shift.grid_bounds()))?; + + Some(Self::new(self.geo_transform, intersection_bounds)) + } + + /// Creates a new spatial grid that has the same origin as self. + /// The pixel sizes are changed and the grid bounds are adapted to cover the same spatial area. + /// Note: if the new resolution is not a multiple of the old resolution the new grid might cover a larger spatial area then self. + #[must_use] + pub fn with_changed_resolution(&self, new_res: SpatialResolution) -> Self { + let geo_transform = + GeoTransform::new(self.geo_transform.origin_coordinate, new_res.x, -new_res.y); + let grid_bounds = geo_transform.spatial_to_grid_bounds(&self.spatial_partition()); + SpatialGridDefinition::new(geo_transform, grid_bounds) + } + + pub fn generate_coord_grid_upper_left_edge(&self) -> Grid { + let map_fn = |idx: GridIdx2D| { + self.geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(idx) + }; + + Grid::from_index_fn(&self.grid_bounds, map_fn) + } + + pub fn generate_coord_grid_pixel_center(&self) -> Grid { + let map_fn = |idx: GridIdx2D| { + self.geo_transform + .grid_idx_to_pixel_center_coordinate_2d(idx) + }; + + Grid::from_index_fn(&self.grid_bounds, map_fn) + } + + #[must_use] + pub fn spatial_bounds_to_compatible_spatial_grid( + &self, + spatial_partition: SpatialPartition2D, + ) -> Self { + let grid_bounds = self + .geo_transform + .spatial_to_grid_bounds(&spatial_partition); + Self::new(self.geo_transform, grid_bounds) + } + + #[must_use] + pub fn flip_axis_y(&self) -> Self { + let geo_transform = GeoTransform::new( + self.geo_transform.origin_coordinate, + self.geo_transform.x_pixel_size(), + -self.geo_transform.y_pixel_size(), + ); + + let y_min = -(self.grid_bounds.y_max() + 1); // since grid bounds are inclusive + let y_max = -(self.grid_bounds.y_min() + 1); + + let grid_bounds = GridBoundingBox2D::new_unchecked( + [y_min, self.grid_bounds.x_min()], + [y_max, self.grid_bounds.x_max()], + ); + + Self::new(geo_transform, grid_bounds) + } +} + +impl SpatialPartitioned for SpatialGridDefinition { + fn spatial_partition(&self) -> SpatialPartition2D { + self.spatial_partition() + } +} + +impl GridBounds for SpatialGridDefinition { + type IndexArray = [isize; 2]; + + fn min_index(&self) -> GridIdx { + self.grid_bounds.min_index() + } + + fn max_index(&self) -> GridIdx { + self.grid_bounds.max_index() + } +} + +impl GeoTransformAccess for SpatialGridDefinition { + fn geo_transform(&self) -> GeoTransform { + self.geo_transform() + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct TilingSpatialGridDefinition { + // Don't make this public to avoid leaking inner + element_grid_definition: SpatialGridDefinition, + tiling_specification: TilingSpecification, +} + +impl TilingSpatialGridDefinition { + pub fn new( + element_grid_definition: SpatialGridDefinition, + tiling_specification: TilingSpecification, + ) -> Self { + Self { + element_grid_definition, + tiling_specification, + } + } + + pub fn tiling_spatial_grid_definition(&self) -> SpatialGridDefinition { + // TODO: maybe do this in new and store it? + self.element_grid_definition + .with_moved_origin_to_nearest_grid_edge( + self.tiling_specification.tiling_origin_reference(), + ) + } + + pub fn tiling_geo_transform(&self) -> GeoTransform { + self.tiling_spatial_grid_definition().geo_transform() + } + + pub fn tiling_grid_bounds(&self) -> GridBoundingBox2D { + self.tiling_spatial_grid_definition().grid_bounds() + } + + pub fn is_compatible_grid_generic(&self, g: &G) -> bool { + // TODO: use tiling_spatial_grid_definition? + self.element_grid_definition.is_compatible_grid_generic(g) + } + + pub fn is_same_tiled_grid(&self, other: &TilingSpatialGridDefinition) -> bool { + // TODO: re-implement when decided how to model struct + let a = self.tiling_spatial_grid_definition(); + let b = other.tiling_spatial_grid_definition(); + approx_eq!(GeoTransform, a.geo_transform(), b.geo_transform()) + } + + /// Returns the data tiling strategy for the given tile size in pixels. + #[must_use] + pub fn generate_data_tiling_strategy(&self) -> TilingStrategy { + TilingStrategy { + geo_transform: self.tiling_geo_transform(), + tile_size_in_pixels: self.tiling_specification.tile_size_in_pixels, + } + } + + #[must_use] + pub fn with_other_bounds(&self, new_bounds: GridBoundingBox2D) -> Self { + let new_grid = SpatialGridDefinition::new(self.tiling_geo_transform(), new_bounds); + Self::new(new_grid, self.tiling_specification) + } +} + +impl SpatialPartitioned for TilingSpatialGridDefinition { + fn spatial_partition(&self) -> SpatialPartition2D { + // TODO: use tiling bounds and geotransform? must be equal!!! + self.element_grid_definition.spatial_partition() + } +} + +impl Reproject

for SpatialGridDefinition { + type Out = Self; + + fn reproject(&self, projector: &P) -> Result { + suggest_output_spatial_grid_like_gdal(self, projector) + } +} + +impl ReprojectClipped

for SpatialGridDefinition { + type Out = Self; + + fn reproject_clipped(&self, projector: &P) -> Result> { + let target_bounds_in_source_srs: Option = projector + .source_srs() + .area_of_use_intersection(&projector.target_srs())?; + if target_bounds_in_source_srs.is_none() { + return Ok(None); + } + let target_bounds_in_source_srs = target_bounds_in_source_srs.expect("case checked above"); + let intersection_grid_bounds = + target_bounds_in_source_srs.intersection(&self.spatial_partition()); + if intersection_grid_bounds.is_none() { + return Ok(None); + } + let intersection_grid_bounds = intersection_grid_bounds.expect("case checked above"); + let intersecting_grid = + self.spatial_bounds_to_compatible_spatial_grid(intersection_grid_bounds); + let compatible_intersecting_grid = if target_bounds_in_source_srs + .contains_coordinate(&self.geo_transform().origin_coordinate()) + { + intersecting_grid + } else { + intersecting_grid.with_moved_origin_to_nearest_grid_edge( + intersecting_grid.spatial_partition().upper_left(), + ) + }; + compatible_intersecting_grid + .reproject(projector) + .map(Option::Some) + } +} + +#[cfg(test)] +mod tests { + use crate::{ + operations::reproject::suggest_output_spatial_grid_like_gdal_helper, + primitives::AxisAlignedRectangle, + raster::{BoundedGrid, GridShape}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority}, + test_data, + util::gdal::gdal_open_dataset, + }; + + use super::*; + + #[test] + fn shift_bounds_relative_by_pixel_offset() { + let s = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + let shifted_s = s.shift_bounds_relative_by_pixel_offset(GridIdx2D::new([1, 1])); + assert_eq!( + shifted_s.geo_transform(), + GeoTransform::new_with_coordinate_x_y(-1., 1.0, 1., -1.0) + ); + assert_eq!(shifted_s.grid_bounds().min_index(), GridIdx2D::new([-1, 1])); + assert_eq!(shifted_s.grid_bounds().max_index(), GridIdx2D::new([1, 3])); + + assert_eq!(s.spatial_partition(), shifted_s.spatial_partition()); + } + + #[test] + fn merge() { + let s = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + let s_2 = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(1.0, 1.0, -1.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + let merged = s.merge(&s_2).unwrap(); + + assert_eq!(s.geo_transform, merged.geo_transform); + assert_eq!( + GridBoundingBox2D::new_min_max(-2, 1, 0, 3).unwrap(), + merged.grid_bounds + ); + + let s_s2_spatial_partition = s.spatial_partition().extended(&s_2.spatial_partition()); + let merged_partition = merged.spatial_partition(); + + assert!(approx_eq!( + Coordinate2D, + s_s2_spatial_partition.upper_left(), + merged_partition.upper_left() + )); + assert!(approx_eq!( + Coordinate2D, + s_s2_spatial_partition.lower_right(), + merged_partition.lower_right() + )); + } + + #[test] + fn no_merge_origin() { + let s = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + let s_2 = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(1.1, 1.0, -1.1, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + assert!(s.merge(&s_2).is_none()); + } + + #[test] + fn no_merge_pixel_size() { + let s = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + let s_2 = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(1.0, 1.1, -1.0, -1.1), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + assert!(s.merge(&s_2).is_none()); + } + + #[test] + fn source_resolution() { + let epsg_4326 = SpatialReference::epsg_4326(); + let epsg_3857 = SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857); + + // use ndvi dataset that was reprojected using gdal as ground truth + let dataset_4326 = gdal_open_dataset(test_data!( + "raster/modis_ndvi/MOD13A2_M_NDVI_2014-04-01.TIFF" + )) + .unwrap(); + let geotransform_4326 = dataset_4326.geo_transform().unwrap(); + let res_4326 = SpatialResolution::new(geotransform_4326[1], -geotransform_4326[5]).unwrap(); + + let dataset_3857 = gdal_open_dataset(test_data!( + "raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01.TIFF" + )) + .unwrap(); + let geotransform_3857 = dataset_3857.geo_transform().unwrap(); + + // ndvi was projected from 4326 to 3857. The calculated source_resolution for getting the raster in 3857 with `res_3857` + // should thus roughly be like the original `res_4326` + + let spatial_grid_3857 = SpatialGridDefinition::new( + geotransform_3857.into(), + GridShape::new_2d(dataset_3857.raster_size().1, dataset_3857.raster_size().0) + .bounding_box(), + ); + + let result_res = + suggest_output_spatial_grid_like_gdal_helper(&spatial_grid_3857, epsg_3857, epsg_4326) + .unwrap(); + + assert!(1. - (result_res.geo_transform().x_pixel_size() / res_4326.x).abs() < 0.02); + assert!(1. - (result_res.geo_transform().y_pixel_size() / res_4326.y).abs() < 0.02); + } + + #[test] + fn flip_axis_y() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(20.0, 20.0), 3., 2.), + GridBoundingBox2D::new_min_max(10, 25, 1, 2).unwrap(), + ); + + let fliped = spatial_grid.flip_axis_y(); + + assert_eq!( + fliped.geo_transform, + GeoTransform::new(Coordinate2D::new(20.0, 20.0), 3., -2.) + ); + + assert_eq!( + fliped.grid_bounds, + GridBoundingBox2D::new_min_max(-26, -11, 1, 2).unwrap() + ); + } +} diff --git a/datatypes/src/raster/grid_traits.rs b/datatypes/src/raster/grid_traits.rs index d6b8e3204..aa1439209 100644 --- a/datatypes/src/raster/grid_traits.rs +++ b/datatypes/src/raster/grid_traits.rs @@ -82,6 +82,10 @@ where pub trait GridIntersection { // Returns true if Self intesects Rhs fn intersection(&self, other: &Rhs) -> Option; + + fn intersects(&self, other: &Rhs) -> bool { + self.intersection(other).is_some() + } } /// Provides the methods needed to map an n-dimensional `GridIdx` to linear space. @@ -148,13 +152,17 @@ pub trait GridShapeAccess { } /// Change the bounds of gridded data. -pub trait ChangeGridBounds: BoundedGrid +pub trait ChangeGridBounds: + BoundedGrid + GridShapeAccess where I: AsRef<[isize]> + Into> + Clone, - GridBoundingBox: GridSize, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, GridIdx: Add> + From, { - type Output; + type BoundedOutput; + type UnboundedOutput; fn shift_bounding_box(&self, offset: GridIdx) -> GridBoundingBox { let bounds = self.bounding_box(); @@ -165,10 +173,13 @@ where } /// shift using an offset - fn shift_by_offset(self, offset: GridIdx) -> Self::Output; + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput; /// set new bounds. will fail if the axis sizes do not match. - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result; + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result; + + /// remove the bounds. Keep the shape + fn unbounded(self) -> Self::UnboundedOutput; } pub trait GridStep: GridSpaceToLinearSpace diff --git a/datatypes/src/raster/masked_grid.rs b/datatypes/src/raster/masked_grid.rs index bf6b50a7e..eb2cf9330 100644 --- a/datatypes/src/raster/masked_grid.rs +++ b/datatypes/src/raster/masked_grid.rs @@ -272,29 +272,39 @@ where } } -impl ChangeGridBounds for MaskedGrid +impl ChangeGridBounds for MaskedGrid where - I: AsRef<[isize]> + Clone, - D: GridBounds + Clone, - T: Clone, - GridBoundingBox: GridSize, - GridIdx: Add> + From + Clone, + D: GridBounds + GridSize, + I: AsRef<[isize]> + Into> + Clone, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, + GridIdx: Add> + From, + T: Copy, { - type Output = MaskedGrid, T>; + type BoundedOutput = MaskedGrid, T>; + type UnboundedOutput = MaskedGrid, T>; - fn shift_by_offset(self, offset: GridIdx) -> Self::Output { + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput { MaskedGrid { inner_grid: self.inner_grid.shift_by_offset(offset.clone()), validity_mask: self.validity_mask.shift_by_offset(offset), } } - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { Ok(MaskedGrid { inner_grid: self.inner_grid.set_grid_bounds(bounds.clone())?, validity_mask: self.validity_mask.set_grid_bounds(bounds)?, }) } + + fn unbounded(self) -> Self::UnboundedOutput { + MaskedGrid { + inner_grid: self.inner_grid.unbounded(), + validity_mask: self.validity_mask.unbounded(), + } + } } impl ByteSize for MaskedGrid diff --git a/datatypes/src/raster/mod.rs b/datatypes/src/raster/mod.rs index 90e9cbc5e..f499d7d68 100755 --- a/datatypes/src/raster/mod.rs +++ b/datatypes/src/raster/mod.rs @@ -9,7 +9,7 @@ pub use self::grid::{ grid_idx_iter_2d, }; pub use self::grid_bounds::{ - GridBoundingBox, GridBoundingBox1D, GridBoundingBox2D, GridBoundingBox3D, + GridBoundingBox, GridBoundingBox1D, GridBoundingBox2D, GridBoundingBox3D, GridBoundingBoxExt, }; pub use self::grid_index::{GridIdx, GridIdx1D, GridIdx2D, GridIdx3D}; pub use self::grid_or_empty::{GridOrEmpty, GridOrEmpty1D, GridOrEmpty2D, GridOrEmpty3D}; @@ -19,7 +19,7 @@ pub use self::grid_traits::{ }; pub use self::grid_typed::{TypedGrid, TypedGrid2D, TypedGrid3D}; pub use self::operations::{ - blit::Blit, convert_data_type::ConvertDataType, convert_data_type::ConvertDataTypeParallel, + convert_data_type::ConvertDataType, convert_data_type::ConvertDataTypeParallel, grid_blit::GridBlit, interpolation::Bilinear, interpolation::InterpolationAlgorithm, interpolation::NearestNeighbor, }; @@ -32,6 +32,8 @@ pub use self::typed_raster_conversion::TypedRasterConversion; pub use self::typed_raster_tile::{TypedRasterTile2D, TypedRasterTile3D}; pub use self::{grid_traits::ChangeGridBounds, grid_traits::GridShapeAccess}; pub use arrow_conversion::raster_tile_2d_to_arrow_ipc_file; +pub use db_types::GridBoundingBox2DDbType; +pub use grid_spatial::{SpatialGridDefinition, TilingSpatialGridDefinition}; pub use masked_grid::{MaskedGrid, MaskedGrid1D, MaskedGrid2D, MaskedGrid3D}; pub use no_data_value_grid::{ NoDataValueGrid, NoDataValueGrid1D, NoDataValueGrid2D, NoDataValueGrid3D, @@ -43,6 +45,7 @@ pub use operations::checked_scaling::{ pub use operations::from_index_fn::{FromIndexFn, FromIndexFnParallel}; pub use operations::map_elements::{MapElements, MapElementsParallel}; pub use operations::map_indexed_elements::{MapIndexedElements, MapIndexedElementsParallel}; +pub use operations::sample_points::SamplePoints; pub use operations::update_elements::{UpdateElements, UpdateElementsParallel}; pub use operations::update_indexed_elements::{ UpdateIndexedElements, UpdateIndexedElementsParallel, @@ -55,12 +58,14 @@ pub use raster_traits::{CoordinatePixelAccess, GeoTransformAccess, Raster}; mod arrow_conversion; mod band_names; mod data_type; +mod db_types; mod empty_grid; mod geo_transform; mod grid; mod grid_bounds; mod grid_index; mod grid_or_empty; +mod grid_spatial; mod grid_traits; mod grid_typed; mod macros_raster; diff --git a/datatypes/src/raster/operations/blit.rs b/datatypes/src/raster/operations/blit.rs index 804cac7c2..8b1378917 100644 --- a/datatypes/src/raster/operations/blit.rs +++ b/datatypes/src/raster/operations/blit.rs @@ -1,294 +1 @@ -use crate::error; -use crate::raster::{ - ChangeGridBounds, GeoTransformAccess, GridBlit, GridIdx2D, MaterializedRasterTile2D, Pixel, - RasterTile2D, -}; -use crate::util::Result; -use snafu::ensure; - -pub trait Blit { - fn blit(&mut self, source: R) -> Result<()>; -} - -impl Blit> for MaterializedRasterTile2D { - /// Copy `source` raster pixels into this raster, fails if the rasters do not overlap - #[allow(clippy::float_cmp)] - fn blit(&mut self, source: RasterTile2D) -> Result<()> { - // TODO: same crs - // TODO: allow approximately equal pixel sizes? - // TODO: ensure pixels are aligned - - let into_geo_transform = self.geo_transform(); - let from_geo_transform = source.geo_transform(); - - ensure!( - (self.geo_transform().x_pixel_size() == source.geo_transform().x_pixel_size()) - && (self.geo_transform().y_pixel_size() == source.geo_transform().y_pixel_size()), - error::Blit { - details: "Incompatible pixel size" - } - ); - - let offset = from_geo_transform.origin_coordinate - into_geo_transform.origin_coordinate; - - let offset_x_pixels = (offset.x / into_geo_transform.x_pixel_size()).round() as isize; - let offset_y_pixels = (offset.y / into_geo_transform.y_pixel_size()).round() as isize; - - /* - ensure!( - offset_x_pixels.abs() <= self.grid_array.axis_size_x() as isize - && offset_y_pixels.abs() <= self.grid_array.axis_size_y() as isize, - error::Blit { - details: "No overlapping region", - } - ); - */ - - let origin_offset_pixels = GridIdx2D::new([offset_y_pixels, offset_x_pixels]); - - let tile_offset_pixels = source.tile_information().global_upper_left_pixel_idx() - - self.tile_information().global_upper_left_pixel_idx(); - let global_offset_pixels = origin_offset_pixels + tile_offset_pixels; - - let shifted_source = source.grid_array.shift_by_offset(global_offset_pixels); - - self.grid_array.grid_blit_from(&shifted_source); - - self.cache_hint.merge_with(&source.cache_hint); - - Ok(()) - } -} - -impl Blit> for RasterTile2D { - /// Copy `source` raster pixels into this raster, fails if the rasters do not overlap - #[allow(clippy::float_cmp)] - fn blit(&mut self, source: RasterTile2D) -> Result<()> { - // TODO: same crs - // TODO: allow approximately equal pixel sizes? - // TODO: ensure pixels are aligned - - let into_geo_transform = self.geo_transform(); - let from_geo_transform = source.geo_transform(); - - ensure!( - (self.geo_transform().x_pixel_size() == source.geo_transform().x_pixel_size()) - && (self.geo_transform().y_pixel_size() == source.geo_transform().y_pixel_size()), - error::Blit { - details: "Incompatible pixel size" - } - ); - - let offset = from_geo_transform.origin_coordinate - into_geo_transform.origin_coordinate; - - let offset_x_pixels = (offset.x / into_geo_transform.x_pixel_size()).round() as isize; - let offset_y_pixels = (offset.y / into_geo_transform.y_pixel_size()).round() as isize; - - /* - ensure!( - offset_x_pixels.abs() <= self.grid_array.axis_size_x() as isize - && offset_y_pixels.abs() <= self.grid_array.axis_size_y() as isize, - error::Blit { - details: "No overlapping region", - } - ); - */ - - let origin_offset_pixels = GridIdx2D::new([offset_y_pixels, offset_x_pixels]); - - let tile_offset_pixels = source.tile_information().global_upper_left_pixel_idx() - - self.tile_information().global_upper_left_pixel_idx(); - let global_offset_pixels = origin_offset_pixels + tile_offset_pixels; - - let shifted_source = source.grid_array.shift_by_offset(global_offset_pixels); - - self.grid_array.grid_blit_from(&shifted_source); - - self.cache_hint.merge_with(&source.cache_hint); - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::{ - primitives::{CacheHint, TimeInterval}, - raster::{Blit, GeoTransform, Grid2D, RasterTile2D}, - }; - - #[test] - fn test_blit_ur_materialized() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::default(), - ) - .into_materialized_tile(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((5.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let t2 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r2, - CacheHint::default(), - ); - - t1.blit(t2).unwrap(); - - assert_eq!( - t1.grid_array.inner_grid.data, - vec![0, 0, 8, 9, 0, 0, 12, 13, 0, 0, 0, 0, 0, 0, 0, 0] - ); - } - - #[test] - fn test_blit_ul() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::default(), - ) - .into_materialized_tile(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((-5.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let t2 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r2, - CacheHint::default(), - ); - - t1.blit(t2).unwrap(); - - assert_eq!( - t1.grid_array.inner_grid.data, - vec![10, 11, 0, 0, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - ); - } - - #[test] - fn test_blit_ll() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::default(), - ) - .into_materialized_tile(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((-5.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let t2 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r2, - CacheHint::default(), - ); - - t1.blit(t2).unwrap(); - - assert_eq!( - t1.grid_array.inner_grid.data, - vec![0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 6, 7, 0, 0] - ); - } - - #[test] - fn test_blit_ur() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::default(), - ); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((5.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let t2 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r2, - CacheHint::default(), - ); - - t1.blit(t2).unwrap(); - - assert!(!t1.is_empty()); - - let masked_grid = match t1.grid_array { - crate::raster::GridOrEmpty::Grid(g) => g, - crate::raster::GridOrEmpty::Empty(_) => panic!("exppected a materialized grid"), - }; - - assert_eq!( - masked_grid.inner_grid.data, - vec![0, 0, 8, 9, 0, 0, 12, 13, 0, 0, 0, 0, 0, 0, 0, 0] - ); - } - - #[test] - fn it_attaches_cache_hint() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::max_duration(), - ) - .into_materialized_tile(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((-5.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let cache_hint = CacheHint::seconds(1234); - let t2 = RasterTile2D::new_without_offset(temporal_bounds, geo_transform, r2, cache_hint); - - t1.blit(t2).unwrap(); - - assert_eq!(t1.cache_hint.expires(), cache_hint.expires()); - } -} diff --git a/datatypes/src/raster/operations/grid_blit.rs b/datatypes/src/raster/operations/grid_blit.rs index f9cc8f8dc..e0b661326 100644 --- a/datatypes/src/raster/operations/grid_blit.rs +++ b/datatypes/src/raster/operations/grid_blit.rs @@ -1,7 +1,7 @@ use crate::raster::{ - BoundedGrid, Grid, Grid1D, Grid2D, Grid3D, GridBoundingBox, GridBounds, GridIdx, - GridIndexAccessMut, GridIntersection, GridOrEmpty, GridSize, GridSpaceToLinearSpace, - empty_grid::EmptyGrid, masked_grid::MaskedGrid, + BoundedGrid, Grid, Grid1D, Grid3D, GridBoundingBox, GridBounds, GridIdx, GridIndexAccessMut, + GridIntersection, GridOrEmpty, GridSize, GridSpaceToLinearSpace, empty_grid::EmptyGrid, + masked_grid::MaskedGrid, }; pub trait GridBlit @@ -37,11 +37,14 @@ where } } -impl GridBlit, T> for Grid2D +impl GridBlit, T> for Grid where D: GridSize + GridBounds + GridSpaceToLinearSpace, + D2: GridSize + + GridBounds + + GridSpaceToLinearSpace, T: Copy + Sized, { fn grid_blit_from(&mut self, other: &Grid) { @@ -145,11 +148,14 @@ where } } -impl GridBlit, T> for Grid2D +impl GridBlit, T> for Grid where D: GridSize + GridBounds + GridSpaceToLinearSpace, + D2: GridSize + + GridBounds + + GridSpaceToLinearSpace, T: Copy + Sized, { fn grid_blit_from(&mut self, other: &EmptyGrid) { diff --git a/datatypes/src/raster/operations/interpolation.rs b/datatypes/src/raster/operations/interpolation.rs index 768076995..afc30dfd6 100644 --- a/datatypes/src/raster/operations/interpolation.rs +++ b/datatypes/src/raster/operations/interpolation.rs @@ -1,70 +1,60 @@ use super::from_index_fn::FromIndexFnParallel; -use crate::primitives::{AxisAlignedRectangle, SpatialPartitioned}; use crate::raster::{ - EmptyGrid, GridIdx, GridIdx2D, GridIndexAccess, GridOrEmpty, Pixel, RasterTile2D, - TileInformation, + GeoTransform, GridBounds, GridIdx, GridIdx2D, GridIndexAccess, GridOrEmpty, GridShapeAccess, + GridSize, GridSpaceToLinearSpace, Pixel, }; use crate::util::Result; -pub trait InterpolationAlgorithm: Send + Sync + Clone + 'static { +pub trait InterpolationAlgorithm: Send + Sync + Clone + 'static { /// interpolate the given input tile into the output tile /// the output must be fully contained in the input tile and have an additional row and column in order /// to have all the required neighbor pixels. /// Also the output must have a finer resolution than the input fn interpolate( - input: &RasterTile2D

, - output_tile_info: &TileInformation, - ) -> Result>; + in_geo_transform: GeoTransform, + input: &GridOrEmpty, + out_geo_transform: GeoTransform, + out_bounds: D, + ) -> Result>; } #[derive(Clone, Debug)] pub struct NearestNeighbor {} -impl

InterpolationAlgorithm

for NearestNeighbor +impl InterpolationAlgorithm for NearestNeighbor where + D: GridShapeAccess + + Clone + + GridSize + + GridBounds + + PartialEq + + Send + + Sync + + GridSpaceToLinearSpace, P: Pixel, + GridOrEmpty: + GridIndexAccess, GridIdx<::IndexArray>>, { - fn interpolate(input: &RasterTile2D

, info_out: &TileInformation) -> Result> { + fn interpolate( + in_geo_transform: GeoTransform, + input: &GridOrEmpty, + out_geo_transform: GeoTransform, + out_bounds: D, + ) -> Result> { if input.is_empty() { - return Ok(RasterTile2D::new_with_tile_info( - input.time, - *info_out, - input.band, - EmptyGrid::new(info_out.tile_size_in_pixels).into(), - input.cache_hint.clone_with_current_datetime(), - )); + return Ok(GridOrEmpty::new_empty_shape(out_bounds)); } - let info_in = input.tile_information(); - let in_upper_left = info_in.spatial_partition().upper_left(); - let in_x_size = info_in.global_geo_transform.x_pixel_size(); - let in_y_size = info_in.global_geo_transform.y_pixel_size(); - - let out_upper_left = info_out.spatial_partition().upper_left(); - let out_x_size = info_out.global_geo_transform.x_pixel_size(); - let out_y_size = info_out.global_geo_transform.y_pixel_size(); - let map_fn = |gidx: GridIdx2D| { - let GridIdx([y, x]) = gidx; - let out_y_coord = out_upper_left.y + y as f64 * out_y_size; - let out_x_coord = out_upper_left.x + x as f64 * out_x_size; - let nearest_in_y_idx = ((out_y_coord - in_upper_left.y) / in_y_size).round() as isize; - let nearest_in_x_idx = ((out_x_coord - in_upper_left.x) / in_x_size).round() as isize; - input.get_at_grid_index_unchecked([nearest_in_y_idx, nearest_in_x_idx]) - }; + let coordinate = out_geo_transform.grid_idx_to_pixel_center_coordinate_2d(gidx); // use center coordinate similar to ArgGIS + let pixel_in_input = in_geo_transform.coordinate_to_grid_idx_2d(coordinate); - let out_data = GridOrEmpty::from_index_fn_parallel(&info_out.tile_size_in_pixels, map_fn); // TODO: this will check for empty tiles. Change to MaskedGrid::from.. to avoid this. + input.get_at_grid_index_unchecked(pixel_in_input) + }; - let out_tile = RasterTile2D::new( - input.time, - info_out.global_tile_position, - input.band, - info_out.global_geo_transform, - out_data, - input.cache_hint.clone_with_current_datetime(), - ); + let out_data = GridOrEmpty::from_index_fn_parallel(&out_bounds, map_fn); // TODO: this will check for empty tiles. Change to MaskedGrid::from.. to avoid this. - Ok(out_tile) + Ok(out_data) } } @@ -99,57 +89,59 @@ impl Bilinear { } } -impl

InterpolationAlgorithm

for Bilinear +impl InterpolationAlgorithm for Bilinear where + D: GridShapeAccess + + Clone + + GridSize + + GridBounds + + PartialEq + + Send + + Sync + + GridSpaceToLinearSpace, P: Pixel, + GridOrEmpty: + GridIndexAccess, GridIdx<::IndexArray>>, { - fn interpolate(input: &RasterTile2D

, info_out: &TileInformation) -> Result> { + fn interpolate( + in_geo_transform: GeoTransform, + input: &GridOrEmpty, + out_geo_transform: GeoTransform, + out_bounds: D, + ) -> Result> { if input.is_empty() { - return Ok(RasterTile2D::new_with_tile_info( - input.time, - *info_out, - input.band, - EmptyGrid::new(info_out.tile_size_in_pixels).into(), - input.cache_hint.clone_with_current_datetime(), - )); + return Ok(GridOrEmpty::new_empty_shape(out_bounds)); } - let info_in = input.tile_information(); - let in_upper_left = info_in.spatial_partition().upper_left(); - let in_x_size = info_in.global_geo_transform.x_pixel_size(); - let in_y_size = info_in.global_geo_transform.y_pixel_size(); - - let out_upper_left = info_out.spatial_partition().upper_left(); - let out_x_size = info_out.global_geo_transform.x_pixel_size(); - let out_y_size = info_out.global_geo_transform.y_pixel_size(); + let map_fn = |out_g_idx: GridIdx2D| { + let out_coord = out_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(out_g_idx); - let map_fn = |g_idx: GridIdx2D| { - let GridIdx([y_idx, x_idx]) = g_idx; + let in_g_idx = in_geo_transform.coordinate_to_grid_idx_2d(out_coord); - let out_y = out_upper_left.y + y_idx as f64 * out_y_size; - let in_y_idx = ((out_y - in_upper_left.y) / in_y_size).floor() as isize; + let in_a_idx = in_g_idx; + let in_b_idx = in_a_idx + [1, 0]; + let in_c_idx = in_a_idx + [0, 1]; + let in_d_idx = in_a_idx + [1, 1]; - let a_y = in_upper_left.y + in_y_size * in_y_idx as f64; - let b_y = a_y + in_y_size; + let in_a_coord = in_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(in_a_idx); + let a_y = in_a_coord.y; + let b_y = a_y + in_geo_transform.y_pixel_size(); - let out_x = out_upper_left.x + x_idx as f64 * out_x_size; - let in_x_idx = ((out_x - in_upper_left.x) / in_x_size).floor() as isize; + let a_x = in_a_coord.x; + let c_x = a_x + in_geo_transform.x_pixel_size(); - let a_x = in_upper_left.x + in_x_size * in_x_idx as f64; - let c_x = a_x + in_x_size; + let a_v = input.get_at_grid_index(in_a_idx).unwrap_or(None); - let a_v = input.get_at_grid_index_unchecked([in_y_idx, in_x_idx]); + let b_v = input.get_at_grid_index(in_b_idx).unwrap_or(None); - let b_v = input.get_at_grid_index_unchecked([in_y_idx + 1, in_x_idx]); + let c_v = input.get_at_grid_index(in_c_idx).unwrap_or(None); - let c_v = input.get_at_grid_index_unchecked([in_y_idx, in_x_idx + 1]); - - let d_v = input.get_at_grid_index_unchecked([in_y_idx + 1, in_x_idx + 1]); + let d_v = input.get_at_grid_index(in_d_idx).unwrap_or(None); let value = match (a_v, b_v, c_v, d_v) { (Some(a), Some(b), Some(c), Some(d)) => Some(Self::bilinear_interpolation( - out_x, - out_y, + out_coord.x, + out_coord.y, a_x, a_y, a.as_(), @@ -165,18 +157,9 @@ where value.map(|v| P::from_(v)) }; - let out_data = GridOrEmpty::from_index_fn_parallel(&info_out.tile_size_in_pixels, map_fn); // TODO: this will check for empty tiles. Change to MaskedGrid::from.. to avoid this. + let out_data = GridOrEmpty::from_index_fn_parallel(&out_bounds, map_fn); - let out_tile = RasterTile2D::new( - input.time, - info_out.global_tile_position, - input.band, - info_out.global_geo_transform, - out_data, - input.cache_hint.clone_with_current_datetime(), - ); - - Ok(out_tile) + Ok(out_data) } } @@ -187,7 +170,10 @@ mod tests { use super::*; use crate::{ primitives::CacheHint, - raster::{GeoTransform, Grid2D, GridOrEmpty, MaskedGrid, RasterTile2D, TileInformation}, + raster::{ + GeoTransform, GeoTransformAccess, Grid2D, GridOrEmpty, MaskedGrid, RasterTile2D, + TileInformation, + }, }; #[test] @@ -206,42 +192,48 @@ mod tests { CacheHint::default(), ); + let input_geo_transform = input.geo_transform(); + let input_grid = input.into_inner_positioned_grid(); + let output_info = TileInformation { global_tile_position: [0, 0].into(), - tile_size_in_pixels: [4, 4].into(), + tile_size_in_pixels: [3, 3].into(), global_geo_transform: GeoTransform::new((0.0, 2.0).into(), 0.5, -0.5), }; + let output_geo_transform = output_info.global_geo_transform; + let output_bounds = output_info.global_pixel_bounds(); + let pool = ThreadPoolBuilder::new().num_threads(0).build().unwrap(); let output = pool - .install(|| NearestNeighbor::interpolate(&input, &output_info)) + .install(|| { + NearestNeighbor::interpolate( + input_geo_transform, + &input_grid, + output_geo_transform, + output_bounds, + ) + }) .unwrap(); assert!(!output.is_empty()); - let output_data = output.grid_array.as_masked_grid().unwrap(); + let output_data = output.as_masked_grid().unwrap(); assert_eq!( output_data .masked_element_deref_iterator() .collect::>(), vec![ + Some(1), Some(1), Some(2), + Some(1), + Some(1), Some(2), - Some(3), Some(4), - Some(5), - Some(5), - Some(6), Some(4), - Some(5), - Some(5), - Some(6), - Some(7), - Some(8), - Some(8), - Some(9) + Some(5) ] ); } @@ -285,20 +277,33 @@ mod tests { CacheHint::default(), ); + let input_geo_transform = input.geo_transform(); + let input_grid = input.into_inner_positioned_grid(); + let output_info = TileInformation { global_tile_position: [0, 0].into(), tile_size_in_pixels: [4, 4].into(), global_geo_transform: GeoTransform::new((0.0, 2.0).into(), 0.5, -0.5), }; + let output_geo_transform = output_info.global_geo_transform; + let output_bounds = output_info.global_pixel_bounds(); + let pool = ThreadPoolBuilder::new().num_threads(0).build().unwrap(); let output = pool - .install(|| Bilinear::interpolate(&input, &output_info)) + .install(|| { + Bilinear::interpolate( + input_geo_transform, + &input_grid, + output_geo_transform, + output_bounds, + ) + }) .unwrap(); assert!(!output.is_empty()); - let output_data = output.grid_array.as_masked_grid().unwrap(); + let output_data = output.as_masked_grid().unwrap(); assert_eq!( output_data diff --git a/datatypes/src/raster/operations/mod.rs b/datatypes/src/raster/operations/mod.rs index 3e05af7b5..f67130748 100644 --- a/datatypes/src/raster/operations/mod.rs +++ b/datatypes/src/raster/operations/mod.rs @@ -6,5 +6,6 @@ pub mod grid_blit; pub mod interpolation; pub mod map_elements; pub mod map_indexed_elements; +pub mod sample_points; pub mod update_elements; pub mod update_indexed_elements; diff --git a/datatypes/src/raster/operations/sample_points.rs b/datatypes/src/raster/operations/sample_points.rs new file mode 100644 index 000000000..9c9c212bf --- /dev/null +++ b/datatypes/src/raster/operations/sample_points.rs @@ -0,0 +1,253 @@ +use num::range_inclusive; + +use crate::{ + primitives::Coordinate2D, + raster::{GridBoundingBox2D, GridIdx2D, GridSize, SpatialGridDefinition}, +}; + +pub trait SamplePoints { + type Coord; + + fn sample_outline(&self, step: usize) -> Vec; + fn sample_cross(&self, step: usize) -> Vec; + fn sample_diagonals(&self, step: usize) -> Vec; +} + +impl SamplePoints for GridBoundingBox2D { + type Coord = GridIdx2D; + + fn sample_outline(&self, step: usize) -> Vec { + let [y_min, y_max] = self.y_bounds(); + let [x_min, x_max] = self.x_bounds(); + + let x_range = range_inclusive(x_min, x_max); + let y_range = range_inclusive(y_min, y_max); + + let capacity = (self.axis_size_x() / step) * 2 + (self.axis_size_y() / step) * 2; + + let mut collected: Vec = Vec::with_capacity(capacity); + + for x in x_range.step_by(step) { + collected.push(GridIdx2D::new_y_x(y_min, x)); + collected.push(GridIdx2D::new_y_x(y_max, x)); + } + + for y in y_range.step_by(step) { + collected.push(GridIdx2D::new_y_x(y, x_min)); + collected.push(GridIdx2D::new_y_x(y, x_max)); + } + + collected + } + + fn sample_cross(&self, step: usize) -> Vec { + let [y_min, y_max] = self.y_bounds(); + let [x_min, x_max] = self.x_bounds(); + let y_mid = y_min + (self.axis_size_y() / 2) as isize; + let x_mid = x_min + (self.axis_size_x() / 2) as isize; + + let x_range = range_inclusive(x_min, x_max); + let y_range = range_inclusive(y_min, y_max); + + let capacity = (self.axis_size_x() / step) + (self.axis_size_y() / step); + + let mut collected: Vec = Vec::with_capacity(capacity); + + for x in x_range.step_by(step) { + collected.push(GridIdx2D::new_y_x(y_mid, x)); + } + + for y in y_range.step_by(step) { + collected.push(GridIdx2D::new_y_x(y, x_mid)); + } + + collected + } + + fn sample_diagonals(&self, step: usize) -> Vec { + enum LongAxis { + X, + Y, + } + + let [y_min, y_max] = self.y_bounds(); + let [x_min, x_max] = self.x_bounds(); + + let x_range = range_inclusive(x_min, x_max); + let y_range = range_inclusive(y_min, y_max); + + let capacity = (self.axis_size_x() / step) * 2 + (self.axis_size_y() / step) * 2; + + let (long_range_id, long_range, b, b_max, m) = if self.axis_size_x() > self.axis_size_y() { + ( + LongAxis::X, + x_range, + y_min, + y_max, + (self.axis_size_y() as f32 / self.axis_size_x() as f32), + ) + } else { + ( + LongAxis::Y, + y_range, + x_min, + x_max, + (self.axis_size_x() as f32 / self.axis_size_y() as f32), + ) + }; + + let mut collected: Vec = Vec::with_capacity(capacity); + + for l in long_range { + let s = (l as f32 * m) as isize + b; + let s_inv = b_max - (l as f32 * m) as isize; + + match long_range_id { + LongAxis::X => { + debug_assert!(l >= x_min); + debug_assert!(l <= x_max); + debug_assert!(s >= y_min); + debug_assert!(s <= y_max); + collected.push(GridIdx2D::new_y_x(s, l)); + collected.push(GridIdx2D::new_y_x(s_inv, l)); + } + LongAxis::Y => { + debug_assert!(s >= x_min); + debug_assert!(s <= x_max); + debug_assert!(l >= y_min); + debug_assert!(l <= y_max); + collected.push(GridIdx2D::new_y_x(l, s)); + collected.push(GridIdx2D::new_y_x(l, s_inv)); + } + } + } + + collected + } +} + +impl SamplePoints for SpatialGridDefinition { + type Coord = Coordinate2D; + + fn sample_outline(&self, step: usize) -> Vec { + let px = self.grid_bounds.sample_outline(step); + px.iter() + .map(|gidx| { + self.geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(*gidx) + }) + .collect() + } + + fn sample_cross(&self, step: usize) -> Vec { + let px = self.grid_bounds.sample_cross(step); + px.iter() + .map(|gidx| { + self.geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(*gidx) + }) + .collect() + } + + fn sample_diagonals(&self, step: usize) -> Vec { + let px = self.grid_bounds.sample_diagonals(step); + px.iter() + .map(|gidx| { + self.geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(*gidx) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_sample_outline() { + let gb = GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(); + let ds = gb.sample_outline(1); + let exp = vec![ + GridIdx2D::new_y_x(0, 0), + GridIdx2D::new_y_x(4, 0), + GridIdx2D::new_y_x(0, 1), + GridIdx2D::new_y_x(4, 1), + GridIdx2D::new_y_x(0, 2), + GridIdx2D::new_y_x(4, 2), + GridIdx2D::new_y_x(0, 3), + GridIdx2D::new_y_x(4, 3), + GridIdx2D::new_y_x(0, 4), + GridIdx2D::new_y_x(4, 4), + GridIdx2D::new_y_x(0, 0), + GridIdx2D::new_y_x(0, 4), + GridIdx2D::new_y_x(1, 0), + GridIdx2D::new_y_x(1, 4), + GridIdx2D::new_y_x(2, 0), + GridIdx2D::new_y_x(2, 4), + GridIdx2D::new_y_x(3, 0), + GridIdx2D::new_y_x(3, 4), + GridIdx2D::new_y_x(4, 0), + GridIdx2D::new_y_x(4, 4), + ]; + assert_eq!(ds, exp); + } + + #[test] + fn test_sample_cross() { + let gb = GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(); + let ds = gb.sample_cross(1); + let exp = vec![ + GridIdx2D::new_y_x(2, 0), + GridIdx2D::new_y_x(2, 1), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(2, 3), + GridIdx2D::new_y_x(2, 4), + GridIdx2D::new_y_x(0, 2), + GridIdx2D::new_y_x(1, 2), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(3, 2), + GridIdx2D::new_y_x(4, 2), + ]; + assert_eq!(ds, exp); + } + + #[test] + fn test_sample_diagnals() { + let gb = GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(); + let ds = gb.sample_diagonals(1); + let exp = vec![ + GridIdx2D::new_y_x(0, 0), + GridIdx2D::new_y_x(0, 4), + GridIdx2D::new_y_x(1, 1), + GridIdx2D::new_y_x(1, 3), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(3, 3), + GridIdx2D::new_y_x(3, 1), + GridIdx2D::new_y_x(4, 4), + GridIdx2D::new_y_x(4, 0), + ]; + assert_eq!(ds, exp); + } + + #[test] + fn test_sample_diagnals_non_symetric() { + let gb = GridBoundingBox2D::new_min_max(0, 2, 0, 4).unwrap(); + let ds = gb.sample_diagonals(1); + let exp = vec![ + GridIdx2D::new_y_x(0, 0), + GridIdx2D::new_y_x(2, 0), + GridIdx2D::new_y_x(0, 1), + GridIdx2D::new_y_x(2, 1), + GridIdx2D::new_y_x(1, 2), + GridIdx2D::new_y_x(1, 2), + GridIdx2D::new_y_x(1, 3), + GridIdx2D::new_y_x(1, 3), + GridIdx2D::new_y_x(2, 4), + GridIdx2D::new_y_x(0, 4), + ]; + assert_eq!(ds, exp); + } +} diff --git a/datatypes/src/raster/raster_tile.rs b/datatypes/src/raster/raster_tile.rs index 266a64669..3f021bd4c 100644 --- a/datatypes/src/raster/raster_tile.rs +++ b/datatypes/src/raster/raster_tile.rs @@ -1,10 +1,13 @@ use super::masked_grid::MaskedGrid; +use super::{ + BoundedGrid, ChangeGridBounds, GridBoundingBox2D, GridIndexAccessMut, RasterProperties, + SpatialGridDefinition, +}; use super::{ GeoTransform, GeoTransformAccess, GridBounds, GridIdx2D, GridIndexAccess, GridShape, GridShape2D, GridShape3D, GridShapeAccess, GridSize, Raster, TileInformation, grid_or_empty::GridOrEmpty, }; -use super::{GridIndexAccessMut, RasterProperties}; use crate::primitives::CacheHint; use crate::primitives::{ SpatialBounded, SpatialPartition2D, SpatialPartitioned, SpatialResolution, TemporalBounded, @@ -12,6 +15,7 @@ use crate::primitives::{ }; use crate::raster::Pixel; use crate::util::{ByteSize, Result}; +use float_cmp::approx_eq; use serde::{Deserialize, Serialize}; use std::fmt::Write; @@ -68,6 +72,26 @@ where ) } + pub fn global_pixel_spatial_grid_definition(&self) -> SpatialGridDefinition { + let global_upper_left_idx = self.tile_position + * [ + self.grid_array.axis_size_y() as isize, + self.grid_array.axis_size_x() as isize, + ]; + + SpatialGridDefinition::new( + self.global_geo_transform, + GridBoundingBox2D::new_unchecked( + global_upper_left_idx, + global_upper_left_idx + + [ + self.grid_array.axis_size_y() as isize, + self.grid_array.axis_size_x() as isize, + ], + ), + ) + } + /// Use this geo transform to transform `Coordinate2D` into local grid indices and vice versa. #[inline] pub fn tile_geo_transform(&self) -> GeoTransform { @@ -142,21 +166,35 @@ impl IterableBaseTile for [BaseTile; N] { } } -impl> TilesEqualIgnoringCacheHint for I { +impl> TilesEqualIgnoringCacheHint for I +where + G: GridSize, +{ fn tiles_equal_ignoring_cache_hint(&self, other: &dyn IterableBaseTile) -> bool { let mut iter_self = self.iter_tiles(); let mut iter_other = other.iter_tiles(); - loop { match (iter_self.next(), iter_other.next()) { (Some(a), Some(b)) => { - if a.time != b.time - || a.tile_position != b.tile_position - || a.band != b.band - || a.global_geo_transform != b.global_geo_transform - || a.grid_array != b.grid_array - || a.properties != b.properties - { + if a.time != b.time { + return false; + } + if a.tile_position != b.tile_position { + return false; + } + if a.band != b.band { + return false; + } + if !approx_eq!(GeoTransform, a.global_geo_transform, b.global_geo_transform) { + return false; + } + if a.global_geo_transform != b.global_geo_transform { + return false; + } + if a.properties != b.properties { + return false; + } + if a.grid_array != b.grid_array { return false; } } @@ -340,6 +378,43 @@ where } } +impl RasterTile2D +where + T: Pixel, +{ + /// Converts the tile into a grid with the global pixel bounds of the tile. + /// + /// # Panics + /// Only if the tile was invalid before... + /// + pub fn into_inner_positioned_grid(self) -> GridOrEmpty { + let b = self.bounding_box(); + let g = self.grid_array; + g.set_grid_bounds(b).expect("tile was valid before") + } +} + +impl BoundedGrid for RasterTile2D +where + T: Pixel, +{ + type IndexArray = [isize; 2]; + + fn bounding_box(&self) -> GridBoundingBox2D { + let shape = self.grid_array.shape_ref(); + let offset = + self.tile_position * [shape.axis_size_y() as isize, shape.axis_size_x() as isize]; + GridBoundingBox2D::new_unchecked( + offset, + offset + + [ + shape.axis_size_y() as isize - 1, + shape.axis_size_x() as isize - 1, + ], + ) + } +} + impl TemporalBounded for BaseTile { fn temporal_bounds(&self) -> TimeInterval { self.time diff --git a/datatypes/src/raster/tiling.rs b/datatypes/src/raster/tiling.rs index 50deaf228..e66ef9dec 100644 --- a/datatypes/src/raster/tiling.rs +++ b/datatypes/src/raster/tiling.rs @@ -1,35 +1,32 @@ -use crate::{ - primitives::{AxisAlignedRectangle, Coordinate2D, SpatialPartition2D, SpatialPartitioned}, - util::test::TestDefault, -}; - use super::{ GeoTransform, GridBoundingBox2D, GridIdx, GridIdx2D, GridShape2D, GridShapeAccess, GridSize, + SpatialGridDefinition, +}; +use crate::{ + primitives::{ + Coordinate2D, RasterSpatialQueryRectangle, SpatialPartition2D, SpatialPartitioned, + }, + raster::GridBounds, + util::test::TestDefault, }; - use serde::{Deserialize, Serialize}; -/// The static parameters of a `TilingStrategy` +/// The static parameters required to create a `TilingStrategy` #[derive(Debug, Serialize, Deserialize, Clone, Copy)] pub struct TilingSpecification { - pub origin_coordinate: Coordinate2D, pub tile_size_in_pixels: GridShape2D, } impl TilingSpecification { - pub fn new(origin_coordinate: Coordinate2D, tile_size_in_pixels: GridShape2D) -> Self { + pub fn new(tile_size_in_pixels: GridShape2D) -> Self { Self { - origin_coordinate, tile_size_in_pixels, } } - /// create a `TilingStrategy` from self and pixel sizes - pub fn strategy(self, x_pixel_size: f64, y_pixel_size: f64) -> TilingStrategy { - debug_assert!(x_pixel_size > 0.0); - debug_assert!(y_pixel_size < 0.0); - - TilingStrategy::new_with_tiling_spec(self, x_pixel_size, y_pixel_size) + #[allow(clippy::unused_self)] + pub fn tiling_origin_reference(&self) -> Coordinate2D { + Coordinate2D::new(0., 0.) } } @@ -45,10 +42,15 @@ impl GridShapeAccess for TilingSpecification { } } +impl From for GridShape2D { + fn from(val: TilingSpecification) -> Self { + val.tile_size_in_pixels + } +} + impl TestDefault for TilingSpecification { fn test_default() -> Self { Self { - origin_coordinate: Coordinate2D::new(0., 0.), tile_size_in_pixels: GridShape2D::new([512, 512]), } } @@ -69,26 +71,13 @@ impl TilingStrategy { } } - pub fn new_with_tiling_spec( - tiling_specification: TilingSpecification, - x_pixel_size: f64, - y_pixel_size: f64, - ) -> Self { - Self { - tile_size_in_pixels: tiling_specification.tile_size_in_pixels, - geo_transform: GeoTransform::new( - tiling_specification.origin_coordinate, - x_pixel_size, - y_pixel_size, - ), - } - } - pub fn pixel_idx_to_tile_idx(&self, pixel_idx: GridIdx2D) -> GridIdx2D { let GridIdx([y_pixel_idx, x_pixel_idx]) = pixel_idx; let [y_tile_size, x_tile_size] = self.tile_size_in_pixels.into_inner(); - let y_tile_idx = (y_pixel_idx as f64 / y_tile_size as f64).floor() as isize; - let x_tile_idx = (x_pixel_idx as f64 / x_tile_size as f64).floor() as isize; + //let y_tile_idx = (y_pixel_idx as f64 / y_tile_size as f64).floor() as isize; + //let x_tile_idx = (x_pixel_idx as f64 / x_tile_size as f64).floor() as isize; + let y_tile_idx = num::integer::div_floor(y_pixel_idx, y_tile_size as isize); + let x_tile_idx = num::integer::div_floor(x_pixel_idx, x_tile_size as isize); [y_tile_idx, x_tile_idx].into() } @@ -98,33 +87,59 @@ impl TilingStrategy { GridBoundingBox2D::new_unchecked(start, end) } - /// generates the tile idx in \[z,y,x\] order for the tiles intersecting the bounding box - /// the iterator moves once along the x-axis and then increases the y-axis - pub fn tile_idx_iterator( + pub fn global_pixel_grid_bounds_to_tile_grid_bounds( &self, - partition: SpatialPartition2D, - ) -> impl Iterator + use<> { - let GridIdx([upper_left_tile_y, upper_left_tile_x]) = - self.pixel_idx_to_tile_idx(self.geo_transform.upper_left_pixel_idx(&partition)); + global_pixel_grid_bounds: GridBoundingBox2D, + ) -> GridBoundingBox2D { + let start = self.pixel_idx_to_tile_idx(global_pixel_grid_bounds.min_index()); + let end = self.pixel_idx_to_tile_idx(global_pixel_grid_bounds.max_index()); + GridBoundingBox2D::new_unchecked(start, end) + } + + /// Transforms a tile position into a global pixel position + pub fn tile_idx_to_global_pixel_idx(&self, tile_idx: GridIdx2D) -> GridIdx2D { + let GridIdx([y_tile_idx, x_tile_idx]) = tile_idx; + GridIdx::new([ + y_tile_idx * self.tile_size_in_pixels.axis_size_y() as isize, + x_tile_idx * self.tile_size_in_pixels.axis_size_x() as isize, + ]) + } - let GridIdx([lower_right_tile_y, lower_right_tile_x]) = - self.pixel_idx_to_tile_idx(self.geo_transform.lower_right_pixel_idx(&partition)); + /// Returns the tile grid bounds for the given `raster_spatial_query`. + /// The query must match the tiling strategy's geo transform for now. + /// + /// # Panics + /// If the query's geo transform does not match the tiling strategy's geo transform. + /// + pub fn raster_spatial_query_to_tiling_grid_box( + &self, + raster_spatial_query: &RasterSpatialQueryRectangle, + ) -> GridBoundingBox2D { + self.global_pixel_grid_bounds_to_tile_grid_bounds(raster_spatial_query.grid_bounds()) + } + + /// Returns an iterator over all tile indices that intersect with the given `grid_bounds`. + pub fn tile_idx_iterator_from_grid_bounds( + &self, + grid_bounds: GridBoundingBox2D, + ) -> impl Iterator + use<> { + let tile_bounds = self.global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds); - let y_range = upper_left_tile_y..=lower_right_tile_y; - let x_range = upper_left_tile_x..=lower_right_tile_x; + let y_range = tile_bounds.y_min()..=tile_bounds.y_max(); + let x_range = tile_bounds.x_min()..=tile_bounds.x_max(); - y_range.flat_map(move |y_tile| x_range.clone().map(move |x_tile| [y_tile, x_tile].into())) + y_range.flat_map(move |y| x_range.clone().map(move |x| [y, x].into())) } /// generates the tile information for the tiles intersecting the bounding box /// the iterator moves once along the x-axis and then increases the y-axis - pub fn tile_information_iterator( + pub fn tile_information_iterator_from_grid_bounds( &self, - partition: SpatialPartition2D, + grid_bounds: GridBoundingBox2D, ) -> impl Iterator + use<> { let tile_pixel_size = self.tile_size_in_pixels; let geo_transform = self.geo_transform; - self.tile_idx_iterator(partition) + self.tile_idx_iterator_from_grid_bounds(grid_bounds) .map(move |idx| TileInformation::new(idx, tile_pixel_size, geo_transform)) } } @@ -150,18 +165,6 @@ impl TileInformation { } } - pub fn with_partition_and_shape(partition: SpatialPartition2D, shape: GridShape2D) -> Self { - Self { - tile_size_in_pixels: shape, - global_tile_position: [0, 0].into(), - global_geo_transform: GeoTransform::new( - partition.upper_left(), - partition.size_x() / shape.axis_size_x() as f64, - -partition.size_y() / shape.axis_size_y() as f64, - ), - } - } - #[allow(clippy::unused_self)] pub fn local_upper_left_pixel_idx(&self) -> GridIdx2D { [0, 0].into() @@ -202,6 +205,13 @@ impl TileInformation { self.global_upper_left_pixel_idx() + self.local_lower_left_pixel_idx() } + pub fn global_pixel_bounds(&self) -> GridBoundingBox2D { + GridBoundingBox2D::new_unchecked( + self.global_upper_left_pixel_idx(), + self.global_lower_right_pixel_idx(), + ) + } + pub fn tile_size_in_pixels(&self) -> GridShape2D { self.tile_size_in_pixels } @@ -221,6 +231,14 @@ impl TileInformation { self.global_geo_transform.y_pixel_size(), ) } + + pub fn spatial_grid_definition(&self) -> SpatialGridDefinition { + SpatialGridDefinition::new(self.global_geo_transform, self.global_pixel_bounds()) + } + + pub fn tiling_strategy(&self) -> TilingStrategy { + TilingStrategy::new(self.tile_size_in_pixels, self.global_geo_transform) + } } impl SpatialPartitioned for TileInformation { @@ -239,32 +257,138 @@ impl SpatialPartitioned for TileInformation { mod tests { use super::*; + use crate::raster::GridIntersection; #[test] fn it_generates_only_intersected_tiles() { + let origin_coordinate = (0., 0.).into(); + + let geo_transform = GeoTransform::new( + origin_coordinate, + 2.095_475_792_884_826_7E-8, + -2.095_475_792_884_826_7E-8, + ); + let strat = TilingStrategy { tile_size_in_pixels: [600, 600].into(), - geo_transform: GeoTransform::new( - (0., 0.).into(), - 2.095_475_792_884_826_7E-8, - -2.095_475_792_884_826_7E-8, - ), + geo_transform, }; - let partition = SpatialPartition2D::new( - (12.477_738_261_222_84, 43.881_293_535_232_544).into(), - (12.477_743_625_640_87, 43.881_288_170_814_514).into(), - ) - .unwrap(); + let ul_idx = strat + .geo_transform + .coordinate_to_grid_idx_2d((12.477_738_261_222_84, 43.881_293_535_232_544).into()); + + let lr_idx = strat + .geo_transform + .coordinate_to_grid_idx_2d((12.477_743_625_640_87, 43.881_288_170_814_514).into()); + + let grid_bounds = GridBoundingBox2D::new_unchecked(ul_idx, lr_idx); let tiles = strat - .tile_information_iterator(partition) + .tile_information_iterator_from_grid_bounds(grid_bounds) .collect::>(); assert_eq!(tiles.len(), 2); for tile in tiles { - assert!(partition.intersects(&tile.spatial_partition())); + assert!(grid_bounds.intersects(&tile.global_pixel_bounds())); } } + + #[test] + fn it_generates_all_interesected_tiles() { + let strat = TilingStrategy { + tile_size_in_pixels: [512, 512].into(), + geo_transform: GeoTransform::new((0., -0.).into(), 10., -10.), + }; + + let bounds = + GridBoundingBox2D::new(GridIdx2D::new([-513, -513]), GridIdx2D::new([512, 512])) + .unwrap(); + + let tiles_idxs = strat + .tile_idx_iterator_from_grid_bounds(bounds) + .collect::>(); + + assert_eq!(tiles_idxs.len(), 4 * 4); + assert_eq!(tiles_idxs[0], [-2, -2].into()); + assert_eq!(tiles_idxs[1], [-2, -1].into()); + assert_eq!(tiles_idxs[14], [1, 0].into()); + assert_eq!(tiles_idxs[15], [1, 1].into()); + } + + #[test] + fn tiling_tile_tile() { + let geo_transform = GeoTransform::new( + (-1_234_567_890., 1_234_567_890.).into(), + 0.000_033_337_4, + -0.000_033_337_4, + ); + + let tile_pixel_size = GridShape2D::new_2d(512, 512); + let tiling_strat = TilingStrategy::new(tile_pixel_size, geo_transform); + + let tiling_origin_reference = Coordinate2D::new(0., 0.); // This is the _currently_ fixed tiling origin reference. + let nearest_to_tiling_origin = geo_transform.nearest_pixel_edge(tiling_origin_reference); + + let tile_idx = tiling_strat.pixel_idx_to_tile_idx(nearest_to_tiling_origin); + let expected_near_tiling_origin_idx = GridIdx::new([72_329_138_149, 72_329_138_149]); + assert_eq!(tile_idx, expected_near_tiling_origin_idx); + + let pixel_distance_reverse = nearest_to_tiling_origin * -1; + + let origin_pixel_tile = tiling_strat.pixel_idx_to_tile_idx(pixel_distance_reverse); + let origin_pixel_offset = + tiling_strat.tile_idx_to_global_pixel_idx(origin_pixel_tile) - pixel_distance_reverse; + + let expected_origin_in_tiling_based_pixels = + GridIdx::new([-72_329_138_150, -72_329_138_150]); + let expected_tile_offset_from_tiling = GridIdx::new([-85, -85]); + assert_eq!(origin_pixel_tile, expected_origin_in_tiling_based_pixels); + assert_eq!(origin_pixel_offset, expected_tile_offset_from_tiling); + } + + #[test] + fn pixel_idx_to_tile_idx() { + let geo_transform = GeoTransform::new((123., 321.).into(), 1.0, -1.0); + let tile_pixel_size = GridShape2D::new_2d(100, 100); + + let tiling_strat = TilingStrategy::new(tile_pixel_size, geo_transform); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(0, 0)); + assert_eq!(GridIdx2D::new_y_x(0, 0), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(1, 1)); + assert_eq!(GridIdx2D::new_y_x(0, 0), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(57, 57)); + assert_eq!(GridIdx2D::new_y_x(0, 0), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(100, 100)); + assert_eq!(GridIdx2D::new_y_x(1, 1), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(200, 200)); + assert_eq!(GridIdx2D::new_y_x(2, 2), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(1000, 1000)); + assert_eq!(GridIdx2D::new_y_x(10, 10), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(-57, -57)); + assert_eq!(GridIdx2D::new_y_x(-1, -1), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(-300, -300)); + assert_eq!(GridIdx2D::new_y_x(-3, -3), pixels); + } + + #[test] + fn tile_idx_to_pixel_idx() { + let geo_transform = GeoTransform::new((123., 321.).into(), 1.0, -1.0); + let tile_pixel_size = GridShape2D::new_2d(100, 100); + + let tiling_strat = TilingStrategy::new(tile_pixel_size, geo_transform); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(0, 0)); + assert_eq!(GridIdx2D::new_y_x(0, 0), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(1, 1)); + assert_eq!(GridIdx2D::new_y_x(100, 100), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(2, 2)); + assert_eq!(GridIdx2D::new_y_x(200, 200), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(3, 3)); + assert_eq!(GridIdx2D::new_y_x(300, 300), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(10, 10)); + assert_eq!(GridIdx2D::new_y_x(1000, 1000), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(-3, -3)); + assert_eq!(GridIdx2D::new_y_x(-300, -300), pixels); + } } diff --git a/datatypes/src/spatial_reference.rs b/datatypes/src/spatial_reference.rs index d7d541992..832126f1e 100644 --- a/datatypes/src/spatial_reference.rs +++ b/datatypes/src/spatial_reference.rs @@ -260,6 +260,13 @@ impl SpatialReferenceOption { pub fn is_unreferenced(self) -> bool { !self.is_spatial_ref() } + + pub fn as_option(self) -> Option { + match self { + SpatialReferenceOption::SpatialReference(s) => Some(s), + SpatialReferenceOption::Unreferenced => None, + } + } } impl ToSql for SpatialReferenceOption { diff --git a/datatypes/src/util/gdal.rs b/datatypes/src/util/gdal.rs index fc5fdb1d4..3066972b6 100644 --- a/datatypes/src/util/gdal.rs +++ b/datatypes/src/util/gdal.rs @@ -1,10 +1,33 @@ +use gdal::{Dataset, DatasetOptions}; use serde::{Deserialize, Serialize}; -use std::fmt::Display; +use snafu::ResultExt; +use std::{fmt::Display, path::Path}; + +use crate::error; +use crate::util::Result; pub fn hide_gdal_errors() { gdal::config::set_error_handler(|_, _, _| {}); } +/// Opens a Gdal Dataset with the given `path`. +/// Other crates should use this method for Gdal Dataset access as a workaround to avoid strange errors. +pub fn gdal_open_dataset(path: &Path) -> Result { + gdal_open_dataset_ex(path, DatasetOptions::default()) +} + +/// Opens a Gdal Dataset with the given `path` and `dataset_options`. +/// Other crates should use this method for Gdal Dataset access as a workaround to avoid strange errors. +pub fn gdal_open_dataset_ex(path: &Path, dataset_options: DatasetOptions) -> Result { + let dataset_options = { + let mut dataset_options = dataset_options; + dataset_options.open_flags |= gdal::GdalOpenFlags::GDAL_OF_VERBOSE_ERROR; + dataset_options + }; + + Dataset::open_ex(path, dataset_options).context(error::Gdal) +} + // TODO: push to `rust-gdal` #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] diff --git a/datatypes/src/util/ranges.rs b/datatypes/src/util/ranges.rs index dbe43cf56..0236c67ec 100644 --- a/datatypes/src/util/ranges.rs +++ b/datatypes/src/util/ranges.rs @@ -4,14 +4,14 @@ pub fn value_in_range(value: T, min: T, max: T) -> bool where T: PartialOrd + Copy, { - (value >= min) && (value < max) + (min..max).contains(&value) } pub fn value_in_range_inclusive(value: T, min: T, max: T) -> bool where T: PartialOrd + Copy, { - (value >= min) && (value <= max) + (min..=max).contains(&value) } pub fn value_in_range_inv(value: T, min: T, max: T) -> bool diff --git a/datatypes/src/util/test.rs b/datatypes/src/util/test.rs index 837a42e03..6e858eb8a 100644 --- a/datatypes/src/util/test.rs +++ b/datatypes/src/util/test.rs @@ -1,4 +1,9 @@ -use crate::raster::{EmptyGrid, Grid, GridOrEmpty, GridSize, MaskedGrid}; +use float_cmp::approx_eq; + +use crate::raster::{ + EmptyGrid, GeoTransform, Grid, GridIndexAccess, GridOrEmpty, GridSize, MaskedGrid, + RasterTile2D, grid_idx_iter_2d, +}; use std::panic; pub trait TestDefault { @@ -77,6 +82,100 @@ pub fn save_test_bytes(bytes: &[u8], filename: &str) { .expect("it should be possible to write this file for testing"); } +/// Method that compares two lists of tiles and panics with a message why there is a difference. +/// +/// # Panics +/// If there is a difference between two tiles or the length of the lists +pub fn assert_eq_two_list_of_tiles_u8( + list_a: &[RasterTile2D], + list_b: &[RasterTile2D], + compare_cache_hint: bool, +) { + assert_eq!( + list_a.len(), + list_b.len(), + "len() of input_a: {}, len of input_b: {}", + list_a.len(), + list_b.len() + ); + + list_a + .iter() + .zip(list_b) + .enumerate() + .for_each(|(i, (a, b))| { + assert_eq!( + a.time, b.time, + "time of tile {} input_a: {}, input_b: {}", + i, a.time, b.time + ); + assert_eq!( + a.band, b.band, + "band of tile {} input_a: {}, input_b: {}", + i, a.band, b.band + ); + assert_eq!( + a.tile_position, b.tile_position, + "tile position of tile {} input_a: {:?}, input_b: {:?}", + i, a.tile_position, b.tile_position + ); + + let spatial_grid_a = a.global_pixel_spatial_grid_definition(); + let spatial_grid_b = b.global_pixel_spatial_grid_definition(); + assert_eq!( + spatial_grid_a.grid_bounds(), + spatial_grid_b.grid_bounds(), + "grid bounds of tile {} input_a: {:?}, input_b {:?}", + i, + spatial_grid_a.grid_bounds(), + spatial_grid_b.grid_bounds() + ); + assert!( + approx_eq!( + GeoTransform, + spatial_grid_a.geo_transform(), + spatial_grid_b.geo_transform() + ), + "geo transform of tile {} input_a: {:?}, input_b: {:?}", + i, + spatial_grid_a.geo_transform(), + spatial_grid_b.geo_transform() + ); + assert_eq!( + a.grid_array.is_empty(), + b.grid_array.is_empty(), + "grid shape of tile {} input_a is_empty: {:?}, input_b is_empty: {:?}", + i, + a.grid_array.is_empty(), + b.grid_array.is_empty(), + ); + if !a.grid_array.is_empty() { + let mat_a = a.grid_array.clone().into_materialized_masked_grid(); + let mat_b = b.grid_array.clone().into_materialized_masked_grid(); + + for (pi, idx) in grid_idx_iter_2d(&mat_a).enumerate() { + let a_v = mat_a + .get_at_grid_index(idx) + .expect("tile a must contain idx inside tile bounds"); + let b_v = mat_b + .get_at_grid_index(idx) + .expect("tile b must contain idx inside tile bounds"); + assert_eq!( + a_v, b_v, + "tile {i} pixel {pi} at {idx:?} input_a: {a_v:?}, input_b: {b_v:?}", + ); + } + } + if compare_cache_hint { + assert_eq!( + a.cache_hint, b.cache_hint, + "cache hint of tile {} input_a: {:?}, input_b: {:?}", + i, a.cache_hint, b.cache_hint + ); + } + }); +} + #[cfg(test)] mod tests { use crate::{ diff --git a/openapi.json b/openapi.json index a68008f2a..24c8d924b 100644 --- a/openapi.json +++ b/openapi.json @@ -5196,14 +5196,6 @@ "type": "string" } }, - { - "name": "spatialResolution", - "in": "query", - "required": true, - "schema": { - "$ref": "#/components/schemas/SpatialResolution" - } - }, { "name": "attributes", "in": "query", @@ -5460,6 +5452,15 @@ "eastNorth" ] }, + "BTreeMap": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "propertyNames": { + "type": "string" + } + }, "BandSelection": { "type": "array", "items": { @@ -5512,15 +5513,7 @@ ], "properties": { "classes": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "propertyNames": { - "type": "integer", - "format": "int32", - "minimum": 0 - } + "$ref": "#/components/schemas/BTreeMap" }, "measurement": { "type": "string" @@ -6461,6 +6454,27 @@ } } }, + "GeoTransform": { + "type": "object", + "required": [ + "originCoordinate", + "xPixelSize", + "yPixelSize" + ], + "properties": { + "originCoordinate": { + "$ref": "#/components/schemas/Coordinate2D" + }, + "xPixelSize": { + "type": "number", + "format": "double" + }, + "yPixelSize": { + "type": "number", + "format": "double" + } + } + }, "GetCapabilitiesFormat": { "type": "string", "enum": [ @@ -6516,6 +6530,36 @@ "GetMap" ] }, + "GridBoundingBox2D": { + "type": "object", + "required": [ + "topLeftIdx", + "bottomRightIdx" + ], + "properties": { + "bottomRightIdx": { + "$ref": "#/components/schemas/GridIdx2D" + }, + "topLeftIdx": { + "$ref": "#/components/schemas/GridIdx2D" + } + } + }, + "GridIdx2D": { + "type": "object", + "required": [ + "yIdx", + "xIdx" + ], + "properties": { + "xIdx": { + "type": "integer" + }, + "yIdx": { + "type": "integer" + } + } + }, "InternalDataId": { "type": "object", "required": [ @@ -7794,26 +7838,6 @@ "ImagePng" ] }, - "PlotQueryRectangle": { - "type": "object", - "description": "A spatio-temporal rectangle with a specified resolution", - "required": [ - "spatialBounds", - "timeInterval", - "spatialResolution" - ], - "properties": { - "spatialBounds": { - "$ref": "#/components/schemas/BoundingBox2D" - }, - "spatialResolution": { - "$ref": "#/components/schemas/SpatialResolution" - }, - "timeInterval": { - "$ref": "#/components/schemas/TimeInterval" - } - } - }, "PlotResultDescriptor": { "type": "object", "description": "A `ResultDescriptor` for plot queries", @@ -8300,7 +8324,7 @@ ] }, "query": { - "$ref": "#/components/schemas/RasterQueryRectangle" + "$ref": "#/components/schemas/RasterToDatasetQueryRectangle" } }, "example": { @@ -8321,10 +8345,6 @@ "timeInterval": { "start": 1388534400000, "end": 1388534401000 - }, - "spatialResolution": { - "x": 0.1, - "y": 0.1 } } } @@ -8369,60 +8389,24 @@ } } }, - "RasterQueryRectangle": { - "type": "object", - "description": "A spatio-temporal rectangle with a specified resolution", - "required": [ - "spatialBounds", - "timeInterval", - "spatialResolution" - ], - "properties": { - "spatialBounds": { - "$ref": "#/components/schemas/SpatialPartition2D" - }, - "spatialResolution": { - "$ref": "#/components/schemas/SpatialResolution" - }, - "timeInterval": { - "$ref": "#/components/schemas/TimeInterval" - } - } - }, "RasterResultDescriptor": { "type": "object", "description": "A `ResultDescriptor` for raster queries", "required": [ "dataType", "spatialReference", + "spatialGrid", "bands" ], "properties": { "bands": { "$ref": "#/components/schemas/RasterBandDescriptors" }, - "bbox": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/SpatialPartition2D" - } - ] - }, "dataType": { "$ref": "#/components/schemas/RasterDataType" }, - "resolution": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/SpatialResolution" - } - ] + "spatialGrid": { + "$ref": "#/components/schemas/SpatialGridDescriptor" }, "spatialReference": { "type": "string" @@ -8469,6 +8453,22 @@ } } }, + "RasterToDatasetQueryRectangle": { + "type": "object", + "description": "A spatio-temporal rectangle with a specified resolution", + "required": [ + "spatialBounds", + "timeInterval" + ], + "properties": { + "spatialBounds": { + "$ref": "#/components/schemas/SpatialPartition2D" + }, + "timeInterval": { + "$ref": "#/components/schemas/TimeInterval" + } + } + }, "Resource": { "oneOf": [ { @@ -8657,6 +8657,43 @@ } } }, + "SpatialGridDefinition": { + "type": "object", + "required": [ + "geoTransform", + "gridBounds" + ], + "properties": { + "geoTransform": { + "$ref": "#/components/schemas/GeoTransform" + }, + "gridBounds": { + "$ref": "#/components/schemas/GridBoundingBox2D" + } + } + }, + "SpatialGridDescriptor": { + "type": "object", + "required": [ + "spatialGrid", + "descriptor" + ], + "properties": { + "descriptor": { + "$ref": "#/components/schemas/SpatialGridDescriptorState" + }, + "spatialGrid": { + "$ref": "#/components/schemas/SpatialGridDefinition" + } + } + }, + "SpatialGridDescriptorState": { + "type": "string", + "enum": [ + "source", + "derived" + ] + }, "SpatialPartition2D": { "type": "object", "description": "A partition of space that include the upper left but excludes the lower right coordinate", @@ -9756,26 +9793,6 @@ "MultiPolygon" ] }, - "VectorQueryRectangle": { - "type": "object", - "description": "A spatio-temporal rectangle with a specified resolution", - "required": [ - "spatialBounds", - "timeInterval", - "spatialResolution" - ], - "properties": { - "spatialBounds": { - "$ref": "#/components/schemas/BoundingBox2D" - }, - "spatialResolution": { - "$ref": "#/components/schemas/SpatialResolution" - }, - "timeInterval": { - "$ref": "#/components/schemas/TimeInterval" - } - } - }, "VectorResultDescriptor": { "type": "object", "required": [ diff --git a/operators/Cargo.toml b/operators/Cargo.toml index bd181df6f..62552b82f 100644 --- a/operators/Cargo.toml +++ b/operators/Cargo.toml @@ -8,8 +8,6 @@ license-file.workspace = true documentation.workspace = true repository.workspace = true -[features] - [dependencies] arrow = { workspace = true } async-trait = { workspace = true } diff --git a/operators/benches/bands.rs b/operators/benches/bands.rs index 28064dc31..64246fd88 100644 --- a/operators/benches/bands.rs +++ b/operators/benches/bands.rs @@ -2,11 +2,8 @@ use futures::{Future, StreamExt}; use geoengine_datatypes::{ - primitives::{ - BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - TimeStep, - }, - raster::{RasterDataType, RasterTile2D, RenameBands}, + primitives::{BandSelection, RasterQueryRectangle, TimeInterval, TimeStep}, + raster::{GridBoundingBox2D, RasterDataType, RasterTile2D, RenameBands}, util::test::TestDefault, }; use geoengine_operators::{ @@ -51,13 +48,13 @@ fn ndvi_source(execution_context: &mut MockExecutionContext) -> Box>().try_into().unwrap(), - }; + let qrect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + (0..bands).collect::>().try_into().unwrap(), + ); let mut times = NumberStatistics::default(); @@ -183,25 +178,14 @@ async fn all_bands_at_once(runs: usize, bands: u32, resolution: SpatialResolutio async fn main() { const RUNS: usize = 5; const BANDS: u32 = 8; - const RESOLUTION: f64 = 0.1; println!("one band at a time"); - one_band_at_a_time( - RUNS, - BANDS, - SpatialResolution::new(RESOLUTION, RESOLUTION).unwrap(), - ) - .await; + one_band_at_a_time(RUNS, BANDS).await; println!("all bands at once"); - all_bands_at_once( - RUNS, - BANDS, - SpatialResolution::new(RESOLUTION, RESOLUTION).unwrap(), - ) - .await; + all_bands_at_once(RUNS, BANDS).await; } async fn time_it(f: F) -> (f64, Vec>) diff --git a/operators/benches/cache.rs b/operators/benches/cache.rs index 499ec7227..eb447d343 100644 --- a/operators/benches/cache.rs +++ b/operators/benches/cache.rs @@ -2,17 +2,15 @@ use futures::StreamExt; use geoengine_datatypes::{ - primitives::{ - BandSelection, QueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - raster::TilesEqualIgnoringCacheHint, + primitives::{BandSelection, RasterQueryRectangle, TimeInterval}, + raster::{GridBoundingBox2D, TilesEqualIgnoringCacheHint}, util::test::TestDefault, }; use geoengine_operators::{ cache::{cache_operator::InitializedCacheOperator, shared_cache::SharedCache}, engine::{ - ChunkByteSize, InitializedRasterOperator, MockExecutionContext, MockQueryContext, - QueryProcessor, RasterOperator, SingleRasterSource, WorkflowOperatorPath, + ChunkByteSize, InitializedRasterOperator, MockExecutionContext, QueryProcessor, + RasterOperator, SingleRasterSource, WorkflowOperatorPath, }, processing::{ AggregateFunctionParams, NeighborhoodAggregate, NeighborhoodAggregateParams, @@ -41,7 +39,7 @@ async fn main() { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new(ndvi_id), } .boxed(), }, @@ -57,7 +55,7 @@ async fn main() { let tile_cache = Arc::new(SharedCache::test_default()); - let query_ctx = MockQueryContext::new_with_query_extensions( + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( ChunkByteSize::test_default(), Some(tile_cache), None, @@ -68,15 +66,11 @@ async fn main() { let stream = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), &query_ctx, ) .await @@ -92,15 +86,11 @@ async fn main() { let stream_from_cache = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), &query_ctx, ) .await diff --git a/operators/benches/cache_concurrent.rs b/operators/benches/cache_concurrent.rs index cb39949e6..e20eb6850 100644 --- a/operators/benches/cache_concurrent.rs +++ b/operators/benches/cache_concurrent.rs @@ -1,9 +1,9 @@ #![allow(clippy::unwrap_used, clippy::print_stdout, clippy::print_stderr)] // okay in benchmarks use futures::future::join_all; +use geoengine_datatypes::primitives::DateTime; use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::primitives::{DateTime, SpatialPartition2D, SpatialResolution}; -use geoengine_datatypes::raster::RasterProperties; +use geoengine_datatypes::raster::{GridBoundingBox2D, RasterProperties}; use geoengine_datatypes::{ primitives::{RasterQueryRectangle, TimeInterval}, raster::{Grid, RasterTile2D}, @@ -118,12 +118,11 @@ async fn read_cache(tile_cache: &SharedCache, op_no: usize) -> ReadMeasurement { } fn query_rect() -> RasterQueryRectangle { - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - } + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ) } fn op(idx: usize) -> CanonicOperatorName { diff --git a/operators/benches/expression.rs b/operators/benches/expression.rs index f8ed69c40..102a4bf49 100644 --- a/operators/benches/expression.rs +++ b/operators/benches/expression.rs @@ -2,10 +2,9 @@ use futures::{Future, StreamExt}; use geoengine_datatypes::{ - primitives::{ - BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - raster::{RasterDataType, RasterTile2D, RenameBands}, + primitives::{BandSelection, RasterQueryRectangle, TimeInterval}, + raster::RenameBands, + raster::{GridBoundingBox2D, RasterDataType, RasterTile2D}, util::test::TestDefault, }; use geoengine_operators::{ @@ -55,7 +54,7 @@ fn ndvi_source(execution_context: &mut MockExecutionContext) -> Box (StatisticsWrappingMockExecutionContext, MockQueryContext let workflow = uuid::Uuid::new_v4(); let computation = uuid::Uuid::new_v4(); - let query_ctx = MockQueryContext::new_with_query_extensions( + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( ChunkByteSize::test_default(), None, Some(QuotaTracking::new( @@ -105,23 +105,17 @@ fn setup_benchmarks(exe_ctx: &mut StatisticsWrappingMockExecutionContext) -> Vec }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: ndvi_id.clone(), - }, + params: GdalSourceParameters::new(ndvi_id.clone()), } .boxed(), }, } .boxed(), - query_rectangle: QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + query_rectangle: RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1800, -900], [1799, 899]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), }, Benchmark::Vector { name: "raster_vector_join".to_string(), @@ -144,22 +138,18 @@ fn setup_benchmarks(exe_ctx: &mut StatisticsWrappingMockExecutionContext) -> Vec .boxed(), rasters: vec![ GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new(ndvi_id), } .boxed(), ], }, } .boxed(), - query_rectangle: QueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + query_rectangle: VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked([-180., -90.].into(), [180., 90.].into()), + TimeInterval::default(), + ColumnSelection::all(), + ), }, ] } diff --git a/operators/benches/sources.rs b/operators/benches/sources.rs index afde6a93b..2b8016c26 100644 --- a/operators/benches/sources.rs +++ b/operators/benches/sources.rs @@ -1,13 +1,12 @@ #![allow(clippy::unwrap_used, clippy::print_stdout, clippy::print_stderr)] // okay in benchmarks -use std::time::Instant; -use std::{hint::black_box, marker::PhantomData}; - use futures::StreamExt; use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::raster::RasterDataType; +use geoengine_datatypes::raster::{ + BoundedGrid, GridBoundingBox2D, GridShapeAccess, RasterDataType, +}; use geoengine_datatypes::{ - primitives::{RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::{RasterQueryRectangle, TimeInterval}, raster::{ GeoTransform, Grid2D, GridOrEmpty2D, GridSize, Pixel, RasterTile2D, TilingSpecification, }, @@ -15,27 +14,29 @@ use geoengine_datatypes::{ }; use geoengine_operators::engine::RasterResultDescriptor; use geoengine_operators::{ - engine::{ChunkByteSize, MockQueryContext, QueryContext, RasterQueryProcessor}, + engine::{ChunkByteSize, MockQueryContext, RasterQueryProcessor}, mock::MockRasterSourceProcessor, source::{GdalMetaDataRegular, GdalSourceProcessor}, util::gdal::create_ndvi_meta_data, }; +use std::time::Instant; +use std::{hint::black_box, marker::PhantomData}; fn setup_gdal_source( meta_data: GdalMetaDataRegular, tiling_specification: TilingSpecification, ) -> GdalSourceProcessor { GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + produced_result_descriptor: meta_data.result_descriptor.clone(), tiling_specification, + overview_level: 0, meta_data: Box::new(meta_data), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, } } +#[allow(clippy::too_many_lines)] fn setup_mock_source(tiling_spec: TilingSpecification) -> MockRasterSourceProcessor { let grid: GridOrEmpty2D = Grid2D::new( tiling_spec.tile_size_in_pixels, @@ -44,6 +45,18 @@ fn setup_mock_source(tiling_spec: TilingSpecification) -> MockRasterSourceProces .unwrap() .into(); let geo_transform = GeoTransform::test_default(); + let grid_bounds = grid.grid_shape().bounding_box(); + let grid_bounds = GridBoundingBox2D::new( + [ + grid_bounds.y_min() - grid.axis_size_y() as isize, + grid_bounds.x_min() - grid.axis_size_x() as isize, + ], + [ + grid_bounds.y_min() + 2 * grid.axis_size_y() as isize, + grid_bounds.x_min() + 2 * grid.axis_size_x() as isize, + ], + ) + .unwrap(); let time = TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(); @@ -51,6 +64,8 @@ fn setup_mock_source(tiling_spec: TilingSpecification) -> MockRasterSourceProces result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( RasterDataType::U8, 1, + grid_bounds, + geo_transform, ), data: vec![ RasterTile2D::new( @@ -135,23 +150,24 @@ fn bench_raster_processor< T: Pixel, F: Fn(TilingSpecification) -> S, S: RasterQueryProcessor, - C: QueryContext, >( bench_id: &'static str, list_of_named_querys: &[(&str, RasterQueryRectangle)], list_of_tiling_specs: &[TilingSpecification], tile_producing_operator_builderr: F, - ctx: &C, run_time: &tokio::runtime::Runtime, ) { for tiling_spec in list_of_tiling_specs { + let ctx = + MockQueryContext::with_chunk_size_and_thread_count(ChunkByteSize::MAX, *tiling_spec, 8); + let operator = (tile_producing_operator_builderr)(*tiling_spec); for &(qrect_name, ref qrect) in list_of_named_querys { run_time.block_on(async { // query the operator let start_query = Instant::now(); - let query = operator.raster_query(qrect.clone(), ctx).await.unwrap(); + let query = operator.raster_query(qrect.clone(), &ctx).await.unwrap(); let query_elapsed = start_query.elapsed(); let start = Instant::now(); @@ -184,72 +200,55 @@ fn bench_no_data_tiles() { let qrects = vec![ ( "1 tile", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 60.).into(), (60., 0.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-60, 0], [-1, 59]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "2 tiles", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 50.).into(), (60., -10.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-50, 0], [9, 59]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "4 tiles", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-5., 50.).into(), (55., -10.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-55, -5], [9, 54]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "2 tiles, 2 no-data tiles", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((130., 120.).into(), (190., 60.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-120, 130], [59, 189]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "empty tiles", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-5., 50.).into(), (55., -10.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_000_000_000_000, 1_000_000_000_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-50, -5], [-9, 54]).unwrap(), + TimeInterval::new(1_000_000_000_000, 1_000_000_000_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ]; - let tiling_specs = vec![TilingSpecification::new((0., 0.).into(), [600, 600].into())]; + let tiling_specs = vec![TilingSpecification::new([600, 600].into())]; let run_time = tokio::runtime::Runtime::new().unwrap(); - let ctx = MockQueryContext::with_chunk_size_and_thread_count(ChunkByteSize::MAX, 8); bench_raster_processor( "no_data_tiles", &qrects, &tiling_specs, setup_mock_source, - &ctx, &run_time, ); bench_raster_processor( @@ -257,7 +256,6 @@ fn bench_no_data_tiles() { &qrects, &tiling_specs, |ts| setup_gdal_source(create_ndvi_meta_data(), ts), - &ctx, &run_time, ); } @@ -265,30 +263,27 @@ fn bench_no_data_tiles() { fn bench_tile_size() { let qrects = vec![( "World in 36000x18000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), )]; let run_time = tokio::runtime::Runtime::new().unwrap(); - let ctx = MockQueryContext::with_chunk_size_and_thread_count(ChunkByteSize::MAX, 8); let tiling_specs = vec![ - TilingSpecification::new((0., 0.).into(), [32, 32].into()), - TilingSpecification::new((0., 0.).into(), [64, 64].into()), - TilingSpecification::new((0., 0.).into(), [128, 128].into()), - TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), - TilingSpecification::new((0., 0.).into(), [600, 600].into()), - TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), - TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), + TilingSpecification::new([32, 32].into()), + TilingSpecification::new([64, 64].into()), + TilingSpecification::new([128, 128].into()), + TilingSpecification::new([256, 256].into()), + TilingSpecification::new([512, 512].into()), + TilingSpecification::new([600, 600].into()), + TilingSpecification::new([900, 900].into()), + TilingSpecification::new([1024, 1024].into()), + TilingSpecification::new([2048, 2048].into()), + TilingSpecification::new([4096, 4096].into()), + TilingSpecification::new([9000, 9000].into()), ]; bench_raster_processor( @@ -296,7 +291,6 @@ fn bench_tile_size() { &qrects, &tiling_specs, |ts| setup_gdal_source(create_ndvi_meta_data(), ts), - &ctx, &run_time, ); } diff --git a/operators/benches/workflows.rs b/operators/benches/workflows.rs index 815229aa4..c04bb590a 100644 --- a/operators/benches/workflows.rs +++ b/operators/benches/workflows.rs @@ -5,38 +5,39 @@ clippy::missing_panics_doc )] // okay in benchmarks -use std::hint::black_box; -use std::time::{Duration, Instant}; - use futures::TryStreamExt; -use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; + +use geoengine_datatypes::primitives::Coordinate2D; +use geoengine_datatypes::primitives::RasterQueryRectangle; use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::primitives::{QueryRectangle, RasterQueryRectangle, SpatialPartitioned}; -use geoengine_datatypes::raster::{Grid2D, RasterDataType, RenameBands}; +use geoengine_datatypes::raster::RenameBands; +use geoengine_datatypes::raster::{ + GeoTransform, Grid2D, GridBoundingBox2D, RasterDataType, TilingStrategy, +}; use geoengine_datatypes::spatial_reference::SpatialReference; +use geoengine_operators::engine::SingleRasterOrVectorSource; +use geoengine_operators::engine::SpatialGridDescriptor; +use geoengine_operators::processing::Reprojection; +use geoengine_operators::processing::ReprojectionParams; +use std::hint::black_box; +use std::time::{Duration, Instant}; -use geoengine_datatypes::util::Identifier; use geoengine_datatypes::{ - primitives::{SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::TimeInterval, raster::{GridSize, RasterTile2D, TilingSpecification}, }; use geoengine_operators::call_on_generic_raster_processor; use geoengine_operators::engine::{ - MetaData, MultipleRasterSources, RasterBandDescriptors, RasterResultDescriptor, - SingleRasterOrVectorSource, SingleRasterSource, WorkflowOperatorPath, + ChunkByteSize, MockExecutionContext, RasterOperator, RasterQueryProcessor, +}; +use geoengine_operators::engine::{ + MultipleRasterSources, RasterBandDescriptors, RasterResultDescriptor, SingleRasterSource, + WorkflowOperatorPath, }; use geoengine_operators::mock::{MockRasterSource, MockRasterSourceParams}; use geoengine_operators::processing::{ - Expression, ExpressionParams, RasterStacker, RasterStackerParams, Reprojection, - ReprojectionParams, + Expression, ExpressionParams, RasterStacker, RasterStackerParams, }; -use geoengine_operators::source::GdalSource; -use geoengine_operators::{ - engine::{ChunkByteSize, MockExecutionContext, RasterOperator, RasterQueryProcessor}, - source::GdalSourceParameters, - util::gdal::create_ndvi_meta_data, -}; - use serde::{Serialize, Serializer}; pub struct BenchmarkCollector { @@ -65,7 +66,7 @@ pub trait BenchmarkRunner { pub struct WorkflowSingleBenchmark { bench_id: &'static str, query_name: &'static str, - query_rect: QueryRectangle, + query_rect: RasterQueryRectangle, tiling_spec: TilingSpecification, chunk_byte_size: ChunkByteSize, num_threads: usize, @@ -76,10 +77,7 @@ pub struct WorkflowSingleBenchmark { impl WorkflowSingleBenchmark where F: Fn(TilingSpecification, usize) -> MockExecutionContext, - O: Fn( - TilingSpecification, - QueryRectangle, - ) -> Box, + O: Fn(TilingSpecification, RasterQueryRectangle) -> Box, { #[inline(never)] pub fn run_bench(&self) -> WorkflowBenchmarkResult { @@ -139,10 +137,7 @@ where impl BenchmarkRunner for WorkflowSingleBenchmark where F: Fn(TilingSpecification, usize) -> MockExecutionContext, - O: Fn( - TilingSpecification, - QueryRectangle, - ) -> Box, + O: Fn(TilingSpecification, RasterQueryRectangle) -> Box, { fn run_all_benchmarks(self, bencher: &mut BenchmarkCollector) { bencher.add_benchmark_result(WorkflowSingleBenchmark::run_bench(&self)); @@ -166,11 +161,7 @@ where T: IntoIterator + Clone, B: IntoIterator + Clone, F: Clone + Fn(TilingSpecification, usize) -> MockExecutionContext, - O: Clone - + Fn( - TilingSpecification, - QueryRectangle, - ) -> Box, + O: Clone + Fn(TilingSpecification, RasterQueryRectangle) -> Box, { pub fn new( bench_id: &'static str, @@ -250,11 +241,7 @@ where T: IntoIterator + Clone, B: IntoIterator + Clone, F: Clone + Fn(TilingSpecification, usize) -> MockExecutionContext, - O: Clone - + Fn( - TilingSpecification, - QueryRectangle, - ) -> Box, + O: Clone + Fn(TilingSpecification, RasterQueryRectangle) -> Box, { fn run_all_benchmarks(self, bencher: &mut BenchmarkCollector) { self.into_benchmark_iterator() @@ -289,16 +276,18 @@ where serializer.serialize_u128(duration.as_millis()) } +#[allow(clippy::needless_pass_by_value)] // must match signature fn bench_mock_source_operator(bench_collector: &mut BenchmarkCollector) { - #[allow(clippy::needless_pass_by_value)] // must match signature fn operator_builder( tiling_spec: TilingSpecification, query_rect: RasterQueryRectangle, ) -> Box { - let query_resolution = query_rect.spatial_resolution; - let query_time = query_rect.time_interval; - let tileing_strategy = tiling_spec.strategy(query_resolution.x, -1. * query_resolution.y); - let tile_iter = tileing_strategy.tile_information_iterator(query_rect.spatial_partition()); + let tileing_strategy = TilingStrategy::new( + tiling_spec.tile_size_in_pixels, + GeoTransform::new(Coordinate2D::new(0.0, 0.), 0.01, -0.01), + ); + let tile_iter = tileing_strategy + .tile_information_iterator_from_grid_bounds(query_rect.spatial_query().grid_bounds()); let mock_data = tile_iter .enumerate() @@ -309,7 +298,7 @@ fn bench_mock_source_operator(bench_collector: &mut BenchmarkCollector) { ) .unwrap(); RasterTile2D::new_with_tile_info( - query_time, + query_rect.time_interval, tile_info, 0, data.into(), @@ -321,26 +310,27 @@ fn bench_mock_source_operator(bench_collector: &mut BenchmarkCollector) { MockRasterSource { params: MockRasterSourceParams { data: mock_data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: RasterResultDescriptor::new( + RasterDataType::U8, + SpatialReference::epsg_4326().into(), + None, + SpatialGridDescriptor::source_from_parts( + tileing_strategy.geo_transform, + query_rect.spatial_query().grid_bounds(), + ), + RasterBandDescriptors::new_single_band(), + ), }, } .boxed() } - let qrect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }; - let tiling_spec = TilingSpecification::new((0., 0.).into(), [512, 512].into()); + let qrect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-18000, -9000], [17999, 8999]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ); + let tiling_spec = TilingSpecification::new([512, 512].into()); let qrects = vec![("World in 36000x18000 pixels", qrect)]; let tiling_specs = vec![tiling_spec]; @@ -359,16 +349,19 @@ fn bench_mock_source_operator(bench_collector: &mut BenchmarkCollector) { .run_all_benchmarks(bench_collector); } +#[allow(clippy::too_many_lines)] fn bench_mock_source_operator_with_expression(bench_collector: &mut BenchmarkCollector) { #[allow(clippy::needless_pass_by_value)] // must match signature fn operator_builder( tiling_spec: TilingSpecification, query_rect: RasterQueryRectangle, ) -> Box { - let query_resolution = query_rect.spatial_resolution; - let query_time = query_rect.time_interval; - let tileing_strategy = tiling_spec.strategy(query_resolution.x, -1. * query_resolution.y); - let tile_iter = tileing_strategy.tile_information_iterator(query_rect.spatial_partition()); + let tileing_strategy = TilingStrategy::new( + tiling_spec.tile_size_in_pixels, + GeoTransform::new(Coordinate2D::new(0.0, 0.), 0.01, -0.01), + ); + let tile_iter = tileing_strategy + .tile_information_iterator_from_grid_bounds(query_rect.spatial_query().grid_bounds()); let mock_data = tile_iter .enumerate() @@ -379,7 +372,7 @@ fn bench_mock_source_operator_with_expression(bench_collector: &mut BenchmarkCol ) .unwrap(); RasterTile2D::new_with_tile_info( - query_time, + query_rect.time_interval, tile_info, 0, data.into(), @@ -391,14 +384,16 @@ fn bench_mock_source_operator_with_expression(bench_collector: &mut BenchmarkCol let mock_raster_operator = MockRasterSource { params: MockRasterSourceParams { data: mock_data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: RasterResultDescriptor::new( + RasterDataType::U8, + SpatialReference::epsg_4326().into(), + None, + SpatialGridDescriptor::source_from_parts( + tileing_strategy.geo_transform, + query_rect.spatial_query().grid_bounds(), + ), + RasterBandDescriptors::new_single_band(), + ), }, }; @@ -427,21 +422,20 @@ fn bench_mock_source_operator_with_expression(bench_collector: &mut BenchmarkCol .boxed() } - let qrect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.005, 0.005).unwrap(), - attributes: BandSelection::first(), - }; + let qrect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-18000, -9000], [17999, 8999]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ); let qrects = vec![("World in 72000x36000 pixels", qrect)]; let tiling_specs = vec![ - TilingSpecification::new((0., 0.).into(), [512, 512].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), - TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), - TilingSpecification::new((0., 0.).into(), [18000, 18000].into()), + TilingSpecification::new([512, 512].into()), + TilingSpecification::new([1024, 1024].into()), + TilingSpecification::new([2048, 2048].into()), + TilingSpecification::new([4096, 4096].into()), + TilingSpecification::new([9000, 9000].into()), + TilingSpecification::new([18000, 18000].into()), ]; WorkflowMultiBenchmark::new( @@ -464,10 +458,12 @@ fn bench_mock_source_operator_with_identity_reprojection(bench_collector: &mut B tiling_spec: TilingSpecification, query_rect: RasterQueryRectangle, ) -> Box { - let query_resolution = query_rect.spatial_resolution; - let query_time = query_rect.time_interval; - let tileing_strategy = tiling_spec.strategy(query_resolution.x, -1. * query_resolution.y); - let tile_iter = tileing_strategy.tile_information_iterator(query_rect.spatial_partition()); + let tileing_strategy = TilingStrategy::new( + tiling_spec.tile_size_in_pixels, + GeoTransform::new(Coordinate2D::new(0.0, 0.), 0.01, -0.01), + ); + let tile_iter = tileing_strategy + .tile_information_iterator_from_grid_bounds(query_rect.spatial_query().grid_bounds()); let mock_data = tile_iter .enumerate() .map(|(id, tile_info)| { @@ -477,7 +473,7 @@ fn bench_mock_source_operator_with_identity_reprojection(bench_collector: &mut B ) .unwrap(); RasterTile2D::new_with_tile_info( - query_time, + query_rect.time_interval, tile_info, 0, data.into(), @@ -489,41 +485,44 @@ fn bench_mock_source_operator_with_identity_reprojection(bench_collector: &mut B let mock_raster_operator = MockRasterSource { params: MockRasterSourceParams { data: mock_data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: RasterResultDescriptor::new( + RasterDataType::U8, + SpatialReference::epsg_4326().into(), + None, + SpatialGridDescriptor::source_from_parts( + tileing_strategy.geo_transform, + query_rect.spatial_query().grid_bounds(), + ), + RasterBandDescriptors::new_single_band(), + ), }, }; Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: + geoengine_operators::processing::DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource::from(mock_raster_operator.boxed()), } .boxed() } - let qrect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }; + let qrect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-18000, -9000], [17999, 8999]).unwrap(), // TODO: should be output bounds? + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ); let qrects = vec![("World in 36000x18000 pixels", qrect)]; let tiling_specs = vec![ - TilingSpecification::new((0., 0.).into(), [512, 512].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), - TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), - TilingSpecification::new((0., 0.).into(), [18000, 18000].into()), + TilingSpecification::new([512, 512].into()), + TilingSpecification::new([1024, 1024].into()), + TilingSpecification::new([2048, 2048].into()), + TilingSpecification::new([4096, 4096].into()), + TilingSpecification::new([9000, 9000].into()), + TilingSpecification::new([18000, 18000].into()), ]; WorkflowMultiBenchmark::new( @@ -540,18 +539,38 @@ fn bench_mock_source_operator_with_identity_reprojection(bench_collector: &mut B .run_all_benchmarks(bench_collector); } +/* fn bench_mock_source_operator_with_4326_to_3857_reprojection( bench_collector: &mut BenchmarkCollector, ) { + let qrect = SpatialPartition2D::new( + (-20_037_508.342_789_244, 20_048_966.104_014_594).into(), + (20_037_508.342_789_244, -20_048_966.104_014_594).into(), + ) + .unwrap(); + + let qtime = TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(); + let qband = BandSelection::first(); + + let tiling_spec = TilingSpecification::new([512, 512].into()); + + let qrects = vec![("World in 36000x18000 pixels", qrect)]; + let tiling_specs = vec![tiling_spec]; + #[allow(clippy::needless_pass_by_value)] // must match signature fn operator_builder( tiling_spec: TilingSpecification, query_rect: RasterQueryRectangle, ) -> Box { - let query_resolution = query_rect.spatial_resolution; - let query_time = query_rect.time_interval; - let tileing_strategy = tiling_spec.strategy(query_resolution.x, -1. * query_resolution.y); - let tile_iter = tileing_strategy.tile_information_iterator(query_rect.spatial_partition()); + // FIXME: The query origin must match the tiling strategy's origin for now. Also use grid bounds not spatial bounds. + + let tileing_strategy = TilingStrategy::new( + tiling_spec.tile_size_in_pixels, + GeoTransform::new(Coordinate2D::new(0., 0.), 0.01, -0.01), + ); + + let tile_iter = tileing_strategy + .tile_information_iterator_from_grid_bounds(query_rect.spatial_query().grid_bounds()); let mock_data = tile_iter .enumerate() .map(|(id, tile_info)| { @@ -561,7 +580,7 @@ fn bench_mock_source_operator_with_4326_to_3857_reprojection( ) .unwrap(); RasterTile2D::new_with_tile_info( - query_time, + query_rect.time_interval, tile_info, 0, data.into(), @@ -572,20 +591,21 @@ fn bench_mock_source_operator_with_4326_to_3857_reprojection( let mock_raster_operator = MockRasterSource { params: MockRasterSourceParams { data: mock_data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: RasterResultDescriptor::new( + RasterDataType::U8, + SpatialReference::epsg_4326().into(), + None, + tileing_strategy.geo_transform, + query_rect.spatial_query().grid_bounds(), + RasterBandDescriptors::new_single_band(), + ), }, }; Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource::from(mock_raster_operator.boxed()), } @@ -622,42 +642,42 @@ fn bench_mock_source_operator_with_4326_to_3857_reprojection( } fn bench_gdal_source_operator_tile_size(bench_collector: &mut BenchmarkCollector) { + let tiling_origin = Coordinate2D::new(0., 0.); + let qrects = vec![ ( "World in 36000x18000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + SpatialResolution::new(0.01, 0.01).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "World in 72000x36000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new(0.005, 0.005).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + SpatialResolution::new(0.005, 0.005).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ]; let tiling_specs = vec![ - TilingSpecification::new((0., 0.).into(), [32, 32].into()), - TilingSpecification::new((0., 0.).into(), [64, 64].into()), - TilingSpecification::new((0., 0.).into(), [128, 128].into()), - TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), + TilingSpecification::new(tiling_origin, [32, 32].into()), + TilingSpecification::new(tiling_origin, [64, 64].into()), + TilingSpecification::new(tiling_origin, [128, 128].into()), + TilingSpecification::new(tiling_origin, [256, 256].into()), + TilingSpecification::new(tiling_origin, [512, 512].into()), // TilingSpecification::new((0., 0.).into(), [600, 600].into()), // TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), + TilingSpecification::new(tiling_origin, [1024, 1024].into()), + TilingSpecification::new(tiling_origin, [2048, 2048].into()), + TilingSpecification::new(tiling_origin, [4096, 4096].into()), // TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), ]; @@ -666,7 +686,7 @@ fn bench_gdal_source_operator_tile_size(bench_collector: &mut BenchmarkCollector let meta_data = create_ndvi_meta_data(); let gdal_operator = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), } .boxed(); @@ -688,28 +708,30 @@ fn bench_gdal_source_operator_tile_size(bench_collector: &mut BenchmarkCollector } fn bench_gdal_source_operator_with_expression_tile_size(bench_collector: &mut BenchmarkCollector) { + let tiling_origin = Coordinate2D::new(0., 0.); + let qrects = vec![( "World in 36000x18000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + SpatialResolution::new(0.01, 0.01).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), )]; let tiling_specs = vec![ // TilingSpecification::new((0., 0.).into(), [32, 32].into()), - TilingSpecification::new((0., 0.).into(), [64, 64].into()), - TilingSpecification::new((0., 0.).into(), [128, 128].into()), - TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), + TilingSpecification::new(tiling_origin, [64, 64].into()), + TilingSpecification::new(tiling_origin, [128, 128].into()), + TilingSpecification::new(tiling_origin, [256, 256].into()), + TilingSpecification::new(tiling_origin, [512, 512].into()), // TilingSpecification::new((0., 0.).into(), [600, 600].into()), // TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), + TilingSpecification::new(tiling_origin, [1024, 1024].into()), + TilingSpecification::new(tiling_origin, [2048, 2048].into()), + TilingSpecification::new(tiling_origin, [4096, 4096].into()), // TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), ]; @@ -718,7 +740,7 @@ fn bench_gdal_source_operator_with_expression_tile_size(bench_collector: &mut Be let meta_data = create_ndvi_meta_data(); let gdal_operator = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), }; let expression_operator = Expression { @@ -760,28 +782,30 @@ fn bench_gdal_source_operator_with_expression_tile_size(bench_collector: &mut Be } fn bench_gdal_source_operator_with_identity_reprojection(bench_collector: &mut BenchmarkCollector) { + let tiling_origin = Coordinate2D::new(0., 0.); + let qrects = vec![( "World in 36000x18000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + SpatialResolution::new(0.01, 0.01).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), )]; let tiling_specs = vec![ // TilingSpecification::new((0., 0.).into(), [32, 32].into()), // TilingSpecification::new((0., 0.).into(), [64, 64].into()), // TilingSpecification::new((0., 0.).into(), [128, 128].into()), - TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), + TilingSpecification::new(tiling_origin, [256, 256].into()), + TilingSpecification::new(tiling_origin, [512, 512].into()), // TilingSpecification::new((0., 0.).into(), [600, 600].into()), // TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), + TilingSpecification::new(tiling_origin, [1024, 1024].into()), + TilingSpecification::new(tiling_origin, [2048, 2048].into()), + TilingSpecification::new(tiling_origin, [4096, 4096].into()), // TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), ]; @@ -790,12 +814,13 @@ fn bench_gdal_source_operator_with_identity_reprojection(bench_collector: &mut B let meta_data = create_ndvi_meta_data(); let gdal_operator = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), }; let projection_operator = Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource::from(gdal_operator.boxed()), } @@ -821,18 +846,21 @@ fn bench_gdal_source_operator_with_identity_reprojection(bench_collector: &mut B fn bench_gdal_source_operator_with_4326_to_3857_reprojection( bench_collector: &mut BenchmarkCollector, ) { + let tiling_origin = Coordinate2D::new(0., 0.); + let qrects = vec![( "World in EPSG:3857 ~ 40000 x 20000 px", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new( + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new( (-20_037_508.342_789_244, 20_048_966.104_014_594).into(), (20_037_508.342_789_244, -20_048_966.104_014_594).into(), ) .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(1050., 2100.).unwrap(), - attributes: BandSelection::first(), - }, + SpatialResolution::new(1050., 2100.).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), )]; let tiling_specs = vec![ @@ -840,12 +868,12 @@ fn bench_gdal_source_operator_with_4326_to_3857_reprojection( // TilingSpecification::new((0., 0.).into(), [64, 64].into()), // TilingSpecification::new((0., 0.).into(), [128, 128].into()), // TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), + TilingSpecification::new(tiling_origin, [512, 512].into()), // TilingSpecification::new((0., 0.).into(), [600, 600].into()), // TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), + TilingSpecification::new(tiling_origin, [1024, 1024].into()), + TilingSpecification::new(tiling_origin, [2048, 2048].into()), + TilingSpecification::new(tiling_origin, [4096, 4096].into()), // TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), ]; @@ -854,7 +882,7 @@ fn bench_gdal_source_operator_with_4326_to_3857_reprojection( let meta_data = create_ndvi_meta_data(); let gdal_operator = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), }; let projection_operator = Reprojection { @@ -863,6 +891,7 @@ fn bench_gdal_source_operator_with_4326_to_3857_reprojection( geoengine_datatypes::spatial_reference::SpatialReferenceAuthority::Epsg, 3857, ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource::from(gdal_operator.boxed()), } @@ -885,15 +914,19 @@ fn bench_gdal_source_operator_with_4326_to_3857_reprojection( .run_all_benchmarks(bench_collector); } +*/ + fn main() { let mut bench_collector = BenchmarkCollector::default(); bench_mock_source_operator(&mut bench_collector); bench_mock_source_operator_with_expression(&mut bench_collector); bench_mock_source_operator_with_identity_reprojection(&mut bench_collector); + /* bench_mock_source_operator_with_4326_to_3857_reprojection(&mut bench_collector); bench_gdal_source_operator_tile_size(&mut bench_collector); bench_gdal_source_operator_with_expression_tile_size(&mut bench_collector); bench_gdal_source_operator_with_identity_reprojection(&mut bench_collector); bench_gdal_source_operator_with_4326_to_3857_reprojection(&mut bench_collector); + */ } diff --git a/operators/src/adapters/feature_collection_merger.rs b/operators/src/adapters/feature_collection_merger.rs index e71b79f5b..c20bb6db7 100644 --- a/operators/src/adapters/feature_collection_merger.rs +++ b/operators/src/adapters/feature_collection_merger.rs @@ -147,15 +147,13 @@ mod tests { use crate::mock::{MockFeatureCollectionSource, MockPointSource, MockPointSourceParams}; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::collections::ChunksEqualIgnoringCacheHint; + use geoengine_datatypes::collections::{DataCollection, MultiPointCollection}; use geoengine_datatypes::primitives::{ BoundingBox2D, Coordinate2D, MultiPoint, TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; + use geoengine_datatypes::raster::TilingSpecification; use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{ - collections::{DataCollection, MultiPointCollection}, - primitives::SpatialResolution, - }; #[tokio::test] async fn simple() { @@ -165,9 +163,7 @@ mod tests { .collect(); let source = MockPointSource { - params: MockPointSourceParams { - points: coordinates.clone(), - }, + params: MockPointSourceParams::new(coordinates.clone()), }; let source = source @@ -184,13 +180,15 @@ mod tests { unreachable!(); }; - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, 0.0).into(), (10.0, 10.0).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let cx = MockQueryContext::new((std::mem::size_of::() * 2).into()); + let qrect = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, 0.0).into(), (10.0, 10.0).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ); + let cx = MockQueryContext::new( + (std::mem::size_of::() * 2).into(), + TilingSpecification::test_default(), + ); let number_of_source_chunks = processor .query(qrect.clone(), &cx) @@ -262,13 +260,12 @@ mod tests { unreachable!(); }; - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, 0.0).into(), (0.0, 0.0).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let cx = MockQueryContext::new((0).into()); + let qrect = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, 0.0).into(), (0.0, 0.0).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ); + let cx = MockQueryContext::new((0).into(), TilingSpecification::test_default()); let collections = FeatureCollectionChunkMerger::new(processor.query(qrect, &cx).await.unwrap().fuse(), 0) diff --git a/operators/src/adapters/mod.rs b/operators/src/adapters/mod.rs index e9d759011..45cd3cb5b 100644 --- a/operators/src/adapters/mod.rs +++ b/operators/src/adapters/mod.rs @@ -13,7 +13,7 @@ pub use feature_collection_merger::FeatureCollectionChunkMerger; pub use raster_stacker::{RasterStackerAdapter, RasterStackerSource}; pub use raster_subquery::{ FoldTileAccu, FoldTileAccuMut, RasterSubQueryAdapter, SubQueryTileAggregator, - TileReprojectionSubQuery, fold_by_coordinate_lookup_future, + TileReprojectionSubQuery, TileReprojectionSubqueryGridInfo, fold_by_coordinate_lookup_future, }; pub use raster_time::{QueryWrapper, Queryable, RasterArrayTimeAdapter, RasterTimeAdapter}; pub use simple_raster_stacker::{ diff --git a/operators/src/adapters/raster_stacker.rs b/operators/src/adapters/raster_stacker.rs index 41b3bc357..08040d9d2 100644 --- a/operators/src/adapters/raster_stacker.rs +++ b/operators/src/adapters/raster_stacker.rs @@ -3,10 +3,10 @@ use futures::future::JoinAll; use futures::stream::{Fuse, FusedStream, Stream}; use futures::{Future, StreamExt, ready}; use geoengine_datatypes::primitives::{ - BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, + BandSelection, RasterQueryRectangle, SpatialGridQueryRectangle, TimeInterval, }; use geoengine_datatypes::raster::{ - GridIdx2D, GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy, + GridBoundingBox2D, GridIdx2D, GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy, }; use pin_project::pin_project; use std::pin::Pin; @@ -66,17 +66,15 @@ impl From<(Q, Vec)> for RasterStackerSource { #[derive(Debug)] pub struct PartialQueryRect { - pub spatial_bounds: SpatialPartition2D, + pub spatial_query: SpatialGridQueryRectangle, pub time_interval: TimeInterval, - pub spatial_resolution: SpatialResolution, } impl PartialQueryRect { fn raster_query_rectangle(&self, attributes: BandSelection) -> RasterQueryRectangle { RasterQueryRectangle { - spatial_bounds: self.spatial_bounds, + spatial_query: self.spatial_query, time_interval: self.time_interval, - spatial_resolution: self.spatial_resolution, attributes, } } @@ -85,9 +83,8 @@ impl PartialQueryRect { impl From for PartialQueryRect { fn from(value: RasterQueryRectangle) -> Self { Self { - spatial_bounds: value.spatial_bounds, + spatial_query: value.spatial_query, time_interval: value.time_interval, - spatial_resolution: value.spatial_resolution, } } } @@ -127,22 +124,22 @@ where } } - fn number_of_tiles_in_partition( + fn number_of_tiles_in_grid_bounds( tile_info: &TileInformation, - partition: SpatialPartition2D, + grid_bounds: GridBoundingBox2D, ) -> usize { - // TODO: get tiling strategy from stream or execution context instead of creating it here let strat = TilingStrategy { tile_size_in_pixels: tile_info.tile_size_in_pixels, geo_transform: tile_info.global_geo_transform, }; - - strat.tile_grid_box(partition).number_of_elements() + strat + .global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds) + .number_of_elements() } fn grid_idx_for_nth_tile( tile_info: &TileInformation, - partition: SpatialPartition2D, + pixel_bounds: GridBoundingBox2D, n: usize, ) -> Option { let strat = TilingStrategy { @@ -150,7 +147,9 @@ where geo_transform: tile_info.global_geo_transform, }; - strat.tile_idx_iterator(partition).nth(n) + strat + .tile_idx_iterator_from_grid_bounds(pixel_bounds) + .nth(n) } } @@ -273,9 +272,9 @@ where }); } - *num_spatial_tiles = Some(Self::number_of_tiles_in_partition( + *num_spatial_tiles = Some(Self::number_of_tiles_in_grid_bounds( &ok_tiles[0].tile_information(), - query_rect.spatial_bounds, + query_rect.spatial_query.grid_bounds(), //TODO: use direct mehtod instead of conversion )); *stream_state = StreamState::ProducingTimeSlice { @@ -332,13 +331,13 @@ where Some(tile.tile_position), Self::grid_idx_for_nth_tile( &tile.tile_information(), - query_rect.spatial_bounds, + query_rect.spatial_query.grid_bounds(), *current_spatial_tile ), "RasteStacker got tile with unexpected tile_position: expected {:?}, got {:?} for source {}", Self::grid_idx_for_nth_tile( &tile.tile_information(), - query_rect.spatial_bounds, + query_rect.spatial_query.grid_bounds(), *current_spatial_tile ), tile.tile_position, @@ -393,7 +392,6 @@ where state.set(State::Initial); } } - return Poll::Ready(Some(Ok(tile))); } }, @@ -408,8 +406,11 @@ where mod tests { use futures::StreamExt; use geoengine_datatypes::{ - primitives::{CacheHint, Measurement, SpatialResolution, TimeInterval}, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, + primitives::{CacheHint, Measurement, TimeInterval}, + raster::{ + GeoTransform, Grid, GridBoundingBox2D, GridShape, RasterDataType, + TilesEqualIgnoringCacheHint, + }, spatial_reference::SpatialReference, util::test::TestDefault, }; @@ -418,7 +419,7 @@ mod tests { adapters::QueryWrapper, engine::{ MockExecutionContext, MockQueryContext, RasterBandDescriptor, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, WorkflowOperatorPath, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, WorkflowOperatorPath, }, mock::{MockRasterSource, MockRasterSourceParams}, }; @@ -428,6 +429,17 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_stacks() { + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [0, 4]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -519,14 +531,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -534,14 +539,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -591,9 +589,10 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_query: SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); @@ -615,6 +614,17 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_keeps_single_band_input() { + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 4]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -659,14 +669,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -699,9 +702,10 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_query: SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); @@ -714,6 +718,38 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_stacks_stacks() { + let result_descriptor_1 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + + let result_descriptor_2 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs2 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs2 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -887,19 +923,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_1.clone(), }, } .boxed(); @@ -907,19 +931,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs2 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs2 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_2, }, } .boxed(); @@ -969,9 +981,10 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_query: SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); @@ -1000,6 +1013,39 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_aligns_temporally_while_stacking_stacks() { + let result_descriptor_1 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + + bands: vec![ + RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + + let result_descriptor_2 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs2 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs2 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -1173,19 +1219,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_1.clone(), }, } .boxed(); @@ -1193,19 +1227,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs2 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs2 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_2, }, } .boxed(); @@ -1255,9 +1277,10 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_query: SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); @@ -1517,6 +1540,55 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_stacks_more() { + let result_descriptor_1 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs1 band3".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + + let result_descriptor_2 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![RasterBandDescriptor::new( + "mrs2 band2".to_string(), + Measurement::Unitless, + )] + .try_into() + .unwrap(), + }; + + let result_descriptor_3 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs3 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs3 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + // input 1: 3 bands let data: Vec> = vec![ RasterTile2D { @@ -1784,20 +1856,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs1 band3".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_1.clone(), }, } .boxed(); @@ -1805,19 +1864,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![RasterBandDescriptor::new( - "mrs2 band1".to_string(), - Measurement::Unitless, - )] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_2, }, } .boxed(); @@ -1825,19 +1872,7 @@ mod tests { let mrs3 = MockRasterSource { params: MockRasterSourceParams { data: data3.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs3 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs3 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_3, }, } .boxed(); @@ -1904,9 +1939,10 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_query: SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); diff --git a/operators/src/adapters/raster_subquery/mod.rs b/operators/src/adapters/raster_subquery/mod.rs index 260c950be..41566f6e6 100644 --- a/operators/src/adapters/raster_subquery/mod.rs +++ b/operators/src/adapters/raster_subquery/mod.rs @@ -6,5 +6,5 @@ pub use raster_subquery_adapter::{ }; pub use raster_subquery_reprojection::{ - TileReprojectionSubQuery, fold_by_coordinate_lookup_future, + TileReprojectionSubQuery, TileReprojectionSubqueryGridInfo, fold_by_coordinate_lookup_future, }; diff --git a/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs b/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs index 36e24a01b..64d6a6bc6 100644 --- a/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs +++ b/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs @@ -5,6 +5,7 @@ use crate::adapters::sparse_tiles_fill_adapter::{ use crate::engine::{QueryContext, QueryProcessor, RasterQueryProcessor, RasterResultDescriptor}; use crate::error; use crate::util::Result; +use async_trait::async_trait; use futures::future::BoxFuture; use futures::{Future, stream::FusedStream}; use futures::{ @@ -12,28 +13,23 @@ use futures::{ stream::{BoxStream, TryFold}, }; use futures::{Stream, StreamExt, TryFutureExt}; +use geoengine_datatypes::primitives::TimeInterval; use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::primitives::{ - RasterQueryRectangle, SpatialPartition2D, SpatialPartitioned, +use geoengine_datatypes::primitives::{RasterQueryRectangle, RasterSpatialQueryRectangle}; +use geoengine_datatypes::raster::{ + EmptyGrid2D, GridBoundingBox2D, GridBounds, GridStep, TilingStrategy, }; -use geoengine_datatypes::raster::{EmptyGrid2D, GridBoundingBox2D, GridBounds, GridStep}; use geoengine_datatypes::{ primitives::TimeInstance, - raster::{Blit, Pixel, RasterTile2D, TileInformation}, + raster::{Pixel, RasterTile2D, TileInformation}, }; -use geoengine_datatypes::{primitives::TimeInterval, raster::TilingSpecification}; - use pin_project::pin_project; use rayon::ThreadPool; - use std::marker::PhantomData; +use std::pin::Pin; use std::sync::Arc; use std::task::Poll; -use std::pin::Pin; - -use async_trait::async_trait; - #[async_trait] pub trait FoldTileAccu { type RasterType: Pixel; @@ -42,7 +38,8 @@ pub trait FoldTileAccu { } pub trait FoldTileAccuMut: FoldTileAccu { - fn tile_mut(&mut self) -> &mut RasterTile2D; + fn set_time(&mut self, new_time: TimeInterval); + fn set_cache_hint(&mut self, new_cache_hint: CacheHint); } pub type RasterFold<'a, T, FoldFuture, FoldMethod, FoldTileAccu> = @@ -95,7 +92,7 @@ where /// The `QueryRectangle` the adapter is queried with query_rect_to_answer: RasterQueryRectangle, /// The `GridBoundingBox2D` that defines the tile grid space of the query. - grid_bounds: GridBoundingBox2D, + tile_grid_bounds: GridBoundingBox2D, // the selected bands from the source bands: Vec, // the band being currently processed @@ -131,23 +128,17 @@ where pub fn new( source_processor: &'a RasterProcessor, query_rect_to_answer: RasterQueryRectangle, - tiling_spec: TilingSpecification, + tiling_strategy: TilingStrategy, query_ctx: &'a dyn QueryContext, sub_query: SubQuery, ) -> Self { - debug_assert!(query_rect_to_answer.spatial_resolution.y > 0.); - - let tiling_strat = tiling_spec.strategy( - query_rect_to_answer.spatial_resolution.x, - -query_rect_to_answer.spatial_resolution.y, - ); - - let grid_bounds = tiling_strat.tile_grid_box(query_rect_to_answer.spatial_partition()); + let grid_bounds = query_rect_to_answer.spatial_query.grid_bounds(); + let tile_bounds = tiling_strategy.global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds); let first_tile_spec = TileInformation { - global_geo_transform: tiling_strat.geo_transform, - global_tile_position: grid_bounds.min_index(), - tile_size_in_pixels: tiling_strat.tile_size_in_pixels, + global_geo_transform: tiling_strategy.geo_transform, + global_tile_position: tile_bounds.min_index(), + tile_size_in_pixels: tiling_strategy.tile_size_in_pixels, }; Self { @@ -155,7 +146,7 @@ where current_time_end: None, current_time_start: query_rect_to_answer.time_interval.start(), current_band_index: 0, - grid_bounds, + tile_grid_bounds: tile_bounds, bands: query_rect_to_answer.attributes.as_vec(), query_ctx, query_rect_to_answer, @@ -174,7 +165,7 @@ where where Self: Stream>>> + 'a, { - let grid_bounds = self.grid_bounds.clone(); + let grid_bounds = self.tile_grid_bounds; let global_geo_transform = self.current_tile_spec.global_geo_transform; let tile_shape = self.current_tile_spec.tile_size_in_pixels; let num_bands = self.bands.len() as u32; @@ -218,7 +209,7 @@ where PixelType: Pixel, RasterProcessorType: QueryProcessor< Output = RasterTile2D, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, @@ -235,7 +226,7 @@ where PixelType: Pixel, RasterProcessorType: QueryProcessor< Output = RasterTile2D, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, @@ -404,7 +395,7 @@ where } else { // all bands for the current tile are processed, we can go to the next tile in space, if there is one *this.current_band_index = 0; - this.grid_bounds + this.tile_grid_bounds .inc_idx_unchecked(this.current_tile_spec.global_tile_position, 1) }; @@ -425,7 +416,7 @@ where } (None, Some(end_time)) if end_time == *this.current_time_start => { // Only for time instants: reset the spatial idx to the first tile of the grid AND increase the request time by 1. - this.current_tile_spec.global_tile_position = this.grid_bounds.min_index(); + this.current_tile_spec.global_tile_position = this.tile_grid_bounds.min_index(); *this.current_time_start = end_time + 1; *this.current_time_end = None; @@ -436,7 +427,7 @@ where } (None, Some(end_time)) => { // reset the spatial idx to the first tile of the grid AND move the requested time to the last known time. - this.current_tile_spec.global_tile_position = this.grid_bounds.min_index(); + this.current_tile_spec.global_tile_position = this.tile_grid_bounds.min_index(); *this.current_time_start = end_time; *this.current_time_end = None; @@ -491,13 +482,13 @@ where source: &'a S, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, - tiling_specification: TilingSpecification, + tiling_strategy: TilingStrategy, ) -> RasterSubQueryAdapter<'a, T, S, Self> where S: RasterQueryProcessor, Self: Sized, { - RasterSubQueryAdapter::<'a, T, S, Self>::new(source, query, tiling_specification, ctx, self) + RasterSubQueryAdapter::<'a, T, S, Self>::new(source, query, tiling_strategy, ctx, self) } } @@ -527,8 +518,12 @@ impl FoldTileAccu for RasterTileAccu2D { } impl FoldTileAccuMut for RasterTileAccu2D { - fn tile_mut(&mut self) -> &mut RasterTile2D { - &mut self.tile + fn set_time(&mut self, new_time: TimeInterval) { + self.tile.time = new_time; + } + + fn set_cache_hint(&mut self, new_cache_hint: CacheHint) { + self.tile.cache_hint = new_cache_hint; } } @@ -563,16 +558,15 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, + _query_rect: RasterQueryRectangle, start_time: TimeInstance, band_idx: u32, ) -> Result> { - Ok(Some(RasterQueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - time_interval: TimeInterval::new_instant(start_time)?, - spatial_resolution: query_rect.spatial_resolution, - attributes: band_idx.into(), - })) + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + tile_info.global_pixel_bounds(), + TimeInterval::new_instant(start_time)?, + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -599,162 +593,3 @@ pub fn identity_accu( }) .map_err(From::from) } - -pub fn fold_by_blit_impl( - accu: RasterTileAccu2D, - tile: RasterTile2D, -) -> Result> -where - T: Pixel, -{ - let mut accu_tile = accu.tile; - let pool = accu.pool; - let t_union = accu_tile.time.union(&tile.time)?; - - accu_tile.time = t_union; - - if tile.grid_array.is_empty() && accu_tile.grid_array.is_empty() { - // only skip if both tiles are empty. There might be valid data in one otherwise. - return Ok(RasterTileAccu2D::new(accu_tile, pool)); - } - - let mut materialized_tile = accu_tile.into_materialized_tile(); - - materialized_tile.blit(tile)?; - - Ok(RasterTileAccu2D::new(materialized_tile.into(), pool)) -} - -#[allow(dead_code)] -pub fn fold_by_blit_future( - accu: RasterTileAccu2D, - tile: RasterTile2D, -) -> impl Future>> -where - T: Pixel, -{ - crate::util::spawn_blocking(|| fold_by_blit_impl(accu, tile)).then(|x| async move { - match x { - Ok(r) => r, - Err(e) => Err(e.into()), - } - }) -} - -#[cfg(test)] -mod tests { - use geoengine_datatypes::{ - primitives::{SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, - spatial_reference::SpatialReference, - util::test::TestDefault, - }; - - use super::*; - use crate::engine::{ - MockExecutionContext, MockQueryContext, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, WorkflowOperatorPath, - }; - use crate::mock::{MockRasterSource, MockRasterSourceParams}; - use futures::StreamExt; - - #[tokio::test] - async fn identity() { - let data: Vec> = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let mrs1 = MockRasterSource { - params: MockRasterSourceParams { - data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mut exe_ctx = MockExecutionContext::test_default(); - exe_ctx.tiling_specification.tile_size_in_pixels = GridShape { - shape_array: [2, 2], - }; - - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - - let query_ctx = MockQueryContext::test_default(); - let tiling_strat = exe_ctx.tiling_specification; - - let op = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap(); - - let qp = op.query_processor().unwrap().get_u8().unwrap(); - - let a = RasterSubQueryAdapter::new( - &qp, - query_rect, - tiling_strat, - &query_ctx, - TileSubQueryIdentity { - fold_fn: fold_by_blit_future, - _phantom_pixel_type: PhantomData, - }, - ); - let res = a - .map(Result::unwrap) - .map(Option::unwrap) - .collect::>>() - .await; - assert!(data.tiles_equal_ignoring_cache_hint(&res)); - } -} diff --git a/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs b/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs index cf4765f7e..30e3168d3 100644 --- a/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs +++ b/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs @@ -7,16 +7,17 @@ use async_trait::async_trait; use futures::future::BoxFuture; use futures::{Future, FutureExt, TryFuture, TryFutureExt}; use geoengine_datatypes::operations::reproject::Reproject; -use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ - RasterQueryRectangle, SpatialPartition2D, SpatialPartitioned, + AxisAlignedRectangle, BoundingBox2D, CacheHint, RasterQueryRectangle, SpatialPartition2D, + SpatialPartitioned, }; use geoengine_datatypes::raster::{ - Grid2D, GridIndexAccess, GridSize, UpdateIndexedElementsParallel, + Grid2D, GridIndexAccess, GridIntersection, GridSize, SpatialGridDefinition, + UpdateIndexedElementsParallel, }; use geoengine_datatypes::{ operations::reproject::{CoordinateProjection, CoordinateProjector}, - primitives::{SpatialResolution, TimeInterval}, + primitives::TimeInterval, raster::EmptyGrid, spatial_reference::SpatialReference, }; @@ -24,22 +25,26 @@ use geoengine_datatypes::{ primitives::{Coordinate2D, TimeInstance}, raster::{CoordinatePixelAccess, GridIdx2D, Pixel, RasterTile2D, TileInformation}, }; -use log::debug; use num; use rayon::ThreadPool; use rayon::iter::{IndexedParallelIterator, ParallelIterator}; -use rayon::slice::ParallelSliceMut; +use rayon::slice::{ParallelSlice, ParallelSliceMut}; +use tracing::debug; use super::{FoldTileAccu, FoldTileAccuMut, SubQueryTileAggregator}; +#[derive(Debug, Clone, Copy)] +pub struct TileReprojectionSubqueryGridInfo { + pub in_spatial_grid: SpatialGridDefinition, + pub out_spatial_grid: SpatialGridDefinition, +} + #[derive(Debug)] pub struct TileReprojectionSubQuery { pub in_srs: SpatialReference, pub out_srs: SpatialReference, pub fold_fn: F, - pub in_spatial_res: SpatialResolution, - pub valid_bounds_in: SpatialPartition2D, - pub valid_bounds_out: SpatialPartition2D, + pub state: TileReprojectionSubqueryGridInfo, pub _phantom_data: PhantomData, } @@ -66,13 +71,11 @@ where query_rect: RasterQueryRectangle, pool: &Arc, ) -> Self::TileAccuFuture { - // println!("new_fold_accu {:?}", &tile_info.global_tile_position); - build_accu( &query_rect, pool.clone(), tile_info, - self.valid_bounds_out, + self.state, self.out_srs, self.in_srs, ) @@ -86,22 +89,39 @@ where start_time: TimeInstance, band_idx: u32, ) -> Result> { - // this is the spatial partition we are interested in - let valid_spatial_bounds = self - .valid_bounds_out - .intersection(&tile_info.spatial_partition()) - .and_then(|vo| vo.intersection(&query_rect.spatial_partition())); + // this are the pixels we are interested in + debug_assert_eq!( + tile_info.global_geo_transform, + self.state.out_spatial_grid.geo_transform() + ); + + let valid_pixel_bounds = self + .state + .out_spatial_grid + .grid_bounds() + .intersection(&tile_info.global_pixel_bounds()) + .and_then(|b| b.intersection(&query_rect.spatial_query.grid_bounds())); + + let valid_spatial_bounds = valid_pixel_bounds.map(|pb| { + self.state + .out_spatial_grid + .geo_transform() + .grid_to_spatial_bounds(&pb) + }); + if let Some(bounds) = valid_spatial_bounds { let proj = CoordinateProjector::from_known_srs(self.out_srs, self.in_srs)?; let projected_bounds = bounds.reproject(&proj); match projected_bounds { - Ok(pb) => Ok(Some(RasterQueryRectangle { - spatial_bounds: pb, - time_interval: TimeInterval::new_instant(start_time)?, - spatial_resolution: self.in_spatial_res, - attributes: band_idx.into(), - })), + Ok(pb) => Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + self.state + .in_spatial_grid + .geo_transform() + .spatial_to_grid_bounds(&pb), + TimeInterval::new_instant(start_time)?, + band_idx.into(), + ))), // In some strange cases the reprojection can return an empty box. // We ignore it since it contains no pixels. Err(geoengine_datatypes::error::Error::OutputBboxEmpty { bbox: _ }) => Ok(None), @@ -122,7 +142,7 @@ fn build_accu( query_rect: &RasterQueryRectangle, pool: Arc, tile_info: TileInformation, - valid_bounds_out: SpatialPartition2D, + state: TileReprojectionSubqueryGridInfo, out_srs: SpatialReference, in_srs: SpatialReference, ) -> impl Future>> + use { @@ -138,7 +158,7 @@ fn build_accu( tile_info, out_srs, in_srs, - &valid_bounds_out, + &state.out_spatial_grid.spatial_partition(), )?; Ok(TileWithProjectionCoordinates { @@ -181,10 +201,12 @@ fn projected_coordinate_grid_parallel( &tile_info.global_tile_position ); - let mut coord_grid: Grid2D> = + let mut in_coord_grid: Grid2D> = Grid2D::new_filled(tile_info.tile_size_in_pixels, None); - let tile_geo_transform = tile_info.tile_geo_transform(); + let out_coords = tile_info + .spatial_grid_definition() + .generate_coord_grid_pixel_center(); let parallelism = pool.current_num_threads(); let par_chunk_split = @@ -198,65 +220,53 @@ fn projected_coordinate_grid_parallel( par_chunk_size ); - let axis_size_x = tile_info.tile_size_in_pixels.axis_size_x(); - - let res = coord_grid + in_coord_grid .data .par_chunks_mut(par_chunk_size) - .enumerate() - .try_for_each(|(chunk_idx, opt_coord_slice)| { - let chunk_start_y = chunk_idx * par_chunk_split; - let chunk_len = opt_coord_slice.len(); - let chunk_end_y = chunk_start_y + (chunk_len / axis_size_x) - 1; - let out_coords = (0..chunk_len) - .map(|lin_idx| { - let x_idx = lin_idx % axis_size_x; - let y_idx = lin_idx / axis_size_x + chunk_start_y; - let grid_idx = GridIdx2D::from([y_idx as isize, x_idx as isize]); - tile_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(grid_idx) - }) - .collect::>(); - - // the output bounds start at the top left corner of the chunk. - let ul_grid_idx = GridIdx2D::from([chunk_start_y as isize, 0_isize]); - let ul_coord = - tile_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(ul_grid_idx); - - // the output bounds must cover the whole chunk pixels. - let lr_grid_idx = - GridIdx2D::from([chunk_end_y as isize, (axis_size_x - 1) as isize]); - let lr_coord = - tile_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(lr_grid_idx + 1); - - let chunk_bounds = SpatialPartition2D::new_unchecked(ul_coord, lr_coord); + .zip(out_coords.data.par_chunks(par_chunk_size)) + .try_for_each(|(in_coord_slice, out_coord_slice)| { + debug_assert_eq!( + in_coord_slice.len(), + out_coord_slice.len(), + "slices must be equal" + ); + let chunk_bounds = BoundingBox2D::from_coord_ref_iter(out_coord_slice.iter()); + + if chunk_bounds.is_none() { + debug!("reprojection early exit"); + return Ok(()); + } + + let chunk_bounds = chunk_bounds.expect("checked above"); + let valid_out_area = valid_out_area.as_bbox(); let proj = CoordinateProjector::from_known_srs(out_srs, in_srs)?; - if valid_out_area.contains(&chunk_bounds) { + if valid_out_area.contains_bbox(&chunk_bounds) { debug!("reproject whole tile chunk"); - let in_coords = proj.project_coordinates(&out_coords)?; - opt_coord_slice + let in_coords = proj.project_coordinates(out_coord_slice)?; + in_coord_slice .iter_mut() - .zip(in_coords) - .for_each(|(opt_coord, in_coord)| *opt_coord = Some(in_coord)); - } else if valid_out_area.intersects(&chunk_bounds) { + .zip(in_coords.into_iter()) + .for_each(|(a, b)| *a = Some(b)); + } else if valid_out_area.intersects_bbox(&chunk_bounds) { debug!("reproject part of tile chunk"); - opt_coord_slice.iter_mut().zip(out_coords).for_each( - |(opt_coord, idx_coord)| { - let in_coord = if valid_out_area.contains_coordinate(&idx_coord) { - proj.project_coordinate(idx_coord).ok() + in_coord_slice + .iter_mut() + .zip(out_coord_slice.iter()) + .for_each(|(in_coord, out_coord)| { + *in_coord = if valid_out_area.contains_coordinate(out_coord) { + proj.project_coordinate(*out_coord).ok() } else { None }; - *opt_coord = in_coord; - }, - ); + }); } else { - debug!("reproject empty tile chunk"); + // do nothing. Should be unreachable } Result::<(), crate::error::Error>::Ok(()) - }); - res.map(|()| coord_grid) + })?; + Ok(in_coord_grid) }); debug!( "projected_coordinate_grid_parallel took {} (ns)", @@ -296,8 +306,8 @@ where let mut accu = accu; let t_union = accu.accu_tile.time.union(&tile.time)?; - accu.tile_mut().time = t_union; - accu.tile_mut().cache_hint.merge_with(&tile.cache_hint); + accu.set_time(t_union); + accu.set_cache_hint(accu.accu_tile.cache_hint.merged(&tile.cache_hint)); if tile.grid_array.is_empty() { return Ok(accu); @@ -351,8 +361,12 @@ impl FoldTileAccu for TileWithProjectionCoordinates { } impl FoldTileAccuMut for TileWithProjectionCoordinates { - fn tile_mut(&mut self) -> &mut RasterTile2D { - &mut self.accu_tile + fn set_time(&mut self, time: TimeInterval) { + self.accu_tile.time = time; + } + + fn set_cache_hint(&mut self, cache_hint: CacheHint) { + self.accu_tile.cache_hint = cache_hint; } } @@ -361,15 +375,19 @@ mod tests { use futures::StreamExt; use geoengine_datatypes::{ primitives::BandSelection, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, + raster::{ + BoundedGrid, GeoTransform, Grid, GridBoundingBox2D, GridShape, GridShape2D, + RasterDataType, SpatialGridDefinition, TilesEqualIgnoringCacheHint, + TilingSpecification, + }, util::test::TestDefault, }; use crate::{ adapters::RasterSubQueryAdapter, engine::{ - MockExecutionContext, MockQueryContext, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, WorkflowOperatorPath, + MockExecutionContext, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + SpatialGridDescriptor, WorkflowOperatorPath, }, mock::{MockRasterSource, MockRasterSourceParams}, }; @@ -386,7 +404,9 @@ mod tests { tile_position: [-1, 0].into(), band: 0, global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), + grid_array: Grid::new([2, 2].into(), vec![1_u8, 2, 3, 4]) + .unwrap() + .into(), properties: Default::default(), cache_hint: CacheHint::default(), }, @@ -423,59 +443,63 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 2.), 1., -1.), + GridShape::new_2d(2, 4).bounding_box(), + )), + bands: RasterBandDescriptors::new_single_band(), + }; + + let tiling_spec = TilingSpecification::new(GridShape2D::new([2, 2])); + + let tiling_grid = result_descriptor.tiling_grid_definition(tiling_spec); + let tiling_strat = tiling_grid.generate_data_tiling_strategy(); + + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_spec); + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); - let mut exe_ctx = MockExecutionContext::test_default(); - exe_ctx.tiling_specification.tile_size_in_pixels = GridShape { - shape_array: [2, 2], - }; - - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); - let query_ctx = MockQueryContext::test_default(); - let tiling_strat = exe_ctx.tiling_specification; + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); let op = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); - let qp = op.query_processor().unwrap().get_u8().unwrap(); - - let valid_bounds = projection.area_of_use_projected().unwrap(); + let qp = op.query_processor().unwrap().get_u8(); + let qp = qp.unwrap(); let state_gen = TileReprojectionSubQuery { in_srs: projection, out_srs: projection, fold_fn: fold_by_coordinate_lookup_future, - in_spatial_res: query_rect.spatial_resolution, - valid_bounds_in: valid_bounds, - valid_bounds_out: valid_bounds, + state: TileReprojectionSubqueryGridInfo { + in_spatial_grid: tiling_grid.tiling_spatial_grid_definition(), + out_spatial_grid: tiling_grid.tiling_spatial_grid_definition(), + }, _phantom_data: PhantomData, }; let a = RasterSubQueryAdapter::new(&qp, query_rect, tiling_strat, &query_ctx, state_gen); let res = a .map(Result::unwrap) .map(Option::unwrap) - .collect::>>() + .collect::>() .await; assert!(data.tiles_equal_ignoring_cache_hint(&res)); } diff --git a/operators/src/adapters/raster_time.rs b/operators/src/adapters/raster_time.rs index 0a14ec545..1ab20d253 100644 --- a/operators/src/adapters/raster_time.rs +++ b/operators/src/adapters/raster_time.rs @@ -3,10 +3,11 @@ use crate::util::Result; use crate::util::stream_zip::StreamArrayZip; use futures::future::{self, BoxFuture, Join, JoinAll}; use futures::stream::{BoxStream, FusedStream, Zip}; -use futures::{Future, Stream}; -use futures::{StreamExt, ready}; -use geoengine_datatypes::primitives::{RasterQueryRectangle, SpatialPartition2D, TimeInterval}; -use geoengine_datatypes::raster::{GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy}; +use futures::{Future, Stream, StreamExt, ready}; +use geoengine_datatypes::primitives::{RasterQueryRectangle, TimeInterval}; +use geoengine_datatypes::raster::{ + GridBoundingBox2D, GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy, +}; use pin_project::pin_project; use std::cmp::min; use std::pin::Pin; @@ -110,17 +111,17 @@ where (tile_a, tile_b) } - fn number_of_tiles_in_partition( + fn number_of_tiles_in_grid_bounds( tile_info: &TileInformation, - partition: SpatialPartition2D, + grid_bounds: GridBoundingBox2D, ) -> usize { - // TODO: get tiling strategy from stream or execution context instead of creating it here let strat = TilingStrategy { tile_size_in_pixels: tile_info.tile_size_in_pixels, geo_transform: tile_info.global_geo_transform, }; - - strat.tile_grid_box(partition).number_of_elements() + strat + .global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds) + .number_of_elements() } } @@ -212,11 +213,11 @@ where tiles } - fn number_of_tiles_in_partition( + fn number_of_tiles_in_grid_bounds( tile_info: &TileInformation, - partition: SpatialPartition2D, + grid_bounds: GridBoundingBox2D, ) -> usize { - RasterTimeAdapter::::number_of_tiles_in_partition(tile_info, partition) + RasterTimeAdapter::::number_of_tiles_in_grid_bounds(tile_info, grid_bounds) } } @@ -284,9 +285,9 @@ where Some((Ok(tile_a), Ok(tile_b))) => { // TODO: calculate at start when tiling info is available before querying first tile let num_spatial_tiles = *num_spatial_tiles.get_or_insert_with(|| { - Self::number_of_tiles_in_partition( + Self::number_of_tiles_in_grid_bounds( &tile_a.tile_information(), - query_rect.spatial_bounds, + query_rect.spatial_query().grid_bounds(), // TODO: this should be calculated from the tile grid bounds and not the spatial bounds. ) }); @@ -437,9 +438,9 @@ where // TODO: calculate at start when tiling info is available before querying first tile let num_spatial_tiles = *num_spatial_tiles.get_or_insert_with(|| { - Self::number_of_tiles_in_partition( + Self::number_of_tiles_in_grid_bounds( &tiles[0].tile_information(), - query_rect.spatial_bounds, + query_rect.spatial_query().grid_bounds(), ) }); @@ -570,15 +571,17 @@ mod tests { use super::*; use crate::engine::{ MockExecutionContext, MockQueryContext, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, WorkflowOperatorPath, + RasterResultDescriptor, SpatialGridDescriptor, WorkflowOperatorPath, }; use crate::mock::{MockRasterSource, MockRasterSourceParams}; use futures::StreamExt; use geoengine_datatypes::primitives::{BandSelection, CacheHint}; - use geoengine_datatypes::raster::{EmptyGrid, Grid, RasterDataType, RasterProperties}; + use geoengine_datatypes::raster::{ + BoundedGrid, EmptyGrid, GeoTransform, Grid, GridShape2D, RasterDataType, RasterProperties, + SpatialGridDefinition, TilingSpecification, + }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{primitives::SpatialResolution, raster::TilingSpecification}; #[tokio::test] #[allow(clippy::too_many_lines)] @@ -635,8 +638,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + )), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -717,24 +722,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + )), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp1 = mrs1 @@ -867,8 +871,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -949,24 +955,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp1 = mrs1 @@ -1099,8 +1104,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1159,24 +1166,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp1 = mrs1 @@ -1293,8 +1299,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1353,24 +1361,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp1 = mrs1 @@ -1465,8 +1472,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1521,24 +1530,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 4), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 4), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp1 = mrs1 @@ -1628,8 +1636,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1688,24 +1698,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 4), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 4), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp1 = mrs1 @@ -1812,8 +1821,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1868,24 +1879,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 8), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 8), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp1 = mrs1 @@ -1997,8 +2007,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -2057,25 +2069,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 8), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 8), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp1 = mrs1 @@ -2186,8 +2196,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -2224,24 +2236,23 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(1, 3), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(1, 3), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let query_processor_a = raster_source_a diff --git a/operators/src/adapters/sparse_tiles_fill_adapter.rs b/operators/src/adapters/sparse_tiles_fill_adapter.rs index 8404fb902..b37eab04e 100644 --- a/operators/src/adapters/sparse_tiles_fill_adapter.rs +++ b/operators/src/adapters/sparse_tiles_fill_adapter.rs @@ -1,13 +1,10 @@ use crate::util::Result; use futures::{Stream, ready}; use geoengine_datatypes::{ - primitives::{ - CacheExpiration, CacheHint, RasterQueryRectangle, SpatialPartitioned, TimeInstance, - TimeInterval, - }, + primitives::{CacheExpiration, CacheHint, RasterQueryRectangle, TimeInstance, TimeInterval}, raster::{ EmptyGrid2D, GeoTransform, GridBoundingBox2D, GridBounds, GridIdx2D, GridShape2D, GridStep, - Pixel, RasterTile2D, TilingSpecification, + Pixel, RasterTile2D, TilingStrategy, }, }; use pin_project::pin_project; @@ -307,6 +304,11 @@ impl StateContainer { } fn update_current_time(&mut self, new_time: TimeInterval) { + debug_assert!( + !new_time.is_instant(), + "Tile time is the data validity and must not be an instant!" + ); + if let Some(old_time) = self.current_time { if old_time == new_time { return; @@ -359,6 +361,21 @@ impl StateContainer { true } + + fn store_tile(&mut self, tile: RasterTile2D) { + debug_assert!(self.next_tile.is_none()); + let current_time = self + .current_time + .expect("Time must be set when the first tile arrives"); + debug_assert!(current_time.start() <= tile.time.start()); + debug_assert!( + current_time.start() < tile.time.start() + || (self.current_idx.y() < tile.tile_position.y() + || (self.current_idx.y() == tile.tile_position.y() + && self.current_idx.x() < tile.tile_position.x())) + ); + self.next_tile = Some(tile); + } } #[pin_project(project=SparseTilesFillAdapterProjection)] @@ -413,27 +430,28 @@ where } } + /// Creates a new `SparseTilesFillAdapter` that fills the gaps of the input stream with empty tiles. + /// The input stream must be sorted by `GridIdx` and `TimeInterval`. + /// The adaper will fill the gaps within the `query_rect_to_answer` with empty tiles. + /// + /// # Panics + /// If the `query_rect_to_answer` has a different `origin_coordinate` than the `tiling_spec`. + /// pub fn new_like_subquery( stream: S, query_rect_to_answer: &RasterQueryRectangle, - tiling_spec: TilingSpecification, + tiling_strat: TilingStrategy, cache_expiration: FillerTileCacheExpirationStrategy, time_bounds: FillerTimeBounds, ) -> Self { - debug_assert!(query_rect_to_answer.spatial_resolution.y > 0.); - - let tiling_strat = tiling_spec.strategy( - query_rect_to_answer.spatial_resolution.x, - -query_rect_to_answer.spatial_resolution.y, - ); - - let grid_bounds = tiling_strat.tile_grid_box(query_rect_to_answer.spatial_partition()); + let grid_bounds = tiling_strat + .raster_spatial_query_to_tiling_grid_box(&query_rect_to_answer.spatial_query()); Self::new( stream, grid_bounds, query_rect_to_answer.attributes.count(), tiling_strat.geo_transform, - tiling_spec.tile_size_in_pixels, + tiling_strat.tile_size_in_pixels, cache_expiration, query_rect_to_answer.time_interval, time_bounds, @@ -479,7 +497,7 @@ where this.sc.state = State::PollingForNextTile; // return the received tile and set state to polling for the next tile tile } else { - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; // save the tile and go to fill mode this.sc.current_no_data_tile() } @@ -585,7 +603,7 @@ where tile } else { // the tile is not the next to produce. Save it and go to fill mode. - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } @@ -606,13 +624,13 @@ where } else { // save the tile and go to fill mode. this.sc.update_current_time(tile.time); - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } } else { // the received tile is in a new TimeInterval but we still need to finish the current one. Store tile and go to fill mode. - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } @@ -629,12 +647,12 @@ where .end(), tile.time.start(), )?); - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } else { // the received tile is in a new TimeInterval but we still need to finish the current one. Store tile and go to fill mode. - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } diff --git a/operators/src/cache/cache_chunks.rs b/operators/src/cache/cache_chunks.rs index 73db18733..0c859e16b 100644 --- a/operators/src/cache/cache_chunks.rs +++ b/operators/src/cache/cache_chunks.rs @@ -184,7 +184,7 @@ where // If the chunk has no spatial bounds it is either an empty collection or a no geometry collection. let spatial_hit = self .spatial_bounds - .is_none_or(|sb| sb.intersects_bbox(&query.spatial_bounds)); + .is_none_or(|sb| sb.intersects_bbox(&query.spatial_query.spatial_bounds)); temporal_hit && spatial_hit } @@ -281,9 +281,11 @@ macro_rules! impl_cache_result_check { &self, query_rect: &VectorQueryRectangle, ) -> Result { + let query_spatial_query = query_rect.spatial_query(); + let geoms_filter_bools = self.geometries().map(|g| { g.bbox() - .map(|bbox| bbox.intersects_bbox(&query_rect.spatial_bounds)) + .map(|bbox| bbox.intersects_bbox(&query_spatial_query.spatial_bounds)) .unwrap_or(false) }); @@ -496,8 +498,8 @@ mod tests { use geoengine_datatypes::{ collections::MultiPointCollection, primitives::{ - BoundingBox2D, CacheHint, ColumnSelection, FeatureData, MultiPoint, SpatialResolution, - TimeInterval, VectorQueryRectangle, + BoundingBox2D, CacheHint, ColumnSelection, FeatureData, MultiPoint, TimeInterval, + VectorQueryRectangle, }, }; use std::{collections::HashMap, sync::Arc}; @@ -561,12 +563,11 @@ mod tests { #[test] fn landing_zone_to_cache_entry() { let cols = create_test_collection(); - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((0., 0.).into(), (1., 1.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((0., 0.).into(), (1., 1.).into()), + Default::default(), + ColumnSelection::all(), + ); let mut lq = VectorLandingQueryEntry::create_empty::>( query.clone(), ); @@ -584,36 +585,33 @@ mod tests { let cols = create_test_collection(); // elemtes are all fully contained - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((0., 0.).into(), (12., 12.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((0., 0.).into(), (12., 12.).into()), + Default::default(), + ColumnSelection::all(), + ); for c in &cols { assert!(c.intersects_query(&query)); } // first element is not contained - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((2., 2.).into(), (10., 10.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((2., 2.).into(), (10., 10.).into()), + Default::default(), + ColumnSelection::all(), + ); assert!(!cols[0].intersects_query(&query)); for c in &cols[1..] { assert!(c.intersects_query(&query)); } // all elements are not contained - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((13., 13.).into(), (26., 26.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((13., 13.).into(), (26., 26.).into()), + Default::default(), + ColumnSelection::all(), + ); for col in &cols { assert!(!col.intersects_query(&query)); } @@ -623,12 +621,11 @@ mod tests { fn cache_entry_matches() { let cols = create_test_collection(); - let cache_entry_bounds = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((1., 1.).into(), (11., 11.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let cache_entry_bounds = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((1., 1.).into(), (11., 11.).into()), + Default::default(), + ColumnSelection::all(), + ); let cache_query_entry = VectorCacheQueryEntry { query: cache_entry_bounds.clone(), @@ -640,21 +637,19 @@ mod tests { assert!(cache_query_entry.query().is_match(&query)); // query is fully contained - let query2 = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((2., 2.).into(), (10., 10.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query2 = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((2., 2.).into(), (10., 10.).into()), + Default::default(), + ColumnSelection::all(), + ); assert!(cache_query_entry.query().is_match(&query2)); // query is exceeds cached bounds - let query3 = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((0., 0.).into(), (8., 8.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query3 = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((0., 0.).into(), (8., 8.).into()), + Default::default(), + ColumnSelection::all(), + ); assert!(!cache_query_entry.query().is_match(&query3)); } } diff --git a/operators/src/cache/cache_operator.rs b/operators/src/cache/cache_operator.rs index 3efbda39d..130053050 100644 --- a/operators/src/cache/cache_operator.rs +++ b/operators/src/cache/cache_operator.rs @@ -15,7 +15,7 @@ use futures::stream::{BoxStream, FusedStream}; use futures::{Stream, StreamExt, TryStreamExt, ready}; use geoengine_datatypes::collections::{FeatureCollection, FeatureCollectionInfos}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, Geometry, QueryAttributeSelection, QueryRectangle, VectorQueryRectangle, + Geometry, QueryAttributeSelection, QueryRectangle, VectorQueryRectangle, }; use geoengine_datatypes::raster::{Pixel, RasterTile2D}; use geoengine_datatypes::util::arrow::ArrowTyped; @@ -159,7 +159,7 @@ impl InitializedVectorOperator for InitializedCacheOperator where E: CacheElement + Send + Sync + 'static, - P: QueryProcessor, + P: QueryProcessor, { processor: P, cache_key: CanonicOperatorName, @@ -168,7 +168,7 @@ where impl CacheQueryProcessor where E: CacheElement + Send + Sync + 'static, - P: QueryProcessor + Sized, + P: QueryProcessor + Sized, { pub fn new(processor: P, cache_key: CanonicOperatorName) -> Self { CacheQueryProcessor { @@ -181,8 +181,8 @@ where #[async_trait] impl QueryProcessor for CacheQueryProcessor where - P: QueryProcessor + Sized, - S: AxisAlignedRectangle + Send + Sync + 'static, + P: QueryProcessor + Sized, + S: Clone + Send + Sync + 'static, U: QueryAttributeSelection, E: CacheElement> + Send @@ -195,13 +195,13 @@ where R: ResultDescriptor, { type Output = E; - type SpatialBounds = S; + type SpatialQuery = S; type Selection = U; type ResultDescription = R; async fn _query<'a>( &'a self, - query: QueryRectangle, + query: QueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { let shared_cache = ctx @@ -430,8 +430,8 @@ mod tests { use crate::{ engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, MultipleRasterSources, - RasterOperator, SingleRasterSource, WorkflowOperatorPath, + ChunkByteSize, MockExecutionContext, MultipleRasterSources, RasterOperator, + SingleRasterSource, WorkflowOperatorPath, }, processing::{Expression, ExpressionParams, RasterStacker, RasterStackerParams}, source::{GdalSource, GdalSourceParameters}, @@ -439,8 +439,8 @@ mod tests { }; use futures::StreamExt; use geoengine_datatypes::{ - primitives::{BandSelection, SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::{RasterDataType, RenameBands, TilesEqualIgnoringCacheHint}, + primitives::{BandSelection, RasterQueryRectangle, TimeInterval}, + raster::{GridBoundingBox2D, RasterDataType, RenameBands, TilesEqualIgnoringCacheHint}, util::test::TestDefault, }; use std::sync::Arc; @@ -452,9 +452,7 @@ mod tests { let ndvi_id = add_ndvi_dataset(&mut exe_ctx); let operator = GdalSource { - params: GdalSourceParameters { - data: ndvi_id.clone(), - }, + params: GdalSourceParameters::new(ndvi_id.clone()), } .boxed() .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -467,7 +465,7 @@ mod tests { let tile_cache = Arc::new(SharedCache::test_default()); - let query_ctx = MockQueryContext::new_with_query_extensions( + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( ChunkByteSize::test_default(), Some(tile_cache), None, @@ -476,15 +474,11 @@ mod tests { let stream = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), &query_ctx, ) .await @@ -501,15 +495,11 @@ mod tests { let stream_from_cache = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), &query_ctx, ) .await @@ -539,6 +529,7 @@ mod tests { GdalSource { params: GdalSourceParameters { data: ndvi_id.clone(), + overview_level: None, }, } .boxed(), @@ -553,6 +544,7 @@ mod tests { raster: GdalSource { params: GdalSourceParameters { data: ndvi_id.clone(), + overview_level: None, }, } .boxed(), @@ -573,7 +565,7 @@ mod tests { let tile_cache = Arc::new(SharedCache::test_default()); - let query_ctx = MockQueryContext::new_with_query_extensions( + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( ChunkByteSize::test_default(), Some(tile_cache), None, @@ -583,15 +575,11 @@ mod tests { // query the first two bands let stream = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::new(vec![0, 1]).unwrap(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::new(vec![0, 1]).unwrap(), + ), &query_ctx, ) .await @@ -621,15 +609,11 @@ mod tests { // now query only the second band let stream_from_cache = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::new_single(1), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::new_single(1), + ), &query_ctx, ) .await diff --git a/operators/src/cache/cache_stream.rs b/operators/src/cache/cache_stream.rs index ca5994693..f8f1c65bc 100644 --- a/operators/src/cache/cache_stream.rs +++ b/operators/src/cache/cache_stream.rs @@ -162,8 +162,7 @@ mod tests { collections::MultiPointCollection, primitives::{ BandSelection, BoundingBox2D, CacheHint, ColumnSelection, FeatureData, MultiPoint, - RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - VectorQueryRectangle, + RasterQueryRectangle, TimeInterval, VectorQueryRectangle, }, raster::{GeoTransform, Grid2D, GridIdx2D, RasterTile2D}, }; @@ -173,6 +172,7 @@ mod tests { cache_stream::CacheStreamInner, cache_tiles::{CompressedRasterTile2D, CompressedRasterTileExt}, }; + use geoengine_datatypes::raster::GridBoundingBox2D; fn create_test_raster_data() -> Vec> { let mut data = Vec::new(); @@ -226,12 +226,11 @@ mod tests { #[test] fn test_cache_stream_inner_raster() { let data = Arc::new(create_test_raster_data()); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((2., -2.).into(), (8., -8.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([4, 4], [15, 15]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); let mut res = Vec::new(); let mut inner = CacheStreamInner::new(data, query); @@ -247,12 +246,11 @@ mod tests { #[test] fn test_cache_stream_inner_vector() { let data = Arc::new(create_test_vecor_data()); - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((2.1, 2.1).into(), (7.9, 7.9).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((2.1, 2.1).into(), (7.9, 7.9).into()), + TimeInterval::new_unchecked(0, 10), + ColumnSelection::all(), + ); let mut res = Vec::new(); let mut inner = CacheStreamInner::new(data, query); diff --git a/operators/src/cache/cache_tiles.rs b/operators/src/cache/cache_tiles.rs index a7c63310b..0fba23d2b 100644 --- a/operators/src/cache/cache_tiles.rs +++ b/operators/src/cache/cache_tiles.rs @@ -6,10 +6,9 @@ use super::shared_cache::{ RasterLandingQueryEntry, }; use crate::util::Result; -use geoengine_datatypes::primitives::SpatialPartitioned; use geoengine_datatypes::raster::{ - BaseTile, EmptyGrid, Grid, GridOrEmpty, GridShape2D, GridSize, GridSpaceToLinearSpace, - MaskedGrid, RasterTile, + BaseTile, EmptyGrid, Grid, GridBoundingBoxExt, GridIntersection, GridOrEmpty, GridShape2D, + GridSize, GridSpaceToLinearSpace, MaskedGrid, RasterTile, }; use geoengine_datatypes::{ primitives::RasterQueryRectangle, @@ -197,7 +196,12 @@ where } fn update_stored_query(&self, query: &mut Self::Query) -> Result<(), CacheError> { - query.spatial_bounds.extend(&self.spatial_partition()); + let stored_spatial_query_mut = query.spatial_query_mut(); + + stored_spatial_query_mut + .grid_bounds() + .extend(&self.tile_information().global_pixel_bounds()); + query.time_interval = query .time_interval .union(&self.time) @@ -206,7 +210,9 @@ where } fn intersects_query(&self, query: &Self::Query) -> bool { - self.spatial_partition().intersects(&query.spatial_bounds) + self.tile_information() + .global_pixel_bounds() + .intersects(&query.spatial_query.grid_bounds()) && self.time.intersects(&query.time_interval) && query.attributes.as_slice().contains(&self.band) } @@ -602,10 +608,8 @@ impl TileCompression for Lz4FlexCompression { mod tests { use std::sync::Arc; - use geoengine_datatypes::{ - primitives::{BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution}, - raster::GeoTransform, - util::test::TestDefault, + use super::{ + CompressedGridOrEmpty, CompressedMaskedGrid, CompressedRasterTile2D, LandingZoneQueryTiles, }; use crate::cache::{ @@ -615,9 +619,10 @@ mod tests { RasterLandingQueryEntry, }, }; - - use super::{ - CompressedGridOrEmpty, CompressedMaskedGrid, CompressedRasterTile2D, LandingZoneQueryTiles, + use geoengine_datatypes::{ + primitives::{BandSelection, RasterQueryRectangle}, + raster::{GeoTransform, GridBoundingBox2D}, + util::test::TestDefault, }; fn create_test_tile() -> CompressedRasterTile2D { @@ -665,12 +670,11 @@ mod tests { #[test] fn landing_zone_to_cache_entry() { let tile = create_test_tile(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., 1.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ); let mut lq = RasterLandingQueryEntry::create_empty::>(query.clone()); tile.move_element_into_landing_zone(lq.elements_mut()) @@ -685,47 +689,37 @@ mod tests { let tile = create_test_tile(); // tile is fully contained - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., -1.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(tile.intersects_query(&query)); // tile is partially contained - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0.5, -0.5).into(), - (1.5, -1.5).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, -1], [0, 0]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(tile.intersects_query(&query)); // tile is not contained - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (10., -10.).into(), - (11., -11.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([10, 10], [11, 11]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(!tile.intersects_query(&query)); } #[test] fn cache_entry_matches() { - let cache_entry_bounds = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., -1.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let cache_entry_bounds = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [10, 10]).unwrap(), + Default::default(), + BandSelection::first(), + ); let cache_query_entry = RasterCacheQueryEntry { query: cache_entry_bounds.clone(), elements: CachedTiles::U8(Arc::new(Vec::new())), @@ -736,27 +730,19 @@ mod tests { assert!(cache_query_entry.query().is_match(&query)); // query is fully contained - let query2 = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0.1, -0.1).into(), - (0.9, -0.9).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query2 = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([1, 1], [9, 9]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(cache_query_entry.query().is_match(&query2)); // query is exceeds cached bounds - let query3 = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-0.1, 0.1).into(), - (1.1, -1.1).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query3 = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [11, 11]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(!cache_query_entry.query().is_match(&query3)); } } diff --git a/operators/src/cache/shared_cache.rs b/operators/src/cache/shared_cache.rs index 29d853daa..26b555009 100644 --- a/operators/src/cache/shared_cache.rs +++ b/operators/src/cache/shared_cache.rs @@ -11,7 +11,7 @@ use futures::Stream; use geoengine_datatypes::{ identifier, primitives::{CacheHint, Geometry, RasterQueryRectangle, VectorQueryRectangle}, - raster::Pixel, + raster::{GridContains, Pixel}, util::{ByteSize, Identifier, arrow::ArrowTyped, test::TestDefault}, }; use log::{debug, log_enabled}; @@ -853,23 +853,26 @@ pub trait CacheQueryMatch { impl CacheQueryMatch for RasterQueryRectangle { fn is_match(&self, query: &RasterQueryRectangle) -> bool { - self.spatial_bounds.contains(&query.spatial_bounds) + let cache_spatial_query = self.spatial_query(); + let query_spatial_query = query.spatial_query(); + + cache_spatial_query + .grid_bounds() + .contains(&query_spatial_query.grid_bounds()) && self.time_interval.contains(&query.time_interval) - && self.spatial_resolution == query.spatial_resolution - && query - .attributes - .as_slice() - .iter() - .all(|b| self.attributes.as_slice().contains(b)) } } impl CacheQueryMatch for VectorQueryRectangle { - // TODO: check if that is what we need fn is_match(&self, query: &VectorQueryRectangle) -> bool { - self.spatial_bounds.contains_bbox(&query.spatial_bounds) + let cache_spatial_query = self.spatial_query(); + let query_spatial_query = query.spatial_query(); + + cache_spatial_query + .spatial_bounds + .contains_bbox(&query_spatial_query.spatial_bounds) && self.time_interval.contains(&query.time_interval) - && self.spatial_resolution == query.spatial_resolution + && self.attributes == query.attributes } } @@ -1025,10 +1028,8 @@ where #[cfg(test)] mod tests { use geoengine_datatypes::{ - primitives::{ - BandSelection, CacheHint, DateTime, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - raster::{Grid, RasterProperties, RasterTile2D}, + primitives::{BandSelection, CacheHint, DateTime, TimeInterval}, + raster::{Grid, GridBoundingBox2D, RasterProperties, RasterTile2D}, }; use serde_json::json; use std::sync::Arc; @@ -1108,16 +1109,11 @@ mod tests { } fn query_rect() -> RasterQueryRectangle { - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - } + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ) } fn op(idx: usize) -> CanonicOperatorName { diff --git a/operators/src/engine/execution_context.rs b/operators/src/engine/execution_context.rs index a689cc50e..08f370d2d 100644 --- a/operators/src/engine/execution_context.rs +++ b/operators/src/engine/execution_context.rs @@ -1,12 +1,13 @@ -use super::query::QueryAbortRegistration; use super::{ CreateSpan, InitializedPlotOperator, InitializedRasterOperator, InitializedVectorOperator, MockQueryContext, }; +use crate::cache::shared_cache::SharedCache; use crate::engine::{ ChunkByteSize, RasterResultDescriptor, ResultDescriptor, VectorResultDescriptor, }; use crate::error::Error; +use crate::meta::quota::{QuotaChecker, QuotaTracking}; use crate::meta::wrapper::InitializedOperatorWrapper; use crate::mock::MockDatasetDataSourceLoadingInfo; use crate::source::{GdalLoadingInfo, OgrSourceDataset}; @@ -157,16 +158,35 @@ impl MockExecutionContext { } pub fn mock_query_context(&self, chunk_byte_size: ChunkByteSize) -> MockQueryContext { - let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); - MockQueryContext { + MockQueryContext::new(chunk_byte_size, self.tiling_specification) + } + + pub fn mock_query_context_with_query_extensions( + &self, + chunk_byte_size: ChunkByteSize, + cache: Option>, + quota_tracking: Option, + quota_checker: Option, + ) -> MockQueryContext { + MockQueryContext::new_with_query_extensions( chunk_byte_size, - thread_pool: self.thread_pool.clone(), - cache: None, - quota_checker: None, - quota_tracking: None, - abort_registration, - abort_trigger: Some(abort_trigger), - } + self.tiling_specification, + cache, + quota_tracking, + quota_checker, + ) + } + + pub fn mock_query_context_with_chunk_size_and_thread_count( + &self, + chunk_byte_size: ChunkByteSize, + num_threads: usize, + ) -> MockQueryContext { + MockQueryContext::with_chunk_size_and_thread_count( + chunk_byte_size, + self.tiling_specification, + num_threads, + ) } } @@ -362,6 +382,23 @@ impl TestDefault for StatisticsWrappingMockExecutionContext { } } +impl StatisticsWrappingMockExecutionContext { + pub fn mock_query_context_with_query_extensions( + &self, + chunk_byte_size: ChunkByteSize, + cache: Option>, + quota_tracking: Option, + quota_checker: Option, + ) -> MockQueryContext { + self.inner.mock_query_context_with_query_extensions( + chunk_byte_size, + cache, + quota_tracking, + quota_checker, + ) + } +} + #[async_trait::async_trait] impl ExecutionContext for StatisticsWrappingMockExecutionContext { fn thread_pool(&self) -> &Arc { diff --git a/operators/src/engine/mod.rs b/operators/src/engine/mod.rs index 5b4fe0bf8..b33a61153 100644 --- a/operators/src/engine/mod.rs +++ b/operators/src/engine/mod.rs @@ -7,6 +7,11 @@ pub use execution_context::{ ExecutionContext, MetaData, MetaDataProvider, MockExecutionContext, StaticMetaData, StatisticsWrappingMockExecutionContext, }; +pub use initialized_sources::{ + InitializedMultiRasterOrVectorOperator, InitializedMultiRasterOrVectorSource, + InitializedSingleRasterOrVectorOperator, InitializedSingleRasterOrVectorSource, + InitializedSingleRasterSource, InitializedSingleVectorSource, InitializedSources, +}; pub use operator::{ CanonicOperatorName, InitializedPlotOperator, InitializedRasterOperator, InitializedVectorOperator, OperatorData, OperatorName, PlotOperator, RasterOperator, @@ -27,18 +32,11 @@ pub use query_processor::{ }; pub use result_descriptor::{ PlotResultDescriptor, RasterBandDescriptor, RasterBandDescriptors, RasterResultDescriptor, - ResultDescriptor, TypedResultDescriptor, VectorColumnInfo, VectorResultDescriptor, + ResultDescriptor, SpatialGridDescriptor, SpatialGridDescriptorState, TypedResultDescriptor, + VectorColumnInfo, VectorResultDescriptor, }; use tracing::Span; - pub use workflow_path::WorkflowOperatorPath; - -pub use initialized_sources::{ - InitializedMultiRasterOrVectorOperator, InitializedMultiRasterOrVectorSource, - InitializedSingleRasterOrVectorOperator, InitializedSingleRasterOrVectorSource, - InitializedSingleRasterSource, InitializedSingleVectorSource, InitializedSources, -}; - mod clonable_operator; mod execution_context; mod initialized_sources; diff --git a/operators/src/engine/query.rs b/operators/src/engine/query.rs index f0ab9d9cf..beb032929 100644 --- a/operators/src/engine/query.rs +++ b/operators/src/engine/query.rs @@ -9,7 +9,7 @@ use crate::{ }; use crate::{meta::quota::QuotaTracking, util::Result}; use futures::Stream; -use geoengine_datatypes::util::test::TestDefault; +use geoengine_datatypes::{raster::TilingSpecification, util::test::TestDefault}; use pin_project::pin_project; use rayon::ThreadPool; use serde::{Deserialize, Serialize}; @@ -52,6 +52,7 @@ impl TestDefault for ChunkByteSize { pub trait QueryContext: Send + Sync { fn chunk_byte_size(&self) -> ChunkByteSize; + fn tiling_specification(&self) -> TilingSpecification; fn thread_pool(&self) -> &Arc; fn quota_tracking(&self) -> Option<&QuotaTracking>; @@ -117,6 +118,7 @@ impl QueryAbortTrigger { pub struct MockQueryContext { pub chunk_byte_size: ChunkByteSize, + pub tiling_specification: TilingSpecification, pub thread_pool: Arc, pub cache: Option>, @@ -132,6 +134,7 @@ impl TestDefault for MockQueryContext { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); Self { chunk_byte_size: ChunkByteSize::test_default(), + tiling_specification: TilingSpecification::test_default(), thread_pool: create_rayon_thread_pool(0), cache: None, quota_checker: None, @@ -143,10 +146,11 @@ impl TestDefault for MockQueryContext { } impl MockQueryContext { - pub fn new(chunk_byte_size: ChunkByteSize) -> Self { + pub fn new(chunk_byte_size: ChunkByteSize, tiling_specification: TilingSpecification) -> Self { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); Self { chunk_byte_size, + tiling_specification, thread_pool: create_rayon_thread_pool(0), cache: None, quota_checker: None, @@ -158,6 +162,7 @@ impl MockQueryContext { pub fn new_with_query_extensions( chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, cache: Option>, quota_tracking: Option, quota_checker: Option, @@ -165,6 +170,7 @@ impl MockQueryContext { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); Self { chunk_byte_size, + tiling_specification, thread_pool: create_rayon_thread_pool(0), cache, quota_checker, @@ -176,11 +182,13 @@ impl MockQueryContext { pub fn with_chunk_size_and_thread_count( chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, num_threads: usize, ) -> Self { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); Self { chunk_byte_size, + tiling_specification, thread_pool: create_rayon_thread_pool(num_threads), cache: None, quota_checker: None, @@ -210,6 +218,10 @@ impl QueryContext for MockQueryContext { .ok_or(error::Error::AbortTriggerAlreadyUsed) } + fn tiling_specification(&self) -> TilingSpecification { + self.tiling_specification + } + fn quota_tracking(&self) -> Option<&QuotaTracking> { self.quota_tracking.as_ref() } diff --git a/operators/src/engine/query_processor.rs b/operators/src/engine/query_processor.rs index 52a926186..93ba98cf0 100644 --- a/operators/src/engine/query_processor.rs +++ b/operators/src/engine/query_processor.rs @@ -13,9 +13,9 @@ use geoengine_datatypes::collections::{ }; use geoengine_datatypes::plots::{PlotData, PlotOutputFormat}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, PlotQueryRectangle, - QueryAttributeSelection, QueryRectangle, RasterQueryRectangle, SpatialPartition2D, - VectorQueryRectangle, + AxisAlignedRectangle, BandSelection, ColumnSelection, PlotQueryRectangle, + QueryAttributeSelection, QueryRectangle, RasterQueryRectangle, RasterSpatialQueryRectangle, + VectorQueryRectangle, VectorSpatialQueryRectangle, }; use geoengine_datatypes::raster::{DynamicRasterDataType, Pixel}; use geoengine_datatypes::{collections::MultiPointCollection, raster::RasterTile2D}; @@ -25,23 +25,22 @@ use ouroboros::self_referencing; #[async_trait] pub trait QueryProcessor: Send + Sync { type Output; - type SpatialBounds: AxisAlignedRectangle + Send + Sync; + type SpatialQuery: Send + Sync; type Selection: QueryAttributeSelection; type ResultDescription: ResultDescriptor< - QueryRectangleSpatialBounds = Self::SpatialBounds, + QueryRectangleSpatialBounds = Self::SpatialQuery, QueryRectangleAttributeSelection = Self::Selection, >; - /// inner logic of the processor async fn _query<'a>( &'a self, - query: QueryRectangle, // TODO: query by reference + query: QueryRectangle, // TODO: query by reference ctx: &'a dyn QueryContext, ) -> Result>>; async fn query<'a>( &'a self, - query: QueryRectangle, // TODO: query by reference + query: QueryRectangle, // TODO: query by reference ctx: &'a dyn QueryContext, ) -> Result>> { self.result_descriptor().validate_query(&query)?; @@ -65,7 +64,7 @@ pub trait QueryProcessorExt: QueryProcessor { /// Thus, it can be stored in a struct. async fn query_into_owned_stream( self, - query: QueryRectangle, // TODO: query by reference + query: QueryRectangle, // TODO: query by reference ctx: Box, ) -> Result> where @@ -110,7 +109,7 @@ impl RasterQueryProcessor for S where S: QueryProcessor< Output = RasterTile2D, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, > + Sync @@ -156,7 +155,7 @@ impl VectorQueryProcessor for S where S: QueryProcessor< Output = VD, - SpatialBounds = BoundingBox2D, + SpatialQuery = VectorSpatialQueryRectangle, Selection = ColumnSelection, ResultDescription = VectorResultDescriptor, > + Sync @@ -200,14 +199,14 @@ pub trait PlotQueryProcessor: Sync + Send { #[async_trait] impl QueryProcessor - for Box> + for Box> where S: AxisAlignedRectangle + Send + Sync, U: QueryAttributeSelection, R: ResultDescriptor, { type Output = T; - type SpatialBounds = S; + type SpatialQuery = S; type Selection = U; type ResultDescription = R; @@ -230,7 +229,7 @@ where T: Pixel, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -253,7 +252,7 @@ where V: 'static, { type Output = V; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -530,6 +529,21 @@ impl TypedRasterQueryProcessor { Self::F64(r) => r, } } + + pub fn result_descriptor(&self) -> &RasterResultDescriptor { + match self { + Self::U8(r) => r.raster_result_descriptor(), + Self::U16(r) => r.raster_result_descriptor(), + Self::U32(r) => r.raster_result_descriptor(), + Self::U64(r) => r.raster_result_descriptor(), + Self::I8(r) => r.raster_result_descriptor(), + Self::I16(r) => r.raster_result_descriptor(), + Self::I32(r) => r.raster_result_descriptor(), + Self::I64(r) => r.raster_result_descriptor(), + Self::F32(r) => r.raster_result_descriptor(), + Self::F64(r) => r.raster_result_descriptor(), + } + } } impl From>> for TypedRasterQueryProcessor { diff --git a/operators/src/engine/result_descriptor.rs b/operators/src/engine/result_descriptor.rs index ddd10e80b..10cf09173 100644 --- a/operators/src/engine/result_descriptor.rs +++ b/operators/src/engine/result_descriptor.rs @@ -1,7 +1,17 @@ +use crate::error::{ + Error, RasterBandNameMustNotBeEmpty, RasterBandNameTooLong, RasterBandNamesMustBeUnique, +}; +use crate::util::Result; +use geoengine_datatypes::operations::reproject::{CoordinateProjection, ReprojectClipped}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, FeatureDataType, - Measurement, PlotSeriesSelection, QueryAttributeSelection, QueryRectangle, SpatialPartition2D, - SpatialResolution, TimeInterval, + AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, Coordinate2D, + FeatureDataType, Measurement, PlotSeriesSelection, QueryAttributeSelection, QueryRectangle, + SpatialGridQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, + VectorSpatialQueryRectangle, +}; +use geoengine_datatypes::raster::{ + GeoTransform, GeoTransformAccess, Grid, GridBoundingBox2D, GridShape2D, GridShapeAccess, + SpatialGridDefinition, TilingSpatialGridDefinition, TilingSpecification, }; use geoengine_datatypes::util::ByteSize; use geoengine_datatypes::{ @@ -13,16 +23,11 @@ use snafu::ensure; use std::collections::{HashMap, HashSet}; use std::ops::Index; -use crate::error::{ - Error, RasterBandNameMustNotBeEmpty, RasterBandNameTooLong, RasterBandNamesMustBeUnique, -}; -use crate::util::Result; - /// A descriptor that contains information about the query result, for instance, the data type /// and spatial reference. pub trait ResultDescriptor: Clone + Serialize { type DataType; - type QueryRectangleSpatialBounds: AxisAlignedRectangle; + type QueryRectangleSpatialBounds; type QueryRectangleAttributeSelection: QueryAttributeSelection; // Check the `query` against the `ResultDescriptor` and return `true` if the query is valid @@ -69,197 +74,230 @@ pub trait ResultDescriptor: Clone + Serialize { F: Fn(&Option) -> Option; } -/// A `ResultDescriptor` for raster queries -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSql, FromSql)] +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] -pub struct RasterResultDescriptor { - pub data_type: RasterDataType, - pub spatial_reference: SpatialReferenceOption, - pub time: Option, - pub bbox: Option, - pub resolution: Option, - pub bands: RasterBandDescriptors, +pub enum SpatialGridDescriptorState { + /// The spatial grid represents a native dataset + Source, + /// The spatial grid was created by merging two non equal spatial grids + Merged, } -impl RasterResultDescriptor { - pub fn with_datatype_and_num_bands(data_type: RasterDataType, num_bands: u32) -> Self { - Self { - data_type, - spatial_reference: SpatialReferenceOption::Unreferenced, - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_multiple_bands(num_bands), +impl SpatialGridDescriptorState { + #[must_use] + pub fn merge(self, other: Self) -> Self { + match (self, other) { + (SpatialGridDescriptorState::Source, SpatialGridDescriptorState::Source) => { + SpatialGridDescriptorState::Source + } + _ => SpatialGridDescriptorState::Merged, } } + + pub fn is_source(self) -> bool { + self == SpatialGridDescriptorState::Source + } + + pub fn is_derived(self) -> bool { + !self.is_source() + } } -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct RasterBandDescriptors(Vec); +/// A `ResultDescriptor` for raster queries +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridDescriptor { + spatial_grid: SpatialGridDefinition, + state: SpatialGridDescriptorState, +} -impl RasterBandDescriptors { - pub fn new(bands: Vec) -> Result { - let mut names = HashSet::new(); - for value in &bands { - ensure!(!value.name.is_empty(), RasterBandNameMustNotBeEmpty); - ensure!(value.name.byte_size() <= 256, RasterBandNameTooLong); - ensure!( - names.insert(&value.name), - RasterBandNamesMustBeUnique { - duplicate_key: value.name.clone() - } - ); +impl SpatialGridDescriptor { + pub fn new_source(spatial_grid_def: SpatialGridDefinition) -> Self { + Self { + spatial_grid: spatial_grid_def, + state: SpatialGridDescriptorState::Source, } - - Ok(Self(bands)) } - /// Convenience method to crate a single band result descriptor with no specific name and a unitless measurement for single band rasters - pub fn new_single_band() -> Self { - Self(vec![RasterBandDescriptor { - name: "band".into(), - measurement: Measurement::Unitless, - }]) + pub fn source_from_parts(geo_transform: GeoTransform, grid_bounds: GridBoundingBox2D) -> Self { + Self::new_source(SpatialGridDefinition::new(geo_transform, grid_bounds)) } - /// Convenience method to crate multipe band result descriptors with no specific name and a unitless measurement - pub fn new_multiple_bands(num_bands: u32) -> Self { - Self( - (0..num_bands) - .map(RasterBandDescriptor::new_unitless_with_idx) - .collect(), - ) + #[must_use] + pub fn as_derived(self) -> Self { + Self { + state: SpatialGridDescriptorState::Merged, + ..self + } } - pub fn bands(&self) -> &[RasterBandDescriptor] { - &self.0 + pub fn merge(&self, other: &SpatialGridDescriptor) -> Option { + // TODO: merge directly to tiling origin? + let merged_grid = self.spatial_grid.merge(&other.spatial_grid)?; + let state = if self.spatial_grid.grid_bounds == merged_grid.grid_bounds + && other.spatial_grid.grid_bounds == merged_grid.grid_bounds + { + self.state.merge(other.state) + } else { + SpatialGridDescriptorState::Merged + }; + + Some(Self { + spatial_grid: merged_grid, + state, + }) } - pub fn len(&self) -> usize { - self.0.len() + #[must_use] + pub fn map SpatialGridDefinition>(&self, map_fn: F) -> Self { + Self { + spatial_grid: map_fn(&self.spatial_grid), + ..*self + } } - pub fn count(&self) -> u32 { - self.0.len() as u32 + pub fn try_map Result>( + &self, + map_fn: F, + ) -> Result { + Ok(Self { + spatial_grid: map_fn(&self.spatial_grid)?, + ..*self + }) } - pub fn is_empty(&self) -> bool { - self.len() == 0 + pub fn is_compatible_grid_generic(&self, g: &G) -> bool { + self.spatial_grid.is_compatible_grid_generic(g) } - pub fn iter(&self) -> impl Iterator { - self.0.iter() + pub fn is_compatible_grid(&self, other: &Self) -> bool { + self.is_compatible_grid_generic(&other.spatial_grid) } - pub fn into_vec(self) -> Vec { - self.0 + pub fn tiling_grid_definition( + &self, + tiling_specification: TilingSpecification, + ) -> TilingSpatialGridDefinition { + // TODO: we could also store the tiling_origin_reference and then use that directly? + TilingSpatialGridDefinition::new(self.spatial_grid, tiling_specification) } - // Merge the bands of two descriptors into a new one, fails if there are duplicate names - pub fn merge(&self, other: &Self) -> Result { - let mut bands = self.0.clone(); - bands.extend(other.0.clone()); - Self::new(bands) + pub fn is_source(&self) -> bool { + self.state == SpatialGridDescriptorState::Source } -} -impl TryFrom> for RasterBandDescriptors { - type Error = Error; + pub fn source_spatial_grid_definition(&self) -> Option { + match self.state { + SpatialGridDescriptorState::Source => Some(self.spatial_grid), + SpatialGridDescriptorState::Merged => None, + } + } - fn try_from(value: Vec) -> Result { - RasterBandDescriptors::new(value) + pub fn derived_spatial_grid_definition(&self) -> Option { + match self.state { + SpatialGridDescriptorState::Merged => Some(self.spatial_grid), + SpatialGridDescriptorState::Source => None, + } } -} -impl From<&RasterBandDescriptors> for BandSelection { - fn from(value: &RasterBandDescriptors) -> Self { - Self::new_unchecked((0..value.len() as u32).collect()) + pub fn spatial_partition(&self) -> SpatialPartition2D { + self.spatial_grid.spatial_partition() } -} -impl Index for RasterBandDescriptors { - type Output = RasterBandDescriptor; + pub fn spatial_resolution(&self) -> SpatialResolution { + self.spatial_grid.geo_transform.spatial_resolution() + } - fn index(&self, index: usize) -> &Self::Output { - &self.0[index] + pub fn grid_shape(&self) -> GridShape2D { + self.spatial_grid.grid_bounds().grid_shape() } -} -impl<'de> Deserialize<'de> for RasterBandDescriptors { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let vec = Vec::deserialize(deserializer)?; - RasterBandDescriptors::new(vec).map_err(serde::de::Error::custom) + #[must_use] + pub fn with_changed_resolution(&self, new_res: SpatialResolution) -> Self { + self.map(|x| x.with_changed_resolution(new_res)) } -} -impl<'a> FromSql<'a> for RasterBandDescriptors { - fn from_sql( - ty: &Type, - raw: &'a [u8], - ) -> Result> { - let vec = Vec::::from_sql(ty, raw)?; - Ok(RasterBandDescriptors(vec)) + #[must_use] + pub fn replace_origin(&self, new_origin: Coordinate2D) -> Self { + self.map(|x| x.replace_origin(new_origin)) } - fn accepts(ty: &Type) -> bool { - as FromSql>::accepts(ty) + #[must_use] + pub fn with_moved_origin_to_nearest_grid_edge( + &self, + new_origin_referece: Coordinate2D, + ) -> Self { + self.map(|x| x.with_moved_origin_to_nearest_grid_edge(new_origin_referece)) } -} -impl ToSql for RasterBandDescriptors { - fn to_sql( + pub fn reproject_clipped( &self, - ty: &Type, - w: &mut bytes::BytesMut, - ) -> Result> { - ToSql::to_sql(&self.0, ty, w) + projector: &P, + ) -> Result> { + let projected = self.spatial_grid.reproject_clipped(projector)?; + match projected { + Some(p) => Ok(Some(Self { + spatial_grid: p, + state: SpatialGridDescriptorState::Merged, + })), + None => Ok(None), + } } - fn accepts(ty: &Type) -> bool { - as FromSql>::accepts(ty) + pub fn generate_coord_grid_pixel_center(&self) -> Grid { + self.spatial_grid.generate_coord_grid_pixel_center() } - fn to_sql_checked( + #[must_use] + pub fn spatial_bounds_to_compatible_spatial_grid( &self, - ty: &Type, - w: &mut bytes::BytesMut, - ) -> Result> { - ToSql::to_sql_checked(&self.0, ty, w) + spatial_partition: SpatialPartition2D, + ) -> Self { + self.map(|x| x.spatial_bounds_to_compatible_spatial_grid(spatial_partition)) } -} -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, ToSql, FromSql)] -pub struct RasterBandDescriptor { - pub name: String, - pub measurement: Measurement, -} + pub fn intersection_with_tiling_grid( + &self, + tiling_grid: &TilingSpatialGridDefinition, + ) -> Option { + let tiling_spatial_grid = tiling_grid.tiling_spatial_grid_definition(); + let intersection = self.spatial_grid.intersection(&tiling_spatial_grid)?; + + let descriptor = if self.spatial_grid.grid_bounds == intersection.grid_bounds { + self.state + } else { + SpatialGridDescriptorState::Merged + }; -impl RasterBandDescriptor { - pub fn new(name: String, measurement: Measurement) -> Self { - Self { name, measurement } + Some(Self { + spatial_grid: intersection, + state: descriptor, + }) } - pub fn new_unitless(name: String) -> Self { - Self { - name, - measurement: Measurement::Unitless, - } + pub fn as_parts(&self) -> (SpatialGridDescriptorState, SpatialGridDefinition) { + let SpatialGridDescriptor { + spatial_grid, + state, + } = *self; + (state, spatial_grid) } +} - pub fn new_unitless_with_idx(idx: u32) -> Self { - Self { - name: format!("band {idx}"), - measurement: Measurement::Unitless, - } - } +/// A `ResultDescriptor` for raster queries +#[derive(Debug, Clone, Serialize, Deserialize, ToSql, FromSql, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RasterResultDescriptor { + pub data_type: RasterDataType, + pub spatial_reference: SpatialReferenceOption, + pub time: Option, + pub spatial_grid: SpatialGridDescriptor, + pub bands: RasterBandDescriptors, } impl ResultDescriptor for RasterResultDescriptor { type DataType = RasterDataType; - type QueryRectangleSpatialBounds = SpatialPartition2D; + type QueryRectangleSpatialBounds = SpatialGridQueryRectangle; type QueryRectangleAttributeSelection = BandSelection; fn data_type(&self) -> Self::DataType { @@ -320,7 +358,59 @@ impl ResultDescriptor for RasterResultDescriptor { } } -impl RasterResultDescriptor {} +impl RasterResultDescriptor { + /// create a new `RasterResultDescriptor` + pub fn new( + data_type: RasterDataType, + spatial_reference: SpatialReferenceOption, + time: Option, + spatial_grid: SpatialGridDescriptor, + bands: RasterBandDescriptors, + ) -> Self { + Self { + data_type, + spatial_reference, + time, + spatial_grid, + bands, + } + } + + pub fn spatial_grid_descriptor(&self) -> &SpatialGridDescriptor { + &self.spatial_grid + } + + /// Returns tiling grid definition of the data. + pub fn tiling_grid_definition( + &self, + tiling_specification: TilingSpecification, + ) -> TilingSpatialGridDefinition { + self.spatial_grid + .tiling_grid_definition(tiling_specification) + } + + pub fn spatial_bounds(&self) -> SpatialPartition2D { + self.spatial_grid.spatial_partition() + } + + pub fn with_datatype_and_num_bands( + data_type: RasterDataType, + num_bands: u32, + pixel_bounds: GridBoundingBox2D, + geo_transform: GeoTransform, + ) -> Self { + Self { + data_type, + spatial_reference: SpatialReferenceOption::Unreferenced, + time: None, + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + geo_transform, + pixel_bounds, + )), + bands: RasterBandDescriptors::new_multiple_bands(num_bands), + } + } +} /// A `ResultDescriptor` for vector queries #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -366,7 +456,7 @@ impl VectorResultDescriptor { impl ResultDescriptor for VectorResultDescriptor { type DataType = VectorDataType; - type QueryRectangleSpatialBounds = BoundingBox2D; + type QueryRectangleSpatialBounds = VectorSpatialQueryRectangle; type QueryRectangleAttributeSelection = ColumnSelection; fn data_type(&self) -> Self::DataType { @@ -498,9 +588,7 @@ impl From for PlotResultDescriptor { spatial_reference: descriptor.spatial_reference, time: descriptor.time, // converting `SpatialPartition2D` to `BoundingBox2D` is ok here, because is makes the covered area only larger - bbox: descriptor - .bbox - .and_then(|p| BoundingBox2D::new(p.lower_left(), p.upper_right()).ok()), + bbox: Some(descriptor.spatial_bounds().as_bbox()), } } } @@ -531,6 +619,173 @@ impl From for TypedResultDescriptor { } } +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct RasterBandDescriptors(Vec); + +impl RasterBandDescriptors { + pub fn new(bands: Vec) -> Result { + let mut names = HashSet::new(); + for value in &bands { + ensure!(!value.name.is_empty(), RasterBandNameMustNotBeEmpty); + ensure!(value.name.byte_size() <= 256, RasterBandNameTooLong); + ensure!( + names.insert(&value.name), + RasterBandNamesMustBeUnique { + duplicate_key: value.name.clone() + } + ); + } + + Ok(Self(bands)) + } + + /// Convenience method to crate a single band result descriptor with no specific name and a unitless measurement for single band rasters + pub fn new_single_band() -> Self { + Self(vec![RasterBandDescriptor { + name: "band".into(), + measurement: Measurement::Unitless, + }]) + } + + /// Convenience method to crate multipe band result descriptors with no specific name and a unitless measurement + pub fn new_multiple_bands(num_bands: u32) -> Self { + Self( + (0..num_bands) + .map(RasterBandDescriptor::new_unitless_with_idx) + .collect(), + ) + } + + pub fn bands(&self) -> &[RasterBandDescriptor] { + &self.0 + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn count(&self) -> u32 { + self.0.len() as u32 + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + pub fn into_vec(self) -> Vec { + self.0 + } + + // Merge the bands of two descriptors into a new one, fails if there are duplicate names + pub fn merge(&self, other: &Self) -> Result { + let mut bands = self.0.clone(); + bands.extend(other.0.clone()); + Self::new(bands) + } + + pub fn is_single(&self) -> bool { + self.len() == 1 + } +} + +impl TryFrom> for RasterBandDescriptors { + type Error = Error; + + fn try_from(value: Vec) -> Result { + RasterBandDescriptors::new(value) + } +} + +impl From<&RasterBandDescriptors> for BandSelection { + fn from(value: &RasterBandDescriptors) -> Self { + Self::new_unchecked((0..value.len() as u32).collect()) + } +} + +impl Index for RasterBandDescriptors { + type Output = RasterBandDescriptor; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + +impl<'de> Deserialize<'de> for RasterBandDescriptors { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let vec = Vec::deserialize(deserializer)?; + RasterBandDescriptors::new(vec).map_err(serde::de::Error::custom) + } +} + +impl<'a> FromSql<'a> for RasterBandDescriptors { + fn from_sql( + ty: &Type, + raw: &'a [u8], + ) -> Result> { + let vec = Vec::::from_sql(ty, raw)?; + Ok(RasterBandDescriptors(vec)) + } + + fn accepts(ty: &Type) -> bool { + as FromSql>::accepts(ty) + } +} + +impl ToSql for RasterBandDescriptors { + fn to_sql( + &self, + ty: &Type, + w: &mut bytes::BytesMut, + ) -> Result> { + ToSql::to_sql(&self.0, ty, w) + } + + fn accepts(ty: &Type) -> bool { + as FromSql>::accepts(ty) + } + + fn to_sql_checked( + &self, + ty: &Type, + w: &mut bytes::BytesMut, + ) -> Result> { + ToSql::to_sql_checked(&self.0, ty, w) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, ToSql, FromSql)] +pub struct RasterBandDescriptor { + pub name: String, + pub measurement: Measurement, +} + +impl RasterBandDescriptor { + pub fn new(name: String, measurement: Measurement) -> Self { + Self { name, measurement } + } + + pub fn new_unitless(name: String) -> Self { + Self { + name, + measurement: Measurement::Unitless, + } + } + + pub fn new_unitless_with_idx(idx: u32) -> Self { + Self { + name: format!("band {idx}"), + measurement: Measurement::Unitless, + } + } +} + mod db_types { use super::*; use crate::error::Error; @@ -658,6 +913,59 @@ mod db_types { } } + #[derive(Debug, ToSql, FromSql)] + #[postgres(name = "SpatialGridDescriptorState")] + pub enum SpatialGridDescriptorStateDbType { + /// The spatial grid represents the original data + Source, + /// The spatial grid was created by merging two non equal spatial grids + Merged, + } + + impl From<&SpatialGridDescriptorState> for SpatialGridDescriptorStateDbType { + fn from(value: &SpatialGridDescriptorState) -> Self { + match value { + SpatialGridDescriptorState::Source => SpatialGridDescriptorStateDbType::Source, + SpatialGridDescriptorState::Merged => SpatialGridDescriptorStateDbType::Merged, + } + } + } + + impl From for SpatialGridDescriptorState { + fn from(value: SpatialGridDescriptorStateDbType) -> Self { + match value { + SpatialGridDescriptorStateDbType::Source => SpatialGridDescriptorState::Source, + SpatialGridDescriptorStateDbType::Merged => SpatialGridDescriptorState::Merged, + } + } + } + + #[derive(Debug, ToSql, FromSql)] + #[postgres(name = "SpatialGridDescriptor")] + pub struct SpatialGridDescriptorDbType { + state: SpatialGridDescriptorStateDbType, + spatial_grid: SpatialGridDefinition, + } + + impl From<&SpatialGridDescriptor> for SpatialGridDescriptorDbType { + fn from(value: &SpatialGridDescriptor) -> Self { + Self { + spatial_grid: value.spatial_grid, + state: SpatialGridDescriptorStateDbType::from(&value.state), + } + } + } + + impl From for SpatialGridDescriptor { + fn from(value: SpatialGridDescriptorDbType) -> Self { + Self { + spatial_grid: value.spatial_grid, + state: SpatialGridDescriptorState::from(value.state), + } + } + } + + delegate_from_to_sql!(SpatialGridDescriptor, SpatialGridDescriptorDbType); delegate_from_to_sql!(VectorResultDescriptor, VectorResultDescriptorDbType); delegate_from_to_sql!(TypedResultDescriptor, TypedResultDescriptorDbType); } @@ -707,6 +1015,59 @@ mod tests { ); } + /* FIXME: bring back? + #[test] + fn raster_tiling_origin() { + let descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReferenceOption::Unreferenced, + time: None, + geo_transform_x: GeoTransform::new(Coordinate2D::new(-10., 10.), 0.3, -0.3), + pixel_bounds_x: GridShape2D::new([36, 30]).bounding_box(), + bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "foo".into(), + Measurement::Unitless, + )]) + .unwrap(), + }; + + let to = descriptor.tiling_origin(); + + assert_approx_eq!(f64, to.x, -0.09999, epsilon = 0.00001); // we are only interested in a number thats smaller then the pixel size + assert_approx_eq!(f64, to.y, 0.09999, epsilon = 0.00001); + } + + #[test] + fn raster_tiling_equals() { + let descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReferenceOption::Unreferenced, + time: None, + geo_transform_x: GeoTransform::new(Coordinate2D::new(-15., 15.), 0.5, -0.5), + pixel_bounds_x: GridShape2D::new([50, 50]).bounding_box(), + bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "foo".into(), + Measurement::Unitless, + )]) + .unwrap(), + }; + + let descriptor2 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReferenceOption::Unreferenced, + time: None, + geo_transform_x: GeoTransform::new(Coordinate2D::new(-10., 10.), 0.5, -0.5), + pixel_bounds_x: GridShape2D::new([9, 11]).bounding_box(), + bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "foo".into(), + Measurement::Unitless, + )]) + .unwrap(), + }; + + assert!(descriptor.spatial_tiling_equals(&descriptor2)); + } + */ #[test] fn it_checks_duplicate_bands() { assert!( diff --git a/operators/src/error.rs b/operators/src/error.rs index 692402f74..f8990bf10 100644 --- a/operators/src/error.rs +++ b/operators/src/error.rs @@ -1,3 +1,4 @@ +use crate::engine::SpatialGridDescriptor; use crate::processing::BandNeighborhoodAggregateError; use crate::util::statistics::StatisticsError; use bb8_postgres::bb8; @@ -359,6 +360,10 @@ pub enum Error { InterpolationOperator { source: crate::processing::InterpolationError, }, + #[snafu(context(false))] + DownsampleOperator { + source: crate::processing::DownsamplingError, + }, #[snafu(display("TimeShift error: {source}"), context(false))] TimeShift { source: crate::processing::TimeShiftError, @@ -429,6 +434,12 @@ pub enum Error { source: Box, }, + #[snafu(display("RasterResults are incompatible error: {a:?} vs {b:?}"))] + CantMergeSpatialGridDescriptor { + a: SpatialGridDescriptor, + b: SpatialGridDescriptor, + }, + #[snafu(display( "Input stream {stream_index} is not temporally aligned. Expected {expected:?}, found {found:?}." ))] @@ -498,6 +509,8 @@ pub enum Error { message: String, }, + ReprojectionFailed, + #[snafu(display("PostgresError: {}", source))] Postgres { source: tokio_postgres::Error, diff --git a/operators/src/machine_learning/onnx.rs b/operators/src/machine_learning/onnx.rs index f29d6a916..068c78b1f 100644 --- a/operators/src/machine_learning/onnx.rs +++ b/operators/src/machine_learning/onnx.rs @@ -76,8 +76,7 @@ impl RasterOperator for Onnx { data_type: model_metadata.output_type, spatial_reference: in_descriptor.spatial_reference, time: in_descriptor.time, - bbox: in_descriptor.bbox, - resolution: in_descriptor.resolution, + spatial_grid: in_descriptor.spatial_grid, bands: vec![RasterBandDescriptor::new( "prediction".to_string(), // TODO: parameter of the operator? Measurement::Unitless, // TODO: get output measurement from model metadata @@ -327,6 +326,7 @@ impl_no_data_value_zero!(i8, u8, i16, u16, i32, u32, i64, u64); #[cfg(test)] mod tests { + use crate::engine::SpatialGridDescriptor; use crate::{ engine::{ MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, @@ -337,9 +337,10 @@ mod tests { }; use approx::assert_abs_diff_eq; use geoengine_datatypes::{ - primitives::{CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::{CacheHint, TimeInterval}, raster::{ - GridOrEmpty, GridShape, RasterDataType, RenameBands, TilesEqualIgnoringCacheHint, + GridBoundingBox2D, GridOrEmpty, GridShape, RasterDataType, RenameBands, + TilesEqualIgnoringCacheHint, }, spatial_reference::SpatialReference, test_data, @@ -513,8 +514,10 @@ mod tests { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -528,8 +531,10 @@ mod tests { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -569,12 +574,11 @@ mod tests { load_model_metadata(test_data!("ml/onnx/test_classification.onnx")).unwrap(), ); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: [0].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); let query_ctx = MockQueryContext::test_default(); @@ -706,8 +710,10 @@ mod tests { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -721,8 +727,10 @@ mod tests { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -736,8 +744,10 @@ mod tests { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -777,12 +787,11 @@ mod tests { load_model_metadata(test_data!("ml/onnx/test_regression.onnx")).unwrap(), ); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: [0].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); let query_ctx = MockQueryContext::test_default(); diff --git a/operators/src/meta/wrapper.rs b/operators/src/meta/wrapper.rs index 61561143a..29ee78e44 100644 --- a/operators/src/meta/wrapper.rs +++ b/operators/src/meta/wrapper.rs @@ -9,9 +9,7 @@ use crate::util::Result; use async_trait::async_trait; use futures::StreamExt; use futures::stream::BoxStream; -use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, QueryAttributeSelection, QueryRectangle, -}; +use geoengine_datatypes::primitives::{QueryAttributeSelection, QueryRectangle}; use std::sync::atomic::{AtomicUsize, Ordering}; use tracing::{Level, span}; @@ -240,21 +238,21 @@ where #[async_trait] impl QueryProcessor for QueryProcessorWrapper where - Q: QueryProcessor, - S: AxisAlignedRectangle + Send + Sync + 'static, + Q: QueryProcessor, + S: std::fmt::Debug + Send + Sync + 'static + Clone + Copy, A: QueryAttributeSelection + 'static, R: ResultDescriptor + 'static, T: Send, { type Output = T; - type SpatialBounds = S; + type SpatialQuery = S; type Selection = A; type ResultDescription = R; async fn _query<'a>( &'a self, - query: QueryRectangle, + query: QueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { let qc = self.next_query_count(); @@ -289,18 +287,15 @@ where let _enter = span.enter(); + let spbox = query.spatial_query; + let time = query.time_interval; tracing::trace!( event = %"query_start", path = %self.path, - bbox = %format!("[{},{},{},{}]", - query.spatial_bounds.lower_left().x, - query.spatial_bounds.lower_left().y, - query.spatial_bounds.upper_right().x, - query.spatial_bounds.upper_right().y - ), + bbox = %format!("{:?}", spbox), // FIXME: better format then debug here time = %format!("[{},{}]", - query.time_interval.start().inner(), - query.time_interval.end().inner() + time.start().inner(), + time.end().inner() ) ); diff --git a/operators/src/mock/mock_dataset_data_source.rs b/operators/src/mock/mock_dataset_data_source.rs index 653960204..42551d78d 100644 --- a/operators/src/mock/mock_dataset_data_source.rs +++ b/operators/src/mock/mock_dataset_data_source.rs @@ -184,12 +184,12 @@ impl InitializedVectorOperator #[cfg(test)] mod tests { use super::*; + use crate::engine::MockExecutionContext; use crate::engine::QueryProcessor; - use crate::engine::{MockExecutionContext, MockQueryContext}; use futures::executor::block_on_stream; use geoengine_datatypes::collections::FeatureCollectionInfos; use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; - use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection, SpatialResolution}; + use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection}; use geoengine_datatypes::util::Identifier; use geoengine_datatypes::util::test::TestDefault; @@ -222,13 +222,13 @@ mod tests { panic!() }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new((2 * std::mem::size_of::()).into()); + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = + execution_context.mock_query_context((2 * std::mem::size_of::()).into()); let stream = point_processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/mock/mock_feature_collection_source.rs b/operators/src/mock/mock_feature_collection_source.rs index 6022fc44e..b6ff88830 100644 --- a/operators/src/mock/mock_feature_collection_source.rs +++ b/operators/src/mock/mock_feature_collection_source.rs @@ -252,11 +252,12 @@ mod tests { use crate::engine::QueryProcessor; use crate::engine::{MockExecutionContext, MockQueryContext}; use futures::executor::block_on_stream; - use geoengine_datatypes::collections::ChunksEqualIgnoringCacheHint; - use geoengine_datatypes::primitives::{BoundingBox2D, Coordinate2D, FeatureData, TimeInterval}; - use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; + use geoengine_datatypes::collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection}; + use geoengine_datatypes::primitives::{ + BoundingBox2D, CacheHint, ColumnSelection, Coordinate2D, FeatureData, TimeInterval, + }; + use geoengine_datatypes::raster::TilingSpecification; use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{collections::MultiPointCollection, primitives::SpatialResolution}; #[test] #[allow(clippy::too_many_lines)] @@ -413,13 +414,15 @@ mod tests { panic!() }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new((2 * std::mem::size_of::()).into()); + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = MockQueryContext::new( + (2 * std::mem::size_of::()).into(), + TilingSpecification::test_default(), + ); let stream = processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/mock/mock_point_source.rs b/operators/src/mock/mock_point_source.rs index 67029bfea..9252e2d8e 100644 --- a/operators/src/mock/mock_point_source.rs +++ b/operators/src/mock/mock_point_source.rs @@ -11,8 +11,7 @@ use async_trait::async_trait; use futures::stream::{self, BoxStream, StreamExt}; use geoengine_datatypes::collections::VectorDataType; use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::CacheHint; -use geoengine_datatypes::primitives::VectorQueryRectangle; +use geoengine_datatypes::primitives::{BoundingBox2D, CacheHint, VectorQueryRectangle}; use geoengine_datatypes::{ collections::MultiPointCollection, primitives::{Coordinate2D, TimeInterval}, @@ -35,10 +34,12 @@ impl VectorQueryProcessor for MockPointSourceProcessor { ctx: &'a dyn QueryContext, ) -> Result>> { let chunk_size = usize::from(ctx.chunk_byte_size()) / std::mem::size_of::(); - let bounding_box = query.spatial_bounds; + let spatial_query = query.spatial_query(); Ok(stream::iter(&self.points) - .filter(move |&coord| std::future::ready(bounding_box.contains_coordinate(coord))) + .filter(move |&coord| { + std::future::ready(spatial_query.spatial_bounds.contains_coordinate(coord)) + }) .chunks(chunk_size) .map(move |chunk| { Ok(MultiPointCollection::from_data( @@ -57,8 +58,41 @@ impl VectorQueryProcessor for MockPointSourceProcessor { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "type")] +pub enum SpatialBoundsDerive { + Derive, + Bounds(BoundingBox2D), + None, +} + +impl Default for SpatialBoundsDerive { + fn default() -> Self { + Self::None + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct MockPointSourceParams { pub points: Vec, + #[serde(default = "SpatialBoundsDerive::default")] + pub spatial_bounds: SpatialBoundsDerive, +} + +impl MockPointSourceParams { + pub fn new(points: Vec) -> Self { + MockPointSourceParams { + points, + spatial_bounds: SpatialBoundsDerive::default(), + } + } + + pub fn new_with_bounds(points: Vec, spatial_bounds: SpatialBoundsDerive) -> Self { + MockPointSourceParams { + points, + spatial_bounds, + } + } } pub type MockPointSource = SourceOperator; @@ -79,6 +113,14 @@ impl VectorOperator for MockPointSource { path: WorkflowOperatorPath, _context: &dyn ExecutionContext, ) -> Result> { + let bounds = match self.params.spatial_bounds { + SpatialBoundsDerive::None => None, + SpatialBoundsDerive::Bounds(b) => Some(b), + SpatialBoundsDerive::Derive => { + BoundingBox2D::from_coord_ref_iter(self.params.points.iter()) + } + }; + Ok(InitializedMockPointSource { name: CanonicOperatorName::from(&self), path, @@ -87,7 +129,7 @@ impl VectorOperator for MockPointSource { spatial_reference: SpatialReference::epsg_4326().into(), columns: Default::default(), time: None, - bbox: None, + bbox: bounds, }, points: self.params.points, } @@ -135,11 +177,11 @@ impl InitializedVectorOperator for InitializedMockPointSource { #[cfg(test)] mod tests { use super::*; + use crate::engine::MockExecutionContext; use crate::engine::QueryProcessor; - use crate::engine::{MockExecutionContext, MockQueryContext}; use futures::executor::block_on_stream; use geoengine_datatypes::collections::FeatureCollectionInfos; - use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection, SpatialResolution}; + use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection}; use geoengine_datatypes::util::test::TestDefault; #[test] @@ -147,11 +189,11 @@ mod tests { let points = vec![Coordinate2D::new(1., 2.); 3]; let mps = MockPointSource { - params: MockPointSourceParams { points }, + params: MockPointSourceParams::new(points), } .boxed(); let serialized = serde_json::to_string(&mps).unwrap(); - let expect = "{\"type\":\"MockPointSource\",\"params\":{\"points\":[{\"x\":1.0,\"y\":2.0},{\"x\":1.0,\"y\":2.0},{\"x\":1.0,\"y\":2.0}]}}"; + let expect = "{\"type\":\"MockPointSource\",\"params\":{\"points\":[{\"x\":1.0,\"y\":2.0},{\"x\":1.0,\"y\":2.0},{\"x\":1.0,\"y\":2.0}],\"spatialBounds\":{\"type\":\"none\"}}}"; assert_eq!(serialized, expect); let _operator: Box = serde_json::from_str(&serialized).unwrap(); @@ -163,7 +205,7 @@ mod tests { let points = vec![Coordinate2D::new(1., 2.); 3]; let mps = MockPointSource { - params: MockPointSourceParams { points }, + params: MockPointSourceParams::new(points), } .boxed(); let initialized = mps @@ -176,13 +218,13 @@ mod tests { panic!() }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new((2 * std::mem::size_of::()).into()); + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = + execution_context.mock_query_context((2 * std::mem::size_of::()).into()); let stream = point_processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/mock/mock_raster_source.rs b/operators/src/mock/mock_raster_source.rs index 28d4bda1a..bfb122519 100644 --- a/operators/src/mock/mock_raster_source.rs +++ b/operators/src/mock/mock_raster_source.rs @@ -10,10 +10,11 @@ use crate::util::Result; use async_trait::async_trait; use futures::{stream, stream::StreamExt}; use geoengine_datatypes::dataset::NamedData; +use geoengine_datatypes::primitives::RasterQueryRectangle; use geoengine_datatypes::primitives::{CacheExpiration, TimeInstance}; -use geoengine_datatypes::primitives::{RasterQueryRectangle, SpatialPartitioned}; use geoengine_datatypes::raster::{ - GridShape2D, GridShapeAccess, GridSize, Pixel, RasterTile2D, TilingSpecification, + GridIntersection, GridShape2D, GridShapeAccess, GridSize, Pixel, RasterTile2D, + TilingSpecification, }; use serde::{Deserialize, Serialize}; use snafu::Snafu; @@ -52,11 +53,9 @@ where data: Vec>, tiling_specification: TilingSpecification, ) -> Self { - Self { - result_descriptor, - data, - tiling_specification, - } + // use expect here since the mock source should not be used in production and this provides valuable debug information + Self::_new(result_descriptor, data, tiling_specification) + .expect("can initialize from inputs") } fn _new( @@ -119,29 +118,35 @@ where .inspect(|m| { let time_interval = m.time; - if time_interval.start() <= query.time_interval.start() { - let t = if time_interval.end() > query.time_interval.start() { - time_interval.start() - } else { - time_interval.end() - }; - known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); + if time_interval.contains(&query.time_interval) { + let t1 = time_interval.start(); + let t2 = time_interval.end(); + known_time_start = Some(t1); + known_time_end = Some(t2); + return; } - if time_interval.end() >= query.time_interval.end() { - let t = if time_interval.start() < query.time_interval.end() { - time_interval.end() - } else { - time_interval.start() - }; - known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + if time_interval.end() <= query.time_interval.start() { + let t1 = time_interval.end(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } else if time_interval.start() <= query.time_interval.start() { + let t1 = time_interval.start(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } + + if time_interval.start() >= query.time_interval.end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() >= query.time_interval.end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); } }) .filter(move |t| { t.time.intersects(&query.time_interval) && t.tile_information() - .spatial_partition() - .intersects(&query.spatial_bounds) + .global_pixel_bounds() + .intersects(&query.spatial_query.grid_bounds()) }) .cloned() .collect(); @@ -152,25 +157,17 @@ where let inner_stream = stream::iter(parts.into_iter().map(Result::Ok)); - // TODO: evaluate if there are GeoTransforms with positive y-axis - // The "Pixel-space" starts at the top-left corner of a `GeoTransform`. - // Therefore, the pixel size on the x-axis is always increasing - let spatial_resolution = query.spatial_resolution; - - let pixel_size_x = spatial_resolution.x; - debug_assert!(pixel_size_x.is_sign_positive()); - // and the pixel size on the y-axis is always decreasing - let pixel_size_y = spatial_resolution.y * -1.0; - debug_assert!(pixel_size_y.is_sign_negative()); + let tiling_grid_spec = self + .result_descriptor + .tiling_grid_definition(self.tiling_specification); - let tiling_strategy = self - .tiling_specification - .strategy(pixel_size_x, pixel_size_y); + let tiling_strategy = tiling_grid_spec.generate_data_tiling_strategy(); // use SparseTilesFillAdapter to fill all the gaps Ok(SparseTilesFillAdapter::new( inner_stream, - tiling_strategy.tile_grid_box(query.spatial_partition()), + tiling_strategy + .global_pixel_grid_bounds_to_tile_grid_bounds(query.spatial_query.grid_bounds()), self.result_descriptor.bands.count(), tiling_strategy.geo_transform, tiling_strategy.tile_size_in_pixels, @@ -340,16 +337,15 @@ mod tests { use super::*; use crate::engine::{ MockExecutionContext, MockQueryContext, QueryProcessor, RasterBandDescriptors, + SpatialGridDescriptor, }; - use geoengine_datatypes::primitives::{BandSelection, CacheHint}; - use geoengine_datatypes::primitives::{SpatialPartition2D, SpatialResolution}; - use geoengine_datatypes::raster::{Grid, MaskedGrid, RasterDataType, RasterProperties}; - use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{ - primitives::TimeInterval, - raster::{Grid2D, TileInformation}, - spatial_reference::SpatialReference, + use geoengine_datatypes::primitives::{BandSelection, CacheHint, TimeInterval}; + use geoengine_datatypes::raster::{ + BoundedGrid, GeoTransform, Grid, Grid2D, GridBoundingBox2D, MaskedGrid, RasterDataType, + RasterProperties, TileInformation, }; + use geoengine_datatypes::spatial_reference::SpatialReference; + use geoengine_datatypes::util::test::TestDefault; #[tokio::test] #[allow(clippy::too_many_lines)] @@ -377,8 +373,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., 0.).into(), 1., -1.), + GridShape2D::new_2d(3, 2).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -432,8 +430,23 @@ mod tests { "dataType": "U8", "spatialReference": "EPSG:4326", "time": null, - "bbox": null, - "resolution": null, + "spatialGrid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 0.0, + "y": 0.0 + }, + "xPixelSize": 1.0, + "yPixelSize": -1.0 + }, + "gridBounds": { + "max": [2, 1], + "min": [0, 0] + } + }, + "state": "source", + }, "bands": [ { "name": "band", @@ -451,7 +464,6 @@ mod tests { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -523,17 +535,18 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let query_processor = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -548,12 +561,11 @@ mod tests { // QUERY 1 - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(-3, 7), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 4), + BandSelection::first(), + ); let result_stream = query_processor.query(query_rect, &query_ctx).await.unwrap(); @@ -575,12 +587,11 @@ mod tests { // QUERY 2 - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 4), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 4), + BandSelection::first(), + ); let result_stream = query_processor.query(query_rect, &query_ctx).await.unwrap(); diff --git a/operators/src/plot/box_plot.rs b/operators/src/plot/box_plot.rs index 2760f9212..ff4026a1a 100644 --- a/operators/src/plot/box_plot.rs +++ b/operators/src/plot/box_plot.rs @@ -1,27 +1,25 @@ +use crate::{ + engine::{ + CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedRasterOperator, + InitializedVectorOperator, MultipleRasterOrSingleVectorSource, Operator, OperatorName, + PlotOperator, PlotQueryProcessor, PlotResultDescriptor, QueryContext, QueryProcessor, + TypedPlotQueryProcessor, TypedRasterQueryProcessor, TypedVectorQueryProcessor, + WorkflowOperatorPath, + }, + error::{self, Error}, + util::{Result, input::MultiRasterOrVectorOperator, statistics::PSquareQuantileEstimator}, +}; use async_trait::async_trait; use futures::StreamExt; +use geoengine_datatypes::collections::FeatureCollectionInfos; +use geoengine_datatypes::plots::{BoxPlotAttribute, Plot, PlotData}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, PlotQueryRectangle, RasterQueryRectangle, - partitions_extent, time_interval_extent, + AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, PlotQueryRectangle, + RasterQueryRectangle, VectorQueryRectangle, partitions_extent, time_interval_extent, }; +use geoengine_datatypes::raster::GridOrEmpty; use num_traits::AsPrimitive; use serde::{Deserialize, Serialize}; - -use geoengine_datatypes::collections::FeatureCollectionInfos; -use geoengine_datatypes::plots::{BoxPlotAttribute, Plot, PlotData}; -use geoengine_datatypes::raster::GridOrEmpty; - -use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedRasterOperator, - InitializedVectorOperator, MultipleRasterOrSingleVectorSource, Operator, OperatorName, - PlotOperator, PlotQueryProcessor, PlotResultDescriptor, QueryContext, QueryProcessor, - TypedPlotQueryProcessor, TypedRasterQueryProcessor, TypedVectorQueryProcessor, - WorkflowOperatorPath, -}; -use crate::error::{self, Error}; -use crate::util::Result; -use crate::util::input::MultiRasterOrVectorOperator; -use crate::util::statistics::PSquareQuantileEstimator; use snafu::ensure; pub const BOXPLOT_OPERATOR_NAME: &str = "BoxPlot"; @@ -113,7 +111,7 @@ impl PlotOperator for BoxPlot { .collect::>(); let time = time_interval_extent(in_descriptors.iter().map(|d| d.time)); - let bbox = partitions_extent(in_descriptors.iter().map(|d| d.bbox)); + let bbox = partitions_extent(in_descriptors.iter().map(|d| d.spatial_bounds())); Ok(InitializedBoxPlot::new( name, @@ -274,8 +272,14 @@ impl PlotQueryProcessor for BoxPlotVectorQueryProcessor { .map(|name| BoxPlotAccum::new(name.clone())) .collect(); + let query = VectorQueryRectangle::new( + query.spatial_query, + query.time_interval, + ColumnSelection::all(), // TODO: use columns names? + ); + call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -313,10 +317,19 @@ impl BoxPlotRasterQueryProcessor { query: PlotQueryRectangle, ctx: &dyn QueryContext, ) -> Result> { - call_on_generic_raster_processor!(input, processor => { + let result_descrpitor = input.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::with_spatial_query_and_geo_transform( + &query, + result_descrpitor + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + BandSelection::first(), + ); + call_on_generic_raster_processor!(input, processor => { - let mut stream = processor.query(RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()), ctx).await?; + let mut stream = processor.query(raster_query_rect, ctx).await?; let mut accum = BoxPlotAccum::new(name); while let Some(tile) = stream.next().await { @@ -489,15 +502,16 @@ impl BoxPlotAccum { #[cfg(test)] mod tests { - use geoengine_datatypes::primitives::{CacheHint, PlotSeriesSelection}; + + use geoengine_datatypes::primitives::{CacheHint, Coordinate2D, PlotSeriesSelection}; use serde_json::json; use geoengine_datatypes::primitives::{ - BoundingBox2D, DateTime, FeatureData, NoGeometry, SpatialResolution, TimeInterval, + BoundingBox2D, DateTime, FeatureData, NoGeometry, TimeInterval, }; use geoengine_datatypes::raster::{ - EmptyGrid2D, Grid2D, MaskedGrid2D, RasterDataType, RasterTile2D, TileInformation, - TilingSpecification, + BoundedGrid, EmptyGrid2D, GeoTransform, Grid2D, GridShape2D, MaskedGrid2D, RasterDataType, + RasterTile2D, TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; @@ -505,7 +519,7 @@ mod tests { use crate::engine::{ ChunkByteSize, MockExecutionContext, MockQueryContext, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, VectorOperator, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, VectorOperator, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; @@ -623,14 +637,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -690,14 +702,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -801,14 +811,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -854,14 +862,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -909,14 +915,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -940,11 +944,21 @@ mod tests { #[tokio::test] async fn no_data_raster_exclude_no_data() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = + geoengine_datatypes::raster::TilingSpecification::new(tile_size_in_pixels); + let box_plot = BoxPlot { params: BoxPlotParams { column_names: vec![], @@ -962,14 +976,7 @@ mod tests { EmptyGrid2D::::new(tile_size_in_pixels).into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -990,14 +997,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1009,11 +1014,19 @@ mod tests { #[tokio::test] async fn no_data_raster_include_no_data() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let box_plot = BoxPlot { params: BoxPlotParams { column_names: vec![], @@ -1033,14 +1046,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1061,13 +1067,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1082,11 +1087,19 @@ mod tests { #[tokio::test] async fn empty_tile_raster_exclude_no_data() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let box_plot = BoxPlot { params: BoxPlotParams { column_names: vec![], @@ -1104,14 +1117,7 @@ mod tests { EmptyGrid2D::::new(tile_size_in_pixels).into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1132,14 +1138,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1151,11 +1155,19 @@ mod tests { #[tokio::test] async fn single_value_raster_stream() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = BoxPlot { params: BoxPlotParams { @@ -1174,14 +1186,7 @@ mod tests { Grid2D::new(tile_size_in_pixels, vec![4; 6]).unwrap().into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1200,16 +1205,11 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), &MockQueryContext::test_default(), ) .await @@ -1225,11 +1225,19 @@ mod tests { #[tokio::test] async fn raster_with_no_data_exclude_no_data() { - let tile_size_in_pixels = [4, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(4, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = BoxPlot { @@ -1258,14 +1266,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1284,15 +1285,11 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -4.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -4.).into(), (2., 0.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), &MockQueryContext::test_default(), ) .await @@ -1308,11 +1305,19 @@ mod tests { #[tokio::test] async fn raster_with_no_data_include_no_data() { - let tile_size_in_pixels = [4, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(4, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = BoxPlot { @@ -1334,14 +1339,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1360,15 +1358,11 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -4.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -4.).into(), (2., 0.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), &MockQueryContext::test_default(), ) .await @@ -1384,11 +1378,19 @@ mod tests { #[tokio::test] async fn multiple_rasters_with_no_data_exclude_no_data() { - let tile_size_in_pixels = [4, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(4, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let src = MockRasterSource { @@ -1413,14 +1415,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, }; @@ -1448,16 +1443,11 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), &MockQueryContext::test_default(), ) .await diff --git a/operators/src/plot/class_histogram.rs b/operators/src/plot/class_histogram.rs index d2e043281..25af5546c 100644 --- a/operators/src/plot/class_histogram.rs +++ b/operators/src/plot/class_histogram.rs @@ -14,8 +14,8 @@ use futures::StreamExt; use geoengine_datatypes::collections::FeatureCollectionInfos; use geoengine_datatypes::plots::{BarChart, Plot, PlotData}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, ClassificationMeasurement, FeatureDataType, - Measurement, PlotQueryRectangle, RasterQueryRectangle, + AxisAlignedRectangle, BandSelection, ClassificationMeasurement, ColumnSelection, + FeatureDataType, Measurement, PlotQueryRectangle, RasterQueryRectangle, VectorQueryRectangle, }; use num_traits::AsPrimitive; use serde::{Deserialize, Serialize}; @@ -90,9 +90,7 @@ impl PlotOperator for ClassHistogram { spatial_reference: in_desc.spatial_reference, time: in_desc.time, // converting `SpatialPartition2D` to `BoundingBox2D` is ok here, because is makes the covered area only larger - bbox: in_desc - .bbox - .and_then(|p| BoundingBox2D::new(p.lower_left(), p.upper_right()).ok()), + bbox: Some(in_desc.spatial_bounds().as_bbox()), }, self.params.column_name, source_measurement, @@ -290,8 +288,17 @@ impl ClassHistogramRasterQueryProcessor { .map(|key| (*key, 0)) .collect(); + let rd = self.input.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::with_spatial_query_and_geo_transform( + &query, + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + BandSelection::first(), + ); + call_on_generic_raster_processor!(&self.input, processor => { - let mut query = processor.query(RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()), ctx).await?; + let mut query = processor.query(raster_query_rect, ctx).await?; while let Some(tile) = query.next().await { match tile?.grid_array { @@ -347,8 +354,14 @@ impl ClassHistogramVectorQueryProcessor { .map(|key| (*key, 0)) .collect(); + let query = VectorQueryRectangle::new( + query.spatial_query(), + query.time_interval, + ColumnSelection::all(), // TODO: figure out why this is a vector query? + ); + call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -399,8 +412,8 @@ mod tests { use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterBandDescriptor, - RasterBandDescriptors, RasterOperator, RasterResultDescriptor, StaticMetaData, + ChunkByteSize, MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, StaticMetaData, VectorColumnInfo, VectorOperator, VectorResultDescriptor, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; @@ -410,12 +423,13 @@ mod tests { use crate::test_data; use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; use geoengine_datatypes::primitives::{ - BoundingBox2D, DateTime, FeatureData, NoGeometry, PlotSeriesSelection, SpatialResolution, + BoundingBox2D, Coordinate2D, DateTime, FeatureData, NoGeometry, PlotSeriesSelection, TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds}; use geoengine_datatypes::raster::{ - Grid2D, RasterDataType, RasterTile2D, TileInformation, TilingSpecification, + BoundedGrid, GeoTransform, Grid2D, GridShape2D, RasterDataType, RasterTile2D, + TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::Identifier; @@ -500,8 +514,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new_2d(3, 2).bounding_box(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "bands".into(), Measurement::classification( @@ -529,7 +545,6 @@ mod tests { async fn simple_raster() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -551,13 +566,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -636,14 +650,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -723,14 +735,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -872,17 +882,30 @@ mod tests { #[tokio::test] async fn no_data_raster() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + + let bands = RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "band".into(), + Measurement::Classification(ClassificationMeasurement { + measurement: "foo".to_string(), + classes: [(1, "A".to_string())].into_iter().collect(), + }), + )]) + .unwrap(); + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands, }; - let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); - let measurement = Measurement::classification( - "foo".to_string(), - [(1, "A".to_string())].into_iter().collect(), - ); + let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = ClassHistogram { params: ClassHistogramParams { column_name: None }, @@ -901,18 +924,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( - "band".into(), - measurement, - )]) - .unwrap(), - }, + result_descriptor, }, } .boxed() @@ -931,13 +943,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -995,14 +1006,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1060,14 +1069,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1086,18 +1093,30 @@ mod tests { #[tokio::test] async fn single_value_raster_stream() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + + let bands = RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "band".into(), + Measurement::Classification(ClassificationMeasurement { + measurement: "foo".to_string(), + classes: [(4, "D".to_string())].into_iter().collect(), + }), + )]) + .unwrap(); + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands, }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); - let measurement = Measurement::classification( - "foo".to_string(), - [(4, "D".to_string())].into_iter().collect(), - ); - let histogram = ClassHistogram { params: ClassHistogramParams { column_name: None }, sources: MockRasterSource { @@ -1113,18 +1132,7 @@ mod tests { Grid2D::new(tile_size_in_pixels, vec![4; 6]).unwrap().into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( - "band".into(), - measurement, - )]) - .unwrap(), - }, + result_descriptor, }, } .boxed() @@ -1143,16 +1151,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/histogram.rs b/operators/src/plot/histogram.rs index ace482946..23c8905dd 100644 --- a/operators/src/plot/histogram.rs +++ b/operators/src/plot/histogram.rs @@ -1,10 +1,10 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedRasterOperator, InitializedVectorOperator, Operator, OperatorName, PlotOperator, PlotQueryProcessor, - PlotResultDescriptor, QueryContext, SingleRasterOrVectorSource, TypedPlotQueryProcessor, - TypedRasterQueryProcessor, TypedVectorQueryProcessor, + PlotResultDescriptor, QueryContext, QueryProcessor, SingleRasterOrVectorSource, + TypedPlotQueryProcessor, TypedRasterQueryProcessor, TypedVectorQueryProcessor, + WorkflowOperatorPath, }; -use crate::engine::{QueryProcessor, WorkflowOperatorPath}; use crate::error; use crate::error::Error; use crate::string_token; @@ -16,8 +16,8 @@ use futures::stream::BoxStream; use futures::{StreamExt, TryFutureExt}; use geoengine_datatypes::plots::{Plot, PlotData}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, DataRef, FeatureDataRef, FeatureDataType, - Geometry, Measurement, PlotQueryRectangle, RasterQueryRectangle, + AxisAlignedRectangle, BandSelection, ColumnSelection, DataRef, FeatureDataRef, FeatureDataType, + Geometry, Measurement, PlotQueryRectangle, RasterQueryRectangle, VectorQueryRectangle, }; use geoengine_datatypes::raster::{Pixel, RasterTile2D}; use geoengine_datatypes::{ @@ -117,9 +117,7 @@ impl PlotOperator for Histogram { spatial_reference: in_desc.spatial_reference, time: in_desc.time, // converting `SpatialPartition2D` to `BoundingBox2D` is ok here, because is makes the covered area only larger - bbox: in_desc - .bbox - .and_then(|p| BoundingBox2D::new(p.lower_left(), p.upper_right()).ok()), + bbox: Some(in_desc.spatial_bounds().as_bbox()), }, self.params, raster_source, @@ -371,10 +369,18 @@ impl HistogramRasterQueryProcessor { return Ok(metadata); } + let rd = self.input.result_descriptor(); + // TODO: compute only number of buckets if possible + let raster_query_rect = RasterQueryRectangle::with_spatial_query_and_geo_transform( + &query, + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + BandSelection::new_single(self.band_idx), + ); call_on_generic_raster_processor!(&self.input, processor => { - process_metadata(processor.query(RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::new_single(self.band_idx)), ctx).await?, self.metadata).await + process_metadata(processor.query(raster_query_rect, ctx).await?, self.metadata).await }) } @@ -393,8 +399,17 @@ impl HistogramRasterQueryProcessor { .build() .map_err(Error::from)?; + let rd = self.input.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::with_spatial_query_and_geo_transform( + &query, + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + BandSelection::new_single(self.band_idx), + ); + call_on_generic_raster_processor!(&self.input, processor => { - let mut query = processor.query(RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::new_single(self.band_idx)), ctx).await?; + let mut query = processor.query(raster_query_rect, ctx).await?; while let Some(tile) = query.next().await { @@ -458,8 +473,14 @@ impl HistogramVectorQueryProcessor { // TODO: compute only number of buckets if possible + let query = VectorQueryRectangle::new( + query.spatial_query, + query.time_interval, + ColumnSelection::all(), + ); + call_on_generic_vector_processor!(&self.input, processor => { - process_metadata(processor.query(query.into(), ctx).await?, &self.column_name, self.metadata).await + process_metadata(processor.query(query, ctx).await?, &self.column_name, self.metadata).await }) } @@ -478,8 +499,14 @@ impl HistogramVectorQueryProcessor { .build() .map_err(Error::from)?; + let query = VectorQueryRectangle::new( + query.spatial_query, + query.time_interval, + ColumnSelection::all(), + ); + call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -654,9 +681,9 @@ mod tests { use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, StaticMetaData, VectorColumnInfo, VectorOperator, - VectorResultDescriptor, + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SpatialGridDescriptor, StaticMetaData, VectorColumnInfo, + VectorOperator, VectorResultDescriptor, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use crate::source::{ @@ -665,12 +692,13 @@ mod tests { use crate::test_data; use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; use geoengine_datatypes::primitives::{ - BoundingBox2D, DateTime, FeatureData, NoGeometry, PlotSeriesSelection, SpatialResolution, + BoundingBox2D, Coordinate2D, DateTime, FeatureData, NoGeometry, PlotSeriesSelection, TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds}; use geoengine_datatypes::raster::{ - EmptyGrid2D, Grid2D, RasterDataType, RasterTile2D, TileInformation, TilingSpecification, + BoundedGrid, EmptyGrid2D, GeoTransform, Grid2D, GridShape2D, RasterDataType, RasterTile2D, + TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::Identifier; @@ -817,8 +845,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new_2d(3, 2).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -830,7 +860,6 @@ mod tests { async fn simple_raster() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -857,13 +886,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -883,7 +911,6 @@ mod tests { async fn simple_raster_without_spec() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -912,13 +939,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -976,14 +1002,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1047,14 +1071,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1194,11 +1216,18 @@ mod tests { #[tokio::test] async fn no_data_raster() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = Histogram { params: HistogramParams { @@ -1222,14 +1251,7 @@ mod tests { EmptyGrid2D::::new(tile_size_in_pixels).into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1248,13 +1270,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1307,14 +1328,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1375,14 +1394,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1405,11 +1422,19 @@ mod tests { #[tokio::test] async fn single_value_raster_stream() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = Histogram { params: HistogramParams { @@ -1433,14 +1458,7 @@ mod tests { Grid2D::new(tile_size_in_pixels, vec![4; 6]).unwrap().into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1459,16 +1477,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/pie_chart.rs b/operators/src/plot/pie_chart.rs index 8423d5b3e..cd7e743a2 100644 --- a/operators/src/plot/pie_chart.rs +++ b/operators/src/plot/pie_chart.rs @@ -11,10 +11,12 @@ use async_trait::async_trait; use futures::StreamExt; use geoengine_datatypes::collections::FeatureCollectionInfos; use geoengine_datatypes::plots::{Plot, PlotData}; -use geoengine_datatypes::primitives::{FeatureDataRef, Measurement, PlotQueryRectangle}; +use geoengine_datatypes::primitives::{ + ColumnSelection, FeatureDataRef, Measurement, PlotQueryRectangle, VectorQueryRectangle, +}; use serde::{Deserialize, Serialize}; use snafu::Snafu; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; pub const PIE_CHART_OPERATOR_NAME: &str = "PieChart"; @@ -108,7 +110,7 @@ pub struct InitializedCountPieChart { result_descriptor: PlotResultDescriptor, column_name: String, column_label: String, - class_mapping: Option>, + class_mapping: Option>, donut: bool, } @@ -119,7 +121,7 @@ impl InitializedCountPieChart { result_descriptor: PlotResultDescriptor, column_name: String, column_label: String, - class_mapping: Option>, + class_mapping: Option>, donut: bool, ) -> Self { Self { @@ -161,7 +163,7 @@ pub struct CountPieChartVectorQueryProcessor { input: TypedVectorQueryProcessor, column_label: String, column_name: String, - class_mapping: Option>, + class_mapping: Option>, donut: bool, } @@ -186,7 +188,7 @@ impl PlotQueryProcessor for CountPieChartVectorQueryProcessor { /// Null-values are empty strings. pub fn feature_data_strings_iter<'f>( feature_data: &'f FeatureDataRef, - class_mapping: Option<&'f HashMap>, + class_mapping: Option<&'f BTreeMap>, ) -> Box + 'f> { match (feature_data, class_mapping) { (FeatureDataRef::Category(feature_data_ref), Some(class_mapping)) => { @@ -227,9 +229,16 @@ impl CountPieChartVectorQueryProcessor { let mut slices: HashMap = HashMap::new(); // TODO: parallelize + let query: VectorQueryRectangle = VectorQueryRectangle::new( + query.spatial_query, + query.time_interval, + ColumnSelection::all(), + ); call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + + + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -285,8 +294,8 @@ mod tests { use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, StaticMetaData, VectorColumnInfo, - VectorOperator, VectorResultDescriptor, + ChunkByteSize, MockExecutionContext, StaticMetaData, VectorColumnInfo, VectorOperator, + VectorResultDescriptor, }; use crate::mock::MockFeatureCollectionSource; use crate::source::{ @@ -296,8 +305,8 @@ mod tests { use crate::test_data; use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, FeatureDataType, NoGeometry, PlotSeriesSelection, - SpatialResolution, TimeInterval, + BoundingBox2D, FeatureData, FeatureDataType, NoGeometry, PlotQueryRectangle, + PlotSeriesSelection, TimeInterval, }; use geoengine_datatypes::primitives::{CacheTtlSeconds, VectorQueryRectangle}; use geoengine_datatypes::spatial_reference::SpatialReference; @@ -399,14 +408,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -478,14 +485,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -625,14 +630,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap_err(); @@ -674,14 +677,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -757,14 +758,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -824,14 +823,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/scatter_plot.rs b/operators/src/plot/scatter_plot.rs index c419d6d14..a78b89b2e 100644 --- a/operators/src/plot/scatter_plot.rs +++ b/operators/src/plot/scatter_plot.rs @@ -13,7 +13,9 @@ use crate::engine::{ }; use crate::error::Error; use crate::util::Result; -use geoengine_datatypes::primitives::{Coordinate2D, PlotQueryRectangle}; +use geoengine_datatypes::primitives::{ + ColumnSelection, Coordinate2D, PlotQueryRectangle, VectorQueryRectangle, +}; pub const SCATTERPLOT_OPERATOR_NAME: &str = "ScatterPlot"; @@ -156,8 +158,14 @@ impl PlotQueryProcessor for ScatterPlotQueryProcessor { let mut collector = CollectorKind::Values(Collector::new(self.column_x.clone(), self.column_y.clone())); + let query = VectorQueryRectangle::new( + query.spatial_query, + query.time_interval, + ColumnSelection::all(), + ); + call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -283,17 +291,15 @@ impl CollectorKind { #[cfg(test)] mod tests { - use geoengine_datatypes::util::test::TestDefault; - use serde_json::json; - + use crate::engine::{ChunkByteSize, MockExecutionContext, VectorOperator}; + use crate::mock::MockFeatureCollectionSource; use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, NoGeometry, PlotSeriesSelection, SpatialResolution, + BoundingBox2D, FeatureData, NoGeometry, PlotQueryRectangle, PlotSeriesSelection, TimeInterval, }; + use geoengine_datatypes::util::test::TestDefault; use geoengine_datatypes::{collections::DataCollection, primitives::MultiPoint}; - - use crate::engine::{ChunkByteSize, MockExecutionContext, MockQueryContext, VectorOperator}; - use crate::mock::MockFeatureCollectionSource; + use serde_json::json; use super::*; @@ -379,14 +385,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -458,14 +462,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -645,14 +647,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -700,14 +700,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -757,14 +755,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -839,14 +835,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/statistics.rs b/operators/src/plot/statistics.rs index d8b06deba..d653cb8ee 100644 --- a/operators/src/plot/statistics.rs +++ b/operators/src/plot/statistics.rs @@ -16,8 +16,8 @@ use futures::stream::select_all; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use geoengine_datatypes::collections::FeatureCollectionInfos; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, PlotQueryRectangle, RasterQueryRectangle, - partitions_extent, time_interval_extent, + AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, PlotQueryRectangle, + RasterQueryRectangle, VectorQueryRectangle, partitions_extent, time_interval_extent, }; use geoengine_datatypes::raster::ConvertDataTypeParallel; use geoengine_datatypes::raster::{GridOrEmpty, GridSize}; @@ -115,7 +115,7 @@ impl PlotOperator for Statistics { } let time = time_interval_extent(in_descriptors.iter().map(|d| d.time)); - let bbox = partitions_extent(in_descriptors.iter().map(|d| d.bbox)); + let bbox = partitions_extent(in_descriptors.iter().map(|d| d.spatial_bounds())); let initialized_operator = InitializedStatistics::new( name, @@ -303,8 +303,14 @@ impl PlotQueryProcessor for StatisticsVectorQueryProcessor { }) .collect(); + let query = VectorQueryRectangle::new( + query.spatial_query, + query.time_interval, + ColumnSelection::all(), + ); + call_on_generic_vector_processor!(&self.vector, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -355,12 +361,19 @@ impl PlotQueryProcessor for StatisticsRasterQueryProcessor { ctx: &'a dyn QueryContext, ) -> Result { let mut queries = Vec::with_capacity(self.rasters.len()); - let q: RasterQueryRectangle = - RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()); for (i, raster_processor) in self.rasters.iter().enumerate() { + let rd = raster_processor.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::with_spatial_query_and_geo_transform( + &query, + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + BandSelection::first(), + ); + queries.push( call_on_generic_raster_processor!(raster_processor, processor => { - processor.query(q.clone(), ctx).await? + processor.query(raster_query_rect.clone(), ctx).await? // TODO: avoid cloning query? .and_then(move |tile| crate::util::spawn_blocking_with_thread_pool(ctx.thread_pool().clone(), move || (i, tile.convert_data_type_parallel()) ).map_err(Into::into)) .boxed() }), @@ -542,23 +555,22 @@ impl From<&StatisticsAggregator> for StatisticsOutput { #[cfg(test)] mod tests { use geoengine_datatypes::collections::DataCollection; - use geoengine_datatypes::primitives::{CacheHint, PlotSeriesSelection}; + use geoengine_datatypes::primitives::{CacheHint, Coordinate2D, PlotSeriesSelection}; use geoengine_datatypes::util::test::TestDefault; use serde_json::json; use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterOperator, - RasterResultDescriptor, + ChunkByteSize, MockExecutionContext, RasterOperator, RasterResultDescriptor, + SpatialGridDescriptor, }; use crate::engine::{RasterBandDescriptors, VectorOperator}; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use crate::util::input::MultiRasterOrVectorOperator::Raster; - use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, NoGeometry, SpatialResolution, TimeInterval, - }; + use geoengine_datatypes::primitives::{BoundingBox2D, FeatureData, NoGeometry, TimeInterval}; use geoengine_datatypes::raster::{ - Grid2D, RasterDataType, RasterTile2D, TileInformation, TilingSpecification, + BoundedGrid, GeoTransform, Grid2D, GridBoundingBox2D, GridShape2D, RasterDataType, + RasterTile2D, TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; @@ -590,9 +602,8 @@ mod tests { #[tokio::test] async fn empty_raster_input() { - let tile_size_in_pixels = [3, 2].into(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -616,14 +627,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -633,11 +642,18 @@ mod tests { #[tokio::test] async fn single_raster_implicit_name() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let raster_source = MockRasterSource { params: MockRasterSourceParams { @@ -654,14 +670,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -686,14 +695,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -718,11 +725,18 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn two_rasters_implicit_names() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let raster_source = vec![ MockRasterSource { @@ -740,14 +754,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(), @@ -766,14 +773,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(), @@ -799,14 +799,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -839,11 +837,18 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn two_rasters_explicit_names() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let raster_source = vec![ MockRasterSource { @@ -861,14 +866,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(), @@ -887,14 +885,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(), @@ -920,14 +911,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -959,11 +948,18 @@ mod tests { #[tokio::test] async fn two_rasters_explicit_names_incomplete() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let raster_source = vec![ MockRasterSource { @@ -981,14 +977,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(), @@ -1007,14 +996,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(), @@ -1044,7 +1026,6 @@ mod tests { async fn vector_no_column() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1103,14 +1084,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1144,7 +1123,6 @@ mod tests { async fn vector_single_column() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1203,14 +1181,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1236,7 +1212,6 @@ mod tests { async fn vector_two_columns() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1295,14 +1270,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1336,10 +1309,20 @@ mod tests { async fn raster_percentile() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-90, 89, -180, 179).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![RasterTile2D::new_with_tile_info( @@ -1355,14 +1338,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1387,14 +1363,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1423,7 +1397,6 @@ mod tests { async fn vector_percentiles() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1482,14 +1455,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/temporal_raster_mean_plot.rs b/operators/src/plot/temporal_raster_mean_plot.rs index 664b49261..c1631a85b 100644 --- a/operators/src/plot/temporal_raster_mean_plot.rs +++ b/operators/src/plot/temporal_raster_mean_plot.rs @@ -145,13 +145,17 @@ impl PlotQueryProcessor for MeanRasterPixelValuesOverTimeQueryProcesso query: PlotQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result { + let rd = self.raster.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::with_spatial_query_and_geo_transform( + &query, + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + BandSelection::first(), + ); + let means = Self::calculate_means( - self.raster - .query( - RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()), - ctx, - ) - .await?, + self.raster.query(raster_query_rect, ctx).await?, self.time_position, ) .await?; @@ -262,8 +266,8 @@ mod tests { use crate::{ engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SpatialGridDescriptor, }, source::GdalSource, }; @@ -272,9 +276,15 @@ mod tests { source::GdalSourceParameters, }; use geoengine_datatypes::primitives::{ - BoundingBox2D, CacheHint, Measurement, PlotSeriesSelection, SpatialResolution, TimeInterval, + BoundingBox2D, CacheHint, Coordinate2D, Measurement, PlotSeriesSelection, TimeInterval, + }; + use geoengine_datatypes::raster::GeoTransform; + use geoengine_datatypes::{ + dataset::NamedData, + plots::PlotMetaData, + primitives::DateTime, + raster::{BoundedGrid, GridShape2D}, }; - use geoengine_datatypes::{dataset::NamedData, plots::PlotMetaData, primitives::DateTime}; use geoengine_datatypes::{raster::TilingSpecification, spatial_reference::SpatialReference}; use geoengine_datatypes::{ raster::{Grid2D, RasterDataType, TileInformation}, @@ -291,9 +301,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("test"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name("test")), } .boxed(), }, @@ -329,7 +337,6 @@ mod tests { async fn single_raster() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -367,14 +374,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -444,8 +449,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new_2d(3, 2).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -457,7 +464,6 @@ mod tests { async fn raster_series() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -510,14 +516,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/temporal_vector_line_plot.rs b/operators/src/plot/temporal_vector_line_plot.rs index f6066ff6c..95aec4c07 100644 --- a/operators/src/plot/temporal_vector_line_plot.rs +++ b/operators/src/plot/temporal_vector_line_plot.rs @@ -1,25 +1,20 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedSources, InitializedVectorOperator, Operator, OperatorName, PlotOperator, PlotQueryProcessor, - PlotResultDescriptor, QueryContext, SingleVectorSource, TypedPlotQueryProcessor, - VectorQueryProcessor, WorkflowOperatorPath, + PlotResultDescriptor, QueryContext, QueryProcessor, SingleVectorSource, + TypedPlotQueryProcessor, VectorColumnInfo, VectorQueryProcessor, WorkflowOperatorPath, }; -use crate::engine::{QueryProcessor, VectorColumnInfo}; use crate::error; use crate::util::Result; use async_trait::async_trait; use futures::TryStreamExt; -use geoengine_datatypes::primitives::{FeatureDataType, PlotQueryRectangle}; use geoengine_datatypes::{ - collections::FeatureCollection, - plots::{Plot, PlotData}, -}; -use geoengine_datatypes::{ - collections::FeatureCollectionInfos, - plots::{DataPoint, MultiLineChart}, -}; -use geoengine_datatypes::{ - primitives::{Geometry, Measurement, TimeInterval}, + collections::{FeatureCollection, FeatureCollectionInfos}, + plots::{DataPoint, MultiLineChart, Plot, PlotData}, + primitives::{ + ColumnSelection, FeatureDataType, Geometry, Measurement, PlotQueryRectangle, TimeInterval, + VectorQueryRectangle, + }, util::arrow::ArrowTyped, }; use serde::{Deserialize, Serialize}; @@ -172,9 +167,15 @@ where ) -> Result { let values = FeatureAttributeValues::::default(); + let query = VectorQueryRectangle::new( + query.spatial_query, + query.time_interval, + ColumnSelection::all(), + ); + let values = self .features - .query(query.into(), ctx) + .query(query, ctx) .await? .try_fold(values, |mut acc, features| async move { let ids = features.data(&self.params.id_column)?; @@ -275,22 +276,20 @@ impl FeatureAttributeValues { #[cfg(test)] mod tests { use super::*; + use crate::{ + engine::{ChunkByteSize, MockExecutionContext, VectorOperator}, + mock::MockFeatureCollectionSource, + }; + use geoengine_datatypes::primitives::PlotQueryRectangle; use geoengine_datatypes::primitives::{CacheHint, PlotSeriesSelection}; use geoengine_datatypes::util::test::TestDefault; use geoengine_datatypes::{ collections::MultiPointCollection, plots::PlotMetaData, - primitives::{ - BoundingBox2D, DateTime, FeatureData, MultiPoint, SpatialResolution, TimeInterval, - }, + primitives::{BoundingBox2D, DateTime, FeatureData, MultiPoint, TimeInterval}, }; use serde_json::{Value, json}; - use crate::{ - engine::{ChunkByteSize, MockExecutionContext, MockQueryContext, VectorOperator}, - mock::MockFeatureCollectionSource, - }; - #[tokio::test] #[allow(clippy::too_many_lines)] async fn plot() { @@ -352,14 +351,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -501,14 +498,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -618,7 +613,7 @@ mod tests { ) .boxed(); - let exe_ctc = MockExecutionContext::test_default(); + let exe_ctx = MockExecutionContext::test_default(); let operator = FeatureAttributeValuesOverTime { params: FeatureAttributeValuesOverTimeParams { @@ -630,7 +625,7 @@ mod tests { let operator = operator .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctc) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); @@ -638,14 +633,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &exe_ctx.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/processing/band_neighborhood_aggregate/mod.rs b/operators/src/processing/band_neighborhood_aggregate/mod.rs index c97388c35..870e3f981 100644 --- a/operators/src/processing/band_neighborhood_aggregate/mod.rs +++ b/operators/src/processing/band_neighborhood_aggregate/mod.rs @@ -746,16 +746,16 @@ impl Accu for MovingAverageAccu { mod tests { use futures::StreamExt; use geoengine_datatypes::{ - primitives::{ - BandSelection, CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, + primitives::{BandSelection, CacheHint, TimeInterval}, + raster::{Grid, GridBoundingBox2D, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, spatial_reference::SpatialReference, util::test::TestDefault, }; use crate::{ - engine::{MockExecutionContext, MockQueryContext, RasterBandDescriptors}, + engine::{ + MockExecutionContext, MockQueryContext, RasterBandDescriptors, SpatialGridDescriptor, + }, mock::{MockRasterSource, MockRasterSourceParams}, }; @@ -1182,8 +1182,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_multiple_bands(3), }, }, @@ -1205,12 +1207,11 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::new_unchecked(vec![0, 1, 2]), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + BandSelection::new_unchecked(vec![0, 1, 2]), + ); let query_ctx = MockQueryContext::test_default(); @@ -1324,8 +1325,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_multiple_bands(3), }, }, @@ -1347,12 +1350,11 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::new_unchecked(vec![0]), // only get first band - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + BandSelection::new_unchecked(vec![0]), // only get first band + ); let query_ctx = MockQueryContext::test_default(); diff --git a/operators/src/processing/bandwise_expression/mod.rs b/operators/src/processing/bandwise_expression/mod.rs index 883200e5b..62622efa2 100644 --- a/operators/src/processing/bandwise_expression/mod.rs +++ b/operators/src/processing/bandwise_expression/mod.rs @@ -248,8 +248,11 @@ where #[cfg(test)] mod tests { use geoengine_datatypes::{ - primitives::{CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::{Grid, GridShape, MapElements, RenameBands, TilesEqualIgnoringCacheHint}, + primitives::{CacheHint, TimeInterval}, + raster::{ + Grid, GridBoundingBox2D, GridShape, MapElements, RenameBands, + TilesEqualIgnoringCacheHint, + }, spatial_reference::SpatialReference, util::test::TestDefault, }; @@ -257,6 +260,7 @@ mod tests { use crate::{ engine::{ MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, + SpatialGridDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{RasterStacker, RasterStackerParams}, @@ -355,17 +359,21 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -373,14 +381,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -410,12 +411,11 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 10), + [0, 1].try_into().unwrap(), + ); let query_ctx = MockQueryContext::test_default(); diff --git a/operators/src/processing/circle_merging_quadtree/operator.rs b/operators/src/processing/circle_merging_quadtree/operator.rs index 25fb3440a..dee3f0209 100644 --- a/operators/src/processing/circle_merging_quadtree/operator.rs +++ b/operators/src/processing/circle_merging_quadtree/operator.rs @@ -6,11 +6,11 @@ use futures::stream::{BoxStream, FuturesUnordered}; use geoengine_datatypes::collections::{ BuilderProvider, GeoFeatureCollectionRowBuilder, MultiPointCollection, VectorDataType, }; +use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use geoengine_datatypes::primitives::{ - BoundingBox2D, Circle, FeatureDataType, FeatureDataValue, Measurement, MultiPoint, - MultiPointAccess, VectorQueryRectangle, + Circle, FeatureDataType, FeatureDataValue, Measurement, MultiPoint, MultiPointAccess, + SpatialBounded, VectorQueryRectangle, VectorSpatialQueryRectangle, }; -use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -37,6 +37,7 @@ use super::quadtree::CircleMergingQuadtree; pub struct VisualPointClusteringParams { pub min_radius_px: f64, pub delta_px: f64, + pub radius_model_scale: Option, // TODO: discuss if this should be a parameter radius_column: String, count_column: String, column_aggregates: HashMap, @@ -86,6 +87,16 @@ impl VectorOperator for VisualPointClustering { error::DuplicateOutputColumns ); + let radius_model_scale = self.params.radius_model_scale.unwrap_or(1.0); + + ensure!( + radius_model_scale > 0.0, + error::InputMustBeGreaterThanZero { + scope: "VisualPointClustering", + name: "radius_model_scale" + } + ); + let name = CanonicOperatorName::from(&self); let radius_model = LogScaledRadius::new(self.params.min_radius_px, self.params.delta_px)?; @@ -185,6 +196,7 @@ impl VectorOperator for VisualPointClustering { }, vector_source, radius_model, + radius_model_scale, radius_column: self.params.radius_column, count_column: self.params.count_column, attribute_mapping: self.params.column_aggregates, @@ -201,6 +213,7 @@ pub struct InitializedVisualPointClustering { result_descriptor: VectorResultDescriptor, vector_source: Box, radius_model: LogScaledRadius, + radius_model_scale: f64, radius_column: String, count_column: String, attribute_mapping: HashMap, @@ -214,6 +227,7 @@ impl InitializedVectorOperator for InitializedVisualPointClustering { VisualPointClusteringProcessor::new( source, self.radius_model, + self.radius_model_scale, self.radius_column.clone(), self.count_column.clone(), self.result_descriptor.clone(), @@ -257,6 +271,7 @@ impl InitializedVectorOperator for InitializedVisualPointClustering { pub struct VisualPointClusteringProcessor { source: Box>, radius_model: LogScaledRadius, + radius_model_scale: f64, radius_column: String, count_column: String, result_descriptor: VectorResultDescriptor, @@ -267,6 +282,7 @@ impl VisualPointClusteringProcessor { fn new( source: Box>, radius_model: LogScaledRadius, + radius_model_scale: f64, radius_column: String, count_column: String, result_descriptor: VectorResultDescriptor, @@ -275,6 +291,7 @@ impl VisualPointClusteringProcessor { Self { source, radius_model, + radius_model_scale, radius_column, count_column, result_descriptor, @@ -375,7 +392,7 @@ struct GridFoldState { #[async_trait] impl QueryProcessor for VisualPointClusteringProcessor { type Output = MultiPointCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -392,12 +409,12 @@ impl QueryProcessor for VisualPointClusteringProcessor { .iter() .map(|(name, column_info)| (name.clone(), column_info.data_type)) .collect(); - - let joint_resolution = f64::max(query.spatial_resolution.x, query.spatial_resolution.y); - let scaled_radius_model = self.radius_model.with_scaled_radii(joint_resolution)?; + let scaled_radius_model = self + .radius_model + .with_scaled_radii(self.radius_model_scale)?; let initial_grid_fold_state = Result::::Ok(GridFoldState { - grid: Grid::new(query.spatial_bounds, scaled_radius_model), + grid: Grid::new(query.spatial_query.spatial_bounds(), scaled_radius_model), column_mapping: self.attribute_mapping.clone(), cache_hint: CacheHint::max_duration(), }); @@ -470,7 +487,11 @@ impl QueryProcessor for VisualPointClusteringProcessor { cache_hint, } = grid?; - let mut cmq = CircleMergingQuadtree::new(query.spatial_bounds, *grid.radius_model(), 1); + let mut cmq = CircleMergingQuadtree::new( + query.spatial_query.spatial_bounds(), + *grid.radius_model(), + 1, + ); // TODO: worker thread for circle_of_points in grid.drain() { @@ -481,7 +502,7 @@ impl QueryProcessor for VisualPointClusteringProcessor { cmq.into_iter(), &self.radius_column, &self.count_column, - joint_resolution, + self.radius_model_scale, &column_schema, cache_hint, ) @@ -498,6 +519,7 @@ impl QueryProcessor for VisualPointClusteringProcessor { #[cfg(test)] mod tests { use geoengine_datatypes::collections::ChunksEqualIgnoringCacheHint; + use geoengine_datatypes::primitives::BoundingBox2D; use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::FeatureData; use geoengine_datatypes::primitives::SpatialResolution; @@ -524,10 +546,14 @@ mod tests { ) .unwrap(); + let resolution = SpatialResolution::new(0.1, 0.1).unwrap(); + let radius_model_scale = resolution.x.max(resolution.y); + let operator = VisualPointClustering { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + radius_model_scale: Some(radius_model_scale), radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: Default::default(), @@ -553,12 +579,11 @@ mod tests { let query_context = MockQueryContext::test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); @@ -598,10 +623,14 @@ mod tests { ) .unwrap(); + let resolution = SpatialResolution::new(0.1, 0.1).unwrap(); + let radius_model_scale = resolution.x.max(resolution.y); + let operator = VisualPointClustering { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + radius_model_scale: Some(radius_model_scale), radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: [( @@ -637,12 +666,11 @@ mod tests { let query_context = MockQueryContext::test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); @@ -683,10 +711,14 @@ mod tests { ) .unwrap(); + let resolution = SpatialResolution::new(0.1, 0.1).unwrap(); + let radius_model_scale = resolution.x.max(resolution.y); + let operator = VisualPointClustering { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + radius_model_scale: Some(radius_model_scale), radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: [( @@ -722,12 +754,11 @@ mod tests { let query_context = MockQueryContext::test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); @@ -776,10 +807,14 @@ mod tests { ) .unwrap(); + let resolution = SpatialResolution::new(0.1, 0.1).unwrap(); + let radius_model_scale = resolution.x.max(resolution.y); + let operator = VisualPointClustering { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + radius_model_scale: Some(radius_model_scale), radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: [( @@ -815,12 +850,11 @@ mod tests { let query_context = MockQueryContext::test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); @@ -880,6 +914,9 @@ mod tests { ) .unwrap(); + let resolution = SpatialResolution::new(0.1, 0.1).unwrap(); + let radius_model_scale = resolution.x.max(resolution.y); + let cache_hint = CacheHint::seconds(1234); input.cache_hint = cache_hint; @@ -888,6 +925,7 @@ mod tests { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + radius_model_scale: Some(radius_model_scale), radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: [( @@ -923,12 +961,11 @@ mod tests { let query_context = MockQueryContext::test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); diff --git a/operators/src/processing/column_range_filter.rs b/operators/src/processing/column_range_filter.rs index 136afa6dc..b3e3ae838 100644 --- a/operators/src/processing/column_range_filter.rs +++ b/operators/src/processing/column_range_filter.rs @@ -14,8 +14,8 @@ use geoengine_datatypes::collections::{ FeatureCollection, FeatureCollectionInfos, FeatureCollectionModifications, }; use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, FeatureDataType, FeatureDataValue, Geometry, - VectorQueryRectangle, + ColumnSelection, FeatureDataType, FeatureDataValue, Geometry, VectorQueryRectangle, + VectorSpatialQueryRectangle, }; use geoengine_datatypes::util::arrow::ArrowTyped; use serde::{Deserialize, Serialize}; @@ -130,7 +130,7 @@ where G: Geometry + ArrowTyped + Sync + Send + 'static, { type Output = FeatureCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -199,14 +199,13 @@ where #[cfg(test)] mod tests { use super::*; - use crate::engine::{MockExecutionContext, MockQueryContext}; + use crate::engine::MockExecutionContext; use crate::mock::MockFeatureCollectionSource; use geoengine_datatypes::collections::{ ChunksEqualIgnoringCacheHint, FeatureCollectionModifications, MultiPointCollection, }; - use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ - BoundingBox2D, Coordinate2D, FeatureData, MultiPoint, SpatialResolution, TimeInterval, + BoundingBox2D, CacheHint, Coordinate2D, FeatureData, MultiPoint, TimeInterval, }; use geoengine_datatypes::util::test::TestDefault; @@ -283,11 +282,10 @@ mod tests { } .boxed(); + let exe_ctx = MockExecutionContext::test_default(); + let initialized = filter - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); @@ -297,14 +295,13 @@ mod tests { panic!(); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); - let ctx = MockQueryContext::new((2 * std::mem::size_of::()).into()); + let ctx = exe_ctx.mock_query_context((2 * std::mem::size_of::()).into()); let stream = point_processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/processing/downsample/mod.rs b/operators/src/processing/downsample/mod.rs new file mode 100644 index 000000000..51c5c408e --- /dev/null +++ b/operators/src/processing/downsample/mod.rs @@ -0,0 +1,921 @@ +use crate::adapters::{ + FoldTileAccu, FoldTileAccuMut, RasterSubQueryAdapter, SubQueryTileAggregator, +}; +use crate::engine::{ + CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, + OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, + RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, +}; +use crate::util::Result; +use async_trait::async_trait; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use futures::{Future, FutureExt, TryFuture, TryFutureExt}; +use geoengine_datatypes::primitives::{BandSelection, CacheHint, Coordinate2D}; +use geoengine_datatypes::primitives::{ + RasterQueryRectangle, RasterSpatialQueryRectangle, SpatialResolution, TimeInstance, + TimeInterval, +}; +use geoengine_datatypes::raster::{ + ChangeGridBounds, GeoTransform, GridBoundingBox2D, GridContains, GridIdx2D, GridIndexAccess, + GridOrEmpty, Pixel, RasterTile2D, TileInformation, TilingSpecification, + UpdateIndexedElementsParallel, +}; +use rayon::ThreadPool; +use serde::{Deserialize, Serialize}; +use snafu::{Snafu, ensure}; +use std::marker::PhantomData; +use std::sync::Arc; + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub struct DownsamplingParams { + pub sampling_method: DownsamplingMethod, + pub output_resolution: DownsamplingResolution, + pub output_origin_reference: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase", tag = "type")] +pub enum DownsamplingResolution { + Resolution(SpatialResolution), + Fraction { x: f64, y: f64 }, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub enum DownsamplingMethod { + NearestNeighbor, + // Mean, +} + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)), context(suffix(false)), module(error))] +pub enum DownsamplingError { + #[snafu(display("The fraction used to downsample must be >= 1, was {f}."))] + FractionMustBeOneOrLarger { f: f64 }, + #[snafu(display("The output resolution must be higher than the input resolution."))] + OutputMustBeLowerResolutionThanInput { + input: SpatialResolution, + output: SpatialResolution, + }, +} + +pub type Downsampling = Operator; + +impl OperatorName for Downsampling { + const TYPE_NAME: &'static str = "Downsampling"; +} + +#[typetag::serde] +#[async_trait] +impl RasterOperator for Downsampling { + async fn _initialize( + self: Box, + path: WorkflowOperatorPath, + context: &dyn ExecutionContext, + ) -> Result> { + let name = CanonicOperatorName::from(&self); + let initialized_source = self + .sources + .initialize_sources(path.clone(), context) + .await?; + InitializedDownsampling::new_with_source_and_params( + name, + path, + initialized_source.raster, + self.params, + context.tiling_specification(), + ) + .map(InitializedRasterOperator::boxed) + } + + span_fn!(Downsampling); +} + +pub struct InitializedDownsampling { + name: CanonicOperatorName, + path: WorkflowOperatorPath, + output_result_descriptor: RasterResultDescriptor, + raster_source: O, + sampling_method: DownsamplingMethod, + tiling_specification: TilingSpecification, +} + +impl InitializedDownsampling { + pub fn new_with_source_and_params( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + raster_source: O, + params: DownsamplingParams, + tiling_specification: TilingSpecification, + ) -> Result { + let in_descriptor = raster_source.result_descriptor(); + + let in_spatial_grid = in_descriptor.spatial_grid_descriptor(); + + let output_resolution = match params.output_resolution { + DownsamplingResolution::Resolution(res) => { + ensure!( + res.x.abs() >= in_spatial_grid.spatial_resolution().x.abs(), + error::OutputMustBeLowerResolutionThanInput { + input: in_spatial_grid.spatial_resolution(), + output: res + } + ); + ensure!( + res.y.abs() >= in_spatial_grid.spatial_resolution().y.abs(), // TODO: allow neg y size in SpatialResolution + error::OutputMustBeLowerResolutionThanInput { + input: in_spatial_grid.spatial_resolution(), + output: res + } + ); + res + } + + DownsamplingResolution::Fraction { x, y } => { + ensure!(x >= 1.0, error::FractionMustBeOneOrLarger { f: x }); + ensure!(y >= 1.0, error::FractionMustBeOneOrLarger { f: y }); + + SpatialResolution::new_unchecked( + in_spatial_grid.spatial_resolution().x * x, + in_spatial_grid.spatial_resolution().y.abs() * y, // TODO: allow negative size + ) + } + }; + + let output_gspatial_grid = if let Some(oc) = params.output_origin_reference { + in_spatial_grid + .with_moved_origin_to_nearest_grid_edge(oc) + .replace_origin(oc) + .with_changed_resolution(output_resolution) + } else { + in_spatial_grid.with_changed_resolution(output_resolution) + }; + + let out_descriptor = RasterResultDescriptor { + spatial_reference: in_descriptor.spatial_reference, + data_type: in_descriptor.data_type, // TODO: datatype depends on resample method! + time: in_descriptor.time, + spatial_grid: output_gspatial_grid, + bands: in_descriptor.bands.clone(), + }; + + Ok(InitializedDownsampling { + name, + path, + output_result_descriptor: out_descriptor, + raster_source, + sampling_method: params.sampling_method, + tiling_specification, + }) + } +} + +impl InitializedRasterOperator for InitializedDownsampling { + fn query_processor(&self) -> Result { + let source_processor = self.raster_source.query_processor()?; + + let res = call_on_generic_raster_processor!( + source_processor, p => match self.sampling_method { + DownsamplingMethod::NearestNeighbor => DownsampleProcessor::<_,_>::new( + p, + self.output_result_descriptor.clone(), + self.tiling_specification, + ).boxed() + .into(), + } + ); + + Ok(res) + } + + fn result_descriptor(&self) -> &RasterResultDescriptor { + &self.output_result_descriptor + } + + fn canonic_name(&self) -> CanonicOperatorName { + self.name.clone() + } + + fn name(&self) -> &'static str { + Downsampling::TYPE_NAME + } + + fn path(&self) -> WorkflowOperatorPath { + self.path.clone() + } +} + +pub struct DownsampleProcessor +where + Q: RasterQueryProcessor, + P: Copy, +{ + source: Q, + out_result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, +} + +impl DownsampleProcessor +where + Q: RasterQueryProcessor, + P: Copy, +{ + pub fn new( + source: Q, + out_result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, + ) -> Self { + Self { + source, + out_result_descriptor, + tiling_specification, + } + } +} + +#[async_trait] +impl QueryProcessor for DownsampleProcessor +where + Q: QueryProcessor< + Output = RasterTile2D

, + SpatialQuery = RasterSpatialQueryRectangle, + Selection = BandSelection, + ResultDescription = RasterResultDescriptor, + >, + P: Pixel, +{ + type Output = RasterTile2D

; + type SpatialQuery = RasterSpatialQueryRectangle; + type Selection = BandSelection; + type ResultDescription = RasterResultDescriptor; + + async fn _query<'a>( + &'a self, + query: RasterQueryRectangle, + ctx: &'a dyn QueryContext, + ) -> Result>> { + // do not interpolate if the source resolution is already fine enough + + let in_spatial_grid = self.source.result_descriptor().spatial_grid_descriptor(); + let out_spatial_grid = self.result_descriptor().spatial_grid_descriptor(); + + // if the output resolution is the same as the input resolution, we can just forward the query // TODO: except the origin changes? + if in_spatial_grid == out_spatial_grid { + return self.source.query(query, ctx).await; + } + + let tiling_grid_definition = + out_spatial_grid.tiling_grid_definition(ctx.tiling_specification()); + // This is the tiling strategy we want to fill + let tiling_strategy: geoengine_datatypes::raster::TilingStrategy = + tiling_grid_definition.generate_data_tiling_strategy(); + + let sub_query = DownsampleSubQuery::<_, P> { + input_geo_transform: in_spatial_grid + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + output_geo_transform: tiling_grid_definition.tiling_geo_transform(), + fold_fn: fold_future, + tiling_specification: self.tiling_specification, + _phantom_pixel_type: PhantomData, + }; + + Ok(RasterSubQueryAdapter::<'a, P, _, _>::new( + &self.source, + query, + tiling_strategy, + ctx, + sub_query, + ) + .filter_and_fill( + crate::adapters::FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles, + )) + } + + fn result_descriptor(&self) -> &RasterResultDescriptor { + &self.out_result_descriptor + } +} + +#[derive(Debug, Clone)] +pub struct DownsampleSubQuery { + input_geo_transform: GeoTransform, + output_geo_transform: GeoTransform, + fold_fn: F, + tiling_specification: TilingSpecification, + _phantom_pixel_type: PhantomData, +} + +impl<'a, T, FoldM, FoldF> SubQueryTileAggregator<'a, T> for DownsampleSubQuery +where + T: Pixel, + FoldM: Send + Sync + 'a + Clone + Fn(DownsampleAccu, RasterTile2D) -> FoldF, + FoldF: Send + TryFuture, Error = crate::error::Error>, +{ + type FoldFuture = FoldF; + + type FoldMethod = FoldM; + + type TileAccu = DownsampleAccu; + type TileAccuFuture = BoxFuture<'a, Result>; + + fn new_fold_accu( + &self, + tile_info: TileInformation, + query_rect: RasterQueryRectangle, + pool: &Arc, + ) -> Self::TileAccuFuture { + create_accu( + self.input_geo_transform, + self.output_geo_transform, + tile_info, + &query_rect, + pool.clone(), + self.tiling_specification, + ) + .boxed() + } + + fn tile_query_rectangle( + &self, + tile_info: TileInformation, + _query_rect: RasterQueryRectangle, + start_time: TimeInstance, + band_idx: u32, + ) -> Result> { + let out_tile_pixel_bounds = tile_info.global_pixel_bounds(); + //.intersection(&query_rect.spatial_query.grid_bounds()); + //let out_tile_pixel_bounds = out_tile_pixel_bounds; + let out_tile_spatial_bounds = self + .output_geo_transform + .grid_to_spatial_bounds(&out_tile_pixel_bounds); + let input_pixel_bounds = self + .input_geo_transform + .spatial_to_grid_bounds(&out_tile_spatial_bounds); + + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + input_pixel_bounds, + TimeInterval::new_instant(start_time)?, + BandSelection::new_single(band_idx), + ))) + } + + fn fold_method(&self) -> Self::FoldMethod { + self.fold_fn.clone() + } +} + +#[derive(Clone, Debug)] +pub struct DownsampleAccu { + pub output_tile_info: TileInformation, + pub output_grid: GridOrEmpty, + pub input_global_geo_transform: GeoTransform, + + pub time: Option, + pub cache_hint: CacheHint, + pub pool: Arc, +} + +impl DownsampleAccu { + pub fn new( + output_tile_info: TileInformation, + input_global_geo_transform: GeoTransform, + time: Option, + cache_hint: CacheHint, + pool: Arc, + ) -> Self { + DownsampleAccu { + output_tile_info, + output_grid: GridOrEmpty::new_empty_shape(output_tile_info.global_pixel_bounds()), + input_global_geo_transform, + time, + cache_hint, + pool, + } + } +} + +#[async_trait] +impl FoldTileAccu for DownsampleAccu { + type RasterType = T; + + async fn into_tile(self) -> Result> { + // TODO: later do conversation of accu into tile here + + let output_tile = RasterTile2D::new_with_tile_info( + self.time.expect("there is at least one input"), + self.output_tile_info, + 0, // TODO: need band? + self.output_grid.unbounded(), + self.cache_hint, + ); + + Ok(output_tile) + } + + fn thread_pool(&self) -> &Arc { + &self.pool + } +} + +impl FoldTileAccuMut for DownsampleAccu { + fn set_time(&mut self, time: TimeInterval) { + self.time = Some(time); + } + + fn set_cache_hint(&mut self, cache_hint: CacheHint) { + self.cache_hint = cache_hint; + } +} + +pub fn create_accu( + input_geo_transform: GeoTransform, + _output_geo_transform: GeoTransform, + tile_info: TileInformation, + _query_rect: &RasterQueryRectangle, + pool: Arc, + _tiling_specification: TilingSpecification, +) -> impl Future>> + use { + crate::util::spawn_blocking(move || { + DownsampleAccu::new( + tile_info, + input_geo_transform, + None, + CacheHint::max_duration(), + pool.clone(), + ) + }) + .map_err(From::from) +} + +pub fn fold_future( + accu: DownsampleAccu, + tile: RasterTile2D, +) -> impl Future>> +where + T: Pixel, +{ + crate::util::spawn_blocking_with_thread_pool(accu.pool.clone(), || fold_impl(accu, tile)).then( + |x| async move { + match x { + Ok(r) => Ok(r), + Err(e) => Err(e.into()), + } + }, + ) +} + +pub fn fold_impl(mut accu: DownsampleAccu, tile: RasterTile2D) -> DownsampleAccu +where + T: Pixel, +{ + // get the time now because it is not known when the accu was created + accu.set_time(tile.time); + accu.cache_hint.merge_with(&tile.cache_hint); + + // TODO: add a skip if both tiles are empty? + if tile.is_empty() { + // TODO: and ignore no-data. + return accu; + } + + // copy all input tiles into the accu to have all data for interpolation + let mut accu_tile = accu.output_grid.into_materialized_masked_grid(); + let in_tile_grid = tile.into_inner_positioned_grid(); + let accu_geo_transform = accu.output_tile_info.global_geo_transform; + let in_geo_transform = accu.input_global_geo_transform; + + let map_fn = |grid_idx: GridIdx2D, current_value: Option| -> Option { + let accu_pixel_coord = accu_geo_transform.grid_idx_to_pixel_center_coordinate_2d(grid_idx); // use center coordinate similar to ArcGIS + let source_pixel_idx = in_geo_transform.coordinate_to_grid_idx_2d(accu_pixel_coord); + + if in_tile_grid.contains(&source_pixel_idx) { + in_tile_grid.get_at_grid_index_unchecked(source_pixel_idx) + } else { + current_value + } + }; + + accu_tile.update_indexed_elements_parallel(map_fn); + + accu.output_grid = accu_tile.into(); + + accu +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::{ + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, SpatialGridDescriptor, + }; + use crate::mock::{MockRasterSource, MockRasterSourceParams}; + use futures::StreamExt; + use geoengine_datatypes::raster::{Grid, GridShape2D, RasterDataType}; + use geoengine_datatypes::spatial_reference::SpatialReference; + use geoengine_datatypes::util::test::TestDefault; + + #[allow(clippy::too_many_lines)] + #[tokio::test] + async fn nearest_neighbor_4() { + // In this test, 2x2 tiles with 4x4 pixels are downsampled using nearest neighbor to one tile with 4x4 pixels. The resolution is now 1/2 of the original resolution. + // The test uses the following input: + // + // _1, _2, _3, _4 | 21, 22, 23, 24 + // _5, _6, _7, _8 | 25, 26, 27, 28 + // _9, 10, 11, 12 | 29, 30, 31, 32 + // 13, 14, 15, 16 | 33, 34, 35, 36 + // ---------------+--------------- + // 41, 42, 43, 44 | 61, 62, 63, 64 + // 45, 46, 47, 48 | 65, 66, 67, 68 + // 49, 50, 51, 52 | 69, 70, 71, 72 + // 53, 54, 55, 56 | 73, 74, 75, 76 + // + // The input is downsampled to: + // + // _6. _8, 26, 28 + // 14, 16, 33, 36 + // 46, 48, 66, 68 + // 54, 56, 74, 76 + // + // The center of each pixel is mapped to a coordinate. Then, for this coordinate the nearest pixel center is selected. + // In this case, we have the special case that the pixel center of the target pixel hits the edge between two original pixels. + // The pixel which "owns" the edge is selected because pixels are defiend from the upper left edge. + // E.g. _6 is selected for the first pixel since it owns the edge between _1, _2, _5_ and _6. + + let in_geo_transform = GeoTransform::new(Coordinate2D::new(0.0, 0.0), 1.0, -1.0); + let out_geo_transform = GeoTransform::new(Coordinate2D::new(0.0, 0.0), 2.0, -2.0); + let tile_size_in_pixels = GridShape2D { + shape_array: [4, 4], + }; + + let exe_ctx = MockExecutionContext::new_with_tiling_spec_and_thread_count( + TilingSpecification::new(tile_size_in_pixels), + 8, + ); + + let data: Vec> = vec![ + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![ + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + ], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![ + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + ], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![ + 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, + ], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + ]; + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + in_geo_transform, + GridBoundingBox2D::new_min_max(0, 7, 0, 7).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + + let mrs1 = MockRasterSource { + params: MockRasterSourceParams { + data: data.clone(), + result_descriptor: result_descriptor.clone(), + }, + } + .boxed(); + + let downsampler = Downsampling { + params: DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_origin_reference: None, + output_resolution: DownsamplingResolution::Resolution(SpatialResolution { + x: out_geo_transform.x_pixel_size(), + y: out_geo_transform.y_pixel_size().abs(), + }), + }, + sources: SingleRasterSource { raster: mrs1 }, + } + .boxed(); + + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(0, 3, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); + + let query_ctx = exe_ctx.mock_query_context(ChunkByteSize::test_default()); + + let op = downsampler + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let qp = op.query_processor().unwrap().get_u8().unwrap(); + + let result = qp + .raster_query(query_rect, &query_ctx) + .await + .unwrap() + .collect::>() + .await; + + assert_eq!(result.len(), 1); + + let tile = result[0].as_ref().unwrap(); + let grid = tile.grid_array.clone().into_materialized_masked_grid(); + // _6. _8, 26, 28 + // 14, 16, 33, 36 + // 46, 48, 66, 68 + // 54, 56, 74, 76 + assert_eq!( + grid.inner_grid.data, + &[6, 8, 26, 28, 14, 16, 34, 36, 46, 48, 66, 68, 54, 56, 74, 76] + ); + } + + #[allow(clippy::too_many_lines)] + #[tokio::test] + async fn nearest_neighbor_3() { + // In this test, 3x3 tiles with 3x3 pixels are downsampled using nearest neighbor to one tile with 3x3 pixels. The resolution is now 1/3 of the original resolution. + // The test uses the following input: + // + // _0, _1, _2 | 10, 11, 12 | 20, 21, 22 + // _3, _4, _5 | 13, 14, 15 | 23, 24, 25 + // _6, _7, _8 | 16, 17, 18 | 26, 27, 28 + // -----------+------------+----------- + // 30, 31, 32 | 40, 41, 42 | 50, 51, 52 + // 33, 34, 35 | 43, 44, 45 | 53, 54, 55 + // 36, 37, 38 | 46, 47, 48 | 56, 57, 58 + // -----------+------------+----------- + // 60, 61, 62 | 70, 71, 72 | 80, 81, 82 + // 63, 64, 65 | 73, 74, 75 | 83, 84, 85 + // 66, 67, 68 | 76, 77, 78 | 86, 87, 88 + // + // The input is downsampled to: + // + // _4. 14, 24 + // 34, 44, 54 + // 64, 74, 84 + // + // The center of each pixel is mapped to a coordinate. Then, for this coordinate the nearest pixel center is selected. + // In this case, each pixel corresponds to the center of the corresponding tile. + // E.g. _4 is selected for the first pixel since it is in the center of the tile. + + let in_geo_transform = GeoTransform::new(Coordinate2D::new(0.0, 0.0), 1.0, -1.0); + let out_geo_transform = GeoTransform::new(Coordinate2D::new(0.0, 0.0), 3.0, -3.0); + let tile_size_in_pixels = GridShape2D { + shape_array: [3, 3], + }; + + let exe_ctx = MockExecutionContext::new_with_tiling_spec_and_thread_count( + TilingSpecification::new(tile_size_in_pixels), + 8, + ); + + let data: Vec> = vec![ + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new(tile_size_in_pixels, vec![0, 1, 2, 3, 4, 5, 6, 7, 8]) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![10, 11, 12, 13, 14, 15, 16, 17, 18], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 2].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![20, 21, 22, 23, 24, 25, 26, 27, 28], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![30, 31, 32, 33, 34, 35, 36, 37, 38], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![40, 41, 42, 43, 44, 45, 46, 47, 48], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 2].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![50, 51, 52, 53, 54, 55, 56, 57, 58], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [2, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![60, 61, 62, 63, 64, 65, 66, 67, 68], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [2, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![70, 71, 72, 73, 74, 75, 76, 77, 78], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [2, 2].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![80, 81, 82, 83, 84, 85, 86, 87, 88], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + ]; + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + in_geo_transform, + GridBoundingBox2D::new_min_max(0, 8, 0, 8).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + + let mrs1 = MockRasterSource { + params: MockRasterSourceParams { + data: data.clone(), + result_descriptor: result_descriptor.clone(), + }, + } + .boxed(); + + let downsampler = Downsampling { + params: DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_origin_reference: None, + output_resolution: DownsamplingResolution::Resolution(SpatialResolution { + x: out_geo_transform.x_pixel_size(), + y: out_geo_transform.y_pixel_size().abs(), + }), + }, + sources: SingleRasterSource { raster: mrs1 }, + } + .boxed(); + + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(0, 2, 0, 2).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); + + let query_ctx = exe_ctx.mock_query_context(ChunkByteSize::test_default()); + + let op = downsampler + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let qp = op.query_processor().unwrap().get_u8().unwrap(); + + let result = qp + .raster_query(query_rect, &query_ctx) + .await + .unwrap() + .collect::>() + .await; + + assert_eq!(result.len(), 1); + + let tile = result[0].as_ref().unwrap(); + let grid = tile.grid_array.clone().into_materialized_masked_grid(); + // _4. 14, 24 + // 34, 44, 54 + // 64, 74, 84 + + assert_eq!(grid.inner_grid.data, &[4, 14, 24, 34, 44, 54, 64, 74, 84]); + } +} diff --git a/operators/src/processing/expression/raster_operator.rs b/operators/src/processing/expression/raster_operator.rs index cf9eaaa36..b4eec87ba 100644 --- a/operators/src/processing/expression/raster_operator.rs +++ b/operators/src/processing/expression/raster_operator.rs @@ -108,8 +108,7 @@ impl RasterOperator for Expression { data_type: self.params.output_type, spatial_reference: in_descriptor.spatial_reference, time: in_descriptor.time, - bbox: in_descriptor.bbox, - resolution: in_descriptor.resolution, + spatial_grid: in_descriptor.spatial_grid, bands: RasterBandDescriptors::new(vec![ self.params .output_band @@ -223,18 +222,16 @@ impl InitializedRasterOperator for InitializedExpression { mod tests { use super::*; use crate::engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, QueryProcessor, + MockExecutionContext, MultipleRasterSources, QueryProcessor, SpatialGridDescriptor, }; use crate::mock::{MockRasterSource, MockRasterSourceParams}; use crate::processing::{RasterStacker, RasterStackerParams}; use futures::StreamExt; use geoengine_datatypes::primitives::{BandSelection, CacheHint, CacheTtlSeconds, Measurement}; - use geoengine_datatypes::primitives::{ - RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - }; + use geoengine_datatypes::primitives::{RasterQueryRectangle, TimeInterval}; use geoengine_datatypes::raster::{ - Grid2D, GridOrEmpty, MapElements, MaskedGrid2D, RasterTile2D, RenameBands, TileInformation, - TilingSpecification, + Grid2D, GridBoundingBox2D, GridOrEmpty, MapElements, MaskedGrid2D, RasterTile2D, + RenameBands, TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; @@ -291,7 +288,6 @@ mod tests { async fn basic_unary() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -315,18 +311,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -352,7 +344,6 @@ mod tests { async fn unary_map_no_data() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -376,18 +367,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -413,7 +400,6 @@ mod tests { async fn basic_binary() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -448,18 +434,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -481,7 +463,6 @@ mod tests { async fn basic_coalesce() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -523,18 +504,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -563,7 +540,6 @@ mod tests { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -599,18 +575,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -644,7 +616,6 @@ mod tests { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -688,18 +659,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -721,7 +688,6 @@ mod tests { async fn it_classifies() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -765,18 +731,14 @@ mod tests { let processor = operator.query_processor().unwrap().get_u8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-3, -1, 0, 1).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -805,7 +767,6 @@ mod tests { let no_data_value = 0; let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -839,18 +800,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ectx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -905,8 +862,10 @@ mod tests { data_type: RasterDataType::I8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -919,7 +878,6 @@ mod tests { let no_data_value = 0; let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -954,18 +912,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ectx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -987,7 +941,6 @@ mod tests { let no_data_value = 0; let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1025,18 +978,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ectx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -1057,7 +1006,6 @@ mod tests { let no_data_value = 0; let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1098,18 +1046,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ectx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await diff --git a/operators/src/processing/expression/raster_query_processor.rs b/operators/src/processing/expression/raster_query_processor.rs index 065d17403..e18295b31 100644 --- a/operators/src/processing/expression/raster_query_processor.rs +++ b/operators/src/processing/expression/raster_query_processor.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use futures::{StreamExt, TryStreamExt, stream::BoxStream}; use geoengine_datatypes::{ primitives::{ - BandSelection, CacheHint, RasterQueryRectangle, SpatialPartition2D, TimeInterval, + BandSelection, CacheHint, RasterQueryRectangle, RasterSpatialQueryRectangle, TimeInterval, }, raster::{ ConvertDataType, FromIndexFnParallel, GeoTransform, GridIdx2D, GridIndexAccess, @@ -62,7 +62,7 @@ where Tuple: ExpressionTupleProcessor, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -73,9 +73,8 @@ where ) -> Result>> { // rewrite query to request all input bands from the source. They are all combined in the single output band by means of the expression. let source_query = RasterQueryRectangle { - spatial_bounds: query.spatial_bounds, + spatial_query: query.spatial_query, time_interval: query.time_interval, - spatial_resolution: query.spatial_resolution, attributes: BandSelection::first_n(Tuple::num_bands()), }; @@ -97,7 +96,7 @@ where ) = Tuple::metadata(&rasters); let program = self.program.clone(); - let map_no_data = self.map_no_data; + let map_no_data: bool = self.map_no_data; let out = crate::util::spawn_blocking_with_thread_pool( ctx.thread_pool().clone(), diff --git a/operators/src/processing/expression/vector_operator.rs b/operators/src/processing/expression/vector_operator.rs index 61cfe4efb..07328e4b8 100644 --- a/operators/src/processing/expression/vector_operator.rs +++ b/operators/src/processing/expression/vector_operator.rs @@ -713,10 +713,8 @@ mod tests { ChunksEqualIgnoringCacheHint, IntoGeometryIterator, MultiPointCollection, MultiPolygonCollection, }, - primitives::{ - BoundingBox2D, ColumnSelection, MultiPoint, MultiPolygon, SpatialResolution, - TimeInterval, - }, + primitives::{BoundingBox2D, ColumnSelection, MultiPoint, MultiPolygon, TimeInterval}, + raster::TilingSpecification, util::test::TestDefault, }; @@ -780,6 +778,8 @@ mod tests { let point_source = MockFeatureCollectionSource::single(points.clone()).boxed(); + let exe_ctx = MockExecutionContext::test_default(); + let operator = VectorExpression { params: VectorExpressionParams { input_columns: vec!["foo".into()], @@ -791,22 +791,18 @@ mod tests { sources: point_source.into(), } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); let query_processor = operator.query_processor().unwrap().multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -870,6 +866,7 @@ mod tests { .unwrap(); let point_source = MockFeatureCollectionSource::single(points.clone()).boxed(); + let exe_ctx = MockExecutionContext::test_default(); let operator = VectorExpression { params: VectorExpressionParams { @@ -882,22 +879,18 @@ mod tests { sources: point_source.into(), } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); let query_processor = operator.query_processor().unwrap().multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -982,12 +975,11 @@ mod tests { }, sources: polygons.into(), }, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), ) .await; @@ -1065,12 +1057,11 @@ mod tests { }, sources: polygons.into(), }, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), ) .await; @@ -1162,16 +1153,15 @@ mod tests { .boxed() .into(), }, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + VectorQueryRectangle::with_bounds( + BoundingBox2D::new( (0., 0.).into(), (NUMBER_OF_ROWS as f64, NUMBER_OF_ROWS as f64).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + TimeInterval::default(), + ColumnSelection::all(), + ), ) .await; @@ -1209,7 +1199,7 @@ mod tests { let query_processor: Box> = operator.query_processor().unwrap().try_into().unwrap(); - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let ctx = MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/processing/interpolation/mod.rs b/operators/src/processing/interpolation/mod.rs index e897c7337..6a29bd256 100644 --- a/operators/src/processing/interpolation/mod.rs +++ b/operators/src/processing/interpolation/mod.rs @@ -14,14 +14,15 @@ use async_trait::async_trait; use futures::future::BoxFuture; use futures::stream::BoxStream; use futures::{Future, FutureExt, TryFuture, TryFutureExt}; +use geoengine_datatypes::primitives::{BandSelection, CacheHint, Coordinate2D}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, Coordinate2D, RasterQueryRectangle, SpatialPartition2D, - SpatialPartitioned, SpatialResolution, TimeInstance, TimeInterval, + RasterQueryRectangle, RasterSpatialQueryRectangle, SpatialResolution, TimeInstance, + TimeInterval, }; -use geoengine_datatypes::primitives::{BandSelection, CacheHint}; use geoengine_datatypes::raster::{ - Bilinear, Blit, EmptyGrid2D, GeoTransform, GridOrEmpty, GridSize, InterpolationAlgorithm, - NearestNeighbor, Pixel, RasterTile2D, TileInformation, TilingSpecification, + Bilinear, ChangeGridBounds, GeoTransform, GridBlit, GridBoundingBox2D, GridOrEmpty, + InterpolationAlgorithm, NearestNeighbor, Pixel, RasterTile2D, TileInformation, + TilingSpecification, }; use rayon::ThreadPool; use serde::{Deserialize, Serialize}; @@ -31,14 +32,15 @@ use snafu::{Snafu, ensure}; #[serde(rename_all = "camelCase")] pub struct InterpolationParams { pub interpolation: InterpolationMethod, - pub input_resolution: InputResolution, + pub output_resolution: InterpolationResolution, + pub output_origin_reference: Option, } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] #[serde(rename_all = "camelCase", tag = "type")] -pub enum InputResolution { - Value(SpatialResolution), - Source, +pub enum InterpolationResolution { + Resolution(SpatialResolution), + Fraction { x: f64, y: f64 }, } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] @@ -51,10 +53,13 @@ pub enum InterpolationMethod { #[derive(Debug, Snafu)] #[snafu(visibility(pub(crate)), context(suffix(false)), module(error))] pub enum InterpolationError { - #[snafu(display( - "The input resolution was defined as `source` but the source resolution is unknown.", - ))] - UnknownInputResolution, + #[snafu(display("The fraction used to interpolate must be >= 1, was {f}."))] + FractionMustBeOneOrLarger { f: f64 }, + #[snafu(display("The output resolution must be higher than the input resolution."))] + OutputMustBeHigherResolutionThanInput { + input: SpatialResolution, + output: SpatialResolution, + }, } pub type Interpolation = Operator; @@ -78,56 +83,100 @@ impl RasterOperator for Interpolation { .initialize_sources(path.clone(), context) .await?; let raster_source = initialized_sources.raster; + InitializedInterpolation::new_with_source_and_params( + name, + path, + raster_source, + &self.params, + context.tiling_specification(), + ) + .map(InitializedRasterOperator::boxed) + } + + span_fn!(Interpolation); +} + +pub struct InitializedInterpolation { + name: CanonicOperatorName, + output_result_descriptor: RasterResultDescriptor, + raster_source: O, + path: WorkflowOperatorPath, + interpolation_method: InterpolationMethod, + tiling_specification: TilingSpecification, +} + +impl InitializedInterpolation { + pub fn new_with_source_and_params( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + raster_source: O, + params: &InterpolationParams, + tiling_specification: TilingSpecification, + ) -> Result { let in_descriptor = raster_source.result_descriptor(); + let in_spatial_grid = in_descriptor.spatial_grid_descriptor(); + + let output_resolution = match params.output_resolution { + InterpolationResolution::Resolution(res) => { + ensure!( + res.x.abs() <= in_spatial_grid.spatial_resolution().x.abs(), + error::OutputMustBeHigherResolutionThanInput { + input: in_spatial_grid.spatial_resolution(), + output: res + } + ); + ensure!( + res.y.abs() <= in_spatial_grid.spatial_resolution().y.abs(), + error::OutputMustBeHigherResolutionThanInput { + input: in_spatial_grid.spatial_resolution(), + output: res + } + ); + res + } - ensure!( - matches!(self.params.input_resolution, InputResolution::Value(_)) - || in_descriptor.resolution.is_some(), - error::UnknownInputResolution - ); + InterpolationResolution::Fraction { x, y } => { + ensure!(x >= 1.0, error::FractionMustBeOneOrLarger { f: x }); + ensure!(y >= 1.0, error::FractionMustBeOneOrLarger { f: y }); - let input_resolution = if let InputResolution::Value(res) = self.params.input_resolution { - res + SpatialResolution::new_unchecked( + in_spatial_grid.spatial_resolution().x / x, + in_spatial_grid.spatial_resolution().y.abs() / y, + ) + } + }; + + let out_spatial_grid = if let Some(oc) = params.output_origin_reference { + in_spatial_grid + .with_changed_resolution(output_resolution) + .with_moved_origin_to_nearest_grid_edge(oc) + .replace_origin(oc) } else { - in_descriptor.resolution.expect("checked in ensure") + in_spatial_grid.with_changed_resolution(output_resolution) }; let out_descriptor = RasterResultDescriptor { spatial_reference: in_descriptor.spatial_reference, data_type: in_descriptor.data_type, - bbox: in_descriptor.bbox, time: in_descriptor.time, - resolution: None, // after interpolation the resolution is uncapped + spatial_grid: out_spatial_grid, bands: in_descriptor.bands.clone(), }; let initialized_operator = InitializedInterpolation { name, path, - result_descriptor: out_descriptor, + output_result_descriptor: out_descriptor, raster_source, - interpolation_method: self.params.interpolation, - input_resolution, - tiling_specification: context.tiling_specification(), + interpolation_method: params.interpolation, + tiling_specification, }; - Ok(initialized_operator.boxed()) + Ok(initialized_operator) } - - span_fn!(Interpolation); -} - -pub struct InitializedInterpolation { - name: CanonicOperatorName, - path: WorkflowOperatorPath, - result_descriptor: RasterResultDescriptor, - raster_source: Box, - interpolation_method: InterpolationMethod, - input_resolution: SpatialResolution, - tiling_specification: TilingSpecification, } -impl InitializedRasterOperator for InitializedInterpolation { +impl InitializedRasterOperator for InitializedInterpolation { fn query_processor(&self) -> Result { let source_processor = self.raster_source.query_processor()?; @@ -135,15 +184,13 @@ impl InitializedRasterOperator for InitializedInterpolation { source_processor, p => match self.interpolation_method { InterpolationMethod::NearestNeighbor => InterploationProcessor::<_,_, NearestNeighbor>::new( p, - self.result_descriptor.clone(), - self.input_resolution, + self.output_result_descriptor.clone(), self.tiling_specification, ).boxed() .into(), InterpolationMethod::BiLinear =>InterploationProcessor::<_,_, Bilinear>::new( p, - self.result_descriptor.clone(), - self.input_resolution, + self.output_result_descriptor.clone(), self.tiling_specification, ).boxed() .into(), @@ -154,7 +201,7 @@ impl InitializedRasterOperator for InitializedInterpolation { } fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor + &self.output_result_descriptor } fn canonic_name(&self) -> CanonicOperatorName { @@ -174,11 +221,10 @@ pub struct InterploationProcessor where Q: RasterQueryProcessor, P: Pixel, - I: InterpolationAlgorithm

, + I: InterpolationAlgorithm, { source: Q, - result_descriptor: RasterResultDescriptor, - input_resolution: SpatialResolution, + out_result_descriptor: RasterResultDescriptor, tiling_specification: TilingSpecification, interpolation: PhantomData, } @@ -187,18 +233,16 @@ impl InterploationProcessor where Q: RasterQueryProcessor, P: Pixel, - I: InterpolationAlgorithm

, + I: InterpolationAlgorithm, { pub fn new( source: Q, - result_descriptor: RasterResultDescriptor, - input_resolution: SpatialResolution, + out_result_descriptor: RasterResultDescriptor, tiling_specification: TilingSpecification, ) -> Self { Self { source, - result_descriptor, - input_resolution, + out_result_descriptor, tiling_specification, interpolation: PhantomData, } @@ -210,15 +254,15 @@ impl QueryProcessor for InterploationProcessor where Q: QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, P: Pixel, - I: InterpolationAlgorithm

, + I: InterpolationAlgorithm, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -228,15 +272,31 @@ where ctx: &'a dyn QueryContext, ) -> Result>> { // do not interpolate if the source resolution is already fine enough - if query.spatial_resolution.x >= self.input_resolution.x - && query.spatial_resolution.y >= self.input_resolution.y - { - // TODO: should we use the query or the input resolution here? + + let in_spatial_grid = self.source.result_descriptor().spatial_grid_descriptor(); + let out_spatial_grid = self.result_descriptor().spatial_grid_descriptor(); + + // if the output resolution is the same as the input resolution, we can just forward the query // TODO: except the origin changes? + if in_spatial_grid == out_spatial_grid { return self.source.query(query, ctx).await; } + let tiling_grid_definition = + out_spatial_grid.tiling_grid_definition(ctx.tiling_specification()); + + // This is the tiling strategy we want to fill + let tiling_strategy: geoengine_datatypes::raster::TilingStrategy = + tiling_grid_definition.generate_data_tiling_strategy(); + + let input_geo_transform = in_spatial_grid + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(); + + let output_geo_transform = tiling_grid_definition.tiling_geo_transform(); + let sub_query = InterpolationSubQuery::<_, P, I> { - input_resolution: self.input_resolution, + input_geo_transform, + output_geo_transform, fold_fn: fold_future, tiling_specification: self.tiling_specification, phantom: PhantomData, @@ -246,7 +306,7 @@ where Ok(RasterSubQueryAdapter::<'a, P, _, _>::new( &self.source, query, - self.tiling_specification, + tiling_strategy, ctx, sub_query, ) @@ -256,13 +316,14 @@ where } fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor + &self.out_result_descriptor } } #[derive(Debug, Clone)] pub struct InterpolationSubQuery { - input_resolution: SpatialResolution, + input_geo_transform: GeoTransform, + output_geo_transform: GeoTransform, // TODO remove because in adapter? fold_fn: F, tiling_specification: TilingSpecification, phantom: PhantomData, @@ -274,7 +335,7 @@ where T: Pixel, FoldM: Send + Sync + 'a + Clone + Fn(InterpolationAccu, RasterTile2D) -> FoldF, FoldF: Send + TryFuture, Error = crate::error::Error>, - I: InterpolationAlgorithm, + I: InterpolationAlgorithm, { type FoldFuture = FoldF; @@ -290,6 +351,8 @@ where pool: &Arc, ) -> Self::TileAccuFuture { create_accu( + self.input_geo_transform, + self.output_geo_transform, tile_info, &query_rect, pool.clone(), @@ -306,19 +369,31 @@ where band_idx: u32, ) -> Result> { // enlarge the spatial bounds in order to have the neighbor pixels for the interpolation - let spatial_bounds = tile_info.spatial_partition(); - let enlarge: Coordinate2D = (self.input_resolution.x, -self.input_resolution.y).into(); - let spatial_bounds = SpatialPartition2D::new( - spatial_bounds.upper_left(), - spatial_bounds.lower_right() + enlarge, - )?; - - Ok(Some(RasterQueryRectangle { - spatial_bounds, - time_interval: TimeInterval::new_instant(start_time)?, - spatial_resolution: self.input_resolution, - attributes: band_idx.into(), - })) + + let tile_pixel_bounds = tile_info.global_pixel_bounds(); + let tile_spatial_bounds = self + .output_geo_transform + .grid_to_spatial_bounds(&tile_pixel_bounds); + let input_pixel_bounds = self + .input_geo_transform + .spatial_to_grid_bounds(&tile_spatial_bounds); + let enlarged_input_pixel_bounds = GridBoundingBox2D::new( + [ + input_pixel_bounds.y_min() - 1, + input_pixel_bounds.x_min() - 1, + ], + [ + input_pixel_bounds.y_max() + 1, + input_pixel_bounds.x_max() + 1, + ], + ) + .expect("max bounds must be larger then min bounds already"); + + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + enlarged_input_pixel_bounds, + TimeInterval::new_instant(start_time)?, + BandSelection::new_single(band_idx), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -327,21 +402,30 @@ where } #[derive(Clone, Debug)] -pub struct InterpolationAccu> { +pub struct InterpolationAccu> { pub output_info: TileInformation, - pub input_tile: RasterTile2D, + pub input_tile: GridOrEmpty, + pub input_geo_transform: GeoTransform, + pub time: TimeInterval, + pub cache_hint: CacheHint, pub pool: Arc, phantom: PhantomData, } -impl> InterpolationAccu { +impl> InterpolationAccu { pub fn new( - input_tile: RasterTile2D, + input_tile: GridOrEmpty, + input_geo_transform: GeoTransform, + time: TimeInterval, + cache_hint: CacheHint, output_info: TileInformation, pool: Arc, ) -> Self { InterpolationAccu { input_tile, + input_geo_transform, + time, + cache_hint, output_info, pool, phantom: Default::default(), @@ -350,17 +434,32 @@ impl> InterpolationAccu { } #[async_trait] -impl> FoldTileAccu for InterpolationAccu { +impl> FoldTileAccu + for InterpolationAccu +{ type RasterType = T; async fn into_tile(self) -> Result> { // now that we collected all the input tile pixels we perform the actual interpolation let output_tile = crate::util::spawn_blocking_with_thread_pool(self.pool, move || { - I::interpolate(&self.input_tile, &self.output_info) + I::interpolate( + self.input_geo_transform, + &self.input_tile, + self.output_info.global_geo_transform, + self.output_info.global_pixel_bounds(), + ) }) .await??; + let output_tile = RasterTile2D::new_with_tile_info( + self.time, + self.output_info, + 0, + output_tile.unbounded(), + self.cache_hint, + ); + Ok(output_tile) } @@ -369,59 +468,58 @@ impl> FoldTileAccu for InterpolationAccu< } } -impl> FoldTileAccuMut for InterpolationAccu { - fn tile_mut(&mut self) -> &mut RasterTile2D { - &mut self.input_tile +impl> FoldTileAccuMut + for InterpolationAccu +{ + fn set_time(&mut self, time: TimeInterval) { + self.time = time; + } + + fn set_cache_hint(&mut self, cache_hint: CacheHint) { + self.cache_hint = cache_hint; } } -pub fn create_accu>( +pub fn create_accu>( + input_geo_transform: GeoTransform, + output_geo_transform: GeoTransform, tile_info: TileInformation, query_rect: &RasterQueryRectangle, pool: Arc, - tiling_specification: TilingSpecification, + _tiling_specification: TilingSpecification, ) -> impl Future>> + use { + let query_rect = query_rect.clone(); + // create an accumulator as a single tile that fits all the input tiles - let spatial_bounds = query_rect.spatial_bounds; - let spatial_resolution = query_rect.spatial_resolution; let time_interval = query_rect.time_interval; crate::util::spawn_blocking(move || { - let tiling = tiling_specification.strategy(spatial_resolution.x, -spatial_resolution.y); - - let origin_coordinate = tiling - .tile_information_iterator(spatial_bounds) - .next() - .expect("a query contains at least one tile") - .spatial_partition() - .upper_left(); - - let geo_transform = GeoTransform::new( - origin_coordinate, - spatial_resolution.x, - -spatial_resolution.y, - ); - - let bbox = tiling.tile_grid_box(spatial_bounds); - - let shape = [ - bbox.axis_size_y() * tiling.tile_size_in_pixels.axis_size_y(), - bbox.axis_size_x() * tiling.tile_size_in_pixels.axis_size_x(), - ]; + let tile_pixel_bounds = tile_info.global_pixel_bounds(); + let tile_spatial_bounds = output_geo_transform.grid_to_spatial_bounds(&tile_pixel_bounds); + let input_pixel_bounds = input_geo_transform.spatial_to_grid_bounds(&tile_spatial_bounds); + let enlarged_input_pixel_bounds = GridBoundingBox2D::new( + [ + input_pixel_bounds.y_min() - 1, + input_pixel_bounds.x_min() - 1, + ], + [ + input_pixel_bounds.y_max() + 1, + input_pixel_bounds.x_max() + 1, + ], + ) + .expect("max bounds must be larger then min bounds already"); // create a non-aligned (w.r.t. the tiling specification) grid by setting the origin to the top-left of the tile and the tile-index to [0, 0] - let grid = EmptyGrid2D::new(shape.into()); + let grid = GridOrEmpty::new_empty_shape(enlarged_input_pixel_bounds); - let input_tile = RasterTile2D::new( + InterpolationAccu::new( + grid, + input_geo_transform, time_interval, - [0, 0].into(), - 0, - geo_transform, - GridOrEmpty::from(grid), CacheHint::max_duration(), - ); - - InterpolationAccu::new(input_tile, tile_info, pool) + tile_info, + pool, + ) }) .map_err(From::from) } @@ -432,11 +530,11 @@ pub fn fold_future( ) -> impl Future>> where T: Pixel, - I: InterpolationAlgorithm, + I: InterpolationAlgorithm, { crate::util::spawn_blocking(|| fold_impl(accu, tile)).then(|x| async move { match x { - Ok(r) => r, + Ok(r) => Ok(r), Err(e) => Err(e.into()), } }) @@ -445,25 +543,23 @@ where pub fn fold_impl( mut accu: InterpolationAccu, tile: RasterTile2D, -) -> Result> +) -> InterpolationAccu where T: Pixel, - I: InterpolationAlgorithm, + I: InterpolationAlgorithm, { // get the time now because it is not known when the accu was created - accu.input_tile.time = tile.time; + accu.set_time(tile.time); + accu.cache_hint.merge_with(&tile.cache_hint); // TODO: add a skip if both tiles are empty? // copy all input tiles into the accu to have all data for interpolation - let mut accu_input_tile = accu.input_tile.into_materialized_tile(); - accu_input_tile.blit(tile)?; - - Ok(InterpolationAccu::new( - accu_input_tile.into(), - accu.output_info, - accu.pool, - )) + let in_tile = &tile.into_inner_positioned_grid(); + + accu.input_tile.grid_blit_from(in_tile); + + accu } #[cfg(test)] @@ -471,7 +567,7 @@ mod tests { use super::*; use futures::StreamExt; use geoengine_datatypes::{ - primitives::{RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::{Coordinate2D, RasterQueryRectangle, SpatialResolution, TimeInterval}, raster::{ Grid2D, GridOrEmpty, RasterDataType, RasterTile2D, RenameBands, TileInformation, TilingSpecification, @@ -483,7 +579,7 @@ mod tests { use crate::{ engine::{ MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{RasterStacker, RasterStackerParams}, @@ -491,17 +587,34 @@ mod tests { #[tokio::test] async fn nearest_neighbor_operator() -> Result<()> { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [2, 2].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); + + // test raster: + // [0, 10) + // || 1 | 2 || 3 | 4 || + // || 5 | 6 || 7 | 8 || + // + // [10, 20) + // || 8 | 7 || 6 | 5 || + // || 4 | 3 || 2 | 1 || + + // exptected raster: + // [0, 10) + // ||1 | 1 || 2 | 2 || + // ||1 | 1 || 2 | 2 || + // ||5 | 5 || 6 | 6 || + // ||5 | 5 || 6 | 6 || let raster = make_raster(CacheHint::max_duration()); let operator = Interpolation { params: InterpolationParams { interpolation: InterpolationMethod::NearestNeighbor, - input_resolution: InputResolution::Value(SpatialResolution::one()), + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::zero_point_five(), + ), + output_origin_reference: None, }, sources: SingleRasterSource { raster }, } @@ -511,13 +624,12 @@ mod tests { let processor = operator.query_processor()?.get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 2.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-4, 0], [-1, 7]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); let result_stream = processor.query(query_rect, &query_ctx).await?; @@ -528,48 +640,48 @@ mod tests { times.append(&mut vec![TimeInterval::new_unchecked(10, 20); 8]); let data = vec![ - vec![1, 2, 5, 6], - vec![2, 3, 6, 7], - vec![3, 4, 7, 8], - vec![4, 0, 8, 0], - vec![5, 6, 0, 0], - vec![6, 7, 0, 0], - vec![7, 8, 0, 0], - vec![8, 0, 0, 0], - vec![8, 7, 4, 3], - vec![7, 6, 3, 2], - vec![6, 5, 2, 1], - vec![5, 0, 1, 0], - vec![4, 3, 0, 0], - vec![3, 2, 0, 0], - vec![2, 1, 0, 0], - vec![1, 0, 0, 0], + vec![1; 4], + vec![2; 4], + vec![3; 4], + vec![4; 4], + vec![5; 4], + vec![6; 4], + vec![7; 4], + vec![8; 4], + vec![8; 4], + vec![7; 4], + vec![6; 4], + vec![5; 4], + vec![4; 4], + vec![3; 4], + vec![2; 4], + vec![1; 4], ]; let valid = vec![ vec![true; 4], vec![true; 4], vec![true; 4], - vec![true, false, true, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, false, false, false], vec![true; 4], vec![true; 4], vec![true; 4], - vec![true, false, true, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, false, false, false], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], ]; for (i, tile) in result.into_iter().enumerate() { let tile = tile.into_materialized_tile(); assert_eq!(tile.time, times[i]); - assert_eq!(tile.grid_array.inner_grid.data, data[i]); assert_eq!(tile.grid_array.validity_mask.data, valid[i]); + assert_eq!(tile.grid_array.inner_grid.data, data[i]); } Ok(()) @@ -638,8 +750,10 @@ mod tests { data_type: RasterDataType::I8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1.0, -1.0), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -649,10 +763,8 @@ mod tests { #[tokio::test] async fn it_attaches_cache_hint() -> Result<()> { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [2, 2].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let cache_hint = CacheHint::seconds(1234); let raster = make_raster(cache_hint); @@ -660,7 +772,10 @@ mod tests { let operator = Interpolation { params: InterpolationParams { interpolation: InterpolationMethod::NearestNeighbor, - input_resolution: InputResolution::Value(SpatialResolution::one()), + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::zero_point_five(), + ), + output_origin_reference: None, }, sources: SingleRasterSource { raster }, } @@ -670,12 +785,11 @@ mod tests { let processor = operator.query_processor()?.get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 2.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let result_stream = processor.query(query_rect, &query_ctx).await?; @@ -694,15 +808,15 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_interpolates_multiple_bands() -> Result<()> { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [2, 2].into(), - )); - + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let operator = Interpolation { params: InterpolationParams { interpolation: InterpolationMethod::NearestNeighbor, - input_resolution: InputResolution::Value(SpatialResolution::one()), + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::zero_point_five(), + ), + output_origin_reference: None, }, sources: SingleRasterSource { raster: RasterStacker { @@ -725,13 +839,12 @@ mod tests { let processor = operator.query_processor()?.get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 2.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: [0, 1].try_into().unwrap(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-4, 0], [-1, 7]).unwrap(), + TimeInterval::new_unchecked(0, 20), + [0, 1].try_into().unwrap(), + ); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); let result_stream = processor.query(query_rect, &query_ctx).await?; @@ -749,48 +862,50 @@ mod tests { .collect::>(); let data = vec![ - vec![1, 2, 5, 6], - vec![2, 3, 6, 7], - vec![3, 4, 7, 8], - vec![4, 0, 8, 0], - vec![5, 6, 0, 0], - vec![6, 7, 0, 0], - vec![7, 8, 0, 0], - vec![8, 0, 0, 0], - vec![8, 7, 4, 3], - vec![7, 6, 3, 2], - vec![6, 5, 2, 1], - vec![5, 0, 1, 0], - vec![4, 3, 0, 0], - vec![3, 2, 0, 0], - vec![2, 1, 0, 0], - vec![1, 0, 0, 0], + vec![1; 4], + vec![2; 4], + vec![3; 4], + vec![4; 4], + vec![5; 4], + vec![6; 4], + vec![7; 4], + vec![8; 4], + vec![8; 4], + vec![7; 4], + vec![6; 4], + vec![5; 4], + vec![4; 4], + vec![3; 4], + vec![2; 4], + vec![1; 4], ]; - let data = data - .clone() - .into_iter() - .zip(data) - .flat_map(|(a, b)| vec![a, b]) - .collect::>(); let valid = vec![ vec![true; 4], vec![true; 4], vec![true; 4], - vec![true, false, true, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, false, false, false], vec![true; 4], vec![true; 4], vec![true; 4], - vec![true, false, true, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, false, false, false], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], ]; + + let data = data + .clone() + .into_iter() + .zip(data) + .flat_map(|(a, b)| vec![a, b]) + .collect::>(); + let valid = valid .clone() .into_iter() diff --git a/operators/src/processing/line_simplification.rs b/operators/src/processing/line_simplification.rs index aa389f947..e8ff30032 100644 --- a/operators/src/processing/line_simplification.rs +++ b/operators/src/processing/line_simplification.rs @@ -15,8 +15,8 @@ use geoengine_datatypes::{ }, error::{BoxedResultExt, ErrorSource}, primitives::{ - BoundingBox2D, ColumnSelection, Geometry, MultiLineString, MultiLineStringRef, - MultiPolygon, MultiPolygonRef, SpatialResolution, VectorQueryRectangle, + ColumnSelection, Geometry, MultiLineString, MultiLineStringRef, MultiPolygon, + MultiPolygonRef, SpatialResolution, VectorQueryRectangle, VectorSpatialQueryRectangle, }, util::arrow::ArrowTyped, }; @@ -43,12 +43,16 @@ impl VectorOperator for LineSimplification { path: WorkflowOperatorPath, context: &dyn ExecutionContext, ) -> Result> { - if self - .params - .epsilon - .map_or(false, |e| !e.is_finite() || e <= 0.0) - { - return Err(LineSimplificationError::InvalidEpsilon.into()); + match self.params.epsilon { + EpsilonOrResolution::Epsilon(epsilon) if epsilon <= 0.0 || !epsilon.is_finite() => { + return Err(LineSimplificationError::InvalidEpsilon.into()); + } + EpsilonOrResolution::Epsilon(_eps) => { + //TODO: do something here... + } + EpsilonOrResolution::Resolution(_res) => { + // TODO: validate resolution + } } let name = CanonicOperatorName::from(&self); @@ -80,13 +84,20 @@ impl VectorOperator for LineSimplification { span_fn!(LineSimplification); } +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub enum EpsilonOrResolution { + Epsilon(f64), + Resolution(SpatialResolution), +} + #[derive(Debug, Serialize, Deserialize, Clone, Copy)] #[serde(rename_all = "camelCase")] pub struct LineSimplificationParams { pub algorithm: LineSimplificationAlgorithm, /// The epsilon parameter is used to determine the maximum distance between the original and the simplified geometry. - /// If `None` is provided, the epsilon is derived by the query's [`SpatialResolution`]. - pub epsilon: Option, + /// As alternative, epsilon can be derived from a provided [`SpatialResolution`]. + pub epsilon: EpsilonOrResolution, } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] @@ -102,7 +113,7 @@ pub struct InitializedLineSimplification { result_descriptor: VectorResultDescriptor, source: Box, algorithm: LineSimplificationAlgorithm, - epsilon: Option, + epsilon: EpsilonOrResolution, } impl InitializedVectorOperator for InitializedLineSimplification { @@ -188,7 +199,7 @@ where { source: P, _algorithm: A, - epsilon: Option, + epsilon: EpsilonOrResolution, } pub trait LineSimplificationAlgorithmImpl: Send + Sync { @@ -286,7 +297,7 @@ impl QueryProcessor for LineSimplificationProcessor where P: QueryProcessor< Output = FeatureCollection, - SpatialBounds = BoundingBox2D, + SpatialQuery = VectorSpatialQueryRectangle, Selection = ColumnSelection, ResultDescription = VectorResultDescriptor, >, @@ -298,7 +309,7 @@ where >, { type Output = FeatureCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -309,9 +320,13 @@ where ) -> Result>> { let chunks = self.source.query(query.clone(), ctx).await?; - let epsilon = self - .epsilon - .unwrap_or_else(|| A::derive_epsilon(query.spatial_resolution)); + let epsilon = match self.epsilon { + EpsilonOrResolution::Epsilon(epsilon) if epsilon <= 0.0 || !epsilon.is_finite() => { + return Err(LineSimplificationError::InvalidEpsilon.into()); + } + EpsilonOrResolution::Epsilon(e) => e, + EpsilonOrResolution::Resolution(res) => A::derive_epsilon(res), + }; let simplified_chunks = chunks.and_then(move |chunk| async move { crate::util::spawn_blocking_with_thread_pool(ctx.thread_pool().clone(), move || { @@ -360,7 +375,8 @@ mod tests { }, dataset::{DataId, DatasetId, NamedData}, primitives::{ - FeatureData, MultiLineString, MultiPoint, TimeInterval, {CacheHint, CacheTtlSeconds}, + BoundingBox2D, CacheHint, CacheTtlSeconds, FeatureData, MultiLineString, MultiPoint, + TimeInterval, }, spatial_reference::SpatialReference, test_data, @@ -371,7 +387,7 @@ mod tests { async fn test_ser_de() { let operator = LineSimplification { params: LineSimplificationParams { - epsilon: Some(1.0), + epsilon: EpsilonOrResolution::Epsilon(1.0), algorithm: LineSimplificationAlgorithm::DouglasPeucker, }, sources: MockFeatureCollectionSource::::multiple(vec![]) @@ -387,7 +403,9 @@ mod tests { serde_json::json!({ "type": "LineSimplification", "params": { - "epsilon": 1.0, + "epsilon": { + "epsilon": 1.0 + }, "algorithm": "douglasPeucker", }, "sources": { @@ -412,7 +430,7 @@ mod tests { assert!( LineSimplification { params: LineSimplificationParams { - epsilon: Some(0.0), + epsilon: EpsilonOrResolution::Epsilon(0.0), algorithm: LineSimplificationAlgorithm::DouglasPeucker, }, sources: MockFeatureCollectionSource::::single( @@ -434,7 +452,7 @@ mod tests { assert!( LineSimplification { params: LineSimplificationParams { - epsilon: Some(f64::NAN), + epsilon: EpsilonOrResolution::Epsilon(f64::NAN), algorithm: LineSimplificationAlgorithm::Visvalingam, }, sources: MockFeatureCollectionSource::::single( @@ -456,7 +474,7 @@ mod tests { assert!( LineSimplification { params: LineSimplificationParams { - epsilon: None, + epsilon: EpsilonOrResolution::Epsilon(0.1), algorithm: LineSimplificationAlgorithm::DouglasPeucker, }, sources: MockFeatureCollectionSource::::single( @@ -502,7 +520,7 @@ mod tests { let simplification = LineSimplification { params: LineSimplificationParams { - epsilon: Some(1.0), + epsilon: EpsilonOrResolution::Epsilon(1.0), algorithm: LineSimplificationAlgorithm::DouglasPeucker, }, sources: source.into(), @@ -523,12 +541,11 @@ mod tests { .multi_line_string() .unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query_ctx = MockQueryContext::test_default(); @@ -608,7 +625,7 @@ mod tests { let simplification = LineSimplification { params: LineSimplificationParams { - epsilon: None, + epsilon: EpsilonOrResolution::Resolution(SpatialResolution::new(1., 1.).unwrap()), algorithm: LineSimplificationAlgorithm::Visvalingam, }, sources: OgrSource { @@ -642,12 +659,11 @@ mod tests { let query_context = MockQueryContext::test_default(); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &query_context, ) .await diff --git a/operators/src/processing/map_query.rs b/operators/src/processing/map_query.rs index 3d5e0097d..f2c987bb3 100644 --- a/operators/src/processing/map_query.rs +++ b/operators/src/processing/map_query.rs @@ -52,13 +52,18 @@ where log::debug!("Query was rewritten to empty query. Returning empty / filled stream."); let s = futures::stream::empty(); + let res_desc = self.raster_result_descriptor(); + let tiling_grid_def = res_desc.tiling_grid_definition(self.additional_data); + + let strat = tiling_grid_def.generate_data_tiling_strategy(); + // TODO: The input of the `SparseTilesFillAdapter` is empty here, so we can't derive the expiration, as there are no tiles to derive them from. // As this is the result of the query not being rewritten, we should check if the expiration could also be `max`, because this error // will be persistent and we might as well cache the empty stream. Ok(SparseTilesFillAdapter::new_like_subquery( s, &query, - self.additional_data, + strat, FillerTileCacheExpirationStrategy::NoCache, FillerTimeBounds::from(query.time_interval), // TODO: derive this from the query once the child query can provide this. ) diff --git a/operators/src/processing/meteosat/mod.rs b/operators/src/processing/meteosat/mod.rs index 246f0f7df..9b5826d8d 100644 --- a/operators/src/processing/meteosat/mod.rs +++ b/operators/src/processing/meteosat/mod.rs @@ -38,27 +38,29 @@ mod test_util { use std::str::FromStr; use futures::StreamExt; - use geoengine_datatypes::hashmap; - use geoengine_datatypes::primitives::{BandSelection, CacheHint, CacheTtlSeconds}; - use geoengine_datatypes::util::test::TestDefault; - use num_traits::AsPrimitive; - use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; + use geoengine_datatypes::hashmap; + use geoengine_datatypes::primitives::{ + BandSelection, CacheHint, CacheTtlSeconds, Coordinate2D, + }; use geoengine_datatypes::primitives::{ ContinuousMeasurement, DateTime, DateTimeParseFormat, Measurement, RasterQueryRectangle, - SpatialPartition2D, SpatialResolution, TimeGranularity, TimeInstance, TimeInterval, - TimeStep, + TimeGranularity, TimeInstance, TimeInterval, TimeStep, }; use geoengine_datatypes::raster::{ - Grid2D, GridOrEmpty, GridOrEmpty2D, MaskedGrid2D, Pixel, RasterDataType, RasterProperties, - RasterPropertiesEntry, RasterPropertiesEntryType, RasterTile2D, TileInformation, + BoundedGrid, GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty, GridOrEmpty2D, + GridShape2D, MaskedGrid2D, Pixel, RasterDataType, RasterProperties, RasterPropertiesEntry, + RasterPropertiesEntryType, RasterTile2D, TileInformation, }; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority}; use geoengine_datatypes::util::Identifier; + use geoengine_datatypes::util::test::TestDefault; + use num_traits::AsPrimitive; use crate::engine::{ MockExecutionContext, MockQueryContext, QueryProcessor, RasterBandDescriptor, - RasterBandDescriptors, RasterOperator, RasterResultDescriptor, WorkflowOperatorPath, + RasterBandDescriptors, RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, + WorkflowOperatorPath, }; use crate::mock::{MockRasterSource, MockRasterSourceParams}; use crate::processing::meteosat::{ @@ -122,27 +124,22 @@ mod test_util { } pub(crate) fn _create_gdal_query() -> RasterQueryRectangle { - let sr = SpatialResolution::new_unchecked(3_000.403_165_817_261, 3_000.403_165_817_261); - let ul = (0., 0.).into(); - let lr = (599. * sr.x, -599. * sr.y).into(); - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked(ul, lr), - time_interval: TimeInterval::new_unchecked( + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(0, 599, 0, 599).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 0, 0)), TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 15, 0)), ), - spatial_resolution: sr, - attributes: BandSelection::first(), - } + BandSelection::first(), + ) } pub(crate) fn create_mock_query() -> RasterQueryRectangle { - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (2., 0.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - } + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-3, -1, 0, 1).unwrap(), + Default::default(), + BandSelection::first(), + ) } pub(crate) fn create_mock_source( @@ -192,8 +189,10 @@ mod test_util { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., -3.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [0, 2]).unwrap(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), measurement.unwrap_or_else(|| { @@ -214,6 +213,11 @@ mod test_util { let dataset_name = NamedData::with_system_name("gdal-ds"); let no_data_value = Some(0.); + let origin_coordinate: Coordinate2D = + (-5_570_248.477_339_745, 5_570_248.477_339_745).into(); + let x_pixel_size = 3_000.403_165_817_261; + let y_pixel_size = -3_000.403_165_817_261; + let meta = GdalMetaDataRegular { data_time: TimeInterval::new_unchecked( TimeInstance::from_str("2012-12-12T12:00:00.000Z").unwrap(), @@ -233,9 +237,9 @@ mod test_util { file_path: test_data!("raster/msg/%_START_TIME_%.tif").into(), rasterband_channel: 1, geo_transform: GdalDatasetGeoTransform { - origin_coordinate: (-5_570_248.477_339_745, 5_570_248.477_339_745).into(), - x_pixel_size: 3_000.403_165_817_261, - y_pixel_size: -3_000.403_165_817_261, + origin_coordinate, + x_pixel_size, + y_pixel_size, }, width: 3712, height: 3712, @@ -269,8 +273,10 @@ mod test_util { spatial_reference: SpatialReference::new(SpatialReferenceAuthority::SrOrg, 81) .into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(origin_coordinate, x_pixel_size, y_pixel_size), + GridShape2D::new_2d(3712, 3712).bounding_box(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), Measurement::Continuous(ContinuousMeasurement { @@ -285,7 +291,7 @@ mod test_util { ctx.add_meta_data(dataset_id, dataset_name.clone(), Box::new(meta)); GdalSource { - params: GdalSourceParameters { data: dataset_name }, + params: GdalSourceParameters::new(dataset_name), } } } diff --git a/operators/src/processing/meteosat/radiance.rs b/operators/src/processing/meteosat/radiance.rs index db3dbae9e..a51565dd2 100644 --- a/operators/src/processing/meteosat/radiance.rs +++ b/operators/src/processing/meteosat/radiance.rs @@ -12,7 +12,7 @@ use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::primitives::{ BandSelection, ClassificationMeasurement, ContinuousMeasurement, Measurement, - RasterQueryRectangle, SpatialPartition2D, + RasterQueryRectangle, RasterSpatialQueryRectangle, }; use geoengine_datatypes::raster::{ MapElementsParallel, Pixel, RasterDataType, RasterPropertiesKey, RasterTile2D, @@ -112,8 +112,7 @@ impl RasterOperator for Radiance { spatial_reference: in_desc.spatial_reference, data_type: RasterOut, time: in_desc.time, - bbox: in_desc.bbox, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: RasterBandDescriptors::new( in_desc .bands @@ -250,14 +249,14 @@ impl QueryProcessor for RadianceProcessor where Q: QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, P: Pixel, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -285,7 +284,7 @@ mod tests { ClassificationMeasurement, ContinuousMeasurement, Measurement, }; use geoengine_datatypes::raster::{EmptyGrid2D, Grid2D, MaskedGrid2D, TilingSpecification}; - use std::collections::HashMap; + use std::collections::BTreeMap; // #[tokio::test] // async fn test_msg_raster() { @@ -312,7 +311,6 @@ mod tests { async fn test_ok() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -351,7 +349,6 @@ mod tests { async fn test_empty_raster() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -387,7 +384,6 @@ mod tests { async fn test_missing_offset() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -415,7 +411,6 @@ mod tests { async fn test_missing_slope() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -444,7 +439,6 @@ mod tests { async fn test_invalid_measurement_unitless() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -474,7 +468,6 @@ mod tests { async fn test_invalid_measurement_continuous() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -511,7 +504,6 @@ mod tests { async fn test_invalid_measurement_classification() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -525,7 +517,7 @@ mod tests { None, Some(Measurement::Classification(ClassificationMeasurement { measurement: "invalid".into(), - classes: HashMap::new(), + classes: BTreeMap::new(), })), ); diff --git a/operators/src/processing/meteosat/reflectance.rs b/operators/src/processing/meteosat/reflectance.rs index 1aa12a99c..df179bd67 100644 --- a/operators/src/processing/meteosat/reflectance.rs +++ b/operators/src/processing/meteosat/reflectance.rs @@ -17,7 +17,7 @@ use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::primitives::{ BandSelection, ClassificationMeasurement, ContinuousMeasurement, DateTime, Measurement, - RasterQueryRectangle, SpatialPartition2D, + RasterQueryRectangle, RasterSpatialQueryRectangle, }; use geoengine_datatypes::raster::{ GridIdx2D, MapIndexedElementsParallel, RasterDataType, RasterPropertiesKey, RasterTile2D, @@ -117,8 +117,7 @@ impl RasterOperator for Reflectance { spatial_reference: in_desc.spatial_reference, data_type: RasterOut, time: in_desc.time, - bbox: in_desc.bbox, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: RasterBandDescriptors::new( in_desc .bands @@ -298,13 +297,13 @@ impl QueryProcessor for ReflectanceProcessor where Q: QueryProcessor< Output = RasterTile2D, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -335,7 +334,7 @@ mod tests { use geoengine_datatypes::raster::{ EmptyGrid2D, Grid2D, GridOrEmpty, MaskedGrid2D, RasterTile2D, TilingSpecification, }; - use std::collections::HashMap; + use std::collections::BTreeMap; async fn process_mock( params: ReflectanceParams, @@ -346,7 +345,6 @@ mod tests { ) -> Result> { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -617,7 +615,7 @@ mod tests { false, Some(Measurement::Classification(ClassificationMeasurement { measurement: "invalid".into(), - classes: HashMap::new(), + classes: BTreeMap::new(), })), ) .await; diff --git a/operators/src/processing/meteosat/temperature.rs b/operators/src/processing/meteosat/temperature.rs index e693f1bab..23665a191 100644 --- a/operators/src/processing/meteosat/temperature.rs +++ b/operators/src/processing/meteosat/temperature.rs @@ -1,28 +1,25 @@ -use std::sync::Arc; - use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, OperatorName, QueryContext, QueryProcessor, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, }; +use crate::error::Error; use crate::util::Result; -use async_trait::async_trait; -use rayon::ThreadPool; - use TypedRasterQueryProcessor::F32 as QueryProcessorOut; - -use crate::error::Error; +use async_trait::async_trait; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::primitives::{ BandSelection, ClassificationMeasurement, ContinuousMeasurement, Measurement, - RasterQueryRectangle, SpatialPartition2D, + RasterQueryRectangle, RasterSpatialQueryRectangle, }; use geoengine_datatypes::raster::{ MapElementsParallel, Pixel, RasterDataType, RasterPropertiesKey, RasterTile2D, }; +use rayon::ThreadPool; use serde::{Deserialize, Serialize}; +use std::sync::Arc; // Output type is always f32 type PixelOut = f32; @@ -112,8 +109,7 @@ impl RasterOperator for Temperature { spatial_reference: in_desc.spatial_reference, data_type: RasterOut, time: in_desc.time, - bbox: in_desc.bbox, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: RasterBandDescriptors::new( in_desc .bands @@ -297,14 +293,14 @@ impl QueryProcessor for TemperatureProcessor where Q: QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, P: Pixel, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -332,7 +328,7 @@ mod tests { ClassificationMeasurement, ContinuousMeasurement, Measurement, }; use geoengine_datatypes::raster::{EmptyGrid2D, Grid2D, MaskedGrid2D, TilingSpecification}; - use std::collections::HashMap; + use std::collections::BTreeMap; // #[tokio::test] // async fn test_msg_raster() { @@ -357,7 +353,7 @@ mod tests { #[tokio::test] async fn test_empty_ok() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -390,7 +386,7 @@ mod tests { #[tokio::test] async fn test_ok() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -432,7 +428,7 @@ mod tests { #[tokio::test] async fn test_ok_force_satellite() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -472,7 +468,7 @@ mod tests { #[tokio::test] async fn test_ok_illegal_input_to_masked() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -522,7 +518,7 @@ mod tests { #[tokio::test] async fn test_invalid_force_satellite() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -548,7 +544,7 @@ mod tests { #[tokio::test] async fn test_missing_satellite() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -572,7 +568,7 @@ mod tests { #[tokio::test] async fn test_invalid_satellite() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -596,7 +592,7 @@ mod tests { #[tokio::test] async fn test_missing_channel() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -620,7 +616,7 @@ mod tests { #[tokio::test] async fn test_invalid_channel() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -644,7 +640,7 @@ mod tests { #[tokio::test] async fn test_missing_slope() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -668,7 +664,7 @@ mod tests { #[tokio::test] async fn test_missing_offset() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -692,7 +688,7 @@ mod tests { #[tokio::test] async fn test_invalid_measurement_unitless() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -717,7 +713,7 @@ mod tests { #[tokio::test] async fn test_invalid_measurement_continuous() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -749,7 +745,7 @@ mod tests { #[tokio::test] async fn test_invalid_measurement_classification() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -761,7 +757,7 @@ mod tests { None, Some(Measurement::Classification(ClassificationMeasurement { measurement: "invalid".into(), - classes: HashMap::new(), + classes: BTreeMap::new(), })), ); diff --git a/operators/src/processing/mod.rs b/operators/src/processing/mod.rs index d80848b55..a85b622e6 100644 --- a/operators/src/processing/mod.rs +++ b/operators/src/processing/mod.rs @@ -2,6 +2,7 @@ mod band_neighborhood_aggregate; mod bandwise_expression; mod circle_merging_quadtree; mod column_range_filter; +mod downsample; mod expression; mod interpolation; mod line_simplification; @@ -26,11 +27,18 @@ pub use band_neighborhood_aggregate::{ pub use circle_merging_quadtree::{ InitializedVisualPointClustering, VisualPointClustering, VisualPointClusteringParams, }; +pub use downsample::{ + Downsampling, DownsamplingError, DownsamplingMethod, DownsamplingParams, + DownsamplingResolution, InitializedDownsampling, +}; pub use expression::{ Expression, ExpressionParams, RasterExpressionError, VectorExpression, VectorExpressionError, VectorExpressionParams, initialize_expression_dependencies, }; -pub use interpolation::{Interpolation, InterpolationError, InterpolationParams}; +pub use interpolation::{ + InitializedInterpolation, Interpolation, InterpolationError, InterpolationMethod, + InterpolationParams, InterpolationResolution, +}; pub use line_simplification::{ LineSimplification, LineSimplificationError, LineSimplificationParams, }; @@ -51,7 +59,8 @@ pub use raster_vector_join::{ TemporalAggregationMethod, }; pub use reprojection::{ - InitializedRasterReprojection, InitializedVectorReprojection, Reprojection, ReprojectionParams, + DeriveOutRasterSpecsSource, InitializedRasterReprojection, InitializedVectorReprojection, + Reprojection, ReprojectionParams, }; pub use temporal_raster_aggregation::{ Aggregation, TemporalRasterAggregation, TemporalRasterAggregationParameters, diff --git a/operators/src/processing/neighborhood_aggregate/aggregate.rs b/operators/src/processing/neighborhood_aggregate/aggregate.rs index 27cb3676d..ad853c04c 100644 --- a/operators/src/processing/neighborhood_aggregate/aggregate.rs +++ b/operators/src/processing/neighborhood_aggregate/aggregate.rs @@ -53,15 +53,6 @@ impl Neighborhood { self.matrix.axis_size_x() / 2 } - pub fn x_width(&self) -> usize { - self.matrix.axis_size_x() - } - - /// Specifies the y extent beginning from the center pixel - pub fn y_width(&self) -> usize { - self.matrix.axis_size_y() - } - /// Specifies the x extent right of one pixel pub fn y_radius(&self) -> usize { self.matrix.axis_size_y() / 2 diff --git a/operators/src/processing/neighborhood_aggregate/mod.rs b/operators/src/processing/neighborhood_aggregate/mod.rs index f183e5d73..e2ce06bb2 100644 --- a/operators/src/processing/neighborhood_aggregate/mod.rs +++ b/operators/src/processing/neighborhood_aggregate/mod.rs @@ -13,7 +13,9 @@ use crate::engine::{ use crate::util::Result; use async_trait::async_trait; use futures::stream::BoxStream; -use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, SpatialPartition2D}; +use geoengine_datatypes::primitives::{ + BandSelection, RasterQueryRectangle, RasterSpatialQueryRectangle, +}; use geoengine_datatypes::raster::{ Grid2D, GridShape2D, GridSize, Pixel, RasterTile2D, TilingSpecification, }; @@ -252,7 +254,7 @@ impl QueryProcessor for NeighborhoodAggregateProcessor where Q: QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, @@ -261,7 +263,7 @@ where A: AggregateFunction + 'static, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -271,15 +273,19 @@ where ctx: &'a dyn QueryContext, ) -> Result>> { stack_individual_aligned_raster_bands(&query, ctx, |query, ctx| async move { - let sub_query = NeighborhoodAggregateTileNeighborhood::::new( - self.neighborhood.clone(), - self.tiling_specification, - ); + let sub_query = + NeighborhoodAggregateTileNeighborhood::::new(self.neighborhood.clone()); + + let tiling_strat = self + .source + .result_descriptor() + .tiling_grid_definition(self.tiling_specification) + .generate_data_tiling_strategy(); Ok(RasterSubQueryAdapter::<'a, P, _, _>::new( &self.source, query, - self.tiling_specification, + tiling_strat, ctx, sub_query, ) @@ -299,11 +305,10 @@ where mod tests { use super::*; - use crate::{ engine::{ MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{RasterStacker, RasterStackerParams}, @@ -315,12 +320,12 @@ mod tests { dataset::NamedData, operations::image::{Colorizer, RgbaColor}, primitives::{ - CacheHint, DateTime, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, + CacheHint, Coordinate2D, DateTime, RasterQueryRectangle, SpatialPartition2D, TimeInstance, TimeInterval, }, raster::{ - Grid2D, GridOrEmpty, RasterDataType, RasterTile2D, RenameBands, TileInformation, - TilesEqualIgnoringCacheHint, TilingSpecification, + GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty, RasterDataType, RasterTile2D, + RenameBands, TileInformation, TilesEqualIgnoringCacheHint, TilingSpecification, }, spatial_reference::SpatialReference, util::test::TestDefault, @@ -337,9 +342,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("matrix-input"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name("matrix-input")), } .boxed(), }, @@ -366,7 +369,8 @@ mod tests { "raster": { "type": "GdalSource", "params": { - "data": "matrix-input" + "data": "matrix-input", + "overviewLevel": null } } } @@ -386,9 +390,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("matrix-input"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name("matrix-input")), } .boxed(), }, @@ -411,7 +413,8 @@ mod tests { "raster": { "type": "GdalSource", "params": { - "data": "matrix-input" + "data": "matrix-input", + "overviewLevel": null } } } @@ -438,10 +441,8 @@ mod tests { #[tokio::test] async fn test_mean_convolution() { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 3].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 3].into())); let raster = make_raster(); @@ -461,12 +462,11 @@ mod tests { let processor = operator.query_processor().unwrap().get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 3.).into(), (6., 0.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 5]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let result_stream = processor.query(query_rect, &query_ctx).await.unwrap(); @@ -501,10 +501,8 @@ mod tests { #[tokio::test] async fn check_make_raster() { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 3].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 3].into())); let raster = make_raster(); @@ -515,12 +513,11 @@ mod tests { let processor = operator.query_processor().unwrap().get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 3.).into(), (6., 0.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 5]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let result_stream = processor.query(query_rect, &query_ctx).await.unwrap(); @@ -662,8 +659,10 @@ mod tests { data_type: RasterDataType::I8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [0, 6]).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -689,7 +688,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new(ndvi_id), } .boxed(), }, @@ -700,15 +699,19 @@ mod tests { .unwrap(); let processor = operator.query_processor().unwrap().get_u8().unwrap(); - - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let result_descriptor = processor.result_descriptor(); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + result_descriptor + .tiling_grid_definition(query_ctx.tiling_specification()) + .tiling_geo_transform() + .spatial_to_grid_bounds( + &SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(), + ), + TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + ); let colorizer = Colorizer::linear_gradient( vec![ @@ -726,8 +729,8 @@ mod tests { processor, query_rect, query_ctx, - 360, - 180, + 600, + 600, None, Some(colorizer), Box::pin(futures::future::pending()), @@ -735,13 +738,13 @@ mod tests { .await .unwrap(); + // Use for getting the image to compare against + // geoengine_datatypes::util::test::save_test_bytes(&bytes, "gaussian_blur_bla.png"); + assert_eq!( bytes, include_bytes!("../../../../test_data/wms/gaussian_blur.png") ); - - // Use for getting the image to compare against - // save_test_bytes(&bytes, "gaussian_blur.png"); } #[ignore] // TODO: remove @@ -759,7 +762,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new(ndvi_id), } .boxed(), }, @@ -770,16 +773,21 @@ mod tests { .unwrap(); let processor = operator.query_processor().unwrap().get_u8().unwrap(); + let result_descriptor = processor.result_descriptor(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + result_descriptor + .tiling_grid_definition(query_ctx.tiling_specification()) + .tiling_geo_transform() + .spatial_to_grid_bounds( + &SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + ), + TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + ); + // let result_stream = processor.query(query_rect, &query_ctx).await.unwrap(); let colorizer = Colorizer::linear_gradient( @@ -818,10 +826,8 @@ mod tests { #[tokio::test] async fn test_mean_convolution_multi_bands() { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 3].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 3].into())); let operator = NeighborhoodAggregate { params: NeighborhoodAggregateParams { @@ -849,12 +855,11 @@ mod tests { let processor = operator.query_processor().unwrap().get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 3.).into(), (6., 0.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::new(vec![0, 2]).unwrap(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 5]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::new(vec![0, 2]).unwrap(), + ); let query_ctx = MockQueryContext::test_default(); let result_stream = processor.query(query_rect, &query_ctx).await.unwrap(); diff --git a/operators/src/processing/neighborhood_aggregate/tile_sub_query.rs b/operators/src/processing/neighborhood_aggregate/tile_sub_query.rs index b81d9f528..52cf2fcba 100644 --- a/operators/src/processing/neighborhood_aggregate/tile_sub_query.rs +++ b/operators/src/processing/neighborhood_aggregate/tile_sub_query.rs @@ -5,16 +5,13 @@ use async_trait::async_trait; use futures::future::BoxFuture; use futures::{FutureExt, TryFutureExt}; use geoengine_datatypes::primitives::CacheHint; -use geoengine_datatypes::primitives::{AxisAlignedRectangle, SpatialPartitioned}; use geoengine_datatypes::raster::{ - Blit, EmptyGrid, EmptyGrid2D, FromIndexFnParallel, GeoTransform, GridIdx, GridIdx2D, - GridIndexAccess, GridOrEmpty, GridSize, + ChangeGridBounds, FromIndexFnParallel, GridBlit, GridBoundingBox2D, GridContains, GridIdx, + GridIdx2D, GridIndexAccess, GridOrEmpty, GridSize, }; use geoengine_datatypes::{ - primitives::{ - Coordinate2D, RasterQueryRectangle, SpatialPartition2D, TimeInstance, TimeInterval, - }, - raster::{Pixel, RasterTile2D, TileInformation, TilingSpecification}, + primitives::{RasterQueryRectangle, TimeInstance, TimeInterval}, + raster::{Pixel, RasterTile2D, TileInformation}, }; use num_traits::AsPrimitive; use rayon::ThreadPool; @@ -42,15 +39,13 @@ use tokio::task::JoinHandle; #[derive(Debug, Clone)] pub struct NeighborhoodAggregateTileNeighborhood { neighborhood: Neighborhood, - tiling_specification: TilingSpecification, _phantom_types: PhantomData<(P, A)>, } impl NeighborhoodAggregateTileNeighborhood { - pub fn new(neighborhood: Neighborhood, tiling_specification: TilingSpecification) -> Self { + pub fn new(neighborhood: Neighborhood) -> Self { Self { neighborhood, - tiling_specification, _phantom_types: PhantomData, } } @@ -77,16 +72,9 @@ where pool: &Arc, ) -> Self::TileAccuFuture { let pool = pool.clone(); - let tiling_specification = self.tiling_specification; let neighborhood = self.neighborhood.clone(); crate::util::spawn_blocking(move || { - create_enlarged_tile( - tile_info, - &query_rect, - pool, - tiling_specification, - neighborhood, - ) + create_enlarged_tile(tile_info, &query_rect, pool, neighborhood) }) .map_err(From::from) .boxed() @@ -96,28 +84,27 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, + _query_rect: RasterQueryRectangle, start_time: TimeInstance, band_idx: u32, ) -> Result> { - let spatial_bounds = tile_info.spatial_partition(); + let pixel_bounds = tile_info.global_pixel_bounds(); - let margin_pixels = Coordinate2D::from(( - self.neighborhood.x_radius() as f64 * tile_info.global_geo_transform.x_pixel_size(), - self.neighborhood.y_radius() as f64 * tile_info.global_geo_transform.y_pixel_size(), - )); + let margin_y = self.neighborhood.y_radius() as isize; + let margin_x = self.neighborhood.x_radius() as isize; - let enlarged_spatial_bounds = SpatialPartition2D::new( - spatial_bounds.upper_left() - margin_pixels, - spatial_bounds.lower_right() + margin_pixels, + let larger_bounds = GridBoundingBox2D::new_min_max( + pixel_bounds.y_min() - margin_y, + pixel_bounds.y_max() + margin_y, + pixel_bounds.x_min() - margin_x, + pixel_bounds.x_max() + margin_x, )?; - Ok(Some(RasterQueryRectangle { - spatial_bounds: enlarged_spatial_bounds, - time_interval: TimeInterval::new_instant(start_time)?, - spatial_resolution: query_rect.spatial_resolution, - attributes: band_idx.into(), - })) + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + larger_bounds, + TimeInterval::new_instant(start_time)?, + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -131,7 +118,10 @@ where #[derive(Clone, Debug)] pub struct NeighborhoodAggregateAccu { pub output_info: TileInformation, - pub input_tile: RasterTile2D

, + pub accu_grid: GridOrEmpty, + pub accu_time: TimeInterval, + pub accu_cache_hint: CacheHint, + pub accu_band: u32, pub pool: Arc, pub neighborhood: Neighborhood, phantom_aggregate_fn: PhantomData, @@ -139,14 +129,20 @@ pub struct NeighborhoodAggregateAccu { impl NeighborhoodAggregateAccu { pub fn new( - input_tile: RasterTile2D

, + accu_grid: GridOrEmpty, + accu_time: TimeInterval, + accu_cache_hint: CacheHint, + accu_band: u32, output_info: TileInformation, pool: Arc, neighborhood: Neighborhood, ) -> Self { NeighborhoodAggregateAccu { output_info, - input_tile, + accu_grid, + accu_time, + accu_cache_hint, + accu_band, pool, neighborhood, phantom_aggregate_fn: PhantomData, @@ -168,7 +164,10 @@ where let neighborhood = self.neighborhood.clone(); let output_tile = crate::util::spawn_blocking_with_thread_pool(self.pool, move || { apply_kernel_for_each_inner_pixel::( - &self.input_tile, + &self.accu_grid, + &self.accu_time, + self.accu_cache_hint, + self.accu_band, &self.output_info, &neighborhood, ) @@ -185,7 +184,10 @@ where /// Apply kernel function to all pixels of the inner input tile in the 9x9 grid fn apply_kernel_for_each_inner_pixel( - input: &RasterTile2D

, + accu_grid: &GridOrEmpty, + accu_time: &TimeInterval, + accu_cache_hint: CacheHint, + accu_band: u32, info_out: &TileInformation, neighborhood: &Neighborhood, ) -> RasterTile2D

@@ -194,13 +196,13 @@ where f64: AsPrimitive

, A: AggregateFunction, { - if input.is_empty() { + if accu_grid.is_empty() { return RasterTile2D::new_with_tile_info( - input.time, + *accu_time, *info_out, 0, // TODO - EmptyGrid::new(info_out.tile_size_in_pixels).into(), - CacheHint::max_duration(), + GridOrEmpty::new_empty_shape(info_out.tile_size_in_pixels), + accu_cache_hint, // TODO: is this correct? Was CacheHint::max_duration() before ); } @@ -210,13 +212,16 @@ where let mut neighborhood_matrix = Vec::>::with_capacity(neighborhood.matrix().number_of_elements()); - let y_stop = y + neighborhood.y_width() as isize; - let x_stop = x + neighborhood.x_width() as isize; + let y_start = y - neighborhood.y_radius() as isize; + let x_start = x - neighborhood.x_radius() as isize; + + let y_stop = y + neighborhood.y_radius() as isize; + let x_stop = x + neighborhood.x_radius() as isize; // copy row-by-row all pixels in x direction into kernel matrix - for y_index in y..y_stop { - for x_index in x..x_stop { + for y_index in y_start..=y_stop { + for x_index in x_start..=x_stop { neighborhood_matrix.push( - input + accu_grid .get_at_grid_index_unchecked([y_index, x_index]) .map(AsPrimitive::as_), ); @@ -226,16 +231,25 @@ where A::apply(&neighborhood.apply(neighborhood_matrix)) }; + let out_pixel_bounds = info_out.global_pixel_bounds(); + + debug_assert!(accu_grid.shape_ref().contains(&out_pixel_bounds)); + // TODO: this will check for empty tiles. Change to MaskedGrid::from(…) to avoid this. - let out_data = GridOrEmpty::from_index_fn_parallel(&info_out.tile_size_in_pixels, map_fn); + let out_data = GridOrEmpty::from_index_fn_parallel(&out_pixel_bounds, map_fn); + + debug_assert_eq!( + out_data.shape_ref().axis_size(), + info_out.tile_size_in_pixels.axis_size() + ); RasterTile2D::new( - input.time, + *accu_time, info_out.global_tile_position, - input.band, + accu_band, info_out.global_geo_transform, - out_data, - input.cache_hint.clone_with_current_datetime(), + out_data.unbounded(), + accu_cache_hint.clone_with_current_datetime(), ) } @@ -243,59 +257,59 @@ fn create_enlarged_tile( tile_info: TileInformation, query_rect: &RasterQueryRectangle, pool: Arc, - tiling_specification: TilingSpecification, neighborhood: Neighborhood, ) -> NeighborhoodAggregateAccu { // create an accumulator as a single tile that fits all the input tiles + some margin for the kernel size - let tiling = tiling_specification.strategy( - query_rect.spatial_resolution.x, - -query_rect.spatial_resolution.y, - ); - - let origin_coordinate = query_rect.spatial_bounds.upper_left(); - - let geo_transform = GeoTransform::new( - origin_coordinate, - query_rect.spatial_resolution.x, - -query_rect.spatial_resolution.y, - ); + let tiling_strategy = tile_info.tiling_strategy(); + + let target_tile_start = + tiling_strategy.tile_idx_to_global_pixel_idx(tile_info.global_tile_position); + let accu_start = target_tile_start + - GridIdx([ + neighborhood.y_radius() as isize, + neighborhood.x_radius() as isize, + ]); + let accu_end = accu_start + + GridIdx2D::new_y_x( + tiling_strategy.tile_size_in_pixels.y() as isize + 2 * neighborhood.y_radius() as isize + - 1, // -1 because the end is inclusive + tiling_strategy.tile_size_in_pixels.x() as isize + 2 * neighborhood.x_radius() as isize + - 1, + ); - let shape = [ - tiling.tile_size_in_pixels.axis_size_y() + 2 * neighborhood.y_radius(), - tiling.tile_size_in_pixels.axis_size_x() + 2 * neighborhood.x_radius(), - ]; + let accu_bounds = GridBoundingBox2D::new(accu_start, accu_end) + .expect("accu bounds must be valid because they are calculated from valid bounds"); // create a non-aligned (w.r.t. the tiling specification) grid by setting the origin to the top-left of the tile and the tile-index to [0, 0] - let grid = EmptyGrid2D::new(shape.into()); + let grid = GridOrEmpty::new_empty_shape(accu_bounds); - let input_tile = RasterTile2D::new( + NeighborhoodAggregateAccu::new( + grid, query_rect.time_interval, - [0, 0].into(), - 0, // TODO - geo_transform, - GridOrEmpty::from(grid), CacheHint::max_duration(), - ); - - NeighborhoodAggregateAccu::new(input_tile, tile_info, pool, neighborhood) + 0, + tile_info, + pool, + neighborhood, + ) } type FoldFutureFn = fn( - Result>, tokio::task::JoinError>, + Result, tokio::task::JoinError>, ) -> Result>; type FoldFuture = - futures::future::Map>>, FoldFutureFn>; + futures::future::Map>, FoldFutureFn>; /// Turn a result of results into a result fn flatten_result( - result: Result>, tokio::task::JoinError>, + result: Result, tokio::task::JoinError>, ) -> Result> where f64: AsPrimitive

, { match result { - Ok(r) => r, + Ok(r) => Ok(r), Err(e) => Err(e.into()), } } @@ -304,30 +318,25 @@ where pub fn merge_tile_into_enlarged_tile( mut accu: NeighborhoodAggregateAccu, tile: RasterTile2D

, -) -> Result> +) -> NeighborhoodAggregateAccu where f64: AsPrimitive

, { // get the time now because it is not known when the accu was created - accu.input_tile.time = tile.time; + accu.accu_time = tile.time; + accu.accu_cache_hint = tile.cache_hint; // if the tile is empty, we can skip it if tile.is_empty() { - return Ok(accu); + return accu; } // copy all input tiles into the accu to have all data for raster kernel - let mut accu_input_tile = accu.input_tile.into_materialized_tile(); - accu_input_tile.blit(tile)?; + let x = tile.into_inner_positioned_grid(); - let accu_input_tile: RasterTile2D

= accu_input_tile.into(); + accu.accu_grid.grid_blit_from(&x); - Ok(NeighborhoodAggregateAccu::new( - accu_input_tile, - accu.output_info, - accu.pool, - accu.neighborhood, - )) + accu } #[cfg(test)] @@ -342,41 +351,45 @@ mod tests { }, }; use geoengine_datatypes::{ - primitives::{BandSelection, SpatialResolution}, - raster::TilingStrategy, + primitives::BandSelection, + raster::{ + GeoTransform, GridBoundingBox2D, SpatialGridDefinition, TilingSpecification, + TilingStrategy, + }, util::test::TestDefault, }; #[test] #[allow(clippy::float_cmp)] fn test_create_enlarged_tile() { - let execution_context = MockExecutionContext::test_default(); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::test_default()); - let spatial_resolution = SpatialResolution::one(); - let tiling_strategy = TilingStrategy::new_with_tiling_spec( - execution_context.tiling_specification, - spatial_resolution.x, - -spatial_resolution.y, + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0., 1., 0., -1.), + GridBoundingBox2D::new([-2, 0], [-1, 1]).unwrap(), + ); + + let tiling_strategy = TilingStrategy::new( + execution_context.tiling_specification.tile_size_in_pixels, + spatial_grid.geo_transform(), ); - let spatial_partition = SpatialPartition2D::new((0., 1.).into(), (1., 0.).into()).unwrap(); let tile_info = tiling_strategy - .tile_information_iterator(spatial_partition) + .tile_information_iterator_from_grid_bounds(spatial_grid.grid_bounds()) .next() .unwrap(); - let qrect = RasterQueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - time_interval: TimeInstance::from_millis(0).unwrap().into(), - spatial_resolution, - attributes: BandSelection::first(), - }; + let qrect = RasterQueryRectangle::new_with_grid_bounds( + tile_info.global_pixel_bounds(), + TimeInstance::from_millis(0).unwrap().into(), + BandSelection::first(), + ); let aggregator = NeighborhoodAggregateTileNeighborhood::::new( NeighborhoodParams::Rectangle { dimensions: [5, 5] } .try_into() .unwrap(), - execution_context.tiling_specification, ); let tile_query_rectangle = aggregator @@ -385,29 +398,26 @@ mod tests { .unwrap(); assert_eq!( - tile_info.spatial_partition(), - SpatialPartition2D::new((0., 512.).into(), (512., 0.).into()).unwrap() + tile_info.global_pixel_bounds(), + GridBoundingBox2D::new([-512, 0], [-1, 511]).unwrap() ); + assert_eq!( - tile_query_rectangle.spatial_bounds, - SpatialPartition2D::new((-2., 514.).into(), (514., -2.).into()).unwrap() + tile_query_rectangle.spatial_query().grid_bounds(), + GridBoundingBox2D::new([-514, -2], [1, 513]).unwrap() ); let accu = create_enlarged_tile::( tile_info, &tile_query_rectangle, execution_context.thread_pool.clone(), - execution_context.tiling_specification, aggregator.neighborhood, ); assert_eq!(tile_info.tile_size_in_pixels.axis_size(), [512, 512]); assert_eq!( - accu.input_tile.grid_array.shape_ref().axis_size(), + accu.accu_grid.shape_ref().axis_size(), [512 + 2 + 2, 512 + 2 + 2] ); - - assert_eq!(accu.input_tile.tile_geo_transform().x_pixel_size(), 1.); - assert_eq!(accu.input_tile.tile_geo_transform().y_pixel_size(), -1.); } } diff --git a/operators/src/processing/point_in_polygon.rs b/operators/src/processing/point_in_polygon.rs index d63fec3e1..3143174df 100644 --- a/operators/src/processing/point_in_polygon.rs +++ b/operators/src/processing/point_in_polygon.rs @@ -301,10 +301,7 @@ impl VectorQueryProcessor for PointInPolygonFilterProcessor { .query(query.clone(), ctx) .await? .and_then(move |points| { - let query: geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - geoengine_datatypes::primitives::ColumnSelection, - > = query.clone(); + let query: VectorQueryRectangle = query.clone(); async move { if points.is_empty() { return Ok(points); @@ -392,7 +389,7 @@ mod tests { use geoengine_datatypes::collections::ChunksEqualIgnoringCacheHint; use geoengine_datatypes::primitives::{ - BoundingBox2D, Coordinate2D, MultiPoint, MultiPolygon, SpatialResolution, TimeInterval, + BoundingBox2D, Coordinate2D, MultiPoint, MultiPolygon, TimeInterval, }; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use geoengine_datatypes::spatial_reference::SpatialReference; @@ -505,6 +502,8 @@ mod tests { let point_source = MockFeatureCollectionSource::single(points.clone()).boxed(); + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); + let polygon_source = MockFeatureCollectionSource::single(MultiPolygonCollection::from_data( vec![MultiPolygon::new(vec![vec![vec![ @@ -528,21 +527,17 @@ mod tests { }, } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = operator.query_processor()?.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -578,6 +573,8 @@ mod tests { )?) .boxed(); + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); + let operator = PointInPolygonFilter { params: PointInPolygonFilterParams {}, sources: PointInPolygonFilterSource { @@ -586,21 +583,17 @@ mod tests { }, } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = operator.query_processor()?.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -647,6 +640,8 @@ mod tests { )?) .boxed(); + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); + let operator = PointInPolygonFilter { params: PointInPolygonFilterParams {}, sources: PointInPolygonFilterSource { @@ -655,21 +650,17 @@ mod tests { }, } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = operator.query_processor()?.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -737,6 +728,8 @@ mod tests { ]) .boxed(); + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); + let operator = PointInPolygonFilter { params: PointInPolygonFilterParams {}, sources: PointInPolygonFilterSource { @@ -745,23 +738,19 @@ mod tests { }, } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = operator.query_processor()?.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); - let ctx_one_chunk = MockQueryContext::new(ChunkByteSize::MAX); - let ctx_minimal_chunks = MockQueryContext::new(ChunkByteSize::MIN); + let ctx_one_chunk = exe_ctx.mock_query_context(ChunkByteSize::MAX); + let ctx_minimal_chunks = exe_ctx.mock_query_context(ChunkByteSize::MIN); let query = query_processor .query(query_rectangle.clone(), &ctx_minimal_chunks) @@ -839,12 +828,11 @@ mod tests { .await .unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-10., -10.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-10., -10.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query_processor = operator.query_processor().unwrap().multi_point().unwrap(); diff --git a/operators/src/processing/raster_scaling.rs b/operators/src/processing/raster_scaling.rs index 173300d2d..2c65ef868 100644 --- a/operators/src/processing/raster_scaling.rs +++ b/operators/src/processing/raster_scaling.rs @@ -104,10 +104,8 @@ impl RasterOperator for RasterScaling { let out_desc = RasterResultDescriptor { spatial_reference: in_desc.spatial_reference, data_type: in_desc.data_type, - - bbox: in_desc.bbox, time: in_desc.time, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: in_desc .bands .iter() @@ -284,35 +282,42 @@ where #[cfg(test)] mod tests { - use geoengine_datatypes::{ - primitives::{ - BandSelection, CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval, + use crate::{ + engine::{ + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, SpatialGridDescriptor, }, + mock::{MockRasterSource, MockRasterSourceParams}, + }; + use geoengine_datatypes::{ + primitives::{BandSelection, CacheHint, Coordinate2D, TimeInterval}, raster::{ - Grid2D, GridOrEmpty2D, GridShape, MaskedGrid2D, RasterDataType, RasterProperties, - TileInformation, TilingSpecification, + BoundedGrid, GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty2D, GridShape, + GridShape2D, MaskedGrid2D, RasterDataType, RasterProperties, TileInformation, + TilingSpecification, }, spatial_reference::SpatialReference, util::test::TestDefault, }; - use crate::{ - engine::{ChunkByteSize, MockExecutionContext, RasterBandDescriptors}, - mock::{MockRasterSource, MockRasterSourceParams}, - }; - use super::*; #[tokio::test] async fn test_unscale() { - let grid_shape = [2, 2].into(); - - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels: grid_shape, + let tile_size_in_pixels = GridShape2D::new_2d(2, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; - let raster = MaskedGrid2D::from(Grid2D::new(grid_shape, vec![7_u8, 7, 7, 6]).unwrap()); + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let raster = + MaskedGrid2D::from(Grid2D::new(tile_size_in_pixels, vec![7_u8, 7, 7, 6]).unwrap()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let query_ctx = ctx.mock_query_context(ChunkByteSize::test_default()); @@ -326,7 +331,7 @@ mod tests { TileInformation { global_geo_transform: TestDefault::test_default(), global_tile_position: [0, 0].into(), - tile_size_in_pixels: grid_shape, + tile_size_in_pixels, }, 0, raster.into(), @@ -334,19 +339,10 @@ mod tests { CacheHint::default(), ); - let spatial_resolution = raster_tile.spatial_resolution(); - let mrs = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - bbox: None, - time: None, - resolution: Some(spatial_resolution), - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -381,12 +377,11 @@ mod tests { let query_processor = initialized_op.query_processor().unwrap(); - let query = geoengine_datatypes::primitives::RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 0.).into(), (2., -2.).into()).unwrap(), - spatial_resolution: SpatialResolution::one(), - time_interval: TimeInterval::default(), - attributes: BandSelection::first(), - }; + let query = geoengine_datatypes::primitives::RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ); let TypedRasterQueryProcessor::U8(typed_processor) = query_processor else { panic!("expected TypedRasterQueryProcessor::U8"); @@ -419,14 +414,22 @@ mod tests { #[tokio::test] async fn test_scale() { - let grid_shape = [2, 2].into(); - - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels: grid_shape, + let tile_size_in_pixels = GridShape2D::new_2d(2, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; - let raster = MaskedGrid2D::from(Grid2D::new(grid_shape, vec![15_u8, 15, 15, 13]).unwrap()); + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let raster = + MaskedGrid2D::from(Grid2D::new(tile_size_in_pixels, vec![15_u8, 15, 15, 13]).unwrap()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let query_ctx = ctx.mock_query_context(ChunkByteSize::test_default()); @@ -440,7 +443,7 @@ mod tests { TileInformation { global_geo_transform: TestDefault::test_default(), global_tile_position: [0, 0].into(), - tile_size_in_pixels: grid_shape, + tile_size_in_pixels, }, 0, raster.into(), @@ -448,19 +451,10 @@ mod tests { CacheHint::default(), ); - let spatial_resolution = raster_tile.spatial_resolution(); - let mrs = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - bbox: None, - time: None, - resolution: Some(spatial_resolution), - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -497,12 +491,11 @@ mod tests { let query_processor = initialized_op.query_processor().unwrap(); - let query = geoengine_datatypes::primitives::RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 0.).into(), (2., -2.).into()).unwrap(), - spatial_resolution: SpatialResolution::one(), - time_interval: TimeInterval::default(), - attributes: BandSelection::first(), - }; + let query = geoengine_datatypes::primitives::RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ); let TypedRasterQueryProcessor::U8(typed_processor) = query_processor else { panic!("expected TypedRasterQueryProcessor::U8"); diff --git a/operators/src/processing/raster_stacker.rs b/operators/src/processing/raster_stacker.rs index b20ebea46..e07d63893 100644 --- a/operators/src/processing/raster_stacker.rs +++ b/operators/src/processing/raster_stacker.rs @@ -11,9 +11,7 @@ use crate::error::{ use crate::util::Result; use async_trait::async_trait; use futures::stream::BoxStream; -use geoengine_datatypes::primitives::{ - BandSelection, RasterQueryRectangle, SpatialResolution, partitions_extent, time_interval_extent, -}; +use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, time_interval_extent}; use geoengine_datatypes::raster::{DynamicRasterDataType, Pixel, RasterTile2D, RenameBands}; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -77,19 +75,17 @@ impl RasterOperator for RasterStacker { } ); - let time = time_interval_extent(in_descriptors.iter().map(|d| d.time)); - let bbox = partitions_extent(in_descriptors.iter().map(|d| d.bbox)); - - let resolution = in_descriptors + let first_spatial_grid = in_descriptors[0].spatial_grid; + let result_spatial_grid = in_descriptors .iter() - .map(|d| d.resolution) - .reduce(|a, b| match (a, b) { - (Some(a), Some(b)) => { - Some(SpatialResolution::new_unchecked(a.x.min(b.x), a.y.min(b.y))) - } - _ => None, - }) - .flatten(); + .skip(1) + .map(|x| x.spatial_grid_descriptor()) + .try_fold(first_spatial_grid, |a, &b| { + a.merge(&b) + .ok_or(crate::error::Error::CantMergeSpatialGridDescriptor { a, b }) + })?; + + let time = time_interval_extent(in_descriptors.iter().map(|d| d.time)); let data_type = in_descriptors[0].data_type; let spatial_reference = in_descriptors[0].spatial_reference; @@ -118,8 +114,7 @@ impl RasterOperator for RasterStacker { data_type, spatial_reference, time, - bbox, - resolution, + spatial_grid: result_spatial_grid, bands: output_band_descriptors, }; @@ -356,8 +351,11 @@ mod tests { use futures::StreamExt; use geoengine_datatypes::{ - primitives::{CacheHint, SpatialPartition2D, TimeInstance, TimeInterval}, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, + primitives::{CacheHint, TimeInstance, TimeInterval}, + raster::{ + GeoTransform, Grid, GridBoundingBox2D, GridShape, RasterDataType, + TilesEqualIgnoringCacheHint, + }, spatial_reference::SpatialReference, util::test::TestDefault, }; @@ -365,7 +363,7 @@ mod tests { use crate::{ engine::{ MockExecutionContext, MockQueryContext, RasterBandDescriptor, RasterBandDescriptors, - SingleRasterSource, + SingleRasterSource, SpatialGridDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{Expression, ExpressionParams}, @@ -488,17 +486,21 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: Some(TimeInterval::new_unchecked(0, 10)), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -506,14 +508,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -533,12 +528,11 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + [0, 1].try_into().unwrap(), + ); let query_ctx = MockQueryContext::test_default(); @@ -742,21 +736,25 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new(vec![ + RasterBandDescriptor::new_unitless("band_0".into()), + RasterBandDescriptor::new_unitless("band_1".into()), + ]) + .unwrap(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![ - RasterBandDescriptor::new_unitless("band_0".into()), - RasterBandDescriptor::new_unitless("band_1".into()), - ]) - .unwrap(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -764,18 +762,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![ - RasterBandDescriptor::new_unitless("band_0".into()), - RasterBandDescriptor::new_unitless("band_1".into()), - ]) - .unwrap(), - }, + result_descriptor, }, } .boxed(); @@ -795,12 +782,11 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1, 2, 3].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 2]).unwrap(), + TimeInterval::new_unchecked(0, 10), + [0, 1, 2, 3].try_into().unwrap(), + ); let query_ctx = MockQueryContext::test_default(); @@ -929,17 +915,21 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -947,14 +937,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -974,12 +957,11 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: 1.into(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 2]).unwrap(), + TimeInterval::new_unchecked(0, 10), + 1.into(), + ); let query_ctx = MockQueryContext::test_default(); @@ -1019,6 +1001,7 @@ mod tests { raster: GdalSource { params: GdalSourceParameters { data: ndvi_id.clone(), + overview_level: None, }, } .boxed(), @@ -1033,7 +1016,7 @@ mod tests { sources: MultipleRasterSources { rasters: vec![ GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new(ndvi_id), } .boxed(), expression, @@ -1057,18 +1040,14 @@ mod tests { let query_ctx = MockQueryContext::test_default(); // query both bands - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_unchecked( + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), ), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1].try_into().unwrap(), - }; + [0, 1].try_into().unwrap(), + ); let result = processor .raster_query(query_rect, &query_ctx) @@ -1081,18 +1060,14 @@ mod tests { assert!(!result.is_empty()); // query only first band - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_unchecked( + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), ), - spatial_resolution: SpatialResolution::one(), - attributes: [0].try_into().unwrap(), - }; + [0].try_into().unwrap(), + ); let result = processor .raster_query(query_rect, &query_ctx) @@ -1105,18 +1080,14 @@ mod tests { assert!(!result.is_empty()); // query only second band - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_unchecked( + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), ), - spatial_resolution: SpatialResolution::one(), - attributes: [1].try_into().unwrap(), - }; + [1].try_into().unwrap(), + ); let result = processor .raster_query(query_rect, &query_ctx) diff --git a/operators/src/processing/raster_type_conversion.rs b/operators/src/processing/raster_type_conversion.rs index fcce1107b..7850c7500 100644 --- a/operators/src/processing/raster_type_conversion.rs +++ b/operators/src/processing/raster_type_conversion.rs @@ -1,17 +1,16 @@ -use async_trait::async_trait; -use futures::{StreamExt, TryFutureExt, TryStreamExt, stream::BoxStream}; -use geoengine_datatypes::{ - primitives::{BandSelection, RasterQueryRectangle, SpatialPartition2D}, - raster::{ConvertDataType, Pixel, RasterDataType, RasterTile2D}, -}; -use serde::{Deserialize, Serialize}; - use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, }; use crate::util::Result; +use async_trait::async_trait; +use futures::{StreamExt, TryFutureExt, TryStreamExt, stream::BoxStream}; +use geoengine_datatypes::{ + primitives::{BandSelection, RasterQueryRectangle, RasterSpatialQueryRectangle}, + raster::{ConvertDataType, Pixel, RasterDataType, RasterTile2D}, +}; +use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -55,9 +54,8 @@ impl RasterOperator for RasterTypeConversion { let out_desc = RasterResultDescriptor { spatial_reference: in_desc.spatial_reference, data_type: out_data_type, - bbox: in_desc.bbox, time: in_desc.time, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: in_desc.bands.clone(), }; @@ -140,7 +138,7 @@ where Q: RasterQueryProcessor, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -165,17 +163,19 @@ where #[cfg(test)] mod tests { use geoengine_datatypes::{ - primitives::{CacheHint, Measurement, SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::{CacheHint, Coordinate2D, Measurement, TimeInterval}, raster::{ - Grid2D, GridOrEmpty2D, MaskedGrid2D, RasterDataType, TileInformation, - TilingSpecification, + BoundedGrid, GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty2D, GridShape2D, + MaskedGrid2D, RasterDataType, TileInformation, TilingSpecification, }, spatial_reference::SpatialReference, util::test::TestDefault, }; use crate::{ - engine::{ChunkByteSize, MockExecutionContext, RasterBandDescriptors}, + engine::{ + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, SpatialGridDescriptor, + }, mock::{MockRasterSource, MockRasterSourceParams}, }; @@ -184,14 +184,22 @@ mod tests { #[tokio::test] #[allow(clippy::float_cmp)] async fn test_type_conversion() { - let grid_shape = [2, 2].into(); - - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels: grid_shape, + let tile_size_in_pixels = GridShape2D::new_2d(2, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); - let raster: MaskedGrid2D = Grid2D::new(grid_shape, vec![7_u8, 7, 7, 6]).unwrap().into(); + let raster: MaskedGrid2D = Grid2D::new(tile_size_in_pixels, vec![7_u8, 7, 7, 6]) + .unwrap() + .into(); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let query_ctx = ctx.mock_query_context(ChunkByteSize::test_default()); @@ -201,7 +209,7 @@ mod tests { TileInformation { global_geo_transform: TestDefault::test_default(), global_tile_position: [0, 0].into(), - tile_size_in_pixels: grid_shape, + tile_size_in_pixels, }, 0, raster.into(), @@ -211,14 +219,7 @@ mod tests { let mrs = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - bbox: None, - time: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -246,12 +247,11 @@ mod tests { let query_processor = initialized_op.query_processor().unwrap(); - let query = geoengine_datatypes::primitives::RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 0.).into(), (2., -2.).into()).unwrap(), - spatial_resolution: SpatialResolution::one(), - time_interval: TimeInterval::default(), - attributes: BandSelection::first(), - }; + let query = geoengine_datatypes::primitives::RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ); let TypedRasterQueryProcessor::F32(typed_processor) = query_processor else { panic!("expected TypedRasterQueryProcessor::F32"); diff --git a/operators/src/processing/raster_vector_join/aggregated.rs b/operators/src/processing/raster_vector_join/aggregated.rs index b77b5ab18..da2facebc 100644 --- a/operators/src/processing/raster_vector_join/aggregated.rs +++ b/operators/src/processing/raster_vector_join/aggregated.rs @@ -1,30 +1,30 @@ -use futures::stream::BoxStream; -use futures::{StreamExt, TryStreamExt}; - -use geoengine_datatypes::collections::{ - FeatureCollection, FeatureCollectionInfos, FeatureCollectionModifications, -}; -use geoengine_datatypes::primitives::{ - BandSelection, CacheHint, ColumnSelection, RasterQueryRectangle, -}; -use geoengine_datatypes::raster::{GridIndexAccess, Pixel, RasterDataType}; -use geoengine_datatypes::util::arrow::ArrowTyped; - -use crate::engine::{ - QueryContext, QueryProcessor, RasterQueryProcessor, VectorQueryProcessor, - VectorResultDescriptor, -}; -use crate::processing::raster_vector_join::TemporalAggregationMethod; -use crate::processing::raster_vector_join::aggregator::{ - Aggregator, FirstValueFloatAggregator, FirstValueIntAggregator, MeanValueAggregator, - TypedAggregator, -}; -use crate::util::Result; -use async_trait::async_trait; -use geoengine_datatypes::primitives::{BoundingBox2D, Geometry, VectorQueryRectangle}; - use super::util::{CoveredPixels, FeatureTimeSpanIter, PixelCoverCreator}; use super::{FeatureAggregationMethod, RasterInput, create_feature_aggregator}; +use crate::{ + engine::{ + QueryContext, QueryProcessor, RasterQueryProcessor, VectorQueryProcessor, + VectorResultDescriptor, + }, + processing::raster_vector_join::{ + TemporalAggregationMethod, + aggregator::{ + Aggregator, FirstValueFloatAggregator, FirstValueIntAggregator, MeanValueAggregator, + TypedAggregator, + }, + }, + util::Result, +}; +use async_trait::async_trait; +use futures::{StreamExt, TryStreamExt, stream::BoxStream}; +use geoengine_datatypes::{ + collections::{FeatureCollection, FeatureCollectionInfos, FeatureCollectionModifications}, + primitives::{ + BandSelection, CacheHint, ColumnSelection, Geometry, RasterQueryRectangle, SpatialBounded, + VectorQueryRectangle, VectorSpatialQueryRectangle, + }, + raster::{GridIndexAccess, Pixel, RasterDataType}, + util::arrow::ArrowTyped, +}; pub struct RasterVectorAggregateJoinProcessor { collection: Box>>, @@ -92,20 +92,23 @@ where let mut cache_hint = CacheHint::max_duration(); + let rd = raster_processor.raster_result_descriptor(); + for time_span in FeatureTimeSpanIter::new(collection.time_intervals()) { - let query = VectorQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: time_span.time_interval, - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }; - - let mut rasters = raster_processor - .raster_query( - RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()), - ctx, - ) - .await?; + let spatial_bounds = query.spatial_query.spatial_bounds(); + + let pixel_bounds = rd + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform() + .bounding_box_2d_to_intersecting_grid_bounds(&spatial_bounds); + + let raster_query = RasterQueryRectangle::new_with_grid_bounds( + pixel_bounds, + time_span.time_interval, + BandSelection::first(), // FIXME: this should prop. use all bands? + ); + + let mut rasters = raster_processor.raster_query(raster_query, ctx).await?; // TODO: optimize geo access (only specific tiles, etc.) @@ -247,7 +250,7 @@ where FeatureCollection: PixelCoverCreator, { type Output = FeatureCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -298,22 +301,23 @@ mod tests { use crate::engine::{ ChunkByteSize, MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, - RasterResultDescriptor, VectorColumnInfo, VectorOperator, WorkflowOperatorPath, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, VectorColumnInfo, + VectorOperator, WorkflowOperatorPath, }; - use crate::engine::{MockQueryContext, RasterOperator}; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use geoengine_datatypes::collections::{ ChunksEqualIgnoringCacheHint, MultiPointCollection, MultiPolygonCollection, VectorDataType, }; - use geoengine_datatypes::primitives::MultiPolygon; - use geoengine_datatypes::primitives::{CacheHint, FeatureData, FeatureDataType, Measurement}; - use geoengine_datatypes::raster::{Grid2D, RasterTile2D, TileInformation}; + + use geoengine_datatypes::primitives::{ + BoundingBox2D, CacheHint, Coordinate2D, FeatureData, FeatureDataRef, FeatureDataType, + Measurement, MultiPoint, MultiPolygon, TimeInterval, + }; + use geoengine_datatypes::raster::{ + GeoTransform, Grid2D, GridBoundingBox2D, RasterTile2D, TileInformation, TilingSpecification, + }; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceOption}; use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{ - primitives::{BoundingBox2D, FeatureDataRef, MultiPoint, SpatialResolution, TimeInterval}, - raster::TilingSpecification, - }; #[tokio::test] async fn extract_raster_values_single_raster() { @@ -331,24 +335,27 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 1]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster_source = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -379,13 +386,12 @@ mod tests { false, TemporalAggregationMethod::First, false, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (2.0, 0.).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, -3.0).into(), (2.0, 0.).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -427,24 +433,27 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 1]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile_a, raster_tile_b], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster_source = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -475,13 +484,12 @@ mod tests { false, TemporalAggregationMethod::Mean, false, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (2.0, 0.0).into()).unwrap(), - time_interval: TimeInterval::new(0, 20).unwrap(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, -3.0).into(), (2.0, 0.0).into()).unwrap(), + TimeInterval::new(0, 20).unwrap(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -549,6 +557,17 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![ @@ -557,21 +576,13 @@ mod tests { raster_tile_b_0, raster_tile_b_1, ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster_source = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -598,13 +609,12 @@ mod tests { false, TemporalAggregationMethod::Mean, false, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), - time_interval: TimeInterval::new(0, 20).unwrap(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new(0, 20).unwrap(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -702,6 +712,17 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 5]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![ @@ -712,21 +733,13 @@ mod tests { raster_tile_b_1, raster_tile_b_2, ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster_source = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -757,13 +770,12 @@ mod tests { false, TemporalAggregationMethod::Mean, false, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), - time_interval: TimeInterval::new(0, 20).unwrap(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new(0, 20).unwrap(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -963,8 +975,10 @@ mod tests { data_type: RasterDataType::U16, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(0, 2, 0, 5).unwrap(), + ), bands: RasterBandDescriptors::new(vec![ RasterBandDescriptor::new_unitless("band_0".into()), RasterBandDescriptor::new_unitless("band_1".into()), @@ -975,9 +989,8 @@ mod tests { } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1042,14 +1055,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()) - .unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new_unchecked(0, 20), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() diff --git a/operators/src/processing/raster_vector_join/mod.rs b/operators/src/processing/raster_vector_join/mod.rs index 6c1f10f21..d46bbe52e 100644 --- a/operators/src/processing/raster_vector_join/mod.rs +++ b/operators/src/processing/raster_vector_join/mod.rs @@ -385,8 +385,8 @@ mod tests { use std::str::FromStr; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, QueryProcessor, - RasterBandDescriptor, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + ChunkByteSize, MockExecutionContext, QueryProcessor, RasterBandDescriptor, + RasterBandDescriptors, RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use crate::source::{GdalSource, GdalSourceParameters}; @@ -396,10 +396,10 @@ mod tests { use geoengine_datatypes::dataset::NamedData; use geoengine_datatypes::primitives::{ BoundingBox2D, ColumnSelection, DataRef, DateTime, FeatureDataRef, MultiPoint, - SpatialResolution, TimeInterval, VectorQueryRectangle, + TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, Measurement}; - use geoengine_datatypes::raster::RasterTile2D; + use geoengine_datatypes::raster::{GeoTransform, GridBoundingBox2D, RasterTile2D}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::{gdal::hide_gdal_errors, test::TestDefault}; use serde_json::json; @@ -451,7 +451,7 @@ mod tests { fn ndvi_source(name: NamedData) -> Box { let gdal_source = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), }; gdal_source.boxed() @@ -488,7 +488,7 @@ mod tests { let operator = RasterVectorJoin { params: RasterVectorJoinParams { - names: ColumnNames::Default, + names: ColumnNames::Names(vec!["ndvi".to_owned()]), feature_aggregation: FeatureAggregationMethod::First, feature_aggregation_ignore_no_data: false, temporal_aggregation: TemporalAggregationMethod::First, @@ -510,14 +510,12 @@ mod tests { let result = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap() @@ -567,7 +565,7 @@ mod tests { let operator = RasterVectorJoin { params: RasterVectorJoinParams { - names: ColumnNames::Default, + names: ColumnNames::Names(vec!["ndvi".to_owned()]), feature_aggregation: FeatureAggregationMethod::First, feature_aggregation_ignore_no_data: false, temporal_aggregation: TemporalAggregationMethod::Mean, @@ -589,14 +587,12 @@ mod tests { let result = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap() @@ -649,7 +645,7 @@ mod tests { let operator = RasterVectorJoin { params: RasterVectorJoinParams { - names: ColumnNames::Default, + names: ColumnNames::Names(vec!["ndvi".to_owned()]), feature_aggregation: FeatureAggregationMethod::First, feature_aggregation_ignore_no_data: false, temporal_aggregation: TemporalAggregationMethod::Mean, @@ -671,14 +667,12 @@ mod tests { let result = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap() @@ -778,8 +772,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new_min_max(0, 0, 2, 2).unwrap(), + ), bands: RasterBandDescriptors::new(vec![ RasterBandDescriptor::new_unitless("band_0".into()), RasterBandDescriptor::new_unitless("band_1".into()), @@ -797,8 +793,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new_min_max(0, 0, 2, 2).unwrap(), + ), bands: RasterBandDescriptors::new(vec![ RasterBandDescriptor::new_unitless("band_0".into()), RasterBandDescriptor::new_unitless("band_1".into()), diff --git a/operators/src/processing/raster_vector_join/non_aggregated.rs b/operators/src/processing/raster_vector_join/non_aggregated.rs index fd1b935cc..66724b999 100644 --- a/operators/src/processing/raster_vector_join/non_aggregated.rs +++ b/operators/src/processing/raster_vector_join/non_aggregated.rs @@ -1,35 +1,32 @@ +use super::aggregator::TypedAggregator; +use super::util::{CoveredPixels, PixelCoverCreator}; +use super::{FeatureAggregationMethod, RasterInput}; use crate::adapters::FeatureCollectionStreamExt; +use crate::engine::{ + QueryContext, QueryProcessor, RasterQueryProcessor, TypedRasterQueryProcessor, + VectorQueryProcessor, VectorResultDescriptor, +}; use crate::processing::raster_vector_join::create_feature_aggregator; +use crate::util::Result; +use crate::{adapters::RasterStreamExt, error::Error}; +use async_trait::async_trait; use futures::stream::{BoxStream, once as once_stream}; use futures::{StreamExt, TryStreamExt}; +use geoengine_datatypes::collections::GeometryCollection; +use geoengine_datatypes::collections::{FeatureCollection, FeatureCollectionInfos}; use geoengine_datatypes::primitives::{ - BandSelection, BoundingBox2D, CacheHint, ColumnSelection, FeatureDataType, Geometry, - RasterQueryRectangle, VectorQueryRectangle, + BandSelection, CacheHint, ColumnSelection, FeatureDataType, Geometry, RasterQueryRectangle, + VectorQueryRectangle, VectorSpatialQueryRectangle, }; -use geoengine_datatypes::util::arrow::ArrowTyped; -use std::marker::PhantomData; -use std::sync::Arc; - use geoengine_datatypes::raster::{ DynamicRasterDataType, GridIdx2D, GridIndexAccess, RasterTile2D, }; +use geoengine_datatypes::util::arrow::ArrowTyped; use geoengine_datatypes::{ collections::FeatureCollectionModifications, primitives::TimeInterval, raster::Pixel, }; - -use super::util::{CoveredPixels, PixelCoverCreator}; -use crate::engine::{ - QueryContext, QueryProcessor, RasterQueryProcessor, TypedRasterQueryProcessor, - VectorQueryProcessor, VectorResultDescriptor, -}; -use crate::util::Result; -use crate::{adapters::RasterStreamExt, error::Error}; -use async_trait::async_trait; -use geoengine_datatypes::collections::GeometryCollection; -use geoengine_datatypes::collections::{FeatureCollection, FeatureCollectionInfos}; - -use super::aggregator::TypedAggregator; -use super::{FeatureAggregationMethod, RasterInput}; +use std::marker::PhantomData; +use std::sync::Arc; pub struct RasterVectorJoinProcessor { collection: Box>>, @@ -112,7 +109,7 @@ where let bbox = collection .bbox() - .and_then(|bbox| bbox.intersection(&query.spatial_bounds)); + .and_then(|bbox| bbox.intersection(&query.spatial_query.spatial_bounds)); let time = collection .time_bounds() @@ -120,7 +117,7 @@ where // TODO: also intersect with raster spatial / time bounds - let (Some(_spatial_bounds), Some(_time_interval)) = (bbox, time) else { + let (Some(spatial_bounds), Some(time_interval)) = (bbox, time) else { log::debug!( "spatial or temporal intersection is empty, returning the same collection, skipping raster query" ); @@ -132,8 +129,16 @@ where ); }; - let query = RasterQueryRectangle::from_qrect_and_bands( - &query, + let rd = raster_processor.result_descriptor(); + + let pixel_bounds = rd + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform() + .bounding_box_2d_to_intersecting_grid_bounds(&spatial_bounds); + + let query = RasterQueryRectangle::new_with_grid_bounds( + pixel_bounds, + time_interval, BandSelection::first_n(column_names.len() as u32), ); @@ -384,7 +389,7 @@ where FeatureCollection: GeometryCollection + PixelCoverCreator, { type Output = FeatureCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -426,8 +431,8 @@ mod tests { use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, QueryProcessor, - RasterBandDescriptor, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + ChunkByteSize, MockExecutionContext, QueryProcessor, RasterBandDescriptor, + RasterBandDescriptors, RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, VectorColumnInfo, VectorOperator, WorkflowOperatorPath, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; @@ -436,12 +441,13 @@ mod tests { use geoengine_datatypes::collections::{ ChunksEqualIgnoringCacheHint, MultiPointCollection, MultiPolygonCollection, VectorDataType, }; - use geoengine_datatypes::primitives::SpatialResolution; - use geoengine_datatypes::primitives::{BoundingBox2D, DateTime, FeatureData, MultiPolygon}; - use geoengine_datatypes::primitives::{CacheHint, Measurement}; - use geoengine_datatypes::primitives::{MultiPoint, TimeInterval}; + use geoengine_datatypes::primitives::{ + BoundingBox2D, CacheHint, Coordinate2D, DateTime, FeatureData, Measurement, MultiPoint, + MultiPolygon, TimeInterval, + }; use geoengine_datatypes::raster::{ - Grid2D, RasterDataType, TileInformation, TilingSpecification, + GeoTransform, Grid2D, GridBoundingBox2D, RasterDataType, TileInformation, + TilingSpecification, }; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceOption}; use geoengine_datatypes::util::test::TestDefault; @@ -472,9 +478,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let raster_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -521,14 +525,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: time_instant, - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + time_instant, + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -582,9 +584,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let raster_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -631,18 +631,16 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2014, 1, 1, 0, 0, 0), DateTime::new_utc(2014, 3, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -704,9 +702,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let raster_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -753,17 +749,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2014, 1, 1, 0, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).unwrap(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -828,9 +819,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let raster_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -877,18 +866,16 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2014, 1, 1, 0, 0, 0), DateTime::new_utc(2014, 3, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -993,6 +980,17 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![ @@ -1001,21 +999,13 @@ mod tests { raster_tile_b_0, raster_tile_b_1, ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1074,14 +1064,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()) - .unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new_unchecked(0, 20), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -1206,6 +1194,17 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![ @@ -1216,21 +1215,13 @@ mod tests { raster_tile_b_1, raster_tile_b_2, ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U16, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1293,14 +1284,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()) - .unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new_unchecked(0, 20), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -1533,8 +1522,10 @@ mod tests { data_type: RasterDataType::U16, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([0, 0], [2, 3]).unwrap(), + ), bands: RasterBandDescriptors::new(vec![ RasterBandDescriptor::new_unitless("band_0".into()), RasterBandDescriptor::new_unitless("band_1".into()), @@ -1545,9 +1536,8 @@ mod tests { } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1610,14 +1600,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()) - .unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new_unchecked(0, 20), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() diff --git a/operators/src/processing/raster_vector_join/util.rs b/operators/src/processing/raster_vector_join/util.rs index d8d975f4d..a66c8c271 100644 --- a/operators/src/processing/raster_vector_join/util.rs +++ b/operators/src/processing/raster_vector_join/util.rs @@ -165,7 +165,7 @@ impl CoveredPixels for MultiPolygonCoveredPixels { for row in 0..height { for col in 0..width { let idx = [row as isize, col as isize].into(); - let coordinate = geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(idx); + let coordinate = geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(idx); // FIXME: should be pixel center? if tester.multi_polygon_contains_coordinate(coordinate, feature_index) { pixels.push(idx); diff --git a/operators/src/processing/rasterization/mod.rs b/operators/src/processing/rasterization/mod.rs index f2e70804c..d72707355 100644 --- a/operators/src/processing/rasterization/mod.rs +++ b/operators/src/processing/rasterization/mod.rs @@ -3,84 +3,43 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, InitializedVectorOperator, Operator, OperatorName, QueryContext, QueryProcessor, RasterBandDescriptors, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, - SingleVectorSource, TypedRasterQueryProcessor, TypedVectorQueryProcessor, WorkflowOperatorPath, + ResultDescriptor, SingleVectorSource, SpatialGridDescriptor, TypedRasterQueryProcessor, + TypedVectorQueryProcessor, WorkflowOperatorPath, }; -use arrow::datatypes::ArrowNativeTypeOp; -use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; - use crate::error; -use crate::processing::rasterization::GridOrDensity::Grid; use crate::util; - +use crate::util::spawn_blocking; use async_trait::async_trait; - use futures::stream::BoxStream; use futures::{StreamExt, stream}; use geoengine_datatypes::collections::GeometryCollection; - use geoengine_datatypes::primitives::{ AxisAlignedRectangle, BoundingBox2D, Coordinate2D, RasterQueryRectangle, SpatialPartition2D, SpatialPartitioned, SpatialResolution, VectorQueryRectangle, }; +use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use geoengine_datatypes::raster::{ - GeoTransform, Grid2D, GridOrEmpty, GridSize, GridSpaceToLinearSpace, RasterDataType, - RasterTile2D, TilingSpecification, + ChangeGridBounds, GeoTransform, Grid as GridWithFlexibleBoundType, Grid2D, GridIdx, + GridOrEmpty, GridSize, RasterDataType, RasterTile2D, TilingSpecification, TilingStrategy, }; - -use num_traits::FloatConst; -use rayon::prelude::*; - +use geoengine_datatypes::spatial_reference::SpatialReference; use serde::{Deserialize, Serialize}; -use snafu::ensure; - -use crate::util::{spawn_blocking, spawn_blocking_with_thread_pool}; - use typetag::serde; /// An operator that rasterizes vector data -pub type Rasterization = Operator; +pub type Rasterization = Operator; impl OperatorName for Rasterization { const TYPE_NAME: &'static str = "Rasterization"; } -#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum GridSizeMode { - /// The spatial resolution is interpreted as a fixed size in coordinate units - Fixed, - /// The spatial resolution is interpreted as a multiplier for the query pixel size - Relative, -} - #[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -#[serde(tag = "type")] -pub enum GridOrDensity { - /// A grid which summarizes points in cells (2D histogram) - Grid(GridParams), - /// A heatmap calculated from a gaussian density function - Density(DensityParams), -} - -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] -pub struct DensityParams { - /// Defines the cutoff (as percentage of maximum density) down to which a point is taken - /// into account for an output pixel density value - cutoff: f64, - /// The standard deviation parameter for the gaussian function - stddev: f64, -} - -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GridParams { +pub struct RasterizationParams { /// The size of grid cells, interpreted depending on the chosen grid size mode spatial_resolution: SpatialResolution, /// The origin coordinate which aligns the grid bounds origin_coordinate: Coordinate2D, - /// Determines how to interpret the grid resolution - grid_size_mode: GridSizeMode, } #[typetag::serde] @@ -102,38 +61,43 @@ impl RasterOperator for Rasterization { let tiling_specification = context.tiling_specification(); + let resolution = self.params.spatial_resolution; + let origin = self.params.origin_coordinate; + + let geo_transform = GeoTransform::new(origin, resolution.x, -resolution.y); + + let spatial_bounds = in_desc + .bbox + .ok_or_else(|| { + in_desc + .spatial_reference() + .as_option() + .map(SpatialReference::area_of_use_projected::) + }) + .map_err(|_| error::Error::NoSpatialBoundsAvailable)?; + + let pixel_bounds = + geo_transform.spatial_to_grid_bounds(&SpatialPartition2D::new_unchecked( + spatial_bounds.upper_left(), + spatial_bounds.lower_right(), + )); + let out_desc = RasterResultDescriptor { spatial_reference: in_desc.spatial_reference, data_type: RasterDataType::F64, - bbox: None, time: in_desc.time, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts(geo_transform, pixel_bounds), bands: RasterBandDescriptors::new_single_band(), }; - match self.params { - Grid(params) => Ok(InitializedGridRasterization { - name, - path, - source: vector_source, - result_descriptor: out_desc, - spatial_resolution: params.spatial_resolution, - grid_size_mode: params.grid_size_mode, - tiling_specification, - origin_coordinate: params.origin_coordinate, - } - .boxed()), - GridOrDensity::Density(params) => InitializedDensityRasterization::new( - name, - path, - vector_source, - out_desc, - tiling_specification, - params.cutoff, - params.stddev, - ) - .map(InitializedRasterOperator::boxed), + Ok(InitializedGridRasterization { + name, + path, + source: vector_source, + result_descriptor: out_desc, + tiling_specification, } + .boxed()) } span_fn!(Rasterization); @@ -144,10 +108,7 @@ pub struct InitializedGridRasterization { path: WorkflowOperatorPath, source: Box, result_descriptor: RasterResultDescriptor, - spatial_resolution: SpatialResolution, - grid_size_mode: GridSizeMode, tiling_specification: TilingSpecification, - origin_coordinate: Coordinate2D, } impl InitializedRasterOperator for InitializedGridRasterization { @@ -160,90 +121,7 @@ impl InitializedRasterOperator for InitializedGridRasterization { GridRasterizationQueryProcessor { input: self.source.query_processor()?, result_descriptor: self.result_descriptor.clone(), - spatial_resolution: self.spatial_resolution, - grid_size_mode: self.grid_size_mode, tiling_specification: self.tiling_specification, - origin_coordinate: self.origin_coordinate, - } - .boxed(), - )) - } - - fn canonic_name(&self) -> CanonicOperatorName { - self.name.clone() - } - - fn name(&self) -> &'static str { - Rasterization::TYPE_NAME - } - - fn path(&self) -> WorkflowOperatorPath { - self.path.clone() - } -} - -pub struct InitializedDensityRasterization { - name: CanonicOperatorName, - path: WorkflowOperatorPath, - source: Box, - result_descriptor: RasterResultDescriptor, - tiling_specification: TilingSpecification, - radius: f64, - stddev: f64, -} - -impl InitializedDensityRasterization { - fn new( - name: CanonicOperatorName, - path: WorkflowOperatorPath, - source: Box, - result_descriptor: RasterResultDescriptor, - tiling_specification: TilingSpecification, - cutoff: f64, - stddev: f64, - ) -> Result { - ensure!( - (0. ..1.).contains(&cutoff), - error::InvalidOperatorSpec { - reason: "The cutoff for density rasterization must be in [0, 1).".to_string() - } - ); - ensure!( - stddev >= 0., - error::InvalidOperatorSpec { - reason: "The standard deviation for density rasterization must be greater than or equal to zero." - .to_string() - } - ); - - // Determine radius from cutoff percentage - let radius = gaussian_inverse(cutoff * gaussian(0., stddev), stddev); - - Ok(InitializedDensityRasterization { - name, - path, - source, - result_descriptor, - tiling_specification, - radius, - stddev, - }) - } -} - -impl InitializedRasterOperator for InitializedDensityRasterization { - fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor - } - - fn query_processor(&self) -> util::Result { - Ok(TypedRasterQueryProcessor::F64( - DensityRasterizationQueryProcessor { - result_descriptor: self.result_descriptor.clone(), - input: self.source.query_processor()?, - tiling_specification: self.tiling_specification, - radius: self.radius, - stddev: self.stddev, } .boxed(), )) @@ -265,10 +143,7 @@ impl InitializedRasterOperator for InitializedDensityRasterization { pub struct GridRasterizationQueryProcessor { input: TypedVectorQueryProcessor, result_descriptor: RasterResultDescriptor, - spatial_resolution: SpatialResolution, - grid_size_mode: GridSizeMode, tiling_specification: TilingSpecification, - origin_coordinate: Coordinate2D, } #[async_trait] @@ -282,241 +157,86 @@ impl RasterQueryProcessor for GridRasterizationQueryProcessor { /// All points within the spatial bounds of the grid are queried and counted in the /// grid cells. /// Finally, the grid resolution is upsampled (if necessary) to the tile resolution. + #[allow(clippy::too_many_lines)] async fn raster_query<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> util::Result>>> { - if let MultiPoint(points_processor) = &self.input { - let grid_resolution = match self.grid_size_mode { - GridSizeMode::Fixed => SpatialResolution { - x: f64::max(self.spatial_resolution.x, query.spatial_resolution.x), - y: f64::max(self.spatial_resolution.y, query.spatial_resolution.y), - }, - GridSizeMode::Relative => SpatialResolution { - x: f64::max( - self.spatial_resolution.x * query.spatial_resolution.x, - query.spatial_resolution.x, - ), - y: f64::max( - self.spatial_resolution.y * query.spatial_resolution.y, - query.spatial_resolution.y, - ), - }, - }; - - let tiling_strategy = self - .tiling_specification - .strategy(query.spatial_resolution.x, -query.spatial_resolution.y); - let tile_shape = tiling_strategy.tile_size_in_pixels; - - let tiles = stream::iter( - tiling_strategy.tile_information_iterator(query.spatial_bounds), - ) - .then(move |tile_info| async move { - let grid_spatial_bounds = tile_info - .spatial_partition() - .snap_to_grid(self.origin_coordinate, grid_resolution); - - let grid_size_x = - f64::ceil(grid_spatial_bounds.size_x() / grid_resolution.x) as usize; - let grid_size_y = - f64::ceil(grid_spatial_bounds.size_y() / grid_resolution.y) as usize; - - let vector_query = VectorQueryRectangle { - spatial_bounds: grid_spatial_bounds.as_bbox(), - time_interval: query.time_interval, - spatial_resolution: grid_resolution, - attributes: ColumnSelection::all(), - }; - - let grid_geo_transform = GeoTransform::new( - grid_spatial_bounds.upper_left(), - grid_resolution.x, - -grid_resolution.y, - ); - - let mut chunks = points_processor.query(vector_query, ctx).await?; - - let mut cache_hint = CacheHint::max_duration(); - - let mut grid_data = vec![0.; grid_size_x * grid_size_y]; - while let Some(chunk) = chunks.next().await { - let chunk = chunk?; + let spatial_grid_desc = self + .result_descriptor + .tiling_grid_definition(ctx.tiling_specification()); - cache_hint.merge_with(&chunk.cache_hint); + let tiling_strategy = spatial_grid_desc.generate_data_tiling_strategy(); + let tiling_geo_transform = spatial_grid_desc.tiling_geo_transform(); - grid_data = spawn_blocking(move || { - for &coord in chunk.coordinates() { - if !grid_spatial_bounds.contains_coordinate(&coord) { - continue; - } - let [y, x] = grid_geo_transform.coordinate_to_grid_idx_2d(coord).0; - grid_data[x as usize + y as usize * grid_size_x] += 1.; - } - grid_data - }) - .await - .expect("Should only forward panics from spawned task"); - } - - let tile_data = spawn_blocking(move || { - let mut tile_data = Vec::with_capacity(tile_shape.number_of_elements()); - for tile_y in 0..tile_shape.axis_size_y() as isize { - for tile_x in 0..tile_shape.axis_size_x() as isize { - let pixel_coordinate = tile_info - .tile_geo_transform() - .grid_idx_to_pixel_center_coordinate_2d([tile_y, tile_x].into()); - if query.spatial_bounds.contains_coordinate(&pixel_coordinate) { - let [grid_y, grid_x] = grid_geo_transform - .coordinate_to_grid_idx_2d(pixel_coordinate) - .0; - tile_data.push( - grid_data[grid_x as usize + grid_y as usize * grid_size_x], - ); - } else { - tile_data.push(0.); - } - } - } - tile_data - }) - .await - .expect("Should only forward panics from spawned task"); - let tile_grid = Grid2D::new(tile_shape, tile_data) - .expect("Data vector length should match the number of pixels in the tile"); - - Ok(RasterTile2D::new_with_tile_info( - query.time_interval, - tile_info, - 0, - GridOrEmpty::Grid(tile_grid.into()), - cache_hint, - )) - }); - Ok(tiles.boxed()) - } else { - Ok(generate_zeroed_tiles(self.tiling_specification, &query)) - } - } - - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor - } -} - -pub struct DensityRasterizationQueryProcessor { - input: TypedVectorQueryProcessor, - result_descriptor: RasterResultDescriptor, - tiling_specification: TilingSpecification, - radius: f64, - stddev: f64, -} - -#[async_trait] -impl RasterQueryProcessor for DensityRasterizationQueryProcessor { - type RasterType = f64; - - /// Performs a gaussian density rasterization. - /// For each tile, the spatial bounds are extended by `radius` in x and y direction. - /// All points within these extended bounds are then queried. For each point, the distance to - /// its surrounding tile pixels (up to `radius` distance) is measured and input into the - /// gaussian density function with the configured standard deviation. The density values - /// for each pixel are then summed to result in the tile pixel grid. - async fn raster_query<'a>( - &'a self, - query: RasterQueryRectangle, - ctx: &'a dyn QueryContext, - ) -> util::Result>>> { if let MultiPoint(points_processor) = &self.input { - let tiling_strategy = self - .tiling_specification - .strategy(query.spatial_resolution.x, -query.spatial_resolution.y); - - let tile_size_x = tiling_strategy.tile_size_in_pixels.axis_size_x(); - let tile_size_y = tiling_strategy.tile_size_in_pixels.axis_size_y(); - - // Use rounding factor calculated from query resolution to extend in full pixel units - let rounding_factor = f64::max( - 1. / query.spatial_resolution.x, - 1. / query.spatial_resolution.y, - ); - let radius = (self.radius * rounding_factor).ceil() / rounding_factor; - - let tiles = stream::iter( - tiling_strategy.tile_information_iterator(query.spatial_bounds), - ) - .then(move |tile_info| async move { - let tile_bounds = tile_info.spatial_partition(); - - let vector_query = VectorQueryRectangle { - spatial_bounds: extended_bounding_box_from_spatial_partition( - tile_bounds, - radius, - ), - time_interval: query.time_interval, - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }; - - let tile_geo_transform = tile_info.tile_geo_transform(); - - let mut chunks = points_processor.query(vector_query, ctx).await?; - - let mut tile_data = vec![0.; tile_size_x * tile_size_y]; - - let mut cache_hint = CacheHint::max_duration(); + let query_grid_bounds = query.spatial_query().grid_bounds(); + let query_spatial_partition = + tiling_geo_transform.grid_to_spatial_bounds(&query_grid_bounds); - while let Some(chunk) = chunks.next().await { - let chunk = chunk?; - - cache_hint.merge_with(&chunk.cache_hint); - - let stddev = self.stddev; - tile_data = - spawn_blocking_with_thread_pool(ctx.thread_pool().clone(), move || { - tile_data.par_iter_mut().enumerate().for_each( - |(linear_index, pixel)| { - let pixel_coordinate = tile_geo_transform - .grid_idx_to_pixel_center_coordinate_2d( - tile_geo_transform - .spatial_to_grid_bounds(&tile_bounds) - .grid_idx_unchecked(linear_index), - ); - - for coord in chunk.coordinates() { - let distance = coord.euclidean_distance(&pixel_coordinate); - - if distance <= radius { - *pixel += gaussian(distance, stddev); - } - } - }, - ); - - tile_data + let tiles = + stream::iter(tiling_strategy.tile_information_iterator_from_grid_bounds( + query.spatial_query().grid_bounds(), + )) + .then(move |tile_info| async move { + let tile_spatial_bounds = tile_info.spatial_partition(); + + let grid_size_x = tile_info.tile_size_in_pixels().axis_size_x(); + + let vector_query = VectorQueryRectangle::with_bounds( + tile_spatial_bounds.as_bbox(), + query.time_interval, + ColumnSelection::all(), // FIXME: should be configurable + ); + + let mut chunks = points_processor.query(vector_query, ctx).await?; + + let mut cache_hint = CacheHint::max_duration(); + + let mut grid_data = + GridWithFlexibleBoundType::new_filled(tile_info.global_pixel_bounds(), 0.); + while let Some(chunk) = chunks.next().await { + let chunk = chunk?; + + cache_hint.merge_with(&chunk.cache_hint); + + grid_data = spawn_blocking(move || { + for &coord in chunk.coordinates() { + if !tile_spatial_bounds.contains_coordinate(&coord) + || !query_spatial_partition.contains_coordinate(&coord) + // TODO: old code checks if the pixel center is in the query bounds. + { + continue; + } + let GridIdx([y, x]) = tiling_geo_transform + .coordinate_to_grid_idx_2d(coord) + - tile_info.global_upper_left_pixel_idx(); + grid_data.data[x as usize + y as usize * grid_size_x] += 1.; + } + grid_data }) - .await?; - } + .await + .expect("Should only forward panics from spawned task"); + } - Ok(RasterTile2D::new_with_tile_info( - query.time_interval, - tile_info, - 0, - GridOrEmpty::Grid( - Grid2D::new(tiling_strategy.tile_size_in_pixels, tile_data) - .expect( - "Data vector length should match the number of pixels in the tile", - ) - .into(), - ), - cache_hint, - )) - }); + let tile_grid = grid_data.unbounded(); + Ok(RasterTile2D::new_with_tile_info( + query.time_interval, + tile_info, + 0, + GridOrEmpty::Grid(tile_grid.into()), + cache_hint, + )) + }); Ok(tiles.boxed()) } else { - Ok(generate_zeroed_tiles(self.tiling_specification, &query)) + Ok(generate_zeroed_tiles( + tiling_geo_transform, + self.tiling_specification, + &query, + )) } } @@ -526,17 +246,18 @@ impl RasterQueryProcessor for DensityRasterizationQueryProcessor { } fn generate_zeroed_tiles<'a>( + tiling_geo_transform: GeoTransform, tiling_specification: TilingSpecification, query: &RasterQueryRectangle, ) -> BoxStream<'a, util::Result>> { - let tiling_strategy = - tiling_specification.strategy(query.spatial_resolution.x, -query.spatial_resolution.y); - let tile_shape = tiling_strategy.tile_size_in_pixels; + let tile_shape = tiling_specification.tile_size_in_pixels; let time_interval = query.time_interval; + let tiling_strategy = TilingStrategy::new(tile_shape, tiling_geo_transform); + stream::iter( tiling_strategy - .tile_information_iterator(query.spatial_bounds) + .tile_information_iterator_from_grid_bounds(query.spatial_query().grid_bounds()) .map(move |tile_info| { let tile_data = vec![0.; tile_shape.number_of_elements()]; let tile_grid = Grid2D::new(tile_shape, tile_data) @@ -554,51 +275,6 @@ fn generate_zeroed_tiles<'a>( .boxed() } -fn extended_bounding_box_from_spatial_partition( - spatial_partition: SpatialPartition2D, - extent: f64, -) -> BoundingBox2D { - BoundingBox2D::new_unchecked( - Coordinate2D::new( - spatial_partition - .lower_left() - .x - .sub_checked(extent) - .unwrap_or(f64::MIN), - spatial_partition - .lower_left() - .y - .sub_checked(extent) - .unwrap_or(f64::MIN), - ), - Coordinate2D::new( - spatial_partition - .upper_right() - .x - .add_checked(extent) - .unwrap_or(f64::MAX), - spatial_partition - .upper_right() - .y - .add_checked(extent) - .unwrap_or(f64::MAX), - ), - ) -} - -/// Calculates the gaussian density value for -/// `x`, the distance from the mean and -/// `stddev`, the standard deviation -fn gaussian(x: f64, stddev: f64) -> f64 { - (1. / (f64::sqrt(2. * f64::PI()) * stddev)) * f64::exp(-(x * x) / (2. * stddev * stddev)) -} - -/// The inverse function of [gaussian](gaussian) -fn gaussian_inverse(x: f64, stddev: f64) -> f64 { - f64::sqrt(2.) - * f64::sqrt(stddev * stddev * f64::ln(1. / (f64::sqrt(2. * f64::PI()) * stddev * x))) -} - #[cfg(test)] mod tests { use crate::engine::{ @@ -606,27 +282,25 @@ mod tests { RasterOperator, SingleVectorSource, VectorOperator, WorkflowOperatorPath, }; use crate::mock::{MockPointSource, MockPointSourceParams}; - use crate::processing::rasterization::GridSizeMode::{Fixed, Relative}; - use crate::processing::rasterization::{ - DensityParams, GridOrDensity, GridParams, Rasterization, gaussian, - }; + use crate::processing::rasterization::{Rasterization, RasterizationParams}; use futures::StreamExt; use geoengine_datatypes::primitives::{ - BandSelection, Coordinate2D, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, + BandSelection, BoundingBox2D, Coordinate2D, RasterQueryRectangle, SpatialResolution, }; - use geoengine_datatypes::raster::TilingSpecification; + use geoengine_datatypes::raster::{GridBoundingBox2D, TilingSpecification}; use geoengine_datatypes::util::test::TestDefault; async fn get_results( rasterization: Box, query: RasterQueryRectangle, + query_ctx: &MockQueryContext, ) -> Vec> { rasterization .query_processor() .unwrap() .get_f64() .unwrap() - .query(query, &MockQueryContext::test_default()) + .query(query, query_ctx) .await .unwrap() .map(|res| { @@ -642,25 +316,30 @@ mod tests { #[tokio::test] async fn fixed_grid_basic() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { + params: RasterizationParams { spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, origin_coordinate: [0., 0.].into(), - grid_size_mode: Fixed, - }), + }, sources: SingleVectorSource { vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ + params: MockPointSourceParams::new_with_bounds( + vec![ (-1., 1.).into(), (1., 1.).into(), (-1., -1.).into(), (1., -1.).into(), ], - }, + crate::mock::SpatialBoundsDerive::Bounds( + BoundingBox2D::new( + Coordinate2D::new(-2., -2.), + Coordinate2D::new(2., 2.), + ) + .unwrap(), + ), + ), } .boxed(), }, @@ -670,14 +349,18 @@ mod tests { .await .unwrap(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-2., 2.].into(), [2., -2.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, -2], [1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ); - let res = get_results(rasterization, query).await; + let res = get_results( + rasterization, + query, + &execution_context.mock_query_context(TestDefault::test_default()), + ) + .await; assert_eq!( res, @@ -692,351 +375,30 @@ mod tests { #[tokio::test] async fn fixed_grid_with_shift() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { + params: RasterizationParams { spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - origin_coordinate: [0.5, -0.5].into(), - grid_size_mode: Fixed, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ - (-1., 1.).into(), - (1., 1.).into(), - (-1., -1.).into(), - (1., -1.).into(), - ], - }, - } - .boxed(), + origin_coordinate: [0.0, 0.0].into(), }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-2., 2.].into(), [2., -2.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![1., 0., 0., 0.], - vec![1., 0., 0., 0.], - vec![1., 0., 0., 0.], - vec![1., 0., 0., 0.], - ] - ); - } - - #[tokio::test] - async fn fixed_grid_with_upsampling() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [3, 3].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { - spatial_resolution: SpatialResolution { x: 2.0, y: 2.0 }, - origin_coordinate: [0., 0.].into(), - grid_size_mode: Fixed, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ - (-1., 1.).into(), - (1., 1.).into(), - (-1., -1.).into(), - (1., -1.).into(), - ], - }, - } - .boxed(), - }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-3., 3.].into(), [3., -3.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![0., 0., 0., 0., 1., 1., 0., 1., 1.], - vec![0., 0., 0., 1., 1., 0., 1., 1., 0.], - vec![0., 1., 1., 0., 1., 1., 0., 0., 0.], - vec![1., 1., 0., 1., 1., 0., 0., 0., 0.], - ] - ); - } - - #[tokio::test] - async fn relative_grid_basic() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [3, 3].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - origin_coordinate: [0., 0.].into(), - grid_size_mode: Relative, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ - (-1., 1.).into(), - (1., 1.).into(), - (-1., -1.).into(), - (1., -1.).into(), - ], - }, - } - .boxed(), - }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-1.5, 1.5].into(), [1.5, -1.5].into()) - .unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 0.5, y: 0.5 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![0., 0., 0., 0., 1., 0., 0., 0., 0.], - vec![0., 0., 0., 0., 0., 1., 0., 0., 0.], - vec![0., 0., 0., 0., 0., 0., 0., 1., 0.], - vec![0., 0., 0., 0., 0., 0., 0., 0., 1.], - ] - ); - } - - #[tokio::test] - async fn relative_grid_with_shift() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [3, 3].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - origin_coordinate: [0.25, -0.25].into(), - grid_size_mode: Relative, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ - (-1., 1.).into(), - (1., 1.).into(), - (-1., -1.).into(), - (1., -1.).into(), - ], - }, - } - .boxed(), - }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-1.5, 1.5].into(), [1.5, -1.5].into()) - .unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 0.5, y: 0.5 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![1., 0., 0., 0., 0., 0., 0., 0., 0.], - vec![0., 1., 0., 0., 0., 0., 0., 0., 0.], - vec![0., 0., 0., 1., 0., 0., 0., 0., 0.], - vec![0., 0., 0., 0., 1., 0., 0., 0., 0.], - ] - ); - } - - #[tokio::test] - async fn relative_grid_with_upsampling() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { - spatial_resolution: SpatialResolution { x: 2.0, y: 2.0 }, - origin_coordinate: [0., 0.].into(), - grid_size_mode: Relative, - }), sources: SingleVectorSource { vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ + params: MockPointSourceParams::new_with_bounds( + vec![ (-1., 1.).into(), (1., 1.).into(), (-1., -1.).into(), (1., -1.).into(), ], - }, - } - .boxed(), - }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-1., 1.].into(), [1., -1.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 0.5, y: 0.5 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![1., 1., 1., 1.], - vec![0., 0., 0., 0.], - vec![0., 0., 0., 0.], - vec![0., 0., 0., 0.] - ] - ); - } - - #[tokio::test] - async fn density_basic() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Density(DensityParams { - cutoff: gaussian(0.99, 1.0) / gaussian(0., 1.0), - stddev: 1.0, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![(-1., 1.).into(), (1., 1.).into()], - }, - } - .boxed(), - }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-2., 2.].into(), [2., 0.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![ - gaussian( - Coordinate2D::new(-1., 1.) - .euclidean_distance(&Coordinate2D::new(-1.5, 1.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(-1., 1.) - .euclidean_distance(&Coordinate2D::new(-0.5, 1.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(-1., 1.) - .euclidean_distance(&Coordinate2D::new(-1.5, 0.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(-1., 1.) - .euclidean_distance(&Coordinate2D::new(-0.5, 0.5)), - 1.0 - ) - ], - vec![ - gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(0.5, 1.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(1.5, 1.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(0.5, 0.5)), - 1.0 + crate::mock::SpatialBoundsDerive::Bounds( + BoundingBox2D::new( + Coordinate2D::new(-2., -2.), + Coordinate2D::new(2., 2.), + ) + .unwrap(), + ), ), - gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(1.5, 0.5)), - 1.0 - ) - ], - ] - ); - } - - #[tokio::test] - async fn density_radius_overlap() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Density(DensityParams { - cutoff: gaussian(1.99, 1.0) / gaussian(0., 1.0), - stddev: 1.0, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![(-1., 1.).into(), (1., 1.).into()], - }, } .boxed(), }, @@ -1046,70 +408,26 @@ mod tests { .await .unwrap(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-2., 2.].into(), [2., 0.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-2, 1, -2, 1).unwrap(), + Default::default(), + BandSelection::first(), + ); - let res = get_results(rasterization, query).await; + let res = get_results( + rasterization, + query, + &execution_context.mock_query_context(TestDefault::test_default()), + ) + .await; assert_eq!( res, vec![ - vec![ - gaussian( - Coordinate2D::new(-1., 1.) - .euclidean_distance(&Coordinate2D::new(-1.5, 1.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(-1., 1.) - .euclidean_distance(&Coordinate2D::new(-0.5, 1.5)), - 1.0 - ) + gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(-0.5, 1.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(-1., 1.) - .euclidean_distance(&Coordinate2D::new(-1.5, 0.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(-1., 1.) - .euclidean_distance(&Coordinate2D::new(-0.5, 0.5)), - 1.0 - ) + gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(-0.5, 0.5)), - 1.0 - ) - ], - vec![ - gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(0.5, 1.5)), - 1.0 - ) + gaussian( - Coordinate2D::new(-1., 1.).euclidean_distance(&Coordinate2D::new(0.5, 1.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(1.5, 1.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(0.5, 0.5)), - 1.0 - ) + gaussian( - Coordinate2D::new(-1., 1.).euclidean_distance(&Coordinate2D::new(0.5, 0.5)), - 1.0 - ), - gaussian( - Coordinate2D::new(1., 1.).euclidean_distance(&Coordinate2D::new(1.5, 0.5)), - 1.0 - ) - ], + vec![0., 0., 0., 1.], + vec![0., 0., 0., 1.], + vec![0., 0., 0., 1.], + vec![0., 0., 0., 1.], ] ); } diff --git a/operators/src/processing/reprojection.rs b/operators/src/processing/reprojection.rs index 3cb8b588c..c492d4ea8 100644 --- a/operators/src/processing/reprojection.rs +++ b/operators/src/processing/reprojection.rs @@ -3,8 +3,8 @@ use std::marker::PhantomData; use super::map_query::MapQueryProcessor; use crate::{ adapters::{ - FillerTileCacheExpirationStrategy, FillerTimeBounds, RasterSubQueryAdapter, - SparseTilesFillAdapter, TileReprojectionSubQuery, fold_by_coordinate_lookup_future, + FillerTileCacheExpirationStrategy, RasterSubQueryAdapter, TileReprojectionSubQuery, + TileReprojectionSubqueryGridInfo, fold_by_coordinate_lookup_future, }, engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, @@ -23,11 +23,12 @@ use geoengine_datatypes::{ collections::FeatureCollection, operations::reproject::{ CoordinateProjection, CoordinateProjector, Reproject, ReprojectClipped, - reproject_and_unify_bbox, reproject_query, suggest_pixel_size_from_diag_cross_projected, + reproject_spatial_query, }, primitives::{ - BandSelection, BoundingBox2D, ColumnSelection, Geometry, RasterQueryRectangle, - SpatialPartition2D, SpatialPartitioned, SpatialResolution, VectorQueryRectangle, + BandSelection, ColumnSelection, Geometry, RasterQueryRectangle, + RasterSpatialQueryRectangle, SpatialGridQueryRectangle, SpatialPartition2D, + VectorQueryRectangle, VectorSpatialQueryRectangle, }, raster::{Pixel, RasterTile2D, TilingSpecification}, spatial_reference::SpatialReference, @@ -37,14 +38,22 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone, Copy)] #[serde(rename_all = "camelCase")] -pub struct ReprojectionParams { - pub target_spatial_reference: SpatialReference, +pub enum DeriveOutRasterSpecsSource { + DataBounds, + ProjectionBounds, +} +impl Default for DeriveOutRasterSpecsSource { + fn default() -> Self { + Self::ProjectionBounds + } } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] -pub struct ReprojectionBounds { - valid_in_bounds: SpatialPartition2D, - valid_out_bounds: SpatialPartition2D, +#[serde(rename_all = "camelCase")] +pub struct ReprojectionParams { + pub target_spatial_reference: SpatialReference, + #[serde(default)] + pub derive_out_spec: DeriveOutRasterSpecsSource, } pub type Reprojection = Operator; @@ -64,15 +73,14 @@ pub struct InitializedVectorReprojection { target_srs: SpatialReference, } -pub struct InitializedRasterReprojection { +pub struct InitializedRasterReprojection { name: CanonicOperatorName, path: WorkflowOperatorPath, result_descriptor: RasterResultDescriptor, - source: Box, - state: Option, + source: O, + state: TileReprojectionSubqueryGridInfo, source_srs: SpatialReference, target_srs: SpatialReference, - tiling_spec: TilingSpecification, } impl InitializedVectorReprojection { @@ -121,12 +129,12 @@ impl InitializedVectorReprojection { } } -impl InitializedRasterReprojection { +impl InitializedRasterReprojection { pub fn try_new_with_input( name: CanonicOperatorName, path: WorkflowOperatorPath, params: ReprojectionParams, - source_raster_operator: Box, + source_raster_operator: O, tiling_spec: TilingSpecification, ) -> Result { let in_desc: RasterResultDescriptor = source_raster_operator.result_descriptor().clone(); @@ -135,71 +143,59 @@ impl InitializedRasterReprojection { .ok_or(Error::AllSourcesMustHaveSameSpatialReference)?; // calculate the intersection of input and output srs in both coordinate systems - let (in_bounds, out_bounds, out_res) = Self::derive_raster_in_bounds_out_bounds_out_res( - in_srs, - params.target_spatial_reference, - in_desc.resolution, - in_desc.bbox, - )?; + let proj_from_to = + CoordinateProjector::from_known_srs(in_srs, params.target_spatial_reference)?; + + let out_spatial_grid = match params.derive_out_spec { + DeriveOutRasterSpecsSource::DataBounds => in_desc + .spatial_grid_descriptor() + .reproject_clipped(&proj_from_to)?, + DeriveOutRasterSpecsSource::ProjectionBounds => { + let in_srs_area: SpatialPartition2D = in_srs.area_of_use_projected()?; // TODO: since we clip in projection anyway, we could use the AOU of the source projection? + let target_proj_total_grid = in_desc + .spatial_grid_descriptor() + .spatial_bounds_to_compatible_spatial_grid(in_srs_area) + .reproject_clipped(&proj_from_to)?; + // jetzt grid mit origin (tl) auf grid vom dataset. dann umprojeziren. Dann intersection mit boundingbox in dataset + let spatial_bounds_proj = + in_desc.spatial_bounds().reproject_clipped(&proj_from_to)?; + target_proj_total_grid.and_then(|x| { + spatial_bounds_proj.map(|spb| x.spatial_bounds_to_compatible_spatial_grid(spb)) + }) + } + }; - let result_descriptor = RasterResultDescriptor { + // Operator will return an error when there is no intersection between data and output projection bounds! + let out_spatial_grid = out_spatial_grid.ok_or(error::Error::ReprojectionFailed)?; // TODO: better error! + + let out_desc = RasterResultDescriptor { spatial_reference: params.target_spatial_reference.into(), data_type: in_desc.data_type, time: in_desc.time, - bbox: out_bounds, - resolution: out_res, + spatial_grid: out_spatial_grid, bands: in_desc.bands.clone(), }; - let state = match (in_bounds, out_bounds) { - (Some(in_bounds), Some(out_bounds)) => Some(ReprojectionBounds { - valid_in_bounds: in_bounds, - valid_out_bounds: out_bounds, - }), - _ => None, + let state = TileReprojectionSubqueryGridInfo { + in_spatial_grid: in_desc + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition(), + out_spatial_grid: out_spatial_grid + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition(), }; Ok(InitializedRasterReprojection { name, path, - result_descriptor, + result_descriptor: out_desc, source: source_raster_operator, state, source_srs: in_srs, target_srs: params.target_spatial_reference, - tiling_spec, }) } - - fn derive_raster_in_bounds_out_bounds_out_res( - source_srs: SpatialReference, - target_srs: SpatialReference, - source_spatial_resolution: Option, - source_bbox: Option, - ) -> Result<( - Option, - Option, - Option, - )> { - let (in_bbox, out_bbox) = if let Some(bbox) = source_bbox { - reproject_and_unify_bbox(bbox, source_srs, target_srs)? - } else { - // use the parts of the area of use that are valid in both spatial references - let valid_bounds_in = source_srs.area_of_use_intersection(&target_srs)?; - let valid_bounds_out = target_srs.area_of_use_intersection(&source_srs)?; - - (valid_bounds_in, valid_bounds_out) - }; - - let out_res = match (source_spatial_resolution, in_bbox, out_bbox) { - (Some(in_res), Some(in_bbox), Some(out_bbox)) => { - suggest_pixel_size_from_diag_cross_projected(in_bbox, out_bbox, in_res).ok() - } - _ => None, - }; - - Ok((in_bbox, out_bbox, out_res)) - } } #[typetag::serde] @@ -250,7 +246,19 @@ impl InitializedVectorOperator for InitializedVectorReprojection { MapQueryProcessor::new( source, self.result_descriptor.clone(), - move |query| reproject_query(query, source_srs, target_srs).map_err(From::from), + move |query: VectorQueryRectangle| { + reproject_spatial_query(query.spatial_query(), source_srs, target_srs) + .map(|sqr| { + sqr.map(|x| { + VectorQueryRectangle::new( + x, + query.time_interval, + ColumnSelection::all(), + ) + }) + }) + .map_err(From::from) + }, (), ) .boxed(), @@ -338,7 +346,7 @@ impl QueryProcessor for VectorReprojectionProcessor where Q: QueryProcessor< Output = FeatureCollection, - SpatialBounds = BoundingBox2D, + SpatialQuery = VectorSpatialQueryRectangle, Selection = ColumnSelection, ResultDescription = VectorResultDescriptor, >, @@ -346,7 +354,7 @@ where G: Geometry + ArrowTyped, { type Output = FeatureCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -355,7 +363,11 @@ where query: VectorQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { - let rewritten_query = reproject_query(query, self.from, self.to)?; + let rewritten_spatial_query = + reproject_spatial_query(query.spatial_query(), self.from, self.to)?; + + let rewritten_query = rewritten_spatial_query + .map(|rwq| VectorQueryRectangle::new(rwq, query.time_interval, query.attributes)); if let Some(rewritten_query) = rewritten_query { Ok(self @@ -417,7 +429,7 @@ impl RasterOperator for Reprojection { span_fn!(Reprojection); } -impl InitializedRasterOperator for InitializedRasterReprojection { +impl InitializedRasterOperator for InitializedRasterReprojection { fn result_descriptor(&self) -> &RasterResultDescriptor { &self.result_descriptor } @@ -437,7 +449,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -450,7 +461,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -464,7 +474,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -477,7 +486,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -490,7 +498,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -503,7 +510,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -516,7 +522,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -529,7 +534,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -542,7 +546,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -555,7 +558,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -583,8 +585,7 @@ where result_descriptor: RasterResultDescriptor, from: SpatialReference, to: SpatialReference, - tiling_spec: TilingSpecification, - state: Option, + state: TileReprojectionSubqueryGridInfo, _phantom_data: PhantomData

, } @@ -592,7 +593,7 @@ impl RasterReprojectionProcessor where Q: QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialQuery = SpatialGridQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, @@ -603,15 +604,13 @@ where result_descriptor: RasterResultDescriptor, from: SpatialReference, to: SpatialReference, - tiling_spec: TilingSpecification, - state: Option, + state: TileReprojectionSubqueryGridInfo, ) -> Self { Self { source, result_descriptor, from, to, - tiling_spec, state, _phantom_data: PhantomData, } @@ -623,14 +622,14 @@ impl QueryProcessor for RasterReprojectionProcessor where Q: QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, P: Pixel, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -639,56 +638,32 @@ where query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { - if let Some(state) = &self.state { - let valid_bounds_in = state.valid_in_bounds; - let valid_bounds_out = state.valid_out_bounds; - - // calculate the spatial resolution the input data should have using the intersection and the requested resolution - let in_spatial_res = suggest_pixel_size_from_diag_cross_projected( - valid_bounds_out, - valid_bounds_in, - query.spatial_resolution, - )?; - - // setup the subquery - let sub_query_spec = TileReprojectionSubQuery { - in_srs: self.from, - out_srs: self.to, - fold_fn: fold_by_coordinate_lookup_future, - in_spatial_res, - valid_bounds_in, - valid_bounds_out, - _phantom_data: PhantomData, - }; - - // return the adapter which will reproject the tiles and uses the fill adapter to inject missing tiles - Ok(RasterSubQueryAdapter::<'a, P, _, _>::new( - &self.source, - query, - self.tiling_spec, - ctx, - sub_query_spec, - ) - .filter_and_fill(FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles)) - } else { - log::debug!("No intersection between source data / srs and target srs"); - - let tiling_strat = self - .tiling_spec - .strategy(query.spatial_resolution.x, -query.spatial_resolution.y); - - let grid_bounds = tiling_strat.tile_grid_box(query.spatial_partition()); - Ok(Box::pin(SparseTilesFillAdapter::new( - stream::empty(), - grid_bounds, - query.attributes.count(), - tiling_strat.geo_transform, - self.tiling_spec.tile_size_in_pixels, - FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles, - query.time_interval, - FillerTimeBounds::from(query.time_interval), // TODO: derive this from the query once the child query can provide this. - ))) - } + let state = self.state; + + // setup the subquery + let sub_query_spec = TileReprojectionSubQuery { + in_srs: self.from, + out_srs: self.to, + fold_fn: fold_by_coordinate_lookup_future, + state, + _phantom_data: PhantomData, + }; + + let tiling_strat = self + .result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(ctx.tiling_specification()) + .generate_data_tiling_strategy(); + + // return the adapter which will reproject the tiles and uses the fill adapter to inject missing tiles + Ok(RasterSubQueryAdapter::<'a, P, _, _>::new( + &self.source, + query, + tiling_strat, + ctx, + sub_query_spec, + ) + .filter_and_fill(FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles)) } fn result_descriptor(&self) -> &RasterResultDescriptor { @@ -699,40 +674,42 @@ where #[cfg(test)] mod tests { use super::*; - use crate::engine::{MockExecutionContext, MockQueryContext, RasterBandDescriptors}; + use crate::engine::{ + MockExecutionContext, MockQueryContext, RasterBandDescriptors, SpatialGridDescriptor, + }; use crate::mock::MockFeatureCollectionSource; use crate::mock::{MockRasterSource, MockRasterSourceParams}; + use crate::source::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, + GdalMetaDataStatic, GdalSourceTimePlaceholder, TimeReference, + }; + use crate::util::gdal::add_ndvi_dataset; use crate::{ engine::{ChunkByteSize, VectorOperator}, - source::{ - FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, - GdalMetaDataRegular, GdalMetaDataStatic, GdalSource, GdalSourceParameters, - GdalSourceTimePlaceholder, TimeReference, - }, + source::{GdalSource, GdalSourceParameters}, test_data, - util::gdal::{add_ndvi_dataset, gdal_open_dataset}, }; - use float_cmp::approx_eq; + use float_cmp::{approx_eq, assert_approx_eq}; use futures::StreamExt; use geoengine_datatypes::collections::IntoGeometryIterator; - use geoengine_datatypes::dataset::NamedData; + use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; + use geoengine_datatypes::hashmap; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, Coordinate2D, DateTimeParseFormat, + CacheHint, CacheTtlSeconds, DateTimeParseFormat, SpatialResolution, TimeGranularity, + TimeInstance, + }; + use geoengine_datatypes::primitives::{Coordinate2D, TimeStep}; + use geoengine_datatypes::raster::{ + GeoTransform, GridBoundingBox2D, GridShape2D, GridSize, SpatialGridDefinition, + TilesEqualIgnoringCacheHint, }; - use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds}; - use geoengine_datatypes::raster::TilesEqualIgnoringCacheHint; use geoengine_datatypes::{ collections::{ GeometryCollection, MultiLineStringCollection, MultiPointCollection, MultiPolygonCollection, }, - dataset::{DataId, DatasetId}, - hashmap, - primitives::{ - BoundingBox2D, MultiLineString, MultiPoint, MultiPolygon, QueryRectangle, - SpatialResolution, TimeGranularity, TimeInstance, TimeInterval, TimeStep, - }, - raster::{Grid, GridShape, GridShape2D, GridSize, RasterDataType, RasterTile2D}, + primitives::{BoundingBox2D, MultiLineString, MultiPoint, MultiPolygon, TimeInterval}, + raster::{Grid, RasterDataType, RasterTile2D}, spatial_reference::SpatialReferenceAuthority, util::{ Identifier, @@ -773,35 +750,34 @@ mod tests { let target_spatial_reference = SpatialReference::new(SpatialReferenceAuthority::Epsg, 900_913); + let exe_ctx = MockExecutionContext::test_default(); + let initialized_operator = VectorOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: point_source.into(), }, }) - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = initialized_operator.query_processor()?; let query_processor = query_processor.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new( (COLOGNE_EPSG_4326.x, MARBURG_EPSG_4326.y).into(), (MARBURG_EPSG_4326.x, HAMBURG_EPSG_4326.y).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -850,35 +826,34 @@ mod tests { let target_spatial_reference = SpatialReference::new(SpatialReferenceAuthority::Epsg, 900_913); + let exe_ctx = MockExecutionContext::test_default(); + let initialized_operator = VectorOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: lines_source.into(), }, }) - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = initialized_operator.query_processor()?; let query_processor = query_processor.multi_line_string().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new( (COLOGNE_EPSG_4326.x, MARBURG_EPSG_4326.y).into(), (MARBURG_EPSG_4326.x, HAMBURG_EPSG_4326.y).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -934,35 +909,34 @@ mod tests { let target_spatial_reference = SpatialReference::new(SpatialReferenceAuthority::Epsg, 900_913); + let exe_ctx = MockExecutionContext::test_default(); + let initialized_operator = VectorOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: polygon_source.into(), }, }) - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = initialized_operator.query_processor()?; let query_processor = query_processor.multi_polygon().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new( (COLOGNE_EPSG_4326.x, MARBURG_EPSG_4326.y).into(), (MARBURG_EPSG_4326.x, HAMBURG_EPSG_4326.y).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -983,12 +957,10 @@ mod tests { Ok(()) } + #[allow(clippy::too_many_lines)] #[tokio::test] async fn raster_identity() -> Result<()> { - let projection = SpatialReference::new( - geoengine_datatypes::spatial_reference::SpatialReferenceAuthority::Epsg, - 4326, - ); + let projection = SpatialReference::epsg_4326(); let data = vec![ RasterTile2D { @@ -996,7 +968,9 @@ mod tests { tile_position: [-1, 0].into(), band: 0, global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), + grid_array: Grid::new([2, 2].into(), vec![1_u8, 2, 3, 4]) + .unwrap() + .into(), properties: Default::default(), cache_hint: CacheHint::default(), }, @@ -1009,6 +983,26 @@ mod tests { properties: Default::default(), cache_hint: CacheHint::default(), }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 0].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([2, 2].into(), vec![1_u8, 2, 3, 4]) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 1].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, RasterTile2D { time: TimeInterval::new_unchecked(5, 10), tile_position: [-1, 0].into(), @@ -1031,34 +1025,60 @@ mod tests { properties: Default::default(), cache_hint: CacheHint::default(), }, + RasterTile2D { + time: TimeInterval::new_unchecked(5, 10), + tile_position: [0, 0].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(5, 10), + tile_position: [0, 1].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, ]; + let geo_transform = GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.); + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + geo_transform, + GridBoundingBox2D::new([-2, 0], [1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); + + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: Some(SpatialResolution::one()), - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); - let mut exe_ctx = MockExecutionContext::test_default(); - exe_ctx.tiling_specification.tile_size_in_pixels = GridShape { - // we need a smaller tile size - shape_array: [2, 2], - }; - - let query_ctx = MockQueryContext::test_default(); - let initialized_operator = RasterOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference: projection, // This test will do a identity reprojection + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: mrs1.into(), @@ -1073,12 +1093,11 @@ mod tests { .get_u8() .unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, 0], [1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); let a = qp.raster_query(query_rect, &query_ctx).await?; @@ -1094,19 +1113,18 @@ mod tests { #[tokio::test] async fn raster_ndvi_3857() -> Result<()> { let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - exe_ctx.tiling_specification = - TilingSpecification::new((0.0, 0.0).into(), [450, 450].into()); - let output_shape: GridShape2D = [900, 1800].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((0., 20_000_000.).into(), (20_000_000., 0.).into()); - let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_388_534_400_001); - // 2014-01-01 + let tile_size = GridShape2D::new_2d(512, 512); + exe_ctx.tiling_specification = TilingSpecification::new(tile_size); + + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + + let time_interval = TimeInterval::new_unchecked(1_396_303_200_000, 1_396_389_600_000); + // 2014-04-01 let gdal_op = GdalSource { - params: GdalSourceParameters { data: id.clone() }, + params: GdalSourceParameters::new(id.clone()), } .boxed(); @@ -1118,6 +1136,7 @@ mod tests { let initialized_operator = RasterOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference: projection, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: gdal_op.into(), @@ -1126,48 +1145,84 @@ mod tests { .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; - let x_query_resolution = output_bounds.size_x() / output_shape.axis_size_x() as f64; - let y_query_resolution = output_bounds.size_y() / (output_shape.axis_size_y() * 2) as f64; // *2 to account for the dataset aspect ratio 2:1 - let spatial_resolution = - SpatialResolution::new_unchecked(x_query_resolution, y_query_resolution); - let qp = initialized_operator .query_processor() .unwrap() .get_u8() .unwrap(); - let qs = qp - .raster_query( - RasterQueryRectangle { - spatial_bounds: output_bounds, - time_interval, - spatial_resolution, - attributes: BandSelection::first(), - }, - &query_ctx, - ) - .await - .unwrap(); + let result_descritptor = qp.result_descriptor(); + + assert_approx_eq!( + f64, + 14_255.015_508_816_849, // TODO: GDAL output is 14228.560819126376373 + result_descritptor + .spatial_grid_descriptor() + .spatial_resolution() + .x, + epsilon = 0.000_001 + ); + + assert_approx_eq!( + f64, + 14_255.015_508_816_849, // TODO: GDAL output is -14233.615370039031404 + result_descritptor + .spatial_grid_descriptor() + .spatial_resolution() + .y, + epsilon = 0.000_001 + ); + + let tlz = result_descritptor + .spatial_grid_descriptor() + .tiling_grid_definition(query_ctx.tiling_specification()) + .generate_data_tiling_strategy(); + let query_tl_pixel = tlz.tile_idx_to_global_pixel_idx([-1, 0].into()); + let query_bounds = + GridBoundingBox2D::new(query_tl_pixel, query_tl_pixel + [511, 511]).unwrap(); + + let qrect = RasterQueryRectangle::new_with_grid_bounds( + query_bounds, + time_interval, + BandSelection::first(), + ); + + let qs = qp.raster_query(qrect.clone(), &query_ctx).await.unwrap(); let res = qs .map(Result::unwrap) .collect::>>() .await; - // Write the tiles to a file - // let mut buffer = File::create("MOD13A2_M_NDVI_2014-04-01_tile-20.rst")?; - // buffer.write(res[8].clone().into_materialized_tile().grid_array.data.as_slice())?; + // get the worldfile + // println!("{}", res[0].tile_geo_transform().worldfile_string()); + + // Write the tile to a file + + /* + let mut buffer = std::fs::File::create("MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst")?; + + std::io::Write::write( + &mut buffer, + res[0] + .clone() + .into_materialized_tile() + .grid_array + .inner_grid + .data + .as_slice(), + )?; + */ // This check is against a tile produced by the operator itself. It was visually validated. TODO: rebuild when open issues are solved. // A perfect validation would be against a GDAL output generated like this: - // gdalwarp -t_srs EPSG:3857 -tr 11111.11111111 11111.11111111 -r near -te 0.0 5011111.111111112 5000000.0 10011111.111111112 -te_srs EPSG:3857 -of GTiff ./MOD13A2_M_NDVI_2014-04-01.TIFF ./MOD13A2_M_NDVI_2014-04-01_tile-20.rst + // gdalwarp -t_srs EPSG:3857 -r near -te_srs EPSG:3857 -of GTiff ./MOD13A2_M_NDVI_2014-04-01.TIFF ./MOD13A2_M_NDVI_2014-04-01.TIFF assert_eq!( include_bytes!( - "../../../test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20.rst" + "../../../test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst" ) as &[u8], - res[8] + res[0] .clone() .into_materialized_tile() .grid_array @@ -1181,20 +1236,19 @@ mod tests { #[test] fn query_rewrite_4326_3857() { - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + ); let expected = BoundingBox2D::new_unchecked( (-20_037_508.342_789_244, -20_048_966.104_014_594).into(), (20_037_508.342_789_244, 20_048_966.104_014_594).into(), ); - let reprojected = reproject_query( - query, + let reprojected = reproject_spatial_query( + query.spatial_query(), SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857), SpatialReference::epsg_4326(), ) @@ -1209,10 +1263,23 @@ mod tests { )); } + #[allow(clippy::too_many_lines)] #[tokio::test] async fn raster_ndvi_3857_to_4326() -> Result<()> { - let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let tile_size_in_pixels = [200, 200].into(); + let data_geo_transform = GeoTransform::new( + Coordinate2D::new(-20_037_508.342_789_244, 19_971_868.880_408_562), + 14_052.950_258_048_738, + -14_057.881_117_788_405, + ); + let data_bounds = GridBoundingBox2D::new([0, 0], [2840, 2850]).unwrap(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857).into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts(data_geo_transform, data_bounds), + bands: RasterBandDescriptors::new_single_band(), + }; let m = GdalMetaDataRegular { data_time: TimeInterval::new_unchecked( @@ -1236,12 +1303,12 @@ mod tests { .into(), rasterband_channel: 1, geo_transform: GdalDatasetGeoTransform { - origin_coordinate: (-20_037_508.342_789_244, 19_971_868.880_408_563).into(), - x_pixel_size: 14_052.950_258_048_739, - y_pixel_size: -14_057.881_117_788_405, + origin_coordinate: data_geo_transform.origin_coordinate, + x_pixel_size: data_geo_transform.x_pixel_size(), + y_pixel_size: data_geo_transform.y_pixel_size(), }, - width: 2851, - height: 2841, + width: data_bounds.axis_size_x(), + height: data_bounds.axis_size_y(), file_not_found_handling: FileNotFoundHandling::Error, no_data_value: Some(0.), properties_mapping: None, @@ -1250,37 +1317,29 @@ mod tests { allow_alphaband_as_mask: true, retry: None, }, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857) - .into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), cache_ttl: CacheTtlSeconds::default(), }; + let mut exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( + tile_size_in_pixels, + )); + let id: DataId = DatasetId::new().into(); let name = NamedData::with_system_name("ndvi"); exe_ctx.add_meta_data(id.clone(), name.clone(), Box::new(m)); - exe_ctx.tiling_specification = TilingSpecification::new((0.0, 0.0).into(), [60, 60].into()); - - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); - let time_interval = TimeInterval::new_unchecked(1_396_310_400_000, 1_396_310_400_000); - // 2014-04-01 + let time_interval = TimeInterval::new_unchecked(1_396_310_400_000, 1_396_310_400_000); // 2014-04-01 let gdal_op = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), } .boxed(); let initialized_operator = RasterOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::DataBounds, }, sources: SingleRasterOrVectorSource { source: gdal_op.into(), @@ -1289,25 +1348,24 @@ mod tests { .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; - let x_query_resolution = output_bounds.size_x() / 480.; // since we request x -180 to 180 and y -90 to 90 with 60x60 tiles this will result in 8 x 4 tiles - let y_query_resolution = output_bounds.size_y() / 240.; // *2 to account for the dataset aspect ratio 2:1 - let spatial_resolution = - SpatialResolution::new_unchecked(x_query_resolution, y_query_resolution); - let qp = initialized_operator .query_processor() .unwrap() .get_u8() .unwrap(); + let qr = qp.result_descriptor(); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + let qs = qp .raster_query( - QueryRectangle { - spatial_bounds: output_bounds, + RasterQueryRectangle::new_with_grid_bounds( + qr.spatial_grid_descriptor() + .tiling_grid_definition(query_ctx.tiling_specification()) + .tiling_grid_bounds(), time_interval, - spatial_resolution, - attributes: BandSelection::first(), - }, + BandSelection::first(), + ), &query_ctx, ) .await @@ -1318,51 +1376,31 @@ mod tests { .collect::>>() .await; - // the test must generate 8x4 tiles - assert_eq!(tiles.len(), 32); + // the test should generate 18x10 tiles. However, since the real procucrd pixel size is < 0.1 we will get 20 tiles on the x-axis + assert_eq!(tiles.len(), /*18*/ 20 * 10); // none of the tiles should be empty assert!(tiles.iter().all(|t| !t.is_empty())); - Ok(()) } - #[test] - fn source_resolution() { - let epsg_4326 = SpatialReference::epsg_4326(); - let epsg_3857 = SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857); - - // use ndvi dataset that was reprojected using gdal as ground truth - let dataset_4326 = gdal_open_dataset(test_data!( - "raster/modis_ndvi/MOD13A2_M_NDVI_2014-04-01.TIFF" - )) - .unwrap(); - let geotransform_4326 = dataset_4326.geo_transform().unwrap(); - let res_4326 = SpatialResolution::new(geotransform_4326[1], -geotransform_4326[5]).unwrap(); - - let dataset_3857 = gdal_open_dataset(test_data!( - "raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01.TIFF" - )) - .unwrap(); - let geotransform_3857 = dataset_3857.geo_transform().unwrap(); - let res_3857 = SpatialResolution::new(geotransform_3857[1], -geotransform_3857[5]).unwrap(); - - // ndvi was projected from 4326 to 3857. The calculated source_resolution for getting the raster in 3857 with `res_3857` - // should thus roughly be like the original `res_4326` - let result_res = suggest_pixel_size_from_diag_cross_projected::( - epsg_3857.area_of_use_projected().unwrap(), - epsg_4326.area_of_use_projected().unwrap(), - res_3857, - ) - .unwrap(); - assert!(1. - (result_res.x / res_4326.x).abs() < 0.02); - assert!(1. - (result_res.y / res_4326.y).abs() < 0.02); - } - #[tokio::test] async fn query_outside_projection_area_of_use_produces_empty_tiles() { - let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let tile_size_in_pixels = [600, 600].into(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32636).into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new( + Coordinate2D::new(166_021.44, 9_329_005.188), + 534_994.66 - 166_021.444, + -9_329_005.18, + ), + GridBoundingBox2D::new_min_max(0, 100, 0, 100).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; let m = GdalMetaDataStatic { time: Some(TimeInterval::default()), @@ -1384,38 +1422,30 @@ mod tests { allow_alphaband_as_mask: true, retry: None, }, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32636) - .into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), cache_ttl: CacheTtlSeconds::default(), }; + let mut exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( + tile_size_in_pixels, + )); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + let id: DataId = DatasetId::new().into(); let name = NamedData::with_system_name("ndvi"); exe_ctx.add_meta_data(id.clone(), name.clone(), Box::new(m)); - exe_ctx.tiling_specification = - TilingSpecification::new((0.0, 0.0).into(), [600, 600].into()); - - let output_shape: GridShape2D = [1000, 1000].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 0.).into(), (180., -90.).into()); let time_interval = TimeInterval::new_instant(1_388_534_400_000).unwrap(); // 2014-01-01 let gdal_op = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), } .boxed(); let initialized_operator = RasterOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: gdal_op.into(), @@ -1425,11 +1455,6 @@ mod tests { .await .unwrap(); - let x_query_resolution = output_bounds.size_x() / output_shape.axis_size_x() as f64; - let y_query_resolution = output_bounds.size_y() / (output_shape.axis_size_y()) as f64; - let spatial_resolution = - SpatialResolution::new_unchecked(x_query_resolution, y_query_resolution); - let qp = initialized_operator .query_processor() .unwrap() @@ -1438,12 +1463,11 @@ mod tests { let result = qp .raster_query( - QueryRectangle { - spatial_bounds: output_bounds, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(500, 1000, 500, 1000).unwrap(), time_interval, - spatial_resolution, - attributes: BandSelection::first(), - }, + BandSelection::first(), + ), &query_ctx, ) .await @@ -1486,6 +1510,7 @@ mod tests { SpatialReferenceAuthority::Epsg, 32636, // utm36n ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: point_source.into(), @@ -1509,12 +1534,11 @@ mod tests { let qs = qp .vector_query( - QueryRectangle { + VectorQueryRectangle::with_bounds( spatial_bounds, - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + TimeInterval::default(), + ColumnSelection::all(), + ), &query_ctx, ) .await @@ -1539,7 +1563,7 @@ mod tests { #[tokio::test] async fn points_from_utm36n_to_wgs84() { let exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); let point_source = MockFeatureCollectionSource::with_collections_and_sref( vec![ @@ -1566,6 +1590,7 @@ mod tests { SpatialReferenceAuthority::Epsg, 4326, // utm36n ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: point_source.into(), @@ -1589,12 +1614,11 @@ mod tests { let qs = qp .vector_query( - QueryRectangle { + VectorQueryRectangle::with_bounds( spatial_bounds, - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + TimeInterval::default(), + ColumnSelection::all(), + ), &query_ctx, ) .await @@ -1647,6 +1671,7 @@ mod tests { SpatialReferenceAuthority::Epsg, 4326, // utm36n ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: point_source.into(), @@ -1670,12 +1695,11 @@ mod tests { let qs = qp .vector_query( - QueryRectangle { + VectorQueryRectangle::with_bounds( spatial_bounds, - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + TimeInterval::default(), + ColumnSelection::all(), + ), &query_ctx, ) .await @@ -1691,39 +1715,40 @@ mod tests { assert!(points.coordinates().is_empty()); } + /* TODO resolve the problem with empty intersections + */ #[test] fn it_derives_raster_result_descriptor() { let in_proj = SpatialReference::epsg_4326(); let out_proj = SpatialReference::from_str("EPSG:3857").unwrap(); - let bbox = Some(SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - )); - let resolution = Some(SpatialResolution::new_unchecked(0.1, 0.1)); + let geo_transform = GeoTransform::new(Coordinate2D::new(0., 0.), 0.1, -0.1); + let grid_bounds = GridBoundingBox2D::new_min_max(-850, 849, -1800, 1799).unwrap(); + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); - let (in_bounds, out_bounds, out_res) = - InitializedRasterReprojection::derive_raster_in_bounds_out_bounds_out_res( - in_proj, out_proj, resolution, bbox, - ) - .unwrap(); + let projector = CoordinateProjector::from_known_srs(in_proj, out_proj).unwrap(); + + let out_spatial_grid = spatial_grid.reproject(&projector).unwrap(); assert_eq!( - in_bounds.unwrap(), - SpatialPartition2D::new_unchecked((-180., 85.06).into(), (180., -85.06).into(),) + out_spatial_grid.geo_transform.origin_coordinate(), + Coordinate2D::new(0., 0.) ); assert_eq!( - out_bounds.unwrap(), - out_proj - .area_of_use_projected::() - .unwrap() + out_spatial_grid.geo_transform.spatial_resolution(), + SpatialResolution::new_unchecked(14_212.246_793_017_477, 14_212.246_793_017_477) ); - // TODO: y resolution should be double the x resolution, but currently we only compute a uniform resolution + /* + Projected bounds: + -20037508.34 -20048966.1 + 20037508.34 20048966.1 + */ + assert_eq!( - out_res.unwrap(), - SpatialResolution::new_unchecked(14_237.781_884_528_267, 14_237.781_884_528_267), + out_spatial_grid.grid_bounds, + GridBoundingBox2D::new_min_max(-1405, 1405, -1410, 1409).unwrap() ); } } diff --git a/operators/src/processing/temporal_raster_aggregation/first_last_subquery.rs b/operators/src/processing/temporal_raster_aggregation/first_last_subquery.rs index a5622f490..257ba55cf 100644 --- a/operators/src/processing/temporal_raster_aggregation/first_last_subquery.rs +++ b/operators/src/processing/temporal_raster_aggregation/first_last_subquery.rs @@ -5,10 +5,7 @@ use crate::{ use async_trait::async_trait; use futures::{Future, FutureExt, TryFuture, TryFutureExt, future::BoxFuture}; use geoengine_datatypes::{ - primitives::{ - CacheHint, QueryRectangle, RasterQueryRectangle, SpatialPartitioned, TimeInstance, - TimeInterval, TimeStep, - }, + primitives::{CacheHint, RasterQueryRectangle, TimeInstance, TimeInterval, TimeStep}, raster::{EmptyGrid2D, Pixel, RasterTile2D, TileInformation}, }; use rayon::ThreadPool; @@ -106,8 +103,12 @@ impl FoldTileAccu for TemporalRasterAggregationTileAccu { } impl FoldTileAccuMut for TemporalRasterAggregationTileAccu { - fn tile_mut(&mut self) -> &mut RasterTile2D { - &mut self.accu_tile + fn set_time(&mut self, time: TimeInterval) { + self.accu_tile.time = time; + } + + fn set_cache_hint(&mut self, new_cache_hint: CacheHint) { + self.accu_tile.cache_hint = new_cache_hint; } } @@ -149,17 +150,16 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, + _query_rect: RasterQueryRectangle, start_time: TimeInstance, band_idx: u32, ) -> Result> { let snapped_start = self.step.snap_relative(self.step_reference, start_time)?; - Ok(Some(QueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - spatial_resolution: query_rect.spatial_resolution, - time_interval: TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, - attributes: band_idx.into(), - })) + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + tile_info.global_pixel_bounds(), + TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -225,17 +225,16 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, + _query_rect: RasterQueryRectangle, start_time: TimeInstance, band_idx: u32, ) -> Result> { - let snapped_start = self.step.snap_relative(self.step_reference, start_time)?; - Ok(Some(QueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - spatial_resolution: query_rect.spatial_resolution, - time_interval: TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, - attributes: band_idx.into(), - })) + let snapped_start_time = self.step.snap_relative(self.step_reference, start_time)?; + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + tile_info.global_pixel_bounds(), + TimeInterval::new(snapped_start_time, (snapped_start_time + self.step)?)?, + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { diff --git a/operators/src/processing/temporal_raster_aggregation/subquery.rs b/operators/src/processing/temporal_raster_aggregation/subquery.rs index 9ee9dae39..f3c6ee3c0 100644 --- a/operators/src/processing/temporal_raster_aggregation/subquery.rs +++ b/operators/src/processing/temporal_raster_aggregation/subquery.rs @@ -6,9 +6,7 @@ use crate::{ use async_trait::async_trait; use futures::TryFuture; use geoengine_datatypes::{ - primitives::{ - CacheHint, RasterQueryRectangle, SpatialPartitioned, TimeInstance, TimeInterval, TimeStep, - }, + primitives::{CacheHint, RasterQueryRectangle, TimeInstance, TimeInterval, TimeStep}, raster::{ EmptyGrid2D, GeoTransform, GridIdx2D, GridIndexAccess, GridOrEmpty, GridOrEmpty2D, GridShapeAccess, Pixel, RasterTile2D, TileInformation, UpdateIndexedElementsParallel, @@ -323,17 +321,16 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, + _query_rect: RasterQueryRectangle, start_time: TimeInstance, band_idx: u32, ) -> Result> { - let snapped_start = self.step.snap_relative(self.step_reference, start_time)?; - Ok(Some(RasterQueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - spatial_resolution: query_rect.spatial_resolution, - time_interval: TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, - attributes: band_idx.into(), - })) + let snapped_start_time = self.step.snap_relative(self.step_reference, start_time)?; + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + tile_info.global_pixel_bounds(), + TimeInterval::new(snapped_start_time, (snapped_start_time + self.step)?)?, + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -383,17 +380,16 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, + _query_rect: RasterQueryRectangle, start_time: TimeInstance, band_idx: u32, ) -> Result> { let snapped_start = self.step.snap_relative(self.step_reference, start_time)?; - Ok(Some(RasterQueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - spatial_resolution: query_rect.spatial_resolution, - time_interval: TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, - attributes: band_idx.into(), - })) + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + tile_info.global_pixel_bounds(), + TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { diff --git a/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs b/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs index 828464add..566d6d3e7 100644 --- a/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs +++ b/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs @@ -26,7 +26,8 @@ use crate::{ }; use async_trait::async_trait; use geoengine_datatypes::primitives::{ - BandSelection, RasterQueryRectangle, SpatialPartition2D, TimeInstance, + BandSelection, RasterQueryRectangle, RasterSpatialQueryRectangle, SpatialGridQueryRectangle, + TimeInstance, }; use geoengine_datatypes::raster::{Pixel, RasterDataType, RasterTile2D}; use geoengine_datatypes::{primitives::TimeStep, raster::TilingSpecification}; @@ -216,7 +217,7 @@ where Q: RasterQueryProcessor + QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialQuery = SpatialGridQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, @@ -307,6 +308,11 @@ where } ); + let grid_desc = self.result_descriptor.spatial_grid_descriptor(); + let tiling_strategy = grid_desc + .tiling_grid_definition(self.tiling_specification) + .generate_data_tiling_strategy(); + Ok(match self.aggregation_type { Aggregation::Min { ignore_no_data: true, @@ -317,7 +323,7 @@ where MinPixelAggregatorIngoringNoData, >, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Min"), Aggregation::Min { ignore_no_data: false, @@ -325,7 +331,7 @@ where .create_subquery( super::subquery::subquery_all_tiles_fold_fn::, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Min"), Aggregation::Max { ignore_no_data: true, @@ -336,7 +342,7 @@ where MaxPixelAggregatorIngoringNoData, >, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Max"), Aggregation::Max { ignore_no_data: false, @@ -344,9 +350,8 @@ where .create_subquery( super::subquery::subquery_all_tiles_fold_fn::, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Max"), - Aggregation::First { ignore_no_data: true, } => self @@ -356,13 +361,13 @@ where FirstPixelAggregatorIngoringNoData, >, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::First"), Aggregation::First { ignore_no_data: false, } => self .create_subquery_first(first_tile_fold_future::

) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::First"), Aggregation::Last { ignore_no_data: true, @@ -373,34 +378,30 @@ where LastPixelAggregatorIngoringNoData, >, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Last"), - Aggregation::Last { ignore_no_data: false, } => self .create_subquery_last(last_tile_fold_future::

) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Last"), - Aggregation::Mean { ignore_no_data: true, } => self .create_subquery( super::subquery::subquery_all_tiles_fold_fn::>, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Mean"), - Aggregation::Mean { ignore_no_data: false, } => self .create_subquery( super::subquery::subquery_all_tiles_fold_fn::>, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Mean"), - Aggregation::Sum { ignore_no_data: true, } => self @@ -410,18 +411,16 @@ where SumPixelAggregatorIngoringNoData, >, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Sum"), - Aggregation::Sum { ignore_no_data: false, } => self .create_subquery( super::subquery::subquery_all_tiles_fold_fn::, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Sum"), - Aggregation::Count { ignore_no_data: true, } => self @@ -431,16 +430,15 @@ where CountPixelAggregatorIngoringNoData, >, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Sum"), - + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) + .expect("no tiles must be skipped in Aggregation::Count"), Aggregation::Count { ignore_no_data: false, } => self .create_subquery( super::subquery::subquery_all_tiles_fold_fn::, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::Sum"), Aggregation::PercentileEstimate { ignore_no_data: true, @@ -450,7 +448,7 @@ where PercentileEstimateAggregator::::new(percentile), super::subquery::subquery_all_tiles_global_state_fold_fn, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::PercentileEstimate"), Aggregation::PercentileEstimate { ignore_no_data: false, @@ -460,7 +458,7 @@ where PercentileEstimateAggregator::::new(percentile), super::subquery::subquery_all_tiles_global_state_fold_fn, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy) .expect("no tiles must be skipped in Aggregation::PercentileEstimate"), }) } @@ -471,14 +469,14 @@ impl QueryProcessor for TemporalRasterAggregationProcessor where Q: QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialQuery = RasterSpatialQueryRectangle, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, P: Pixel, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -500,20 +498,11 @@ where #[cfg(test)] mod tests { - use futures::stream::StreamExt; - use geoengine_datatypes::{ - primitives::{CacheHint, SpatialResolution, TimeInterval}, - raster::{ - EmptyGrid, EmptyGrid2D, Grid2D, GridOrEmpty, MaskedGrid2D, RasterDataType, RenameBands, - TileInformation, TilesEqualIgnoringCacheHint, - }, - spatial_reference::SpatialReference, - util::test::TestDefault, - }; - + use super::*; use crate::{ engine::{ MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, + SpatialGridDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{ @@ -521,25 +510,40 @@ mod tests { raster_stacker::{RasterStacker, RasterStackerParams}, }, }; - - use super::*; + use futures::stream::StreamExt; + use geoengine_datatypes::{ + primitives::{CacheHint, Coordinate2D, TimeInterval}, + raster::{ + EmptyGrid, EmptyGrid2D, GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty, + GridShape2D, MaskedGrid2D, RasterDataType, RenameBands, TileInformation, + TilesEqualIgnoringCacheHint, + }, + spatial_reference::SpatialReference, + util::test::TestDefault, + }; #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_min() { let raster_tiles = make_raster(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [-1, 2]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -560,16 +564,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -652,17 +652,23 @@ mod tests { async fn test_max() { let raster_tiles = make_raster(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -683,16 +689,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -775,17 +777,23 @@ mod tests { async fn test_max_with_no_data() { let raster_tiles = make_raster(); // TODO: switch to make_raster_with_no_data? + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -806,16 +814,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -898,17 +902,23 @@ mod tests { async fn test_max_with_no_data_but_ignoring_it() { let raster_tiles = make_raster(); // TODO: switch to make_raster_with_no_data? + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -929,16 +939,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1019,6 +1025,19 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_only_no_data() { + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new_min_max(-3, -1, 0, 2).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: vec![RasterTile2D::new_with_tile_info( @@ -1032,14 +1051,7 @@ mod tests { GridOrEmpty::from(EmptyGrid2D::::new([3, 2].into())), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1060,16 +1072,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (2., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-3, -1, 0, 1).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1109,17 +1117,23 @@ mod tests { async fn test_first_with_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1140,16 +1154,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1212,21 +1222,26 @@ mod tests { async fn test_last_with_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); - let agg = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Last { @@ -1243,16 +1258,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1315,17 +1326,23 @@ mod tests { async fn test_last() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1346,16 +1363,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1418,17 +1431,23 @@ mod tests { async fn test_first() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1449,16 +1468,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1521,17 +1536,23 @@ mod tests { async fn test_mean_nodata() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1552,16 +1573,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1624,17 +1641,23 @@ mod tests { async fn test_mean_ignore_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1655,16 +1678,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1726,6 +1745,29 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_sum_without_nodata() { + let raster_tiles = make_raster(); + + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let mrs = MockRasterSource { + params: MockRasterSourceParams { + data: raster_tiles, + result_descriptor, + }, + } + .boxed(); + let operator = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Sum { @@ -1738,35 +1780,16 @@ mod tests { window_reference: Some(TimeInstance::from_millis(0).unwrap()), output_type: None, }, - sources: SingleRasterSource { - raster: MockRasterSource { - params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(), - }, + sources: SingleRasterSource { raster: mrs }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let query_processor = operator @@ -1848,17 +1871,23 @@ mod tests { async fn test_sum_nodata() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1879,16 +1908,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -1951,17 +1976,23 @@ mod tests { async fn test_sum_ignore_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1982,16 +2013,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -2053,6 +2080,29 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_sum_with_larger_data_type() { + let raster_tiles = make_raster(); + + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let mrs = MockRasterSource { + params: MockRasterSourceParams { + data: raster_tiles, + result_descriptor: result_descriptor.clone(), + }, + } + .boxed(); + let operator = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Sum { @@ -2073,38 +2123,19 @@ mod tests { output_band: None, map_no_data: true, }, - sources: SingleRasterSource { - raster: MockRasterSource { - params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(), - }, + sources: SingleRasterSource { raster: mrs }, } .boxed(), }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let query_processor = operator @@ -2197,6 +2228,29 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_count_without_nodata() { + let raster_tiles = make_raster(); + + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let mrs = MockRasterSource { + params: MockRasterSourceParams { + data: raster_tiles, + result_descriptor, + }, + } + .boxed(); + let operator = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Count { @@ -2209,35 +2263,16 @@ mod tests { window_reference: Some(TimeInstance::from_millis(0).unwrap()), output_type: None, }, - sources: SingleRasterSource { - raster: MockRasterSource { - params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(), - }, + sources: SingleRasterSource { raster: mrs }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let query_processor = operator @@ -2319,17 +2354,23 @@ mod tests { async fn test_count_nodata() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -2350,16 +2391,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -2422,17 +2459,23 @@ mod tests { async fn test_count_ignore_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -2453,16 +2496,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -2525,17 +2564,23 @@ mod tests { async fn test_query_not_aligned_with_window_reference() { let raster_tiles = make_raster(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -2556,16 +2601,12 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(5, 5), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(5, 5), + BandSelection::first(), + ); let query_ctx = MockQueryContext::test_default(); let qp = agg @@ -2804,6 +2845,18 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_sums_multiple_bands() { + let data = make_raster(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let operator = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Sum { @@ -2825,29 +2878,15 @@ mod tests { rasters: vec![ MockRasterSource { params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + data: data.clone(), + result_descriptor: result_descriptor.clone(), }, } .boxed(), MockRasterSource { params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + data: data.clone(), + result_descriptor, }, } .boxed(), @@ -2859,16 +2898,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1].try_into().unwrap(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + [0, 1].try_into().unwrap(), + ); let query_ctx = MockQueryContext::test_default(); let query_processor = operator @@ -3002,17 +3038,21 @@ mod tests { async fn it_estimates_a_median() { let raster_tiles = make_raster(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -3034,16 +3074,14 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); + let query_ctx = MockQueryContext::test_default(); let qp = agg diff --git a/operators/src/processing/time_projection/mod.rs b/operators/src/processing/time_projection/mod.rs index b9cfb0f23..648a72959 100644 --- a/operators/src/processing/time_projection/mod.rs +++ b/operators/src/processing/time_projection/mod.rs @@ -254,12 +254,11 @@ fn expand_query_rectangle( step_reference: TimeInstance, query: &VectorQueryRectangle, ) -> Result { - Ok(VectorQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: expand_time_interval(step, step_reference, query.time_interval)?, - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }) + Ok(VectorQueryRectangle::new( + query.spatial_query, + expand_time_interval(step, step_reference, query.time_interval)?, + ColumnSelection::all(), + )) } fn expand_time_interval( @@ -293,8 +292,7 @@ mod tests { use geoengine_datatypes::{ collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection, VectorDataType}, primitives::{ - BoundingBox2D, CacheHint, DateTime, MultiPoint, SpatialResolution, TimeGranularity, - TimeInterval, + BoundingBox2D, CacheHint, DateTime, MultiPoint, TimeGranularity, TimeInterval, }, spatial_reference::SpatialReference, util::test::TestDefault, @@ -478,16 +476,15 @@ mod tests { let mut stream = query_processor .vector_query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 4, 3, 0, 0, 0), DateTime::new_utc(2010, 5, 14, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }, + ColumnSelection::all(), + ), &query_context, ) .await @@ -583,16 +580,15 @@ mod tests { let mut stream = query_processor .vector_query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 4, 3, 0, 0, 0), DateTime::new_utc(2010, 5, 14, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }, + ColumnSelection::all(), + ), &query_context, ) .await diff --git a/operators/src/processing/time_shift.rs b/operators/src/processing/time_shift.rs index 8f295b2a4..0e65ba865 100644 --- a/operators/src/processing/time_shift.rs +++ b/operators/src/processing/time_shift.rs @@ -457,12 +457,8 @@ where ) -> Result>> { let (time_interval, state) = self.shift.shift(query.time_interval)?; - let query = VectorQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval, - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }; + let query = + VectorQueryRectangle::new(query.spatial_query, time_interval, ColumnSelection::all()); let stream = self.processor.vector_query(query, ctx).await?; let stream = stream.then(move |collection| async move { @@ -507,12 +503,7 @@ where ctx: &'a dyn QueryContext, ) -> Result>>> { let (time_interval, state) = self.shift.shift(query.time_interval)?; - let query = RasterQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval, - spatial_resolution: query.spatial_resolution, - attributes: query.attributes, - }; + let query = RasterQueryRectangle::new(query.spatial_query, time_interval, query.attributes); // TODO: use grid bounds? let stream = self.processor.raster_query(query, ctx).await?; let stream = stream.map(move |raster| { @@ -539,7 +530,7 @@ mod tests { use crate::{ engine::{ MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, - SingleRasterSource, + SingleRasterSource, SpatialGridDescriptor, }, mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}, processing::{Expression, ExpressionParams, RasterStacker, RasterStackerParams}, @@ -551,12 +542,12 @@ mod tests { collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection}, dataset::NamedData, primitives::{ - BandSelection, BoundingBox2D, CacheHint, DateTime, MultiPoint, SpatialPartition2D, - SpatialResolution, TimeGranularity, + BandSelection, BoundingBox2D, CacheHint, Coordinate2D, DateTime, MultiPoint, + TimeGranularity, }, raster::{ - EmptyGrid2D, GridOrEmpty, RasterDataType, RenameBands, TileInformation, - TilingSpecification, + BoundedGrid, EmptyGrid2D, GeoTransform, GridBoundingBox2D, GridOrEmpty, GridShape2D, + RasterDataType, RenameBands, TileInformation, TilingSpecification, }, spatial_reference::SpatialReference, util::test::TestDefault, @@ -568,9 +559,9 @@ mod tests { sources: SingleRasterOrVectorSource { source: RasterOrVectorOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("test-raster"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name( + "test-raster", + )), } .boxed(), ), @@ -599,7 +590,8 @@ mod tests { "source": { "type": "GdalSource", "params": { - "data": "test-raster" + "data": "test-raster", + "overviewLevel": null } } } @@ -617,9 +609,9 @@ mod tests { sources: SingleRasterOrVectorSource { source: RasterOrVectorOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("test-raster"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name( + "test-raster", + )), } .boxed(), ), @@ -644,7 +636,8 @@ mod tests { "source": { "type": "GdalSource", "params": { - "data": "test-raster" + "data": "test-raster", + "overviewLevel": null } } } @@ -711,16 +704,15 @@ mod tests { let mut stream = query_processor .vector_query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2009, 1, 1, 0, 0, 0), DateTime::new_utc(2012, 1, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }, + ColumnSelection::all(), + ), &query_context, ) .await @@ -802,16 +794,15 @@ mod tests { let mut stream = query_processor .vector_query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 1, 1, 0, 0, 0), DateTime::new_utc(2011, 1, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }, + ColumnSelection::all(), + ), &query_context, ) .await @@ -849,7 +840,20 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_absolute_raster_shift() { - let empty_grid = GridOrEmpty::Empty(EmptyGrid2D::::new([3, 2].into())); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., -3.), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let empty_grid = GridOrEmpty::Empty(EmptyGrid2D::::new(tile_size_in_pixels)); let raster_tiles = vec![ RasterTile2D::new_with_tile_info( TimeInterval::new_unchecked( @@ -940,14 +944,7 @@ mod tests { let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -964,9 +961,8 @@ mod tests { }, }; - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_context = MockQueryContext::test_default(); let query_processor = RasterOperator::boxed(time_shift) @@ -980,19 +976,15 @@ mod tests { let mut stream = query_processor .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (4., 0.).into(), - ), - time_interval: TimeInterval::new( + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 1, 1, 0, 0, 0), DateTime::new_utc(2011, 1, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + BandSelection::first(), + ), &query_context, ) .await @@ -1024,7 +1016,20 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_relative_raster_shift() { - let empty_grid = GridOrEmpty::Empty(EmptyGrid2D::::new([3, 2].into())); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [0, 4]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let empty_grid = GridOrEmpty::Empty(EmptyGrid2D::::new(tile_size_in_pixels)); let raster_tiles = vec![ RasterTile2D::new_with_tile_info( TimeInterval::new_unchecked( @@ -1115,14 +1120,7 @@ mod tests { let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1137,9 +1135,7 @@ mod tests { }, }; - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let query_context = MockQueryContext::test_default(); let query_processor = RasterOperator::boxed(time_shift) @@ -1153,19 +1149,15 @@ mod tests { let mut stream = query_processor .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (4., 0.).into(), - ), - time_interval: TimeInterval::new( + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 1, 1, 0, 0, 0), DateTime::new_utc(2011, 1, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + BandSelection::first(), + ), &query_context, ) .await @@ -1199,9 +1191,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let ndvi_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -1249,18 +1239,11 @@ mod tests { let mut stream = query_processor .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2014, 3, 1, 0, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-90, 89, -180, 179).unwrap(), // Note: this is not the actual bounding box of the NDVI dataset. The pixel size is 0.1! + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ), &query_context, ) .await @@ -1287,9 +1270,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let ndvi_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -1316,18 +1297,11 @@ mod tests { let mut stream = query_processor .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2014, 3, 1, 0, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-90, 89, -180, 179).unwrap(), // Note: this is not the actual bounding box of the NDVI dataset. The pixel size is 0.1! + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ), &query_context, ) .await diff --git a/operators/src/processing/vector_join/equi_data_join.rs b/operators/src/processing/vector_join/equi_data_join.rs index 79b95740f..e302c2199 100644 --- a/operators/src/processing/vector_join/equi_data_join.rs +++ b/operators/src/processing/vector_join/equi_data_join.rs @@ -11,7 +11,8 @@ use geoengine_datatypes::collections::{ GeometryRandomAccess, }; use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, FeatureDataRef, Geometry, TimeInterval, VectorQueryRectangle, + ColumnSelection, FeatureDataRef, Geometry, TimeInterval, VectorQueryRectangle, + VectorSpatialQueryRectangle, }; use geoengine_datatypes::util::arrow::ArrowTyped; @@ -355,7 +356,7 @@ where FeatureCollectionRowBuilder: GeoFeatureCollectionRowBuilder, { type Output = FeatureCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -409,23 +410,20 @@ where #[cfg(test)] mod tests { use futures::executor::block_on_stream; - use geoengine_datatypes::collections::{ ChunksEqualIgnoringCacheHint, MultiPointCollection, VectorDataType, }; - use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, MultiPoint, SpatialResolution, TimeInterval, + BoundingBox2D, CacheHint, FeatureData, MultiPoint, TimeInterval, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; + use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, VectorOperator, WorkflowOperatorPath, + ChunkByteSize, MockExecutionContext, VectorOperator, WorkflowOperatorPath, }; use crate::mock::MockFeatureCollectionSource; - - use super::*; use crate::processing::vector_join::util::translation_table; async fn join_mock_collections( @@ -451,18 +449,13 @@ mod tests { let left_processor = left.query_processor().unwrap().multi_point().unwrap(); let right_processor = right.query_processor().unwrap().data().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( - (f64::MIN, f64::MIN).into(), - (f64::MAX, f64::MAX).into(), - ) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((f64::MIN, f64::MIN).into(), (f64::MAX, f64::MAX).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let ctx = execution_context.mock_query_context(ChunkByteSize::MAX); let processor = EquiGeoToDataJoinProcessor::new( VectorResultDescriptor { diff --git a/operators/src/source/csv.rs b/operators/src/source/csv.rs index ff3e2f108..119cb1e0b 100644 --- a/operators/src/source/csv.rs +++ b/operators/src/source/csv.rs @@ -8,7 +8,9 @@ use futures::stream::BoxStream; use futures::task::{Context, Poll}; use futures::{Stream, StreamExt}; use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::{ColumnSelection, VectorQueryRectangle}; +use geoengine_datatypes::primitives::{ + ColumnSelection, SpatialBounded, VectorQueryRectangle, VectorSpatialQueryRectangle, +}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, ensure}; @@ -374,7 +376,7 @@ struct CsvSourceProcessor { #[async_trait] impl QueryProcessor for CsvSourceProcessor { type Output = MultiPointCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -384,7 +386,12 @@ impl QueryProcessor for CsvSourceProcessor { _ctx: &'a dyn QueryContext, ) -> Result>> { // TODO: properly handle chunk_size - Ok(CsvSourceStream::new(self.params.clone(), query.spatial_bounds, 10)?.boxed()) + Ok(CsvSourceStream::new( + self.params.clone(), + query.spatial_query().spatial_bounds(), + 10, + )? + .boxed()) } fn result_descriptor(&self) -> &VectorResultDescriptor { @@ -407,13 +414,14 @@ struct ParsedRow { #[cfg(test)] mod tests { - use std::io::{Seek, SeekFrom, Write}; - - use geoengine_datatypes::primitives::SpatialResolution; - use super::*; use crate::engine::MockQueryContext; - use geoengine_datatypes::collections::{FeatureCollectionInfos, ToGeoJson}; + use geoengine_datatypes::{ + collections::{FeatureCollectionInfos, ToGeoJson}, + raster::TilingSpecification, + util::test::TestDefault, + }; + use std::io::{Seek, SeekFrom, Write}; #[test] fn it_deserializes() { @@ -590,16 +598,12 @@ x,y }, }; - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - Coordinate2D::new(0., 0.), - Coordinate2D::new(3., 3.), - ), - time_interval: TimeInterval::new_unchecked(0, 1), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new((10 * 8 * 2).into()); + let query = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked(Coordinate2D::new(0., 0.), Coordinate2D::new(3., 3.)), + TimeInterval::new_unchecked(0, 1), + ColumnSelection::all(), + ); + let ctx = MockQueryContext::new((10 * 8 * 2).into(), TilingSpecification::test_default()); let r: Vec> = p.query(query, &ctx).await.unwrap().collect().await; diff --git a/operators/src/source/gdal_source/error.rs b/operators/src/source/gdal_source/error.rs index b7958c52b..ed51774a8 100644 --- a/operators/src/source/gdal_source/error.rs +++ b/operators/src/source/gdal_source/error.rs @@ -7,4 +7,9 @@ use snafu::Snafu; pub enum GdalSourceError { #[snafu(display("Unsupported raster type: {raster_type:?}"))] UnsupportedRasterType { raster_type: RasterDataType }, + + #[snafu(display("Unsupported spatial query: {spatial_query:?}"))] + IncompatibleSpatialQuery { + spatial_query: geoengine_datatypes::primitives::SpatialGridQueryRectangle, + }, } diff --git a/operators/src/source/gdal_source/loading_info.rs b/operators/src/source/gdal_source/loading_info.rs index c03d0c053..9738b2f0d 100644 --- a/operators/src/source/gdal_source/loading_info.rs +++ b/operators/src/source/gdal_source/loading_info.rs @@ -12,7 +12,7 @@ use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, FromSql, ToSql)] +#[derive(Serialize, Deserialize, Debug, Clone, FromSql, ToSql, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GdalMetaDataStatic { pub time: Option, @@ -79,7 +79,7 @@ impl MetaData /// sets `step` time apart. The `time_placeholders` in the file path of the dataset are replaced with the /// specified time `reference` in specified time `format`. Inside the `data_time` the gdal source will load the data /// from the files and outside it will create nodata. -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GdalMetaDataRegular { pub result_descriptor: RasterResultDescriptor, @@ -106,22 +106,28 @@ impl MetaData TimeStepIter::new_with_interval(data_time, step)? .into_intervals(step, data_time.end()) .for_each(|time_interval| { - if time_interval.start() <= query.time_interval.start() { - let t = if time_interval.end() > query.time_interval.start() { - time_interval.start() - } else { - time_interval.end() - }; - known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); + if time_interval.contains(&query.time_interval) { + let t1 = time_interval.start(); + let t2 = time_interval.end(); + known_time_start = Some(t1); + known_time_end = Some(t2); + return; } - if time_interval.end() >= query.time_interval.end() { - let t = if time_interval.start() < query.time_interval.end() { - time_interval.end() - } else { - time_interval.start() - }; - known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + if time_interval.end() <= query.time_interval.start() { + let t1 = time_interval.end(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } else if time_interval.start() <= query.time_interval.start() { + let t1 = time_interval.start(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } + + if time_interval.start() >= query.time_interval.end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() >= query.time_interval.end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); } }); @@ -155,7 +161,7 @@ impl MetaData } /// Meta data for 4D `NetCDF` CF datasets -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GdalMetadataNetCdfCf { pub result_descriptor: RasterResultDescriptor, @@ -232,7 +238,7 @@ impl MetaData } // TODO: custom deserializer that checks that that params are sorted and do not overlap -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, FromSql, ToSql)] +#[derive(Serialize, Deserialize, Debug, Clone, FromSql, ToSql, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GdalMetaDataList { pub result_descriptor: RasterResultDescriptor, @@ -251,22 +257,42 @@ impl MetaData for .inspect(|m| { let time_interval = m.time; - if time_interval.start() <= query.time_interval.start() { - let t = if time_interval.end() > query.time_interval.start() { - time_interval.start() - } else { - time_interval.end() - }; - known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); + debug_assert!( + !time_interval.is_instant(), + "time_interval {time_interval} is an instant!" + ); + + if time_interval.contains(&query.time_interval) { + let t1 = time_interval.start(); + let t2 = time_interval.end(); + known_time_start = Some(t1); + known_time_end = Some(t2); + return; } - if time_interval.end() >= query.time_interval.end() { - let t = if time_interval.start() < query.time_interval.end() { - time_interval.end() - } else { - time_interval.start() - }; - known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + if time_interval.end() <= query.time_interval.start() { + let t1 = time_interval.end(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } else if time_interval.start() <= query.time_interval.start() { + let t1 = time_interval.start(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } + + if query.time_interval.is_instant() { + // be carefull not to use instant ends... + if time_interval.start() > query.time_interval.end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() > query.time_interval.end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } + } else if time_interval.start() >= query.time_interval.end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() >= query.time_interval.end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); } }) .filter(|m| m.time.intersects(&query.time_interval)) @@ -616,17 +642,14 @@ pub struct GdalLoadingInfoTemporalSlice { mod tests { use geoengine_datatypes::{ hashmap, - primitives::{ - BandSelection, DateTime, DateTimeParseFormat, SpatialPartition2D, SpatialResolution, - TimeGranularity, - }, - raster::RasterDataType, + primitives::{BandSelection, DateTime, DateTimeParseFormat, TimeGranularity}, + raster::{BoundedGrid, GeoTransform, GridBoundingBox2D, GridShape2D, RasterDataType}, spatial_reference::SpatialReference, util::test::TestDefault, }; use crate::{ - engine::RasterBandDescriptors, + engine::{RasterBandDescriptors, SpatialGridDescriptor}, source::{FileNotFoundHandling, GdalDatasetGeoTransform, TimeReference}, }; @@ -640,8 +663,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: GdalDatasetParameters { @@ -686,9 +711,11 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band() + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box() + ), + bands: RasterBandDescriptors::new_single_band(), } ); } @@ -699,15 +726,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first() + )) .await .unwrap() .info @@ -742,15 +765,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::default(), + BandSelection::first() + )) .await .unwrap() .info @@ -787,15 +806,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(-10, -5), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(-10, -5), + BandSelection::first() + )) .await .unwrap() .info @@ -817,15 +832,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(50, 55), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(50, 55), + BandSelection::first() + )) .await .unwrap() .info @@ -847,15 +858,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(0, 22), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(0, 22), + BandSelection::first() + )) .await .unwrap() .info @@ -886,15 +893,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first() + )) .await .unwrap() .info @@ -929,8 +932,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: vec![ @@ -997,23 +1002,21 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box() + ), bands: RasterBandDescriptors::new_single_band() } ); assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(0, 3), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(0, 3), + BandSelection::first() + )) .await .unwrap() .info @@ -1044,8 +1047,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., 0.).into(), 1., -1.), + GridShape2D::new_2d(128, 128).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: GdalDatasetParameters { @@ -1073,12 +1078,11 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }; - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 128.).into(), (128., 0.).into()), - time_interval: TimeInterval::new(time_start, time_end).unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-128, 0], [-1, 127]).unwrap(), + TimeInterval::new(time_start, time_end).unwrap(), + BandSelection::first(), + ); let loading_info = metadata.loading_info(query).await.unwrap(); let mut iter = loading_info.info; @@ -1112,8 +1116,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: GdalDatasetParameters { @@ -1141,12 +1147,11 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }; - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 128.).into(), (128., 0.).into()), - time_interval: TimeInterval::new(time_start, time_end).unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-128, 0], [-1, 127]).unwrap(), + TimeInterval::new(time_start, time_end).unwrap(), + BandSelection::first(), + ); let loading_info = metadata.loading_info(query).await.unwrap(); let mut iter = loading_info.info; @@ -1180,8 +1185,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: GdalDatasetParameters { @@ -1209,15 +1216,14 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }; - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 128.).into(), (128., 0.).into()), - time_interval: TimeInterval::new_unchecked( + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-128, 0], [-1, 127]).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from(DateTime::new_utc(2009, 7, 1, 0, 0, 0)), TimeInstance::from(DateTime::new_utc(2013, 3, 1, 0, 0, 0)), ), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + BandSelection::first(), + ); let loading_info = metadata.loading_info(query).await.unwrap(); let mut iter = loading_info.info; diff --git a/operators/src/source/gdal_source/mod.rs b/operators/src/source/gdal_source/mod.rs index 1d381ecde..0b75bfcd2 100644 --- a/operators/src/source/gdal_source/mod.rs +++ b/operators/src/source/gdal_source/mod.rs @@ -2,8 +2,10 @@ use crate::adapters::{ FillerTileCacheExpirationStrategy, FillerTimeBounds, SparseTilesFillAdapter, }; use crate::engine::{ - CanonicOperatorName, MetaData, OperatorData, OperatorName, QueryProcessor, WorkflowOperatorPath, + CanonicOperatorName, MetaData, OperatorData, OperatorName, QueryProcessor, + SpatialGridDescriptor, WorkflowOperatorPath, }; +use crate::source::gdal_source::reader::ReaderState; use crate::util::TemporaryGdalThreadLocalConfigOptions; use crate::util::gdal::gdal_open_dataset_ex; use crate::util::input::float_option_with_nan; @@ -28,23 +30,21 @@ use gdal::errors::GdalError; use gdal::raster::{GdalType, RasterBand as GdalRasterBand}; use gdal::{Dataset as GdalDataset, DatasetOptions, GdalOpenFlags, Metadata as GdalMetadata}; use gdal_sys::VSICurlPartialClearCache; -use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, Coordinate2D, DateTimeParseFormat, RasterQueryRectangle, - SpatialPartition2D, SpatialPartitioned, TimeInstance, -}; -use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::raster::TileInformation; -use geoengine_datatypes::raster::{ - EmptyGrid, GeoTransform, GridIdx2D, GridOrEmpty, GridOrEmpty2D, GridShape2D, GridShapeAccess, - MapElements, MaskedGrid, NoDataValueGrid, Pixel, RasterDataType, RasterProperties, - RasterPropertiesEntry, RasterPropertiesEntryType, RasterPropertiesKey, RasterTile2D, - TilingStrategy, -}; -use geoengine_datatypes::util::test::TestDefault; +use geoengine_datatypes::primitives::SpatialGridQueryRectangle; use geoengine_datatypes::{ - primitives::TimeInterval, - raster::{Grid, GridBlit, GridBoundingBox2D, GridIdx, GridSize, TilingSpecification}, + dataset::NamedData, + primitives::{ + BandSelection, CacheHint, Coordinate2D, DateTimeParseFormat, RasterQueryRectangle, + RasterSpatialQueryRectangle, TimeInstance, TimeInterval, + }, + raster::{ + ChangeGridBounds, EmptyGrid, GeoTransform, Grid, GridBlit, GridBoundingBox2D, + GridIntersection, GridOrEmpty, GridOrEmpty2D, GridShapeAccess, GridSize, MapElements, + MaskedGrid, NoDataValueGrid, Pixel, RasterDataType, RasterProperties, + RasterPropertiesEntry, RasterPropertiesEntryType, RasterPropertiesKey, RasterTile2D, + SpatialGridDefinition, TileInformation, TilingSpecification, TilingStrategy, + }, + util::test::TestDefault, }; use itertools::Itertools; pub use loading_info::{ @@ -52,8 +52,11 @@ pub use loading_info::{ GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, GdalMetadataNetCdfCf, }; use log::debug; -use num::FromPrimitive; +use num::{FromPrimitive, integer::div_ceil, integer::div_floor}; use postgres_types::{FromSql, ToSql}; +use reader::{ + GdalReadAdvise, GdalReadWindow, GdalReaderMode, GridAndProperties, OverviewReaderState, +}; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, ensure}; use std::collections::HashMap; @@ -62,10 +65,10 @@ use std::ffi::CString; use std::marker::PhantomData; use std::path::{Path, PathBuf}; use std::time::Instant; - mod db_types; mod error; mod loading_info; +mod reader; static GDAL_RETRY_INITIAL_BACKOFF_MS: u64 = 1000; static GDAL_RETRY_MAX_BACKOFF_MS: u64 = 60 * 60 * 1000; @@ -73,8 +76,35 @@ static GDAL_RETRY_EXPONENTIAL_BACKOFF_FACTOR: f64 = 2.; /// Parameters for the GDAL Source Operator #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct GdalSourceParameters { pub data: NamedData, + #[serde(default)] + pub overview_level: Option, // TODO: should also allow a resolution? Add resample method? +} + +impl GdalSourceParameters { + #[must_use] + pub fn new(data: NamedData) -> Self { + Self { + data, + overview_level: None, + } + } + + #[must_use] + pub fn new_with_overview_level(data: NamedData, overview_level: u32) -> Self { + Self { + data, + overview_level: Some(overview_level), + } + } + + #[must_use] + pub fn with_overview_level(mut self, overview_level: Option) -> Self { + self.overview_level = overview_level; + self + } } impl OperatorData for GdalSourceParameters { @@ -125,30 +155,41 @@ pub struct GdalDatasetParameters { pub retry: Option, } -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, Copy)] -#[serde(rename_all = "camelCase")] -pub struct GdalRetryOptions { - pub max_retries: usize, -} - -#[derive(Debug, PartialEq, Eq)] -struct GdalReadWindow { - start_x: isize, // pixelspace origin - start_y: isize, - size_x: usize, // pixelspace size - size_y: usize, -} +impl GdalDatasetParameters { + pub fn dataset_bounds(&self) -> GridBoundingBox2D { + GridBoundingBox2D::new_unchecked( + [0, 0], + [self.height as isize - 1, self.width as isize - 1], + ) + } -impl GdalReadWindow { - fn gdal_window_start(&self) -> (isize, isize) { - (self.start_x, self.start_y) + pub fn gdal_geo_transform(&self) -> GdalDatasetGeoTransform { + self.geo_transform } - fn gdal_window_size(&self) -> (usize, usize) { - (self.size_x, self.size_y) + /// Returns the `SpatialGridDefinition` of the Gdal dataset. + /// + /// Note: This allows upside down datasets (where `GeoTransform` `y_pixel_size` is positive)! + /// + /// # Panics + /// Panics if the `GdalDatasetParameters` are faulty. + pub fn spatial_grid_definition(&self) -> SpatialGridDefinition { + let gdal_geo_transform = GeoTransform::new( + self.gdal_geo_transform().origin_coordinate, + self.gdal_geo_transform().x_pixel_size, + self.gdal_geo_transform().y_pixel_size, + ); + + SpatialGridDefinition::new(gdal_geo_transform, self.dataset_bounds()) } } +#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub struct GdalRetryOptions { + pub max_retries: usize, +} + /// A user friendly representation of Gdal's geo transform. In contrast to [`GeoTransform`] this /// geo transform allows arbitrary pixel sizes and can thus also represent rasters where the origin is not located /// in the upper left corner. It should only be used for loading rasters with Gdal and not internally. @@ -162,120 +203,6 @@ pub struct GdalDatasetGeoTransform { pub y_pixel_size: f64, } -impl GdalDatasetGeoTransform { - /// Produce the `SpatialPartition` anchored at the datasets origin with a size of x * y pixels. This method handles non-standard pixel sizes. - pub fn spatial_partition(&self, x_size: usize, y_size: usize) -> SpatialPartition2D { - // the opposite y value (y value of the non origin edge) - let opposite_coord_y = self.origin_coordinate.y + self.y_pixel_size * y_size as f64; - - // if the y-axis is negative then the origin is on the upper side. - let (upper_y, lower_y) = if self.y_pixel_size.is_sign_negative() { - (self.origin_coordinate.y, opposite_coord_y) - } else { - (opposite_coord_y, self.origin_coordinate.y) - }; - - let opposite_coord_x = self.origin_coordinate.x + self.x_pixel_size * x_size as f64; - - // if the y-axis is negative then the origin is on the upper side. - let (left_x, right_x) = if self.x_pixel_size.is_sign_positive() { - (self.origin_coordinate.x, opposite_coord_x) - } else { - (opposite_coord_x, self.origin_coordinate.x) - }; - - SpatialPartition2D::new_unchecked( - Coordinate2D::new(left_x, upper_y), - Coordinate2D::new(right_x, lower_y), - ) - } - - /// Transform a `Coordinate2D` into a `GridIdx2D` - #[inline] - pub fn coordinate_to_grid_idx_2d(&self, coord: Coordinate2D) -> GridIdx2D { - // TODO: use an epsilon here? - let grid_x_index = - ((coord.x - self.origin_coordinate.x) / self.x_pixel_size).floor() as isize; - let grid_y_index = - ((coord.y - self.origin_coordinate.y) / self.y_pixel_size).floor() as isize; - - [grid_y_index, grid_x_index].into() - } - - fn spatial_partition_to_read_window( - &self, - spatial_partition: &SpatialPartition2D, - ) -> GdalReadWindow { - // World coordinates and pixel sizes use float values. Since the float imprecision might cause overflowing into the next pixel we use an epsilon to correct values very close the pixel borders. This logic is the same as used in [`GeoTransform::grid_idx_to_pixel_upper_left_coordinate_2d`]. - const EPSILON: f64 = 0.000_001; - let epsilon: Coordinate2D = - (self.x_pixel_size * EPSILON, self.y_pixel_size * EPSILON).into(); - - /* - The read window is relative to the transform of the gdal dataset. The `SpatialPartition` is oriented at axis of the spatial SRS. This usually causes this situation: - - The gdal data is stored with negative pixel size. The "ul" coordinate of the `SpatialPartition` is neareest to the origin of the gdal raster data. - ul ur - +_______________________+ - |_|_ row 1 | - | |_|_ row 2 | - | |_|_ row ... | - | |_| | - |_______________________| - + * - ll lr - - However, sometimes the data is stored up-side down. Like this: - - The gdal data is stored with a positive pixel size. So the "ll" coordinate is nearest to the reading the raster data needs to starts at this anchor. - - ul ur - +_______________________+ - | _ | - | _|_| row ... | - | _|_| row 3 | - | |_| row 2 | - |_______________________| - + * - ll lr - - Therefore we need to select the raster read start based on the coordinate next to the raster data origin. From there we then calculate the size of the window to read. - */ - let (near_origin_coord, far_origin_coord) = if self.y_pixel_size.is_sign_negative() { - ( - spatial_partition.upper_left(), - spatial_partition.lower_right(), - ) - } else { - ( - spatial_partition.lower_left(), - spatial_partition.upper_right(), - ) - }; - - // Move the coordinate near the origin a bit inside the bbox by adding an epsilon of the pixel size. - let safe_near_coord = near_origin_coord + epsilon; - // Move the coordinate far from the origin a bit inside the bbox by subtracting an epsilon of the pixel size - let safe_far_coord = far_origin_coord - epsilon; - - let GridIdx([near_idx_y, near_idx_x]) = self.coordinate_to_grid_idx_2d(safe_near_coord); - let GridIdx([far_idx_y, far_idx_x]) = self.coordinate_to_grid_idx_2d(safe_far_coord); - - debug_assert!(near_idx_x <= far_idx_x); - debug_assert!(near_idx_y <= far_idx_y); - - let read_size_x = (far_idx_x - near_idx_x) as usize + 1; - let read_size_y = (far_idx_y - near_idx_y) as usize + 1; - - GdalReadWindow { - start_x: near_idx_x, - start_y: near_idx_y, - size_x: read_size_x, - size_y: read_size_y, - } - } -} - /// Default implementation for testing purposes where geo transform doesn't matter impl TestDefault for GdalDatasetGeoTransform { fn test_default() -> Self { @@ -307,8 +234,8 @@ impl TryFrom for GeoTransform { fn try_from(dataset_geo_transform: GdalDatasetGeoTransform) -> Result { ensure!( - dataset_geo_transform.x_pixel_size > 0.0 && dataset_geo_transform.y_pixel_size < 0.0, - crate::error::GeoTransformOrigin + dataset_geo_transform.x_pixel_size != 0.0 && dataset_geo_transform.y_pixel_size != 0.0, + crate::error::GeoTransformOrigin // TODO new name? ); Ok(GeoTransform::new( @@ -329,13 +256,6 @@ impl From for GdalDatasetGeoTransform { } } -impl SpatialPartitioned for GdalDatasetParameters { - fn spatial_partition(&self) -> SpatialPartition2D { - self.geo_transform - .spatial_partition(self.width, self.height) - } -} - impl GridShapeAccess for GdalDatasetParameters { type ShapeArray = [usize; 2]; @@ -398,9 +318,11 @@ pub struct GdalSourceProcessor where T: Pixel, { - pub result_descriptor: RasterResultDescriptor, + pub produced_result_descriptor: RasterResultDescriptor, pub tiling_specification: TilingSpecification, pub meta_data: GdalMetaData, + pub overview_level: u32, + pub original_resolution_spatial_grid: Option, pub _phantom_data: PhantomData, } @@ -412,10 +334,8 @@ impl GdalRasterLoader { /// async fn load_tile_data_async( dataset_params: GdalDatasetParameters, - tile_information: TileInformation, - tile_time: TimeInterval, - cache_hint: CacheHint, - ) -> Result> { + read_advise: GdalReadAdvise, + ) -> Result>> { // TODO: detect usage of vsi curl properly, e.g. also check for `/vsicurl_streaming` and combinations with `/vsizip` let is_vsi_curl = dataset_params.file_path.starts_with("/vsicurl/"); @@ -432,11 +352,10 @@ impl GdalRasterLoader { let file_path = ds.file_path.clone(); async move { - let load_tile_result = crate::util::spawn_blocking(move || { - Self::load_tile_data(&ds, tile_information, tile_time, cache_hint) - }) - .await - .context(crate::error::TokioJoin); + let load_tile_result = + crate::util::spawn_blocking(move || Self::load_tile_data(&ds, read_advise)) + .await + .context(crate::error::TokioJoin); match load_tile_result { Ok(Ok(r)) => Ok(r), @@ -458,29 +377,49 @@ impl GdalRasterLoader { async fn load_tile_async( dataset_params: Option, + reader_mode: GdalReaderMode, tile_information: TileInformation, tile_time: TimeInterval, cache_hint: CacheHint, ) -> Result> { + let tile_spatial_grid = tile_information.spatial_grid_definition(); + match dataset_params { // TODO: discuss if we need this check here. The metadata provider should only pass on loading infos if the query intersects the datasets bounds! And the tiling strategy should only generate tiles that intersect the querys bbox. - Some(ds) - if tile_information - .spatial_partition() - .intersects(&ds.spatial_partition()) => - { + Some(ds) => { debug!( "Loading tile {:?}, from {:?}, band: {}", &tile_information, ds.file_path, ds.rasterband_channel ); - Self::load_tile_data_async(ds, tile_information, tile_time, cache_hint).await - } - Some(_) => { - debug!("Skipping tile not in query rect {:?}", &tile_information); - - Ok(create_no_data_tile(tile_information, tile_time, cache_hint)) + let gdal_read_advise: Option = reader_mode + .tiling_to_dataset_read_advise( + &ds.spatial_grid_definition(), + &tile_spatial_grid, + ); + + let Some(gdal_read_advise) = gdal_read_advise else { + debug!( + "Tile {:?} not intersecting dataset grid or gdal grid {:?}", + &tile_information, ds.file_path + ); + return Ok(create_no_data_tile(tile_information, tile_time, cache_hint)); + }; + + let grid = Self::load_tile_data_async(ds, gdal_read_advise).await?; + + match grid { + Some(grid) => Ok(RasterTile2D::new_with_properties( + tile_time, + tile_information.global_tile_position, + 0, + tile_information.global_geo_transform, + grid.grid, + grid.properties, + cache_hint, + )), + None => Ok(create_no_data_tile(tile_information, tile_time, cache_hint)), + } } - _ => { debug!( "Skipping tile without GdalDatasetParameters {:?}", @@ -497,16 +436,14 @@ impl GdalRasterLoader { /// fn load_tile_data( dataset_params: &GdalDatasetParameters, - tile_information: TileInformation, - tile_time: TimeInterval, - cache_hint: CacheHint, - ) -> Result> { + read_advise: GdalReadAdvise, + ) -> Result>> { let start = Instant::now(); debug!( "GridOrEmpty2D<{:?}> requested for {:?}.", T::TYPE, - &tile_information.spatial_partition() + &read_advise.bounds_of_target, ); let options = dataset_params @@ -533,9 +470,7 @@ impl GdalRasterLoader { let is_file_not_found = error_is_gdal_file_not_found(error); let err_result = match dataset_params.file_not_found_handling { - FileNotFoundHandling::NoData if is_file_not_found => { - Ok(create_no_data_tile(tile_information, tile_time, cache_hint)) - } + FileNotFoundHandling::NoData if is_file_not_found => Ok(None), _ => Err(crate::error::Error::CouldNotOpenGdalDataset { file_path: dataset_params.file_path.to_string_lossy().to_string(), }), @@ -553,36 +488,69 @@ impl GdalRasterLoader { let dataset = dataset_result.expect("checked"); - let result_tile = read_raster_tile_with_properties( - &dataset, - dataset_params, - tile_information, - tile_time, - cache_hint, - )?; + let rasterband = dataset.rasterband(dataset_params.rasterband_channel)?; + + let gdal_dataset_geotransform = GdalDatasetGeoTransform::from(dataset.geo_transform()?); + // check that the dataset geo transform is the same as the one we get from GDAL + debug_assert!(approx_eq!( + Coordinate2D, + gdal_dataset_geotransform.origin_coordinate, + dataset_params.geo_transform.origin_coordinate + )); + + debug_assert!(approx_eq!( + f64, + gdal_dataset_geotransform.x_pixel_size, + dataset_params.geo_transform.x_pixel_size + )); + + debug_assert!(approx_eq!( + f64, + gdal_dataset_geotransform.y_pixel_size, + dataset_params.geo_transform.y_pixel_size + )); + + let (gdal_dataset_pixels_x, gdal_dataset_pixels_y) = dataset.raster_size(); + // check that the dataset pixel size is the same as the one we get from GDAL + debug_assert_eq!(gdal_dataset_pixels_x, dataset_params.width); + debug_assert_eq!(gdal_dataset_pixels_y, dataset_params.height); + + let result_grid = + read_grid_and_handle_edges(&dataset, &rasterband, dataset_params, read_advise)?; + + let properties = read_raster_properties(&dataset, dataset_params, &rasterband); let elapsed = start.elapsed(); debug!("data loaded -> returning data grid, took {elapsed:?}"); - Ok(result_tile) + Ok(Some(GridAndProperties { + grid: result_grid, + properties, + })) } /// /// A stream of futures producing `RasterTile2D` for a single slice in time /// fn temporal_slice_tile_future_stream( - spatial_bounds: SpatialPartition2D, + spatial_query: SpatialGridQueryRectangle, info: GdalLoadingInfoTemporalSlice, tiling_strategy: TilingStrategy, + reader_mode: GdalReaderMode, ) -> impl Stream>>> + use { - stream::iter(tiling_strategy.tile_information_iterator(spatial_bounds)).map(move |tile| { - GdalRasterLoader::load_tile_async( - info.params.clone(), - tile, - info.time, - info.cache_ttl.into(), - ) - }) + stream::iter( + tiling_strategy + .tile_information_iterator_from_grid_bounds(spatial_query.grid_bounds()) + .map(move |tile| { + GdalRasterLoader::load_tile_async( + info.params.clone(), + reader_mode, + tile, + info.time, + info.cache_ttl.into(), + ) + }), + ) } fn loading_info_to_tile_stream< @@ -590,16 +558,17 @@ impl GdalRasterLoader { S: Stream>, >( loading_info_stream: S, - query: &RasterQueryRectangle, + spatial_query: SpatialGridQueryRectangle, tiling_strategy: TilingStrategy, + reader_mode: GdalReaderMode, ) -> impl Stream>> + use { - let spatial_bounds = query.spatial_bounds; loading_info_stream .map_ok(move |info| { GdalRasterLoader::temporal_slice_tile_future_stream( - spatial_bounds, + spatial_query, info, tiling_strategy, + reader_mode, ) .map(Result::Ok) }) @@ -636,7 +605,7 @@ where P: Pixel + gdal::raster::GdalType + FromPrimitive, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialQuery = RasterSpatialQueryRectangle; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -649,50 +618,47 @@ where query.attributes.as_slice() == [0], crate::error::GdalSourceDoesNotSupportQueryingOtherBandsThanTheFirstOneYet ); - - let start = Instant::now(); - debug!( + tracing::debug!( "Querying GdalSourceProcessor<{:?}> with: {:?}.", P::TYPE, &query ); + // this is the result descriptor of the operator. It already incorporates the overview level AND shifts the origin to the tiling origin + let result_descriptor = self.result_descriptor(); - debug!( - "GdalSourceProcessor<{:?}> meta data loaded, took {:?}.", - P::TYPE, - start.elapsed() - ); - - let spatial_resolution = query.spatial_resolution; - + let grid_produced_by_source_desc = result_descriptor.spatial_grid; + let grid_produced_by_source = grid_produced_by_source_desc + .source_spatial_grid_definition() + .expect("the source grid definition should be present in a source..."); // A `GeoTransform` maps pixel space to world space. // Usually a SRS has axis directions pointing "up" (y-axis) and "up" (y-axis). // We are not aware of spatial reference systems where the x-axis points to the right. // However, there are spatial reference systems where the y-axis points downwards. // The standard "pixel-space" starts at the top-left corner of a `GeoTransform` and points down-right. // Therefore, the pixel size on the x-axis is always increasing - let pixel_size_x = spatial_resolution.x; + let pixel_size_x = grid_produced_by_source.geo_transform().x_pixel_size(); debug_assert!(pixel_size_x.is_sign_positive()); // and the y-axis should only be positive if the y-axis of the spatial reference system also "points down". // NOTE: at the moment we do not allow "down pointing" y-axis. - let pixel_size_y = spatial_resolution.y * -1.0; + let pixel_size_y = grid_produced_by_source.geo_transform().y_pixel_size(); debug_assert!(pixel_size_y.is_sign_negative()); - let tiling_strategy = self - .tiling_specification - .strategy(pixel_size_x, pixel_size_y); + // The data origin is not neccessarily the origin of the tileing we want to use. + // TODO: maybe derive tilling origin reference from the data projection + let produced_tiling_grid = + grid_produced_by_source_desc.tiling_grid_definition(self.tiling_specification); + + let tiling_based_pixel_bounds = produced_tiling_grid.tiling_grid_bounds(); - let result_descriptor = self.meta_data.result_descriptor().await?; + let tiling_strategy = produced_tiling_grid.generate_data_tiling_strategy(); + + let query_pixel_bounds = query.spatial_query().grid_bounds(); let mut empty = false; - debug!("result descr bbox: {:?}", result_descriptor.bbox); - debug!("query bbox: {:?}", query.spatial_bounds); - if let Some(data_spatial_bounds) = result_descriptor.bbox { - if !data_spatial_bounds.intersects(&query.spatial_bounds) { - debug!("query does not intersect spatial data bounds"); - empty = true; - } + if !tiling_based_pixel_bounds.intersects(&query_pixel_bounds) { + debug!("query does not intersect spatial data bounds"); + empty = true; } // TODO: use the time bounds to early return. @@ -705,6 +671,17 @@ where } */ + let reader_mode = match self.original_resolution_spatial_grid { + None => GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: grid_produced_by_source, + }), + Some(original_resolution_spatial_grid) => { + GdalReaderMode::OverviewLevel(OverviewReaderState { + original_dataset_grid: original_resolution_spatial_grid, + }) + } + }; + let loading_info = if empty { // TODO: using this shortcut will insert one no-data element with max time validity. However, this does not honor time intervals of data in other areas! GdalLoadingInfo::new( @@ -718,6 +695,13 @@ where self.meta_data.loading_info(query.clone()).await? }; + debug_assert!( + loading_info.start_time_of_output_stream < loading_info.end_time_of_output_stream, + "Data time validity must not be a TimeInstance. Is ({:?}, {:?}]", + loading_info.start_time_of_output_stream, + loading_info.end_time_of_output_stream + ); + let time_bounds = match ( loading_info.start_time_of_output_stream, loading_info.end_time_of_output_stream, @@ -746,17 +730,19 @@ where let query_time = query.time_interval; let skipping_loading_info = loading_info .info - .filter_ok(move |s: &GdalLoadingInfoTemporalSlice| s.time.intersects(&query_time)); + .filter_ok(move |s: &GdalLoadingInfoTemporalSlice| s.time.intersects(&query_time)); // Check that the time slice intersects the query time - let source_stream = stream::iter(skipping_loading_info); - - let source_stream = - GdalRasterLoader::loading_info_to_tile_stream(source_stream, &query, tiling_strategy); + let source_stream = GdalRasterLoader::loading_info_to_tile_stream( + stream::iter(skipping_loading_info), + query.spatial_query(), + tiling_strategy, + reader_mode, + ); // use SparseTilesFillAdapter to fill all the gaps let filled_stream = SparseTilesFillAdapter::new( source_stream, - tiling_strategy.tile_grid_box(query.spatial_partition()), + tiling_strategy.global_pixel_grid_bounds_to_tile_grid_bounds(query_pixel_bounds), query.attributes.count(), tiling_strategy.geo_transform, tiling_strategy.tile_size_in_pixels, @@ -768,7 +754,7 @@ where } fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor + &self.produced_result_descriptor } } @@ -778,6 +764,44 @@ impl OperatorName for GdalSource { const TYPE_NAME: &'static str = "GdalSource"; } +fn overview_level_spatial_grid( + source_spatial_grid: SpatialGridDefinition, + overview_level: u32, +) -> Option { + if overview_level > 0 { + debug!("Using overview level {overview_level}"); + let geo_transform = GeoTransform::new( + source_spatial_grid.geo_transform.origin_coordinate, + source_spatial_grid.geo_transform.x_pixel_size() * f64::from(overview_level), + source_spatial_grid.geo_transform.y_pixel_size() * f64::from(overview_level), + ); + let grid_bounds = GridBoundingBox2D::new_min_max( + div_floor( + source_spatial_grid.grid_bounds.y_min(), + overview_level as isize, + ), + div_ceil( + source_spatial_grid.grid_bounds.y_max(), + overview_level as isize, + ), + div_floor( + source_spatial_grid.grid_bounds.x_min(), + overview_level as isize, + ), + div_ceil( + source_spatial_grid.grid_bounds.x_max(), + overview_level as isize, + ), + ) + .expect("overview level must be a positive integer"); + + Some(SpatialGridDefinition::new(geo_transform, grid_bounds)) + } else { + debug!("Using original resolution (ov = 0)"); + None + } +} + #[typetag::serde] #[async_trait] impl RasterOperator for GdalSource { @@ -792,13 +816,29 @@ impl RasterOperator for GdalSource { debug!("Initializing GdalSource for {:?}.", &self.params.data); debug!("GdalSource path: {:?}", path); - let op = InitializedGdalSourceOperator { - name: CanonicOperatorName::from(&self), - path, - data: self.params.data.to_string(), - result_descriptor: meta_data.result_descriptor().await?, - meta_data, - tiling_specification: context.tiling_specification(), + let meta_data_result_descriptor = meta_data.result_descriptor().await?; + + let op_name = CanonicOperatorName::from(&self); + let op = if self.params.overview_level.is_none() { + InitializedGdalSourceOperator::initialize_original_resolution( + op_name, + path, + self.params.data.to_string(), + meta_data, + meta_data_result_descriptor, + context.tiling_specification(), + ) + } else { + // generate a result descriptor with the overview level + InitializedGdalSourceOperator::initialize_with_overview_level( + op_name, + path, + self.params.data.to_string(), + meta_data, + meta_data_result_descriptor, + context.tiling_specification(), + self.params.overview_level.unwrap_or(0), + ) }; Ok(op.boxed()) @@ -812,40 +852,109 @@ pub struct InitializedGdalSourceOperator { path: WorkflowOperatorPath, data: String, pub meta_data: GdalMetaData, - pub result_descriptor: RasterResultDescriptor, + pub produced_result_descriptor: RasterResultDescriptor, pub tiling_specification: TilingSpecification, + // the overview level to use. 0/1 means the highest resolution + pub overview_level: u32, + pub original_resolution_spatial_grid: Option, +} + +impl InitializedGdalSourceOperator { + pub fn initialize_original_resolution( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + data: String, + meta_data: GdalMetaData, + result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, + ) -> Self { + InitializedGdalSourceOperator { + name, + path, + data, + produced_result_descriptor: result_descriptor, + meta_data, + tiling_specification, + overview_level: 0, + original_resolution_spatial_grid: None, + } + } + + pub fn initialize_with_overview_level( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + data: String, + meta_data: GdalMetaData, + result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, + overview_level: u32, + ) -> Self { + let source_resolution_spatial_grid = result_descriptor + .spatial_grid_descriptor() + .source_spatial_grid_definition() + .expect("Source data must be a source grid definition..."); + + let (result_descriptor, original_grid) = if let Some(ovr_spatial_grid) = + overview_level_spatial_grid(source_resolution_spatial_grid, overview_level) + { + let ovr_res = RasterResultDescriptor { + spatial_grid: SpatialGridDescriptor::new_source(ovr_spatial_grid), + ..result_descriptor + }; + (ovr_res, Some(source_resolution_spatial_grid)) + } else { + (result_descriptor, None) + }; + + InitializedGdalSourceOperator { + name, + path, + data, + produced_result_descriptor: result_descriptor, + meta_data, + tiling_specification, + overview_level, + original_resolution_spatial_grid: original_grid, + } + } } impl InitializedRasterOperator for InitializedGdalSourceOperator { fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor + &self.produced_result_descriptor } fn query_processor(&self) -> Result { Ok(match self.result_descriptor().data_type { RasterDataType::U8 => TypedRasterQueryProcessor::U8( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), ), RasterDataType::U16 => TypedRasterQueryProcessor::U16( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), ), RasterDataType::U32 => TypedRasterQueryProcessor::U32( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), @@ -862,18 +971,22 @@ impl InitializedRasterOperator for InitializedGdalSourceOperator { } RasterDataType::I16 => TypedRasterQueryProcessor::I16( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), ), RasterDataType::I32 => TypedRasterQueryProcessor::I32( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), @@ -885,18 +998,22 @@ impl InitializedRasterOperator for InitializedGdalSourceOperator { } RasterDataType::F32 => TypedRasterQueryProcessor::F32( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), ), RasterDataType::F64 => TypedRasterQueryProcessor::F64( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), @@ -999,144 +1116,61 @@ where Ok(GridOrEmpty::from(masked_grid)) } -/// This method reads the data for a single grid with a specified size from the GDAL dataset. -/// If the tile overlaps the borders of the dataset only the data in the dataset bounds is read. -/// The data read from the dataset is clipped into a grid with the requested size filled with the `no_data_value`. -fn read_partial_grid_from_raster( - rasterband: &GdalRasterBand, - gdal_read_window: &GdalReadWindow, - out_tile_read_bounds: GridBoundingBox2D, - out_tile_shape: GridShape2D, - dataset_params: &GdalDatasetParameters, - flip_y_axis: bool, -) -> Result> -where - T: Pixel + GdalType + Default + FromPrimitive, -{ - let dataset_raster = read_grid_from_raster( - rasterband, - gdal_read_window, - out_tile_read_bounds, - dataset_params, - flip_y_axis, - )?; - - let mut tile_raster = GridOrEmpty::from(EmptyGrid::new(out_tile_shape)); - tile_raster.grid_blit_from(&dataset_raster); - Ok(tile_raster) -} - /// This method reads the data for a single tile with a specified size from the GDAL dataset. /// It handles conversion to grid coordinates. /// If the tile is inside the dataset it uses the `read_grid_from_raster` method. /// If the tile overlaps the borders of the dataset it uses the `read_partial_grid_from_raster` method. fn read_grid_and_handle_edges( - tile_info: TileInformation, - dataset: &GdalDataset, + _dataset: &GdalDataset, rasterband: &GdalRasterBand, dataset_params: &GdalDatasetParameters, + gdal_read_advice: GdalReadAdvise, ) -> Result> where T: Pixel + GdalType + Default + FromPrimitive, { - let gdal_dataset_geotransform = GdalDatasetGeoTransform::from(dataset.geo_transform()?); - let (gdal_dataset_pixels_x, gdal_dataset_pixels_y) = dataset.raster_size(); - - if !approx_eq!( - GdalDatasetGeoTransform, - gdal_dataset_geotransform, - dataset_params.geo_transform - ) { - log::warn!( - "GdalDatasetParameters geo transform is different to the one retrieved from GDAL dataset: {:?} != {:?}", - dataset_params.geo_transform, - gdal_dataset_geotransform, - ); - } - - debug_assert_eq!(gdal_dataset_pixels_x, dataset_params.width); - debug_assert_eq!(gdal_dataset_pixels_y, dataset_params.height); - - let gdal_dataset_bounds = - gdal_dataset_geotransform.spatial_partition(gdal_dataset_pixels_x, gdal_dataset_pixels_y); - - let output_bounds = tile_info.spatial_partition(); - let dataset_intersects_tile = gdal_dataset_bounds.intersection(&output_bounds); - let output_shape = tile_info.tile_size_in_pixels(); - - let Some(dataset_intersection_area) = dataset_intersects_tile else { - return Ok(GridOrEmpty::from(EmptyGrid::new(output_shape))); - }; - - let tile_geo_transform = tile_info.tile_geo_transform(); - - let gdal_read_window = - gdal_dataset_geotransform.spatial_partition_to_read_window(&dataset_intersection_area); - - let is_y_axis_flipped = tile_geo_transform.y_pixel_size().is_sign_negative() - != gdal_dataset_geotransform.y_pixel_size.is_sign_negative(); - - if is_y_axis_flipped { - debug!("The GDAL data has a flipped y-axis. Need to unflip it!"); - } - - let result_grid = if dataset_intersection_area == output_bounds { + let result_grid = if gdal_read_advice.direct_read() { read_grid_from_raster( rasterband, - &gdal_read_window, - output_shape, + &gdal_read_advice.gdal_read_widow, + gdal_read_advice.read_window_bounds.grid_shape(), dataset_params, - is_y_axis_flipped, + gdal_read_advice.flip_y, )? } else { - let partial_tile_grid_bounds = - tile_geo_transform.spatial_to_grid_bounds(&dataset_intersection_area); - - read_partial_grid_from_raster( + let r: GridOrEmpty = read_grid_from_raster( rasterband, - &gdal_read_window, - partial_tile_grid_bounds, - output_shape, + &gdal_read_advice.gdal_read_widow, + gdal_read_advice.read_window_bounds, dataset_params, - is_y_axis_flipped, - )? + gdal_read_advice.flip_y, + )?; + let mut tile_raster = GridOrEmpty::from(EmptyGrid::new(gdal_read_advice.bounds_of_target)); + tile_raster.grid_blit_from(&r); + tile_raster.unbounded() }; Ok(result_grid) } /// This method reads the data for a single tile with a specified size from the GDAL dataset and adds the requested metadata as properties to the tile. -fn read_raster_tile_with_properties( +fn read_raster_properties( dataset: &GdalDataset, dataset_params: &GdalDatasetParameters, - tile_info: TileInformation, - tile_time: TimeInterval, - cache_hint: CacheHint, -) -> Result> { - let rasterband = dataset.rasterband(dataset_params.rasterband_channel)?; - - let result_grid = read_grid_and_handle_edges(tile_info, dataset, &rasterband, dataset_params)?; - + rasterband: &GdalRasterBand, +) -> RasterProperties { let mut properties = RasterProperties::default(); // always read the scale and offset values from the rasterband - properties_from_band(&mut properties, &rasterband); + properties_from_band(&mut properties, rasterband); // read the properties from the dataset and rasterband metadata if let Some(properties_mapping) = dataset_params.properties_mapping.as_ref() { properties_from_gdal_metadata(&mut properties, dataset, properties_mapping); - properties_from_gdal_metadata(&mut properties, &rasterband, properties_mapping); + properties_from_gdal_metadata(&mut properties, rasterband, properties_mapping); } - // TODO: add cache_hint - Ok(RasterTile2D::new_with_tile_info_and_properties( - tile_time, - tile_info, - 0, - result_grid, - properties, - cache_hint, - )) + properties } fn create_no_data_tile( @@ -1232,35 +1266,50 @@ mod tests { use crate::test_data; use crate::util::Result; use crate::util::gdal::add_ndvi_dataset; + use float_cmp::assert_approx_eq; use geoengine_datatypes::hashmap; - use geoengine_datatypes::primitives::{AxisAlignedRectangle, SpatialPartition2D, TimeInstance}; + use geoengine_datatypes::primitives::{ + AxisAlignedRectangle, SpatialGridQueryRectangle, SpatialPartition2D, TimeInstance, + }; + use geoengine_datatypes::raster::{BoundedGrid, GridShape2D, SpatialGridDefinition}; use geoengine_datatypes::raster::{ EmptyGrid2D, GridBounds, GridIdx2D, TilesEqualIgnoringCacheHint, }; use geoengine_datatypes::raster::{TileInformation, TilingStrategy}; use geoengine_datatypes::util::gdal::hide_gdal_errors; - use geoengine_datatypes::{primitives::SpatialResolution, raster::GridShape2D}; use httptest::matchers::request; use httptest::{Expectation, Server, responders}; + use reader::{GdalReadAdvise, GdalReadWindow}; + + fn tile_information_with_partition_and_shape( + partition: SpatialPartition2D, + shape: GridShape2D, + ) -> TileInformation { + let real_geotransform = GeoTransform::new( + partition.upper_left(), + partition.size_x() / shape.axis_size_x() as f64, + -partition.size_y() / shape.axis_size_y() as f64, + ); + + TileInformation { + tile_size_in_pixels: shape, + global_tile_position: [0, 0].into(), + global_geo_transform: real_geotransform, + } + } async fn query_gdal_source( exe_ctx: &MockExecutionContext, query_ctx: &MockQueryContext, name: NamedData, - output_shape: GridShape2D, - output_bounds: SpatialPartition2D, + spatial_query: SpatialGridQueryRectangle, time_interval: TimeInterval, ) -> Vec>> { let op = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), } .boxed(); - let x_query_resolution = output_bounds.size_x() / output_shape.axis_size_x() as f64; - let y_query_resolution = output_bounds.size_y() / output_shape.axis_size_y() as f64; - let spatial_resolution = - SpatialResolution::new_unchecked(x_query_resolution, y_query_resolution); - let o = op .initialize(WorkflowOperatorPath::initialize_root(), exe_ctx) .await @@ -1271,12 +1320,7 @@ mod tests { .get_u8() .unwrap() .raster_query( - RasterQueryRectangle { - spatial_bounds: output_bounds, - time_interval, - spatial_resolution, - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new(spatial_query, time_interval, BandSelection::first()), query_ctx, ) .await @@ -1285,56 +1329,45 @@ mod tests { .await } - fn load_ndvi_jan_2014( - output_shape: GridShape2D, - output_bounds: SpatialPartition2D, - ) -> Result> { - GdalRasterLoader::load_tile_data::( - &GdalDatasetParameters { - file_path: test_data!("raster/modis_ndvi/MOD13A2_M_NDVI_2014-01-01.TIFF").into(), - rasterband_channel: 1, - geo_transform: GdalDatasetGeoTransform { - origin_coordinate: (-180., 90.).into(), - x_pixel_size: 0.1, - y_pixel_size: -0.1, - }, - width: 3600, - height: 1800, - file_not_found_handling: FileNotFoundHandling::NoData, - no_data_value: Some(0.), - properties_mapping: Some(vec![ - GdalMetadataMapping { - source_key: RasterPropertiesKey { - domain: None, - key: "AREA_OR_POINT".to_string(), - }, - target_type: RasterPropertiesEntryType::String, - target_key: RasterPropertiesKey { - domain: None, - key: "AREA_OR_POINT".to_string(), - }, - }, - GdalMetadataMapping { - source_key: RasterPropertiesKey { - domain: Some("IMAGE_STRUCTURE".to_string()), - key: "COMPRESSION".to_string(), - }, - target_type: RasterPropertiesEntryType::String, - target_key: RasterPropertiesKey { - domain: Some("IMAGE_STRUCTURE_INFO".to_string()), - key: "COMPRESSION".to_string(), - }, - }, - ]), - gdal_open_options: None, - gdal_config_options: None, - allow_alphaband_as_mask: true, - retry: None, + // This method loads raster data from a cropped MODIS NDVI raster. + // To inspect the byte values first convert the file to XYZ with GDAL: + // 'gdal_translate -of xyz MOD13A2_M_NDVI_2014-04-01_30X30.tif MOD13A2_M_NDVI_2014-04-01_30x30.xyz' + // Then you can convert them to gruped bytes: + // 'cut -d ' ' -f 1,2 --complement MOD13A2_M_NDVI_2014-04-01_30x30.xyz | xargs -n 30 > MOD13A2_M_NDVI_2014-04-01_30x30_bytes.txt'. + fn load_ndvi_apr_2014_cropped( + gdal_read_advice: GdalReadAdvise, + ) -> Result>> { + let dataset_params = GdalDatasetParameters { + file_path: test_data!("raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif") + .into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: (8.0, 57.4).into(), + x_pixel_size: 0.1, + y_pixel_size: -0.1, }, - TileInformation::with_partition_and_shape(output_bounds, output_shape), - TimeInterval::default(), - CacheHint::default(), - ) + width: 30, + height: 30, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: Some(255.), + properties_mapping: Some(vec![GdalMetadataMapping { + source_key: RasterPropertiesKey { + domain: None, + key: "AREA_OR_POINT".to_string(), + }, + target_type: RasterPropertiesEntryType::String, + target_key: RasterPropertiesKey { + domain: None, + key: "AREA_OR_POINT".to_string(), + }, + }]), + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }; + + GdalRasterLoader::load_tile_data::(&dataset_params, gdal_read_advice) } #[test] @@ -1352,9 +1385,7 @@ mod tests { assert_eq!( operator, GdalSource { - params: GdalSourceParameters { - data: NamedData::with_namespaced_name("ns", "dataset"), - }, + params: GdalSourceParameters::new(NamedData::with_namespaced_name("ns", "dataset")), } ); } @@ -1447,7 +1478,7 @@ mod tests { dataset_y_pixel_size, ); - let partition = SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); + let grid_bounds = GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(); let origin_split_tileing_strategy = TilingStrategy { tile_size_in_pixels: tile_size_in_pixels.into(), @@ -1455,7 +1486,7 @@ mod tests { }; let vres: Vec = origin_split_tileing_strategy - .tile_idx_iterator(partition) + .tile_idx_iterator_from_grid_bounds(grid_bounds) .collect(); assert_eq!(vres.len(), 4 * 6); assert_eq!(vres[0], [-2, -3].into()); @@ -1477,7 +1508,7 @@ mod tests { dataset_y_pixel_size, ); - let partition = SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); + let grid_bounds = GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(); let origin_split_tileing_strategy = TilingStrategy { tile_size_in_pixels: tile_size_in_pixels.into(), @@ -1485,7 +1516,7 @@ mod tests { }; let vres: Vec = origin_split_tileing_strategy - .tile_information_iterator(partition) + .tile_information_iterator_from_grid_bounds(grid_bounds) .collect(); assert_eq!(vres.len(), 4 * 6); assert_eq!( @@ -1564,41 +1595,48 @@ mod tests { } #[test] - fn test_load_tile_data() { - let output_shape: GridShape2D = [8, 8].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + fn test_load_tile_data_top_left() { + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + flip_y: false, + }; - let RasterTile2D { - global_geo_transform: _, - grid_array: grid, - tile_position: _, - band: _, - time: _, - properties, - cache_hint: _, - } = load_ndvi_jan_2014(output_shape, output_bounds).unwrap(); + let GridAndProperties { grid, properties } = load_ndvi_apr_2014_cropped(gdal_read_advice) + .unwrap() + .unwrap(); assert!(!grid.is_empty()); let grid = grid.into_materialized_masked_grid(); assert_eq!(grid.inner_grid.data.len(), 64); + // pixel value are the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt assert_eq!( grid.inner_grid.data, &[ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 75, 37, 255, 44, 34, 39, 32, 255, 86, - 255, 255, 255, 30, 96, 255, 255, 255, 255, 255, 90, 255, 255, 255, 255, 255, 202, - 255, 193, 255, 255, 255, 255, 255, 89, 255, 111, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 127, 107, 255, 255, 255, 255, 255, 164, 185, 182, + 255, 255, 255, 175, 186, 190, 167, 140, 255, 255, 161, 175, 184, 173, 170, 188, + 255, 255, 128, 177, 165, 145, 191, 174, 255, 117, 100, 174, 159, 147, 99, 135 ] ); assert_eq!(grid.validity_mask.data.len(), 64); - assert_eq!(grid.validity_mask.data, &[true; 64]); + // pixel mask is pixel > 0 from the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt + assert_eq!( + grid.validity_mask.data, + &[ + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, true, true, + false, false, false, false, false, true, true, true, false, false, false, true, + true, true, true, true, false, false, true, true, true, true, true, true, false, + false, true, true, true, true, true, true, false, true, true, true, true, true, + true, true + ] + ); - assert!((properties.scale_option()).is_none()); - assert!(properties.offset_option().is_none()); assert_eq!( properties.get_property(&RasterPropertiesKey { key: "AREA_OR_POINT".to_string(), @@ -1606,34 +1644,24 @@ mod tests { }), Some(&RasterPropertiesEntry::String("Area".to_string())) ); - assert_eq!( - properties.get_property(&RasterPropertiesKey { - domain: Some("IMAGE_STRUCTURE_INFO".to_string()), - key: "COMPRESSION".to_string(), - }), - Some(&RasterPropertiesEntry::String("LZW".to_string())) - ); } #[test] - fn test_load_tile_data_overlaps_dataset_bounds() { - let output_shape: GridShape2D = [8, 8].into(); + fn test_load_tile_data_overlaps_dataset_bounds_top_left_out1() { // shift world bbox one pixel up and to the left - let (x_size, y_size) = (45., 22.5); - let output_bounds = SpatialPartition2D::new_unchecked( - (-180. - x_size, 90. + y_size).into(), - (180. - x_size, -90. + y_size).into(), - ); + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [7, 7].into()), // this is the data we can read + read_window_bounds: GridBoundingBox2D::new([1, 1], [7, 7]).unwrap(), // this is the area we can fill in target + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), // this is the area of the target + flip_y: false, + }; - let RasterTile2D { - global_geo_transform: _, - grid_array: grid, - tile_position: _, - band: _, - time: _, - properties: _, - cache_hint: _, - } = load_ndvi_jan_2014(output_shape, output_bounds).unwrap(); + let GridAndProperties { + grid, + properties: _properties, + } = load_ndvi_apr_2014_cropped(gdal_read_advice) + .unwrap() + .unwrap(); assert!(!grid.is_empty()); @@ -1643,19 +1671,34 @@ mod tests { assert_eq!( x.inner_grid.data, &[ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 0, 255, 75, 37, 255, - 44, 34, 39, 0, 255, 86, 255, 255, 255, 30, 96, 0, 255, 255, 255, 255, 90, 255, 255, - 0, 255, 255, 202, 255, 193, 255, 255, 0, 255, 255, 89, 255, 111, 255, 255, 0, 255, - 255, 255, 255, 255, 255, 255 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 0, 255, 255, 255, + 255, 255, 255, 255, 0, 255, 255, 255, 255, 255, 255, 127, 0, 255, 255, 255, 255, + 255, 164, 185, 0, 255, 255, 255, 175, 186, 190, 167, 0, 255, 255, 161, 175, 184, + 173, 170, 0, 255, 255, 128, 177, 165, 145, 191, + ] + ); + + assert_eq!(x.validity_mask.data.len(), 64); + // pixel mask is pixel == 255 from the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt + assert_eq!( + x.validity_mask.data, + &[ + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, true, false, false, false, false, + false, false, true, true, false, false, false, false, true, true, true, true, + false, false, false, true, true, true, true, true, false, false, false, true, true, + true, true, true, ] ); } + /* This test no longer works since we now employ a clipping strategy and this makes us read a lot more data? #[test] fn test_load_tile_data_is_inside_single_pixel() { let output_shape: GridShape2D = [8, 8].into(); // shift world bbox one pixel up and to the left - let (x_size, y_size) = (0.000_000_000_01, 0.000_000_000_01); + let (x_size, y_size) = (0.001, 0.001); let output_bounds = SpatialPartition2D::new( (-116.22222, 66.66666).into(), (-116.22222 + x_size, 66.66666 - y_size).into(), @@ -1679,27 +1722,20 @@ mod tests { assert_eq!(x.inner_grid.data.len(), 64); assert_eq!(x.inner_grid.data, &[1; 64]); } + */ #[tokio::test] async fn test_query_single_time_slice() { let mut exe_ctx = MockExecutionContext::test_default(); let query_ctx = MockQueryContext::test_default(); let id = add_ndvi_dataset(&mut exe_ctx); + let spatial_query = SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(), + ); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_388_534_400_001); // 2014-01-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 4); @@ -1736,20 +1772,13 @@ mod tests { let query_ctx = MockQueryContext::test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + let spatial_query = SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(), + ); + let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_393_632_000_000); // 2014-01-01 - 2014-03-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 8); @@ -1771,20 +1800,12 @@ mod tests { let query_ctx = MockQueryContext::test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + let spatial_query = SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(), + ); let time_interval = TimeInterval::new_unchecked(1_380_585_600_000, 1_380_585_600_000); // 2013-10-01 - 2013-10-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 4); @@ -1801,20 +1822,12 @@ mod tests { let query_ctx = MockQueryContext::test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + let spatial_query = SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(), + ); let time_interval = TimeInterval::new_unchecked(1_420_074_000_000, 1_420_074_000_000); // 2015-01-01 - 2015-01-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 4); @@ -1833,20 +1846,12 @@ mod tests { let query_ctx = MockQueryContext::test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + let spatial_query = SpatialGridQueryRectangle::new( + GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(), + ); let time_interval = TimeInterval::new_unchecked(1_385_856_000_000, 1_388_534_400_000); // 2013-12-01 - 2014-01-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 4); @@ -1867,12 +1872,19 @@ mod tests { SpatialPartition2D::new_unchecked((-90., 90.).into(), (90., -90.).into()); let output_shape: GridShape2D = [256, 256].into(); - let tile_info = TileInformation::with_partition_and_shape(output_bounds, output_shape); + let tile_info = tile_information_with_partition_and_shape(output_bounds, output_shape); let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_391_212_800_000); // 2014-01-01 - 2014-01-15 let params = None; + let reader_mode = GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + tile_info.global_geo_transform, + GridShape2D::new([3600, 1800]).bounding_box(), + ), + }); let tile = GdalRasterLoader::load_tile_async::( params, + reader_mode, tile_info, time_interval, CacheHint::default(), @@ -2048,121 +2060,8 @@ mod tests { } #[test] - fn gdal_geotransform_to_bounds_neg_y_0() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(0., 0.), - x_pixel_size: 1., - y_pixel_size: -1., - }; - - let sb = gt.spatial_partition(10, 10); - - let exp = SpatialPartition2D::new(Coordinate2D::new(0., 0.), Coordinate2D::new(10., -10.)) - .unwrap(); - - assert_eq!(sb, exp); - } - - #[test] - fn gdal_geotransform_to_bounds_neg_y_5() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(5., 5.), - x_pixel_size: 0.5, - y_pixel_size: -0.5, - }; - - let sb = gt.spatial_partition(10, 10); - - let exp = - SpatialPartition2D::new(Coordinate2D::new(5., 5.), Coordinate2D::new(10., 0.)).unwrap(); - - assert_eq!(sb, exp); - } - - #[test] - fn gdal_geotransform_to_bounds_pos_y_0() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(0., 0.), - x_pixel_size: 1., - y_pixel_size: 1., - }; - - let sb = gt.spatial_partition(10, 10); - - let exp = SpatialPartition2D::new(Coordinate2D::new(0., 10.), Coordinate2D::new(10., 0.)) - .unwrap(); - - assert_eq!(sb, exp); - } - - #[test] - fn gdal_geotransform_to_bounds_pos_y_5() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(5., -5.), - x_pixel_size: 0.5, - y_pixel_size: 0.5, - }; - - let sb = gt.spatial_partition(10, 10); - - let exp = SpatialPartition2D::new(Coordinate2D::new(5., 0.), Coordinate2D::new(10., -5.)) - .unwrap(); - - assert_eq!(sb, exp); - } - - #[test] - fn gdal_read_window_data_origin_upper_left() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(5., -5.), - x_pixel_size: 0.5, - y_pixel_size: -0.5, - }; - - let sb = SpatialPartition2D::new(Coordinate2D::new(8., -7.), Coordinate2D::new(10., -10.)) - .unwrap(); - - let rw = gt.spatial_partition_to_read_window(&sb); - - let exp = GdalReadWindow { - size_x: 4, - size_y: 6, - start_x: 6, - start_y: 4, - }; - - assert_eq!(rw, exp); - } - - #[test] - fn gdal_read_window_data_origin_lower_left() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(0., 0.), - x_pixel_size: 1., - y_pixel_size: 1., - }; - - let sb = SpatialPartition2D::new(Coordinate2D::new(0., 10.), Coordinate2D::new(10., 0.)) - .unwrap(); - - let rw = gt.spatial_partition_to_read_window(&sb); - - let exp = GdalReadWindow { - size_x: 10, - size_y: 10, - start_x: 0, - start_y: 0, - }; - - assert_eq!(rw, exp); - } - - #[test] + #[allow(clippy::too_many_lines)] fn read_up_side_down_raster() { - let output_shape: GridShape2D = [8, 8].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); - let up_side_down_params = GdalDatasetParameters { file_path: test_data!( "raster/modis_ndvi/flipped_axis_y/MOD13A2_M_NDVI_2014-01-01_flipped_y.tiff" @@ -2208,37 +2107,78 @@ mod tests { retry: None, }; - let tile_information = - TileInformation::with_partition_and_shape(output_bounds, output_shape); + let ge_global_dataset_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 0.1, -0.1), + GridBoundingBox2D::new_min_max(0, 1799, 0, 3599).unwrap(), + ); - let RasterTile2D { - global_geo_transform: _, - grid_array: grid, - tile_position: _, - band: _, - time: _, - properties, - cache_hint: _, - } = GdalRasterLoader::load_tile_data::( - &up_side_down_params, - tile_information, - TimeInterval::default(), - CacheHint::default(), - ) - .unwrap(); + let gdal_dataset_grid = ge_global_dataset_grid.flip_axis_y(); // first, flip axis + assert_approx_eq!( + Coordinate2D, + gdal_dataset_grid.geo_transform().origin_coordinate, + Coordinate2D::new(-180., 90.) + ); + assert_approx_eq!(f64, gdal_dataset_grid.geo_transform.y_pixel_size(), 0.1); + assert_approx_eq!(f64, gdal_dataset_grid.geo_transform.x_pixel_size(), 0.1); + assert_eq!( + gdal_dataset_grid.grid_bounds, + GridBoundingBox2D::new_min_max(-1800, -1, 0, 3599).unwrap() + ); - assert!(!grid.is_empty()); + let gdal_dataset_grid = gdal_dataset_grid + .with_moved_origin_exact_grid(Coordinate2D::new(-180., -90.)) + .unwrap(); // second, move origin (to other side of axis) + assert_approx_eq!( + Coordinate2D, + gdal_dataset_grid.geo_transform().origin_coordinate, + Coordinate2D::new(-180., -90.) + ); + assert_approx_eq!(f64, gdal_dataset_grid.geo_transform.y_pixel_size(), 0.1); + assert_approx_eq!(f64, gdal_dataset_grid.geo_transform.x_pixel_size(), 0.1); + assert_eq!( + gdal_dataset_grid.grid_bounds, + GridBoundingBox2D::new_min_max(0, 1799, 0, 3599).unwrap() + ); + let ovr = OverviewReaderState { + original_dataset_grid: ge_global_dataset_grid, + }; + + let tile = SpatialGridDefinition::new( + ge_global_dataset_grid.geo_transform, + GridBoundingBox2D::new_min_max(326, 326 + 7, 1880, 1880 + 7).unwrap(), + ); + + let gdal_read_advice = ovr + .tiling_to_dataset_read_advise(&gdal_dataset_grid, &tile) + .unwrap(); + + let exp_gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([1466, 1880].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([326, 1880], [326 + 7, 1880 + 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([326, 1880], [326 + 7, 1880 + 7]).unwrap(), + flip_y: true, + }; + + assert_eq!(gdal_read_advice, exp_gdal_read_advice); + + let GridAndProperties { grid, properties } = + GdalRasterLoader::load_tile_data::(&up_side_down_params, gdal_read_advice) + .unwrap() + .unwrap(); + + assert!(!grid.is_empty()); let grid = grid.into_materialized_masked_grid(); assert_eq!(grid.inner_grid.data.len(), 64); assert_eq!( grid.inner_grid.data, &[ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 75, 37, 255, 44, 34, 39, 32, 255, 86, - 255, 255, 255, 30, 96, 255, 255, 255, 255, 255, 90, 255, 255, 255, 255, 255, 202, - 255, 193, 255, 255, 255, 255, 255, 89, 255, 111, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + // TODO: check in tiff! + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 53, 47, 255, 255, 255, 255, 255, 68, 81, 93, 255, 255, + 255, 97, 102, 91, 73, 72, 255, 255, 91, 97, 100, 86, 78, 106, 255, 255, 59, 95, 85, + 66, 105, 104, 255, 47, 42, 82, 81, 76, 73, 98 ] ); @@ -2251,25 +2191,19 @@ mod tests { #[test] fn read_raster_and_offset_scale() { - let output_shape: GridShape2D = [8, 8].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); - let up_side_down_params = GdalDatasetParameters { - file_path: test_data!( - "raster/modis_ndvi/with_offset_scale/MOD13A2_M_NDVI_2014-01-01.TIFF" - ) - .into(), + file_path: test_data!("raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif") + .into(), rasterband_channel: 1, geo_transform: GdalDatasetGeoTransform { - origin_coordinate: (-180., -90.).into(), + origin_coordinate: (8.0, 57.4).into(), x_pixel_size: 0.1, - y_pixel_size: 0.1, + y_pixel_size: -0.1, }, - width: 3600, - height: 1800, + width: 30, + height: 30, file_not_found_handling: FileNotFoundHandling::NoData, - no_data_value: Some(0.), + no_data_value: Some(255.), properties_mapping: None, gdal_open_options: None, gdal_config_options: None, @@ -2277,52 +2211,56 @@ mod tests { retry: None, }; - let tile_information = - TileInformation::with_partition_and_shape(output_bounds, output_shape); + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + flip_y: false, + }; - let RasterTile2D { - global_geo_transform: _, - grid_array: grid, - tile_position: _, - band: _, - time: _, - properties, - cache_hint: _, - } = GdalRasterLoader::load_tile_data::( - &up_side_down_params, - tile_information, - TimeInterval::default(), - CacheHint::default(), - ) - .unwrap(); + let GridAndProperties { grid, properties } = + GdalRasterLoader::load_tile_data::(&up_side_down_params, gdal_read_advice) + .unwrap() + .unwrap(); assert!(!grid.is_empty()); let grid = grid.into_materialized_masked_grid(); assert_eq!(grid.inner_grid.data.len(), 64); + // pixel value are the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt assert_eq!( grid.inner_grid.data, &[ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 75, 37, 255, 44, 34, 39, 32, 255, 86, - 255, 255, 255, 30, 96, 255, 255, 255, 255, 255, 90, 255, 255, 255, 255, 255, 202, - 255, 193, 255, 255, 255, 255, 255, 89, 255, 111, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 127, 107, 255, 255, 255, 255, 255, 164, 185, 182, + 255, 255, 255, 175, 186, 190, 167, 140, 255, 255, 161, 175, 184, 173, 170, 188, + 255, 255, 128, 177, 165, 145, 191, 174, 255, 117, 100, 174, 159, 147, 99, 135 ] ); assert_eq!(grid.validity_mask.data.len(), 64); - assert_eq!(grid.validity_mask.data, &[true; 64]); + // pixel mask is pixel > 0 from the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt + assert_eq!( + grid.validity_mask.data, + &[ + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, true, true, + false, false, false, false, false, true, true, true, false, false, false, true, + true, true, true, true, false, false, true, true, true, true, true, true, false, + false, true, true, true, true, true, true, false, true, true, true, true, true, + true, true + ] + ); - assert_eq!(properties.offset_option(), Some(37.)); - assert_eq!(properties.scale_option(), Some(3.7)); + assert_eq!(properties.offset_option(), Some(1.)); + assert_eq!(properties.scale_option(), Some(2.)); - assert!(approx_eq!(f64, properties.offset(), 37.)); - assert!(approx_eq!(f64, properties.scale(), 3.7)); + assert!(approx_eq!(f64, properties.offset(), 1.)); + assert!(approx_eq!(f64, properties.scale(), 2.)); } #[test] - #[allow(clippy::too_many_lines)] fn it_creates_no_data_only_for_missing_files() { hide_gdal_errors(); @@ -2341,19 +2279,20 @@ mod tests { retry: None, }; - let tile_info = TileInformation { - tile_size_in_pixels: [100, 100].into(), - global_tile_position: [0, 0].into(), - global_geo_transform: TestDefault::test_default(), + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + flip_y: false, }; - let tile_time = TimeInterval::default(); + let res = GdalRasterLoader::load_tile_data::(&ds, gdal_read_advice); - // file doesn't exist => no data - let result = - GdalRasterLoader::load_tile_data::(&ds, tile_info, tile_time, CacheHint::default()) - .unwrap(); - assert!(matches!(result.grid_array, GridOrEmpty::Empty(_))); + assert!(res.is_ok()); + + let res = res.unwrap(); + + assert!(res.is_none()); let ds = GdalDatasetParameters { file_path: test_data!("raster/modis_ndvi/MOD13A2_M_NDVI_2014-01-01.TIFF").into(), @@ -2371,10 +2310,12 @@ mod tests { }; // invalid channel => error - let result = - GdalRasterLoader::load_tile_data::(&ds, tile_info, tile_time, CacheHint::default()); + let result = GdalRasterLoader::load_tile_data::(&ds, gdal_read_advice); assert!(result.is_err()); + } + #[test] + fn it_creates_no_data_only_for_http_404() { let server = Server::run(); server.expect( @@ -2416,10 +2357,17 @@ mod tests { }; // 404 => no data - let result = - GdalRasterLoader::load_tile_data::(&ds, tile_info, tile_time, CacheHint::default()) - .unwrap(); - assert!(matches!(result.grid_array, GridOrEmpty::Empty(_))); + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + flip_y: false, + }; + + let res = GdalRasterLoader::load_tile_data::(&ds, gdal_read_advice); + assert!(res.is_ok()); + let res = res.unwrap(); + assert!(res.is_none()); let ds = GdalDatasetParameters { file_path: format!("/vsicurl/{}", server.url_str("/internal_error.tif")).into(), @@ -2448,9 +2396,8 @@ mod tests { }; // 500 => error - let result = - GdalRasterLoader::load_tile_data::(&ds, tile_info, tile_time, CacheHint::default()); - assert!(result.is_err()); + let res = GdalRasterLoader::load_tile_data::(&ds, gdal_read_advice); + assert!(res.is_err()); } #[test] @@ -2542,12 +2489,18 @@ mod tests { SpatialPartition2D::new_unchecked((-90., 90.).into(), (90., -90.).into()); let output_shape: GridShape2D = [256, 256].into(); - let tile_info = TileInformation::with_partition_and_shape(output_bounds, output_shape); + let tile_info = tile_information_with_partition_and_shape(output_bounds, output_shape); let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_391_212_800_000); // 2014-01-01 - 2014-01-15 let params = None; let tile = GdalRasterLoader::load_tile_async::( params, + GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + tile_info.global_geo_transform, + GridShape2D::new([3600, 1800]).bounding_box(), + ), + }), tile_info, time_interval, CacheHint::seconds(1234), diff --git a/operators/src/source/gdal_source/reader.rs b/operators/src/source/gdal_source/reader.rs new file mode 100644 index 000000000..b03d47e2b --- /dev/null +++ b/operators/src/source/gdal_source/reader.rs @@ -0,0 +1,1153 @@ +use geoengine_datatypes::raster::{ + GridBoundingBox2D, GridBounds, GridContains, GridIdx2D, GridOrEmpty2D, GridShape2D, + GridShapeAccess, GridSize, RasterProperties, SpatialGridDefinition, +}; + +/// This struct is used to advise the GDAL reader how to read the data from the dataset. +/// The Workflow is as follows: +/// 1. The `gdal_read_window` is the window in the pixel space of the dataset that should be read. +/// 2. The `read_window_bounds` is the area in the target pixel space where the data should be placed. +/// 2.1 The data read in step one is read to the width and height of the `read_window_bounds`. +/// 2.2 if `flip_y` is true the data is flipped in the y direction. And should be unflipped after reading. +/// 3. The `bounds_of_target` is the area in the target pixel space where the data should be placed. +/// 3.1 The `read_window_bounds` might be offset from the `bounds_of_target` or might have a different size. +/// Then, the data needs to be placed in the target pixel space accordingly. Other parts of the target pixel space should be filled with nodata. +#[allow(dead_code)] +#[derive(Copy, Clone, Debug, PartialEq)] +pub struct GdalReadAdvise { + pub gdal_read_widow: GdalReadWindow, + pub read_window_bounds: GridBoundingBox2D, + pub bounds_of_target: GridBoundingBox2D, + pub flip_y: bool, +} + +impl GdalReadAdvise { + pub fn direct_read(&self) -> bool { + self.read_window_bounds == self.bounds_of_target + } +} + +#[allow(dead_code)] +#[derive(Copy, Clone, Debug)] +pub enum GdalReaderMode { + // read the original resolution + OriginalResolution(ReaderState), + // read an overview level of the dataset + OverviewLevel(OverviewReaderState), +} + +impl GdalReaderMode { + /// Returns the read advise for the tiling based bounds + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, + tile: &SpatialGridDefinition, + ) -> Option { + match self { + GdalReaderMode::OriginalResolution(re) => { + re.tiling_to_dataset_read_advise(actual_gdal_dataset_spatial_grid_definition, tile) + } + GdalReaderMode::OverviewLevel(rs) => { + rs.tiling_to_dataset_read_advise(actual_gdal_dataset_spatial_grid_definition, tile) + } + } + } +} + +#[derive(Copy, Clone, Debug)] +pub struct ReaderState { + pub dataset_spatial_grid: SpatialGridDefinition, +} + +impl ReaderState { + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, + tile: &SpatialGridDefinition, + ) -> Option { + // Check if the y_axis is fliped. + let (actual_gdal_dataset_spatial_grid_definition, flip_y) = + if actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .y_axis_is_neg() + == self.dataset_spatial_grid.geo_transform().y_axis_is_neg() + { + (*actual_gdal_dataset_spatial_grid_definition, false) + } else { + ( + actual_gdal_dataset_spatial_grid_definition + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )), + true, + ) + }; + + // Now we can work with a matching dataset. However, we need to reverse the read window later! + + // let's only look at data in the geo engine dataset definition! The intersection is relative to the first elements origin coordinate. + let dataset_gdal_data_intersection = + actual_gdal_dataset_spatial_grid_definition.intersection(&self.dataset_spatial_grid)?; + + // Now, we need the tile in the gdal dataset bounds to identify readable areas + let tile_in_gdal_dataset_bounds = tile.with_moved_origin_exact_grid( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .origin_coordinate, + )?; // TODO: raise error if this fails! + + // Then, calculate the intersection between the datataset and the tile. Again, the intersection is relative to the first elements orrigin coordinate. + let tile_gdal_dataset_intersection = + dataset_gdal_data_intersection.intersection(&tile_in_gdal_dataset_bounds)?; + + // if we need to unflip the dataset grid now is the time to do this. + let tile_intersection_for_read_window = if flip_y { + tile_gdal_dataset_intersection + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )) + } else { + tile_gdal_dataset_intersection + }; + + // generate the read window for GDAL + + let gdal_read_window = GdalReadWindow::new( + tile_intersection_for_read_window.grid_bounds.min_index(), + tile_intersection_for_read_window.grid_bounds.grid_shape(), + ); + + // if the read window has the same shape as the tiling based bounds we can fill that completely + if tile_in_gdal_dataset_bounds == tile_gdal_dataset_intersection { + return Some(GdalReadAdvise { + gdal_read_widow: gdal_read_window, + read_window_bounds: tile.grid_bounds, + bounds_of_target: tile.grid_bounds, + flip_y, + }); + } + + // we need to crop the window to the intersection of the tiling based bounds and the dataset bounds + let crop_tl = + tile_gdal_dataset_intersection.min_index() - tile_in_gdal_dataset_bounds.min_index(); + let crop_lr = + tile_gdal_dataset_intersection.max_index() - tile_in_gdal_dataset_bounds.max_index(); + + let shifted_tl = tile.grid_bounds.min_index() + crop_tl; + let shifted_lr = tile.grid_bounds.max_index() + crop_lr; + + // now we need to adapt the target pixel space read window to the clipped dataset intersection area + let shifted_readable_bounds = GridBoundingBox2D::new_unchecked(shifted_tl, shifted_lr); + debug_assert!( + tile.grid_bounds().contains(&shifted_readable_bounds), + "readable bounds must be contained in tile bounds" + ); + + Some(GdalReadAdvise { + gdal_read_widow: gdal_read_window, + read_window_bounds: shifted_readable_bounds, + bounds_of_target: tile.grid_bounds, + flip_y: false, + }) + } +} + +#[derive(Copy, Clone, Debug)] +pub struct OverviewReaderState { + pub original_dataset_grid: SpatialGridDefinition, +} + +impl OverviewReaderState { + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, // This is the spatial grid of an actual gdal file + tile: &SpatialGridDefinition, // This is a tile inside the grid we use for the global dataset consisting of potentially many gdal files... + ) -> Option { + // Check if the y_axis is fliped. + let (actual_gdal_dataset_spatial_grid_definition, flip_y) = + if actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .y_axis_is_neg() + == self.original_dataset_grid.geo_transform().y_axis_is_neg() + { + (*actual_gdal_dataset_spatial_grid_definition, false) + } else { + ( + actual_gdal_dataset_spatial_grid_definition + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )), + true, + ) + }; + + // This is the intersection of grid of the gdal file and the global grid we use. Usually the dataset is inside the global dataset grid. + // IF the intersection is empty the we return early and load nothing + // The intersection uses the geo_transform of the gdal dataset which enables us to adress gdal pixels starting at 0,0 + let actual_bounds_to_use_original_resolution = actual_gdal_dataset_spatial_grid_definition + .intersection(&self.original_dataset_grid)?; + + // now we map the tile we want to fill to the original grid. First, we set the tile to use the same origin coordinate as the gdal file/dataset + let tile_with_overview_resolution_in_actual_space = tile + .with_moved_origin_exact_grid( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .origin_coordinate(), + ) + .expect("The overview level grid must map to pixel coordinates in the original grid"); // TODO: maybe relax this? + let tile_with_original_resolution_in_actual_space = + tile_with_overview_resolution_in_actual_space.with_changed_resolution( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .spatial_resolution(), + ); + + // Now we need to intersect the tile and the actual bounds to use to identify what we can really read + let tile_intersection_original_resolution_actual_space = + &tile_with_original_resolution_in_actual_space + .intersection(&actual_bounds_to_use_original_resolution)?; + + // if we need to unflip the dataset grid now is the time to do this. + let tile_intersection_for_read_window = if flip_y { + tile_intersection_original_resolution_actual_space + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )) + } else { + *tile_intersection_original_resolution_actual_space + }; + + // generate the read window for GDAL --> This is what we can read in any case. + let read_window = GdalReadWindow::new( + tile_intersection_for_read_window.min_index(), + tile_intersection_for_read_window.grid_bounds().grid_shape(), + ); + + let is_tile_contained = tile_intersection_for_read_window.grid_bounds() + == tile_with_original_resolution_in_actual_space.grid_bounds(); + + if is_tile_contained { + return Some(GdalReadAdvise { + gdal_read_widow: read_window, + read_window_bounds: tile.grid_bounds, + bounds_of_target: tile.grid_bounds, + flip_y, + }); + } + + // IF we can't read the whole tile, we have to find out which area of the tile we can fill. + let readble_area_in_overview_res = tile_intersection_original_resolution_actual_space + .with_changed_resolution(tile.geo_transform().spatial_resolution()); + + // Calculate the intersection of the readable area and the tile, result is in geotransform of the tile! + let readable_tile_area = tile.intersection(&readble_area_in_overview_res).expect( + "Since there was an intersection earlyer, there must be a part of data to read.", + ); + + // we need to crop the window to the intersection of the tiling based bounds and the dataset bounds + let crop_tl = readable_tile_area.min_index() - tile.min_index(); + let crop_lr = readable_tile_area.max_index() - tile.max_index(); + + let shifted_tl = tile.grid_bounds.min_index() + crop_tl; + let shifted_lr = tile.grid_bounds.max_index() + crop_lr; + + // now we need to adapt the target pixel space read window to the clipped dataset intersection area + let shifted_readable_bounds = GridBoundingBox2D::new_unchecked(shifted_tl, shifted_lr); + debug_assert!( + tile.grid_bounds().contains(&shifted_readable_bounds), + "readable bounds must be contained in tile bounds" + ); + + Some(GdalReadAdvise { + gdal_read_widow: read_window, + read_window_bounds: shifted_readable_bounds, + bounds_of_target: tile.grid_bounds, + flip_y, + }) + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct GdalReadWindow { + start_x: isize, // pixelspace origin + start_y: isize, + size_x: usize, // pixelspace size + size_y: usize, +} + +impl GdalReadWindow { + pub fn new(start: GridIdx2D, size: GridShape2D) -> Self { + Self { + start_x: start.x(), + start_y: start.y(), + size_x: size.axis_size_x(), + size_y: size.axis_size_y(), + } + } + + pub fn gdal_window_start(&self) -> (isize, isize) { + (self.start_x, self.start_y) + } + + pub fn gdal_window_size(&self) -> (usize, usize) { + (self.size_x, self.size_y) + } +} + +pub struct GridAndProperties { + pub grid: GridOrEmpty2D, + pub properties: RasterProperties, +} + +#[cfg(test)] +mod tests { + use geoengine_datatypes::{ + primitives::Coordinate2D, + raster::{ + BoundedGrid, GeoTransform, GridBoundingBox2D, GridIdx2D, GridShape2D, + SpatialGridDefinition, + }, + }; + + use crate::source::gdal_source::reader::{GdalReadWindow, OverviewReaderState, ReaderState}; + + #[test] + fn reader_state_dataset_geo_transform() { + let reader_state = ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ), + }; + + assert_eq!( + reader_state.dataset_spatial_grid.geo_transform(), + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.) + ); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_no_change() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 512, + size_y: 512, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 180, + start_y: 90, + size_x: 180, + size_y: 90, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted_and_clipped() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([99, 189])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 190, + start_y: 100, + size_x: 170, + size_y: 80, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([99, 189])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted_flipy() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let spatial_grid_flipy = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., -90.), 1., 1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid_flipy, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 180, + start_y: 0, + size_x: 180, + size_y: 90, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert!(tiling_to_dataset_read_advise.flip_y); + } + + /* + #[test] + fn gdal_geotransform_to_read_bounds() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(0., 0.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + + let ti: TileInformation = TileInformation::new( + GridIdx([1, 1]), + GridShape2D::new([512, 512]), + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + ); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 512, + size_y: 512, + start_x: 512, + start_y: 512, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([512, 512]), GridIdx([1023, 1023])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_half_res() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(0., 0.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + + let ti: TileInformation = TileInformation::new( + GridIdx([0, 0]), + GridShape2D::new([512, 512]), + GeoTransform::new(Coordinate2D::new(0., 0.), 2., -2.), + ); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 1024, + size_y: 1024, + start_x: 0, + start_y: 0, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([0, 0]), GridIdx([511, 511])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_2x_res() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(0., 0.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + + let ti: TileInformation = TileInformation::new( + GridIdx([0, 0]), + GridShape2D::new([512, 512]), + GeoTransform::new(Coordinate2D::new(0., 0.), 0.5, -0.5), + ); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 256, + size_y: 256, + start_x: 0, + start_y: 0, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([0, 0]), GridIdx([511, 511])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_ul_out() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(-3., 3.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + let tile_grid_shape = GridShape2D::new([512, 512]); + let tiling_global_geo_transfom = GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.); + + let ti: TileInformation = + TileInformation::new(GridIdx([0, 0]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + // since the origin of the tile is at -3,3 and the "coordinate nearest to zero" is 0,0 the tile at tile position 0,0 maps to the read window starting at 3,3 with 512x512 pixels + assert_eq!( + read_window, + GdalReadWindow { + size_x: 512, + size_y: 512, + start_x: 3, + start_y: 3, + } + ); + + // the data maps to the complete tile + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([0, 0]), GridIdx([511, 511])).unwrap() + ); + + let ti: TileInformation = + TileInformation::new(GridIdx([1, 1]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + // since the origin of the tile is at -3,3 and the "coordinate nearest to zero" is 0,0 the tile at tile position 1,1 maps to the read window starting at 515,515 (512+3, 512+3) with 512x512 pixels + assert_eq!( + read_window, + GdalReadWindow { + size_x: 509, + size_y: 509, + start_x: 515, + start_y: 515, + } + ); + + // the data maps only to a part of the tile since the data is only 1024x1024 pixels in size. So the tile at tile position 1,1 maps to the data starting at 515,515 (512+3, 512+3) with 509x509 pixels left. + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([512, 512]), GridIdx([1020, 1020])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_ul_in() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(3., -3.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + let tile_grid_shape = GridShape2D::new([512, 512]); + let tiling_global_geo_transfom = GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.); + + let ti: TileInformation = + TileInformation::new(GridIdx([0, 0]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + // in this case the data origin is at 3,-3 which is inside the tile at tile position 0,0. Since the tile starts at the "coordinate nearest to zero, which is 0.0,0.0" we need to read the data starting at data 0,0 with 509x509 pixels (512-3, 512-3). + assert_eq!( + read_window, + GdalReadWindow { + size_x: 509, + size_y: 509, + start_x: 0, + start_y: 0, + } + ); + + // in this case, the data only maps to the last 509x509 pixels of the tile. So the data we read does not fill a whole tile. + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([3, 3]), GridIdx([511, 511])).unwrap() + ); + + let ti: TileInformation = + TileInformation::new(GridIdx([1, 1]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 512, + size_y: 512, + start_x: 509, + start_y: 509, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([512, 512]), GridIdx([1023, 1023])).unwrap() + ); + + let ti: TileInformation = + TileInformation::new(GridIdx([2, 2]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 3, + size_y: 3, + start_x: 1021, + start_y: 1021, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([1024, 1024]), GridIdx([1026, 1026])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_ul_out_frac_res() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(-9., 9.), + x_pixel_size: 9., + y_pixel_size: -9., + }; + let gdal_data_size = GridShape2D::new([1024, 1024]); + let tile_grid_shape = GridShape2D::new([512, 512]); + let tiling_global_geo_transfom = GeoTransform::new(Coordinate2D::new(-0., 0.), 3., -3.); + + let ti: TileInformation = + TileInformation::new(GridIdx([0, 0]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 170, // + size_y: 170, + start_x: 1, + start_y: 1, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([0, 0]), GridIdx([512, 512])).unwrap() + ); // we need to read 683 pixels but we only want 682.6666666666666 pixels. + + let ti: TileInformation = + TileInformation::new(GridIdx([1, 1]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 171, + size_y: 171, + start_x: 171, + start_y: 171, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([510, 510]), GridIdx([1025, 1025])).unwrap() + ); + + let ti: TileInformation = + TileInformation::new(GridIdx([2, 2]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 171, + size_y: 171, + start_x: 342, + start_y: 342, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([1023, 1023]), GridIdx([1535, 1535])).unwrap() + ); + } + */ + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_2() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 2., -2.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 1024, + size_y: 1024, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([2048, 2048]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048, + size_y: 2048, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([4096, 4096]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 2048, + start_y: 2048, + size_x: 2048, + size_y: 2048, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_lrcrop() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 2048, + start_y: 2048, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new( + GridIdx2D::new([512, 512]), + GridIdx2D::new([1023 - 4, 1023 - 4]) + ) + .unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_ulcrop() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(16., -16.), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([4, 4]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_ulcrop_numbers() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(17.123_456, -17.123_456), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(1.123_456, -1.123_456), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([4, 4]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } +} diff --git a/operators/src/source/ogr_source/dataset_iterator.rs b/operators/src/source/ogr_source/dataset_iterator.rs index 443cb3a0c..95f161e34 100644 --- a/operators/src/source/ogr_source/dataset_iterator.rs +++ b/operators/src/source/ogr_source/dataset_iterator.rs @@ -8,7 +8,7 @@ use crate::util::gdal::gdal_open_dataset_ex; use gdal::vector::sql::Dialect; use gdal::vector::{Feature, LayerAccess}; use gdal::{Dataset, DatasetOptions, GdalOpenFlags}; -use geoengine_datatypes::primitives::VectorQueryRectangle; +use geoengine_datatypes::primitives::{SpatialBounded, VectorQueryRectangle}; use log::debug; use ouroboros::self_referencing; use std::cell::Cell; @@ -199,10 +199,10 @@ impl OgrDatasetIterator { if use_ogr_spatial_filter { debug!( "using spatial filter {:?} for layer {:?}", - query_rectangle.spatial_bounds, &dataset_information.layer_name + query_rectangle.spatial_query, &dataset_information.layer_name ); // NOTE: the OGR-filter may be inaccurately allowing more features that should be returned in a "strict" fashion. - features_provider.set_spatial_filter(&query_rectangle.spatial_bounds); + features_provider.set_spatial_filter(&query_rectangle.spatial_query().spatial_bounds()); } Ok((features_provider, time_filter.is_some())) diff --git a/operators/src/source/ogr_source/mod.rs b/operators/src/source/ogr_source/mod.rs index c2e9d3a04..c7332b144 100644 --- a/operators/src/source/ogr_source/mod.rs +++ b/operators/src/source/ogr_source/mod.rs @@ -1,26 +1,10 @@ mod dataset_iterator; -use self::dataset_iterator::OgrDatasetIterator; -use crate::adapters::FeatureCollectionStreamExt; -use crate::engine::{ - CanonicOperatorName, OperatorData, OperatorName, QueryProcessor, WorkflowOperatorPath, -}; -use crate::error::Error; -use crate::util::input::StringOrNumberRange; -use crate::util::{Result, safe_lock_mutex}; -use crate::{ - engine::{ - InitializedVectorOperator, MetaData, QueryContext, SourceOperator, - TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, - }, - error, -}; -use async_trait::async_trait; -use futures::future::BoxFuture; + +use futures::FutureExt; +use futures::future::{BoxFuture, Future}; use futures::stream::{BoxStream, FusedStream}; use futures::task::Context; -use futures::{Future, FutureExt}; use futures::{Stream, StreamExt, ready}; -use gdal::errors::GdalError; use gdal::vector::sql::ResultSet; use gdal::vector::{Feature, FieldValue, Layer, LayerAccess, LayerCaps, OGRwkbGeometryType}; use geoengine_datatypes::collections::{ @@ -28,21 +12,18 @@ use geoengine_datatypes::collections::{ FeatureCollectionModifications, FeatureCollectionRowBuilder, GeoFeatureCollectionRowBuilder, VectorDataType, }; -use geoengine_datatypes::dataset::NamedData; +use geoengine_datatypes::primitives::CacheTtlSeconds; use geoengine_datatypes::primitives::{ AxisAlignedRectangle, BoundingBox2D, Coordinate2D, DateTime, DateTimeParseFormat, FeatureDataType, FeatureDataValue, Geometry, MultiLineString, MultiPoint, MultiPolygon, - NoGeometry, TimeInstance, TimeInterval, TimeStep, TypedGeometry, VectorQueryRectangle, + NoGeometry, SpatialBounded, TimeInstance, TimeInterval, TimeStep, TypedGeometry, + VectorQueryRectangle, VectorSpatialQueryRectangle, }; -use geoengine_datatypes::primitives::{CacheTtlSeconds, ColumnSelection}; use geoengine_datatypes::util::arrow::ArrowTyped; use log::debug; -use pin_project::pin_project; use postgres_protocol::escape::{escape_identifier, escape_literal}; -use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; -use std::convert::{TryFrom, TryInto}; use std::fmt::Debug; use std::marker::PhantomData; use std::ops::{Add, DerefMut}; @@ -53,6 +34,28 @@ use std::sync::Arc; use std::task::Poll; use tokio::sync::Mutex; +use self::dataset_iterator::OgrDatasetIterator; +use crate::adapters::FeatureCollectionStreamExt; +use crate::engine::{ + CanonicOperatorName, OperatorData, OperatorName, QueryProcessor, WorkflowOperatorPath, +}; +use crate::error::Error; +use crate::util::input::StringOrNumberRange; +use crate::util::{Result, safe_lock_mutex}; +use crate::{ + engine::{ + InitializedVectorOperator, MetaData, QueryContext, SourceOperator, + TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, + }, + error, +}; +use async_trait::async_trait; +use gdal::errors::GdalError; +use geoengine_datatypes::dataset::NamedData; +use geoengine_datatypes::primitives::ColumnSelection; +use pin_project::pin_project; +use postgres_types::{FromSql, ToSql}; + #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct OgrSourceParameters { @@ -535,7 +538,7 @@ where FeatureCollectionRowBuilder: FeatureCollectionBuilderGeometryHandler, { type Output = FeatureCollection; - type SpatialBounds = BoundingBox2D; + type SpatialQuery = VectorSpatialQueryRectangle; type Selection = ColumnSelection; type ResultDescription = VectorResultDescriptor; @@ -1311,7 +1314,7 @@ where // filter out geometries that are not contained in the query's bounding box if !was_spatial_filtered_by_ogr - && !geometry.intersects_bbox(&query_rectangle.spatial_bounds) + && !geometry.intersects_bbox(&query_rectangle.spatial_query().spatial_bounds()) { return Ok(()); } @@ -2007,8 +2010,9 @@ mod tests { use geoengine_datatypes::dataset::{DataId, DatasetId}; use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, Measurement, SpatialResolution, TimeGranularity, + BoundingBox2D, FeatureData, Measurement, TimeGranularity, }; + use geoengine_datatypes::raster::TilingSpecification; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceOption}; use geoengine_datatypes::util::Identifier; use geoengine_datatypes::util::test::TestDefault; @@ -2190,15 +2194,15 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2245,15 +2249,15 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ), &context, ) .await; @@ -2294,15 +2298,15 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (5., 5.).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (5., 5.).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2348,15 +2352,15 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (5., 5.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (5., 5.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2434,15 +2438,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2534,15 +2537,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2637,15 +2639,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2791,15 +2792,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2966,18 +2966,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( - (-180.0, -90.0).into(), - (180.0, 90.0).into(), - )?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180.0, -90.0).into(), (180.0, 90.0).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -4155,15 +4151,15 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -4277,15 +4273,15 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 2.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 2.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -4487,15 +4483,14 @@ mod tests { (4.824_087_161, 52.413_055_56), ])?; - let context1 = MockQueryContext::new(ChunkByteSize::MIN); + let context1 = exe_ctx.mock_query_context(ChunkByteSize::MIN); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context1, ) .await @@ -4522,15 +4517,14 @@ mod tests { assert!(!result.last().unwrap().is_empty()); // LARGER CHUNK - let context = MockQueryContext::new((1_650).into()); + let context = exe_ctx.mock_query_context((1_650).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -4640,15 +4634,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (-180.00, -90.0).into()).unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MIN); + let context = exe_ctx.mock_query_context(ChunkByteSize::MIN); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -4730,15 +4723,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -4851,15 +4843,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -4979,15 +4970,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -5105,15 +5095,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -5231,15 +5220,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -5353,15 +5341,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -5488,15 +5475,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -5609,15 +5595,14 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + query_bbox, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -5723,15 +5708,15 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -5853,15 +5838,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -5972,15 +5957,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6091,15 +6076,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6214,15 +6199,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6336,15 +6321,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6473,15 +6458,15 @@ mod tests { ], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6592,15 +6577,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6703,15 +6688,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6800,15 +6785,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6894,15 +6879,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -6988,15 +6973,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -7082,15 +7067,15 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = + MockQueryContext::new(ChunkByteSize::MAX, TilingSpecification::test_default()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await diff --git a/operators/src/util/gdal.rs b/operators/src/util/gdal.rs index 7ff177edf..6c466f32b 100644 --- a/operators/src/util/gdal.rs +++ b/operators/src/util/gdal.rs @@ -1,11 +1,17 @@ -use std::{ - collections::HashSet, - convert::TryInto, - hash::BuildHasher, - path::{Path, PathBuf}, - str::FromStr, +use crate::{ + engine::{ + MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, RasterResultDescriptor, + SpatialGridDescriptor, StaticMetaData, VectorColumnInfo, VectorResultDescriptor, + }, + error::{self, Error}, + source::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, + GdalSourceTimePlaceholder, OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, + OgrSourceErrorSpec, TimeReference, + }, + test_data, + util::Result, }; - use gdal::{Dataset, DatasetOptions, DriverManager}; use geoengine_datatypes::{ collections::VectorDataType, @@ -13,29 +19,21 @@ use geoengine_datatypes::{ hashmap, primitives::{ BoundingBox2D, CacheTtlSeconds, ContinuousMeasurement, DateTimeParseFormat, - FeatureDataType, Measurement, SpatialPartition2D, SpatialResolution, TimeGranularity, - TimeInstance, TimeInterval, TimeStep, VectorQueryRectangle, + FeatureDataType, Measurement, TimeGranularity, TimeInstance, TimeInterval, TimeStep, + VectorQueryRectangle, }, - raster::{GeoTransform, RasterDataType}, + raster::{BoundedGrid, GeoTransform, GridBoundingBox2D, GridShape2D, RasterDataType}, spatial_reference::SpatialReference, util::Identifier, }; use itertools::Itertools; use snafu::ResultExt; - -use crate::{ - engine::{ - MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, RasterResultDescriptor, - StaticMetaData, VectorColumnInfo, VectorResultDescriptor, - }, - error::{self, Error}, - source::{ - FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, - GdalSourceTimePlaceholder, OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, - OgrSourceErrorSpec, TimeReference, - }, - test_data, - util::Result, +use std::{ + collections::HashSet, + convert::TryInto, + hash::BuildHasher, + path::{Path, PathBuf}, + str::FromStr, }; // TODO: move test helper somewhere else? @@ -43,6 +41,10 @@ pub fn create_ndvi_meta_data() -> GdalMetaDataRegular { create_ndvi_meta_data_with_cache_ttl(CacheTtlSeconds::default()) } +pub fn create_ndvi_meta_data_cropped_to_valid_webmercator_bounds() -> GdalMetaDataRegular { + create_ndvi_meta_data_with_cache_ttl(CacheTtlSeconds::default()) +} + #[allow(clippy::missing_panics_doc)] pub fn create_ndvi_meta_data_with_cache_ttl(cache_ttl: CacheTtlSeconds) -> GdalMetaDataRegular { let no_data_value = Some(0.); // TODO: is it really 0? @@ -90,11 +92,78 @@ pub fn create_ndvi_meta_data_with_cache_ttl(cache_ttl: CacheTtlSeconds) -> GdalM TimeInstance::from_str("2014-07-01T00:00:00.000Z") .expect("it should only be used in tests"), )), - bbox: Some(SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., 90.).into(), 0.1, -0.1), + GridBoundingBox2D::new([0, 0], [1799, 3599]).expect("should only be used in tests"), + ), + bands: vec![RasterBandDescriptor { + name: "ndvi".to_string(), + measurement: Measurement::Continuous(ContinuousMeasurement { + measurement: "vegetation".to_string(), + unit: None, + }), + }] + .try_into() + .expect("it should only be used in tests"), + }, + cache_ttl, + } +} + +#[allow(clippy::missing_panics_doc)] +pub fn create_ndvi_meta_data_cropped_to_valid_webmercator_bounds_with_cache_ttl( + cache_ttl: CacheTtlSeconds, +) -> GdalMetaDataRegular { + let no_data_value = Some(0.); // TODO: is it really 0? + GdalMetaDataRegular { + data_time: TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00.000Z") + .expect("should only be used in tests"), + TimeInstance::from_str("2014-07-01T00:00:00.000Z") + .expect("should only be used in tests"), + ), + step: TimeStep { + granularity: TimeGranularity::Months, + step: 1, + }, + time_placeholders: hashmap! { + "%_START_TIME_%".to_string() => GdalSourceTimePlaceholder { + format: DateTimeParseFormat::custom("%Y-%m-%d".to_string()), + reference: TimeReference::Start, + }, + }, + params: GdalDatasetParameters { + file_path: test_data!("raster/modis_ndvi/MOD13A2_M_NDVI_%_START_TIME_%.TIFF").into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: (-180., 85.).into(), + x_pixel_size: 0.1, + y_pixel_size: -0.1, + }, + width: 3600, + height: 1700, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }, + result_descriptor: RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., 0.).into(), 0.1, -0.1), + GridBoundingBox2D::new([-850, -1800], [-845, -1799]) + .expect("should only be used in tests"), + ), + time: Some(TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00.000Z") + .expect("should only be used in tests"), + TimeInstance::from_str("2014-07-01T00:00:00.000Z") + .expect("should only be used in tests"), )), - resolution: Some(SpatialResolution::new_unchecked(0.1, 0.1)), bands: vec![RasterBandDescriptor { name: "ndvi".to_string(), measurement: Measurement::Continuous(ContinuousMeasurement { @@ -117,6 +186,19 @@ pub fn add_ndvi_dataset(ctx: &mut MockExecutionContext) -> NamedData { name } +pub fn add_ndvi_dataset_cropped_to_valid_webmercator_bounds( + ctx: &mut MockExecutionContext, +) -> NamedData { + let id: DataId = DatasetId::new().into(); + let name = NamedData::with_system_name("ndvi_crop_y_85"); + ctx.add_meta_data( + id, + name.clone(), + Box::new(create_ndvi_meta_data_cropped_to_valid_webmercator_bounds()), + ); + name +} + #[allow(clippy::missing_panics_doc)] pub fn create_ports_meta_data() -> StaticMetaData { @@ -240,14 +322,17 @@ pub fn raster_descriptor_from_dataset( let data_type = RasterDataType::from_gdal_data_type(rasterband.band_type()) .map_err(|_| Error::GdalRasterDataTypeNotSupported)?; - let geo_transfrom = GeoTransform::from(dataset.geo_transform()?); + let data_geo_transfrom = GeoTransform::from(dataset.geo_transform()?); + let data_shape = GridShape2D::new([dataset.raster_size().1, dataset.raster_size().0]); Ok(RasterResultDescriptor { data_type, spatial_reference: spatial_ref.into(), time: None, - bbox: None, - resolution: Some(geo_transfrom.spatial_resolution()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + data_geo_transfrom, + data_shape.bounding_box(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), // TODO: derive better name? measurement_from_rasterband(dataset, band)?, @@ -266,14 +351,17 @@ pub fn raster_descriptor_from_dataset_and_sref( let data_type = RasterDataType::from_gdal_data_type(rasterband.band_type()) .map_err(|_| Error::GdalRasterDataTypeNotSupported)?; - let geo_transfrom = GeoTransform::from(dataset.geo_transform()?); + let data_geo_transfrom = GeoTransform::from(dataset.geo_transform()?); + let data_shape = GridShape2D::new([dataset.raster_size().1, dataset.raster_size().0]); Ok(RasterResultDescriptor { data_type, spatial_reference: spatial_ref.into(), time: None, - bbox: None, - resolution: Some(geo_transfrom.spatial_resolution()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + data_geo_transfrom, + data_shape.bounding_box(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), // TODO derive better name? measurement_from_rasterband(dataset, band)?, diff --git a/operators/src/util/input/multi_raster_or_vector.rs b/operators/src/util/input/multi_raster_or_vector.rs index 49109bd4c..95e274093 100644 --- a/operators/src/util/input/multi_raster_or_vector.rs +++ b/operators/src/util/input/multi_raster_or_vector.rs @@ -68,9 +68,7 @@ mod tests { fn it_serializes() { let operator = MultiRasterOrVectorOperator::Raster(vec![ GdalSource { - params: GdalSourceParameters { - data: NamedData::with_namespaced_name("foo", "bar"), - }, + params: GdalSourceParameters::new(NamedData::with_namespaced_name("foo", "bar")), } .boxed(), ]); @@ -80,7 +78,8 @@ mod tests { serde_json::json!([{ "type": "GdalSource", "params": { - "data": "foo:bar" + "data": "foo:bar", + "overviewLevel": null, } }]) ); diff --git a/operators/src/util/input/raster_or_vector.rs b/operators/src/util/input/raster_or_vector.rs index c05854000..7a4cc6d06 100644 --- a/operators/src/util/input/raster_or_vector.rs +++ b/operators/src/util/input/raster_or_vector.rs @@ -67,9 +67,7 @@ mod tests { fn it_serializes() { let operator = RasterOrVectorOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: NamedData::with_namespaced_name("foo", "bar"), - }, + params: GdalSourceParameters::new(NamedData::with_namespaced_name("foo", "bar")), } .boxed(), ); @@ -79,7 +77,8 @@ mod tests { serde_json::json!({ "type": "GdalSource", "params": { - "data": "foo:bar" + "data": "foo:bar", + "overviewLevel": null, } }) ); diff --git a/operators/src/util/mod.rs b/operators/src/util/mod.rs index e0b4cd91f..c1316ac99 100644 --- a/operators/src/util/mod.rs +++ b/operators/src/util/mod.rs @@ -12,6 +12,8 @@ pub mod stream_zip; pub mod string_token; pub mod sunpos; mod temporary_gdal_thread_local_config_options; +pub mod test; +mod wrap_with_projection_and_resample; use crate::error::Error; use std::collections::HashSet; @@ -23,6 +25,7 @@ pub use self::async_util::{ }; pub use self::rayon::create_rayon_thread_pool; pub use self::temporary_gdal_thread_local_config_options::TemporaryGdalThreadLocalConfigOptions; +pub use wrap_with_projection_and_resample::WrapWithProjectionAndResample; pub type Result = std::result::Result; diff --git a/operators/src/util/raster_stream_to_geotiff.rs b/operators/src/util/raster_stream_to_geotiff.rs index d0588990b..bf357ea28 100644 --- a/operators/src/util/raster_stream_to_geotiff.rs +++ b/operators/src/util/raster_stream_to_geotiff.rs @@ -1,3 +1,4 @@ +use crate::engine::QueryProcessor; use crate::error; use crate::source::{ FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, @@ -12,15 +13,12 @@ use futures::future::BoxFuture; use futures::{StreamExt, TryFutureExt}; use gdal::raster::{Buffer, GdalType, RasterBand, RasterCreationOptions}; use gdal::{Dataset, DriverManager, Metadata}; -use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, DateTimeParseFormat, QueryRectangle, RasterQueryRectangle, - SpatialPartition2D, TimeInterval, -}; use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds}; +use geoengine_datatypes::primitives::{DateTimeParseFormat, RasterQueryRectangle, TimeInterval}; use geoengine_datatypes::raster::{ - ChangeGridBounds, EmptyGrid2D, GeoTransform, GridBlit, GridIdx, GridIdx2D, GridSize, - MapElements, MaskedGrid2D, NoDataValueGrid, Pixel, RasterTile2D, TilingSpecification, - TilingStrategy, + ChangeGridBounds, GeoTransform, GridBlit, GridBoundingBox2D, GridBounds, GridIntersection, + GridOrEmpty, GridSize, MapElements, MaskedGrid2D, NoDataValueGrid, Pixel, RasterTile2D, + TilingSpecification, TilingStrategy, }; use geoengine_datatypes::spatial_reference::SpatialReference; use log::debug; @@ -49,6 +47,10 @@ pub async fn raster_stream_to_multiband_geotiff_bytes( tiles: &[RasterTile2D], - query_rect: &QueryRectangle, - tiling_specification: TilingSpecification, + query_rect: &RasterQueryRectangle, + tiling_strategy: TilingStrategy, gdal_tiff_options: GdalGeoTiffOptions, gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, ) -> Result<(TimeInterval, PathBuf, Dataset, GdalDatasetWriter), Error> @@ -129,28 +131,24 @@ where geo_transform: initial_tile_info.global_geo_transform, }; let num_tiles_per_timestep = strat - .tile_grid_box(query_rect.spatial_bounds) + .global_pixel_grid_bounds_to_tile_grid_bounds(query_rect.spatial_query().grid_bounds()) .number_of_elements(); let num_timesteps = tiles.len() / num_tiles_per_timestep; - let x_pixel_size = query_rect.spatial_resolution.x; - let y_pixel_size = query_rect.spatial_resolution.y; - let width = (query_rect.spatial_bounds.size_x() / x_pixel_size).ceil() as usize; - let height = (query_rect.spatial_bounds.size_y() / y_pixel_size).ceil() as usize; - let output_geo_transform = GeoTransform::new( - query_rect.spatial_bounds.upper_left(), - x_pixel_size, - -y_pixel_size, - ); + let x_pixel_size = tiling_strategy.geo_transform.x_pixel_size(); + let y_pixel_size = tiling_strategy.geo_transform.y_pixel_size(); - let global_geo_transform = tiling_specification - .strategy(x_pixel_size, -y_pixel_size) - .geo_transform; - let window_start = - global_geo_transform.coordinate_to_grid_idx_2d(query_rect.spatial_bounds.upper_left()); - let window_end = window_start + GridIdx2D::from([height as isize, width as isize]); + let coordinate_of_ul_query_pixel = strat + .geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d( + query_rect.spatial_query().grid_bounds().min_index(), + ); + let output_geo_transform = + GeoTransform::new(coordinate_of_ul_query_pixel, x_pixel_size, y_pixel_size); + let out_pixel_bounds = query_rect.spatial_query().grid_bounds(); - let uncompressed_byte_size = width * height * std::mem::size_of::(); + let uncompressed_byte_size = + query_rect.spatial_query.grid_bounds().number_of_elements() * std::mem::size_of::(); let use_big_tiff = gdal_tiff_options.force_big_tiff || uncompressed_byte_size >= BIG_TIFF_BYTE_THRESHOLD; @@ -183,8 +181,8 @@ where let mut dataset = driver.create_with_band_type_with_options::( &file_path, - width, - height, + query_rect.spatial_query().grid_bounds().axis_size_x(), + query_rect.spatial_query().grid_bounds().axis_size_y(), num_timesteps, &options, )?; @@ -205,12 +203,10 @@ where let writer = GdalDatasetWriter:: { gdal_tiff_options, gdal_tiff_metadata, - _output_bounds: query_rect.spatial_bounds, + output_pixel_grid_bounds: out_pixel_bounds, output_geo_transform, use_big_tiff, _type: Default::default(), - window_start, - window_end, }; drop(option_vars); @@ -220,7 +216,7 @@ where async fn consume_stream_into_vec( processor: Box>, - query_rect: geoengine_datatypes::primitives::QueryRectangle, + query_rect: geoengine_datatypes::primitives::RasterQueryRectangle, query_ctx: C, tile_limit: Option, ) -> Result>> @@ -253,7 +249,6 @@ pub async fn single_timestep_raster_stream_to_geotiff_bytes, conn_closed: BoxFuture<'_, ()>, - tiling_specification: TilingSpecification, ) -> Result> where T: Pixel + GdalType, @@ -266,7 +261,6 @@ where gdal_tiff_options, tile_limit, conn_closed, - tiling_specification, ) .await?; @@ -291,7 +285,6 @@ pub async fn raster_stream_to_geotiff_bytes( gdal_tiff_options: GdalGeoTiffOptions, tile_limit: Option, conn_closed: BoxFuture<'_, ()>, - tiling_specification: TilingSpecification, ) -> Result>> where T: Pixel + GdalType, @@ -307,7 +300,6 @@ where gdal_tiff_options, tile_limit, conn_closed, - tiling_specification, ) .await? .into_iter() @@ -330,14 +322,13 @@ pub async fn raster_stream_to_geotiff( gdal_tiff_options: GdalGeoTiffOptions, tile_limit: Option, conn_closed: BoxFuture<'_, ()>, - tiling_specification: TilingSpecification, ) -> Result> where P: Pixel + GdalType, { // TODO: support multi band geotiffs ensure!( - query_rect.attributes.count() == 1, + query_rect.attributes.is_single() || processor.result_descriptor().bands.is_single(), crate::error::OperationDoesNotSupportMultiBandQueriesYet { operation: "raster_stream_to_geotiff" } @@ -345,6 +336,12 @@ where let query_abort_trigger = query_ctx.abort_trigger()?; + let tiling_strategy = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(query_ctx.tiling_specification()) + .generate_data_tiling_strategy(); + // TODO: create file path if it doesn't exist let file_path = file_path.to_owned(); @@ -359,14 +356,15 @@ where None }; - let dataset_holder: Result> = Ok(GdalDatasetHolder::new_with_tiling_spec( - tiling_specification, - &file_path, - &query_rect, - gdal_tiff_metadata, - gdal_tiff_options, - gdal_config_options, - )); + let dataset_holder: Result> = + Ok(GdalDatasetHolder::new_with_tiling_strat( + tiling_strategy, + &file_path, + &query_rect, + gdal_tiff_metadata, + gdal_tiff_options, + gdal_config_options, + )); let tile_stream = processor .raster_query(query_rect.clone(), &query_ctx) @@ -498,8 +496,7 @@ impl GdalDatasetHolder

{ gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, gdal_config_options: Option>, - window_start: GridIdx2D, - window_end: GridIdx2D, + tiling_strategy: TilingStrategy, ) -> Self { const INTERMEDIATE_FILE_SUFFIX: &str = "GEO-ENGINE-TMP"; @@ -518,27 +515,35 @@ impl GdalDatasetHolder

{ let file_path = file_path.join("raster.tiff"); let intermediate_file_path = file_path.with_extension(INTERMEDIATE_FILE_SUFFIX); - let x_pixel_size = query_rect.spatial_resolution.x; - let y_pixel_size = query_rect.spatial_resolution.y; - let width = (query_rect.spatial_bounds.size_x() / x_pixel_size).ceil() as u32; - let height = (query_rect.spatial_bounds.size_y() / y_pixel_size).ceil() as u32; + let width = query_rect.spatial_query().grid_bounds().axis_size_x(); + let height = query_rect.spatial_query().grid_bounds().axis_size_y(); + + let output_pixel_grid_bounds = query_rect.spatial_query().grid_bounds(); + + let out_geo_transform_origin = tiling_strategy + .geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d( + query_rect.spatial_query().grid_bounds().min_index(), + ); let output_geo_transform = GeoTransform::new( - query_rect.spatial_bounds.upper_left(), - x_pixel_size, - -y_pixel_size, + out_geo_transform_origin, + tiling_strategy.geo_transform.x_pixel_size(), + tiling_strategy.geo_transform.y_pixel_size(), ); + let output_gdal_geo_transform = GdalDatasetGeoTransform { + origin_coordinate: out_geo_transform_origin, + x_pixel_size: tiling_strategy.geo_transform.x_pixel_size(), + y_pixel_size: tiling_strategy.geo_transform.y_pixel_size(), + }; + let intermediate_dataset_parameters = GdalDatasetParameters { file_path: intermediate_file_path, rasterband_channel: 1, - geo_transform: GdalDatasetGeoTransform { - origin_coordinate: query_rect.spatial_bounds.upper_left(), - x_pixel_size, - y_pixel_size: -y_pixel_size, - }, - width: width as usize, - height: height as usize, + geo_transform: output_gdal_geo_transform, + width, + height, file_not_found_handling: FileNotFoundHandling::Error, no_data_value: None, // `None` will let the GdalSource detect the correct no-data value. properties_mapping: None, // TODO: add properties @@ -565,8 +570,8 @@ impl GdalDatasetHolder

{ intermediate_dataset: None, create_meta: IntermediateDatasetMetadata { raster_band_index: rasterband_index, - width, - height, + width: width as u32, + height: height as u32, use_big_tiff, path_with_placeholder, gdal_config_options, @@ -575,12 +580,10 @@ impl GdalDatasetHolder

{ dataset_writer: GdalDatasetWriter { gdal_tiff_options, gdal_tiff_metadata, - _output_bounds: query_rect.spatial_bounds, + output_pixel_grid_bounds, output_geo_transform, use_big_tiff, _type: Default::default(), - window_start, - window_end, }, result: vec![], } @@ -657,37 +660,21 @@ impl GdalDatasetHolder

{ Ok(()) } - fn new_with_tiling_spec( - tiling_specification: TilingSpecification, + fn new_with_tiling_strat( + tiling_strategy: TilingStrategy, file_path: &Path, query_rect: &RasterQueryRectangle, gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, gdal_config_options: Option>, ) -> Self { - let x_pixel_size = query_rect.spatial_resolution.x; - let y_pixel_size = query_rect.spatial_resolution.y; - - let width = (query_rect.spatial_bounds.size_x() / x_pixel_size).ceil() as u32; - let height = (query_rect.spatial_bounds.size_y() / y_pixel_size).ceil() as u32; - - let global_geo_transform = tiling_specification - .strategy(x_pixel_size, -y_pixel_size) - .geo_transform; - - let window_start = - global_geo_transform.coordinate_to_grid_idx_2d(query_rect.spatial_bounds.upper_left()); - - let window_end = window_start + GridIdx2D::from([height as isize, width as isize]); - Self::new( file_path, query_rect, gdal_tiff_metadata, gdal_tiff_options, gdal_config_options, - window_start, - window_end, + tiling_strategy, ) } @@ -738,82 +725,43 @@ pub struct GdalGeoTiffDatasetMetadata { struct GdalDatasetWriter { gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, - _output_bounds: SpatialPartition2D, // currently unused due to workaround for intersection and contained because of float precision + output_pixel_grid_bounds: GridBoundingBox2D, output_geo_transform: GeoTransform, use_big_tiff: bool, _type: std::marker::PhantomData

, - window_start: GridIdx2D, - window_end: GridIdx2D, } impl GdalDatasetWriter

{ fn write_tile_into_band(&self, tile: RasterTile2D

, raster_band: RasterBand) -> Result<()> { let tile_info = tile.tile_information(); - let tile_start = tile_info.global_upper_left_pixel_idx(); - let [tile_height, tile_width] = tile_info.tile_size_in_pixels.shape_array; - let tile_end = tile_start + GridIdx2D::from([tile_height as isize, tile_width as isize]); - - let GridIdx([tile_start_y, tile_start_x]) = tile_start; - let GridIdx([tile_end_y, tile_end_x]) = tile_end; - let GridIdx([window_start_y, window_start_x]) = self.window_start; - let GridIdx([window_end_y, window_end_x]) = self.window_end; - - // compute the upper left pixel index in the output raster and extract the input data - let (GridIdx([output_ul_y, output_ul_x]), grid_array) = - // TODO: check contains on the `SpatialPartition2D`s once the float precision issue is fixed - if tile_start_x >= window_start_x && tile_start_y >= window_start_y && tile_end_x <= window_end_x && tile_end_y <= window_end_y { - // tile is completely inside the output raster - ( - tile_info.global_upper_left_pixel_idx() - self.window_start, - tile.into_materialized_tile().grid_array, - ) - } else { - // extract relevant data from tile (intersection with output_bounds) - - // TODO: compute the intersection on the `SpatialPartition2D`s once the float precision issue is fixed - - if tile_end_y < window_start_y - || tile_end_x < window_start_x - || tile_start_y >= window_end_y - || tile_start_x >= window_end_x - { - // tile is outside of output bounds - return Ok(()); - } + let tile_grid_bounds = tile_info.global_pixel_bounds(); - let intersection_start = GridIdx2D::from([ - std::cmp::max(tile_start_y, window_start_y), - std::cmp::max(tile_start_x, window_start_x), - ]); - let GridIdx([intersection_start_y, intersection_start_x]) = intersection_start; + let out_data_bounds = self + .output_pixel_grid_bounds + .intersection(&tile_grid_bounds); - let width = std::cmp::min( - tile_info.tile_size_in_pixels.axis_size_x() as isize, - window_end_x - intersection_start_x, - ); + if out_data_bounds.is_none() { + return Ok(()); + } - let height = std::cmp::min( - tile_info.tile_size_in_pixels.axis_size_y() as isize, - window_end_y - intersection_start_y, - ); + let out_data_bounds = out_data_bounds.expect("was checked before"); - let mut output_grid = - MaskedGrid2D::from(EmptyGrid2D::new([height as usize, width as usize].into())); + let mut write_buffer_grid = GridOrEmpty::<_, P>::new_empty_shape(out_data_bounds); - let shift_offset = intersection_start - tile_start; - let shifted_source = tile - .grid_array - .shift_by_offset(GridIdx([-1, -1]) * shift_offset); + write_buffer_grid.grid_blit_from(&tile.into_inner_positioned_grid()); - output_grid.grid_blit_from(&shifted_source); + let window_start = out_data_bounds.min_index() - self.output_pixel_grid_bounds.min_index(); + let window = (window_start.x(), window_start.y()); - (intersection_start - self.window_start, output_grid) - }; + let window_size = ( + write_buffer_grid.shape_ref().axis_size_x(), + write_buffer_grid.shape_ref().axis_size_y(), + ); - let window = (output_ul_x, output_ul_y); - let [shape_y, shape_x] = grid_array.axis_size(); - let window_size = (shape_x, shape_y); + let grid_array = write_buffer_grid + .into_materialized_masked_grid() + .unbounded(); // Check if the gdal_tiff_metadata no-data value is set. // If it is set write a geotiff with no-data values. @@ -905,7 +853,6 @@ fn create_gdal_tiff_options( ) -> Result { let mut options = RasterCreationOptions::new(); options.add_name_value("COMPRESS", COMPRESSION_FORMAT)?; - options.add_name_value("TILED", "YES")?; options.add_name_value("ZLEVEL", COMPRESSION_LEVEL)?; options.add_name_value("NUM_THREADS", compression_num_threads)?; options.add_name_value("INTERLEAVE", "BAND")?; @@ -914,6 +861,8 @@ fn create_gdal_tiff_options( // COGs require a block size of 512x512, so we enforce it now so that we do the work only once. options.add_name_value("BLOCKXSIZE", COG_BLOCK_SIZE)?; options.add_name_value("BLOCKYSIZE", COG_BLOCK_SIZE)?; + } else { + options.add_name_value("TILED", "YES")?; } if as_big_tiff { @@ -992,7 +941,6 @@ fn geotiff_to_cog( let mut options = RasterCreationOptions::new(); options.add_name_value("COMPRESS", COMPRESSION_FORMAT)?; - options.add_name_value("TILED", "YES")?; options.add_name_value("NUM_THREADS", num_threads)?; options.add_name_value("BLOCKSIZE", COG_BLOCK_SIZE)?; @@ -1011,60 +959,51 @@ fn geotiff_to_cog( #[cfg(test)] mod tests { - use super::*; - use crate::engine::RasterResultDescriptor; + use std::marker::PhantomData; + use std::ops::Add; + + use crate::engine::{ChunkByteSize, RasterResultDescriptor}; use crate::mock::MockRasterSourceProcessor; use crate::util::gdal::gdal_open_dataset; use crate::{ engine::MockQueryContext, source::GdalSourceProcessor, util::gdal::create_ndvi_meta_data, }; - use geoengine_datatypes::primitives::CacheHint; - use geoengine_datatypes::primitives::{DateTime, Duration}; - use geoengine_datatypes::raster::{Grid, RasterDataType}; + use geoengine_datatypes::primitives::{ + BandSelection, CacheHint, DateTime, Duration, TimeInterval, + }; + use geoengine_datatypes::raster::{Grid, GridBoundingBox2D, RasterDataType}; use geoengine_datatypes::test_data; + use geoengine_datatypes::util::test::TestDefault; use geoengine_datatypes::util::{ ImageFormat, assert_image_equals, assert_image_equals_with_format, }; - use geoengine_datatypes::{ - primitives::{Coordinate2D, SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::TilingSpecification, - util::test::TestDefault, - }; - use std::marker::PhantomData; - use std::ops::Add; + + use super::*; #[tokio::test] async fn geotiff_with_no_data_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); - + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1077,7 +1016,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, ) .await .unwrap(); @@ -1087,44 +1025,39 @@ mod tests { // "../test_data/raster/geotiff_from_stream_compressed.tiff", // ); - assert_image_equals( - test_data!("raster/geotiff_from_stream_compressed.tiff"), - &bytes, + // FIXME: this will fail since we no longer use scaling sources which causes this to write a subset of the data not a scaled version of all data + assert_eq!( + include_bytes!("../../../test_data/raster/geotiff_from_stream_compressed.tiff") + as &[u8], + bytes.as_slice() ); } #[tokio::test] async fn geotiff_with_mask_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: None, @@ -1137,7 +1070,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, ) .await .unwrap(); @@ -1150,36 +1082,29 @@ mod tests { #[tokio::test] async fn geotiff_big_tiff_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1192,7 +1117,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, ) .await .unwrap(); @@ -1211,36 +1135,29 @@ mod tests { #[tokio::test] async fn cloud_optimized_geotiff_big_tiff_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1253,7 +1170,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, ) .await .unwrap(); @@ -1274,36 +1190,29 @@ mod tests { #[tokio::test] async fn cloud_optimized_geotiff_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1316,7 +1225,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, ) .await .unwrap(); @@ -1336,39 +1244,29 @@ mod tests { #[tokio::test] async fn cloud_optimized_geotiff_multiple_timesteps_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let mut bytes = raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new( - 1_388_534_400_000, - 1_388_534_400_000 + 7_776_000_000, - ) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 7_776_000_000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1381,7 +1279,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, ) .await .unwrap(); @@ -1419,39 +1316,29 @@ mod tests { #[tokio::test] async fn cloud_optimized_geotiff_multiple_timesteps_from_stream_wrong_request() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new( - 1_388_534_400_000, - 1_388_534_400_000 + 7_776_000_000, - ) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 7_776_000_000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1464,7 +1351,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, ) .await; @@ -1473,36 +1359,29 @@ mod tests { #[tokio::test] async fn geotiff_from_stream_limit() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1515,66 +1394,12 @@ mod tests { }, Some(1), Box::pin(futures::future::pending()), - tiling_specification, ) .await; assert!(bytes.is_err()); } - #[tokio::test] - async fn geotiff_from_stream_in_range_of_window() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); - - let metadata = create_ndvi_meta_data(); - - let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, - meta_data: Box::new(metadata), - _phantom_data: PhantomData, - }; - - let query_bbox = - SpatialPartition2D::new((-180., -66.227_224_576_271_84).into(), (180., -90.).into()) - .unwrap(); - - let bytes = single_timestep_raster_stream_to_geotiff_bytes( - gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - 0.228_716_645_489_199_48, - 0.226_407_384_987_887_26, - ), - attributes: BandSelection::first(), - }, - ctx, - GdalGeoTiffDatasetMetadata { - no_data_value: Some(0.), - spatial_reference: SpatialReference::epsg_4326(), - }, - GdalGeoTiffOptions { - as_cog: false, - compression_num_threads: GdalCompressionNumThreads::AllCpus, - force_big_tiff: false, - }, - None, - Box::pin(futures::future::pending()), - tiling_specification, - ) - .await; - - assert!(bytes.is_ok()); - } - fn generate_time_intervals( start_time: DateTime, time_step: Duration, @@ -1622,27 +1447,32 @@ mod tests { }, ]; + let ctx = MockQueryContext::new( + ChunkByteSize::test_default(), + TilingSpecification::new([600, 600].into()), + ); + + let result_descriptor = RasterResultDescriptor::with_datatype_and_num_bands( + RasterDataType::U8, + 1, + GridBoundingBox2D::new([-4, -4], [4, 4]).unwrap(), + GeoTransform::test_default(), + ); + let query_time = TimeInterval::new(data[0].time.start(), data[1].time.end()).unwrap(); - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); let processor = MockRasterSourceProcessor { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + result_descriptor, data, - tiling_specification, + tiling_specification: ctx.tiling_specification(), } .boxed(); - let query_rectangle = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 2.).into(), (2., 0.).into()).unwrap(), - time_interval: query_time, - spatial_resolution: GeoTransform::test_default().spatial_resolution(), - attributes: BandSelection::first(), - }; + let query_rectangle = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-2, -1], [0, 1]).unwrap(), + query_time, + BandSelection::first(), + ); let file_path = PathBuf::from(format!("/vsimem/{}/", uuid::Uuid::new_v4())); let expected_paths = file_suffixes @@ -1665,7 +1495,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, ) .await .unwrap(); @@ -1740,32 +1569,28 @@ mod tests { #[tokio::test] async fn multi_band_geotriff() { let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [512, 512].into()); let metadata = create_ndvi_meta_data(); + let tiling_specification = TilingSpecification::new([512, 512].into()); + let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + produced_result_descriptor: metadata.result_descriptor.clone(), tiling_specification, + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); - let (mut bytes, _) = raster_stream_to_multiband_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(0, 1799, 0, 3599).unwrap(), // 1.1.2014 - 1.4.2014 - time_interval: TimeInterval::new(1_388_534_400_000, 1_396_306_800_000).unwrap(), - spatial_resolution: SpatialResolution::new_unchecked(0.1, 0.1), - attributes: BandSelection::first(), - }, + TimeInterval::new(1_388_534_400_000, 1_396_306_800_000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: None, diff --git a/operators/src/util/raster_stream_to_png.rs b/operators/src/util/raster_stream_to_png.rs index 1614af499..65f63822b 100644 --- a/operators/src/util/raster_stream_to_png.rs +++ b/operators/src/util/raster_stream_to_png.rs @@ -4,12 +4,12 @@ use crate::util::Result; use futures::TryStreamExt; use futures::{StreamExt, future::BoxFuture}; use geoengine_datatypes::error::{BoxedResultExt, ErrorSource}; -use geoengine_datatypes::operations::image::{ColorMapper, RgbParams}; -use geoengine_datatypes::raster::{FromIndexFn, GridIndexAccess, GridShapeAccess}; +use geoengine_datatypes::operations::image::{ColorMapper, RasterColorizer, RgbParams}; +use geoengine_datatypes::raster::{FromIndexFn, GridIndexAccess, GridShapeAccess, RasterTile2D}; use geoengine_datatypes::{ - operations::image::{Colorizer, RasterColorizer, RgbaColor, ToPng}, - primitives::{AxisAlignedRectangle, CacheHint, RasterQueryRectangle, TimeInterval}, - raster::{Blit, ConvertDataType, EmptyGrid2D, GeoTransform, GridOrEmpty, Pixel, RasterTile2D}, + operations::image::{Colorizer, RgbaColor, ToPng}, + primitives::{CacheHint, RasterQueryRectangle, TimeInterval}, + raster::{ChangeGridBounds, GridBlit, GridBoundingBox2D, GridOrEmpty, Pixel}, }; use num_traits::AsPrimitive; use snafu::Snafu; @@ -25,7 +25,7 @@ pub async fn raster_stream_to_png_bytes( mut query_ctx: C, width: u32, height: u32, - time: Option, + _time: Option, raster_colorizer: Option, conn_closed: BoxFuture<'_, ()>, ) -> Result<(Vec, CacheHint)> { @@ -80,31 +80,12 @@ pub async fn raster_stream_to_png_bytes( let query_abort_trigger = query_ctx.abort_trigger()?; - let x_query_resolution = query_rect.spatial_bounds.size_x() / f64::from(width); - let y_query_resolution = query_rect.spatial_bounds.size_y() / f64::from(height); - - // build png - let dim = [height as usize, width as usize]; - let query_geo_transform = GeoTransform::new( - query_rect.spatial_bounds.upper_left(), - x_query_resolution, - -y_query_resolution, // TODO: negative, s.t. geo transform fits... - ); - - let tile_template: RasterTile2D = RasterTile2D::new_without_offset( - time.unwrap_or_default(), - query_geo_transform, - GridOrEmpty::from(EmptyGrid2D::new(dim.into())), - CacheHint::max_duration(), - ); - match raster_colorizer { RasterColorizer::SingleBand { band_colorizer, .. } => { single_band_colorizer_to_png_bytes( processor, query_rect, query_ctx, - tile_template, width, height, band_colorizer, @@ -121,7 +102,6 @@ pub async fn raster_stream_to_png_bytes( processor, query_rect, query_ctx, - tile_template, width, height, rgba_params, @@ -139,7 +119,6 @@ async fn single_band_colorizer_to_png_bytes processor: Box>, query_rect: RasterQueryRectangle, query_ctx: C, - tile_template: RasterTile2D, width: u32, height: u32, colorizer: Colorizer, @@ -148,19 +127,30 @@ async fn single_band_colorizer_to_png_bytes ) -> Result<(Vec, CacheHint)> { debug_assert_eq!(query_rect.attributes.count(), 1); + // the tile stream will allways produce tiles aligned to the tiling origin let tile_stream = processor.query(query_rect.clone(), &query_ctx).await?; + let output_cache_hint = CacheHint::max_duration(); - let output_tile = Box::pin( - tile_stream.fold(Ok(tile_template), |raster, tile| async move { - blit_tile(raster, tile) - }), + let output_grid = GridOrEmpty::::new_empty_shape( + query_rect.spatial_query.grid_bounds(), ); - let result = abortable_query_execution(output_tile, conn_closed, query_abort_trigger).await?; - Ok(( - result.grid_array.to_png(width, height, &colorizer)?, - result.cache_hint, - )) + let accu = Ok((output_grid, output_cache_hint)); + + let output_tile: BoxFuture, CacheHint)>> = + Box::pin(tile_stream.fold(accu, |accu, tile| { + let result: Result<(GridOrEmpty, CacheHint)> = + blit_tile(accu, tile); + + match result { + Ok(updated_raster2d) => futures::future::ok(updated_raster2d), + Err(error) => futures::future::err(error), + } + })); + + let (result, ch) = + abortable_query_execution(output_tile, conn_closed, query_abort_trigger).await?; + Ok((result.unbounded().to_png(width, height, &colorizer)?, ch)) } #[allow(clippy::too_many_arguments)] @@ -168,7 +158,6 @@ async fn multi_band_colorizer_to_png_bytes( processor: Box>, query_rect: RasterQueryRectangle, query_ctx: C, - tile_template: RasterTile2D, width: u32, height: u32, rgb_params: RgbParams, @@ -178,15 +167,18 @@ async fn multi_band_colorizer_to_png_bytes( ) -> Result<(Vec, CacheHint)> { let rgb_channel_count = query_rect.attributes.count() as usize; let no_data_color = rgb_params.no_data_color; - let tile_template: RasterTile2D = tile_template.convert_data_type(); + let tile_template: GridOrEmpty = + GridOrEmpty::new_empty_shape(query_rect.spatial_query.grid_bounds()); + let output_cache_hint = CacheHint::max_duration(); let red_band_index = band_positions[0]; let green_band_index = band_positions[1]; let blue_band_index = band_positions[2]; let tile_stream = processor.query(query_rect.clone(), &query_ctx).await?; + let accu = Ok((tile_template, output_cache_hint)); let output_tile = Box::pin(tile_stream.try_chunks(rgb_channel_count).fold( - Ok(tile_template), + accu, |raster2d, chunk| async move { let chunk = chunk.boxed_context(error::QueryDidNotProduceNextChunk)?; @@ -212,31 +204,30 @@ async fn multi_band_colorizer_to_png_bytes( }, )); - let result = abortable_query_execution(output_tile, conn_closed, query_abort_trigger).await?; + let (result, ch) = + abortable_query_execution(output_tile, conn_closed, query_abort_trigger).await?; Ok(( result - .grid_array + .unbounded() .to_png_with_mapper(width, height, ColorMapper::Rgba, no_data_color)?, - result.cache_hint, + ch, )) } fn blit_tile( - raster2d: Result>, + accu: Result<(GridOrEmpty, CacheHint)>, tile: Result>, -) -> Result> +) -> Result<(GridOrEmpty, CacheHint)> where T: Pixel, { - let result: Result> = match (raster2d, tile) { - (Ok(mut raster2d), Ok(tile)) if tile.is_empty() => { - raster2d.cache_hint.merge_with(&tile.cache_hint); - Ok(raster2d) + let result: Result<(GridOrEmpty, CacheHint)> = match (accu, tile) { + (Ok((empty_grid, ch)), Ok(tile)) if tile.is_empty() => Ok((empty_grid, ch)), + (Ok((mut grid, mut ch)), Ok(tile)) => { + ch.merge_with(&tile.cache_hint); + grid.grid_blit_from(&tile.into_inner_positioned_grid()); + Ok((grid, ch)) } - (Ok(mut raster2d), Ok(tile)) => match raster2d.blit(tile) { - Ok(()) => Ok(raster2d), - Err(error) => Err(error.into()), - }, (Err(error), _) | (_, Err(error)) => Err(error), }; @@ -351,19 +342,17 @@ pub enum PngCreationError { mod tests { use std::marker::PhantomData; + use crate::{ + engine::MockQueryContext, source::GdalSourceProcessor, util::gdal::create_ndvi_meta_data, + }; + use geoengine_datatypes::primitives::{DateTime, TimeInstance}; use geoengine_datatypes::{ - primitives::{BandSelection, Coordinate2D, SpatialPartition2D, SpatialResolution}, - raster::{RasterDataType, TilingSpecification}, + primitives::BandSelection, + raster::TilingSpecification, test_data, util::{assert_image_equals, test::TestDefault}, }; - use crate::{ - engine::{MockQueryContext, RasterResultDescriptor}, - source::GdalSourceProcessor, - util::gdal::create_ndvi_meta_data, - }; - use super::*; #[test] @@ -384,31 +373,28 @@ mod tests { #[tokio::test] async fn png_from_stream() { let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let tiling_specification = TilingSpecification::new([600, 600].into()); + + let meta_data = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + produced_result_descriptor: meta_data.result_descriptor.clone(), tiling_specification, - meta_data: Box::new(create_ndvi_meta_data()), + overview_level: 0, + meta_data: Box::new(meta_data), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_partition = - SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); + let query = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([-800, -100], [-199, 499]).unwrap(), + TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + ); let (image_bytes, _) = raster_stream_to_png_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_partition, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + query, ctx, 600, 600, diff --git a/operators/src/util/test.rs b/operators/src/util/test.rs new file mode 100644 index 000000000..4b7071092 --- /dev/null +++ b/operators/src/util/test.rs @@ -0,0 +1,89 @@ +use super::Result; +use crate::engine::{ExecutionContext, QueryContext, RasterOperator, WorkflowOperatorPath}; +use futures::StreamExt; +use geoengine_datatypes::{ + primitives::RasterQueryRectangle, raster::RasterTile2D, + util::test::assert_eq_two_list_of_tiles_u8, +}; + +pub async fn raster_operator_to_list_of_tiles_u8( + exe_ctx: &E, + query_ctx: &Q, + operator: Box, + query_rectangle: RasterQueryRectangle, +) -> Result>> { + let initialized_operator = operator + .initialize(WorkflowOperatorPath::initialize_root(), exe_ctx) + .await?; + let query_processor = initialized_operator.query_processor()?.get_u8().ok_or( + crate::error::Error::MustNotHappen { + message: "Operator does not produce u8 while this function requires it".to_owned(), + }, + )?; + + let res = query_processor + .raster_query(query_rectangle, query_ctx) + .await? + .collect::>() + .await; + + let res = res.into_iter().collect::, _>>()?; + + Ok(res) +} + +/// Compares the output of a raster operators and a list of tiles and panics with a message if they are not equal +/// +/// # Panics +/// +/// If there are tiles that are not equal +pub async fn assert_eq_raster_operator_res_and_list_of_tiles_u8< + E: ExecutionContext, + Q: QueryContext, +>( + exe_ctx: &E, + query_ctx: &Q, + operator: Box, + query_rectangle: RasterQueryRectangle, + compare_cache_hint: bool, + list_of_tiles: &[RasterTile2D], +) { + let res_a = raster_operator_to_list_of_tiles_u8(exe_ctx, query_ctx, operator, query_rectangle) + .await + .expect("raster operator to list failed!"); + + assert_eq_two_list_of_tiles_u8(&res_a, list_of_tiles, compare_cache_hint); +} + +/// Compares the output of two raster operators and panics with a message if they are not equal +/// +/// # Panics +/// +/// If there are tiles that are not equal +pub async fn assert_eq_two_raster_operator_res_u8( + exe_ctx: &E, + query_ctx: &Q, + operator_a: Box, + operator_b: Box, + query_rectangle: RasterQueryRectangle, + compare_cache_hint: bool, +) { + let res_a = raster_operator_to_list_of_tiles_u8( + exe_ctx, + query_ctx, + operator_a, + query_rectangle.clone(), + ) + .await + .expect("raster operator to list failed for operator_a!"); + + assert_eq_raster_operator_res_and_list_of_tiles_u8( + exe_ctx, + query_ctx, + operator_b, + query_rectangle, + compare_cache_hint, + &res_a, + ) + .await; +} diff --git a/operators/src/util/wrap_with_projection_and_resample.rs b/operators/src/util/wrap_with_projection_and_resample.rs new file mode 100644 index 000000000..4704bcfd8 --- /dev/null +++ b/operators/src/util/wrap_with_projection_and_resample.rs @@ -0,0 +1,222 @@ +use crate::engine::{ + CanonicOperatorName, InitializedRasterOperator, RasterOperator, RasterResultDescriptor, + ResultDescriptor, SingleRasterOrVectorSource, WorkflowOperatorPath, +}; +use crate::error; +use crate::processing::{ + DeriveOutRasterSpecsSource, Downsampling, DownsamplingMethod, DownsamplingParams, + DownsamplingResolution, InitializedDownsampling, InitializedInterpolation, + InitializedRasterReprojection, Interpolation, InterpolationMethod, InterpolationParams, + InterpolationResolution, Reprojection, ReprojectionParams, +}; +use crate::util::Result; +use crate::util::input::RasterOrVectorOperator; +use geoengine_datatypes::primitives::{Coordinate2D, SpatialResolution}; +use geoengine_datatypes::raster::TilingSpecification; +use geoengine_datatypes::spatial_reference::SpatialReference; + +pub struct WrapWithProjectionAndResample { + pub operator: Box, + pub initialized_operator: Box, + pub result_descriptor: RasterResultDescriptor, +} + +impl WrapWithProjectionAndResample { + pub fn new_create_result_descriptor( + operator: Box, + initialized: Box, + ) -> Self { + let result_descriptor = initialized.result_descriptor().clone(); + Self::new(operator, initialized, result_descriptor) + } + + pub fn new( + operator: Box, + initialized: Box, + result_descriptor: RasterResultDescriptor, + ) -> Self { + Self { + operator, + initialized_operator: initialized, + result_descriptor, + } + } + + pub fn wrap_with_projection( + self, + target_sref: SpatialReference, + _target_origin_reference: Option, // TODO: add resampling if origin does not match! Could also do that in projection and avoid extra operation? + tiling_spec: TilingSpecification, + ) -> Result { + let result_sref = self + .result_descriptor + .spatial_reference() + .as_option() + .ok_or(error::Error::SpatialReferenceMustNotBeUnreferenced)?; + + // perform reprojection if necessary + let res = if target_sref == result_sref { + self + } else { + log::debug!( + "Target srs: {target_sref}, workflow srs: {result_sref} --> injecting reprojection" + ); + + let reprojection_params = ReprojectionParams { + target_spatial_reference: target_sref, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, + }; + + // create the reprojection operator in order to get the canonic operator name + let reprojected_workflow = Reprojection { + params: reprojection_params, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster(self.operator), + }, + }; + + // create the inititalized operator directly, to avoid re-initializing everything + let irp = InitializedRasterReprojection::try_new_with_input( + CanonicOperatorName::from(&reprojected_workflow), + WorkflowOperatorPath::initialize_root(), // FIXME: this is not correct since the root is the child operator + reprojection_params, + self.initialized_operator, + tiling_spec, + )?; + let rd = irp.result_descriptor().clone(); + + Self::new(reprojected_workflow.boxed(), irp.boxed(), rd) + }; + Ok(res) + } + + pub fn wrap_with_resample( + self, + target_origin_reference: Option, + target_spatial_resolution: Option, + tiling_spec: TilingSpecification, + ) -> Result { + if target_origin_reference.is_none() && target_spatial_resolution.is_none() { + return Ok(self); + } + + let rd_resolution = self + .result_descriptor + .spatial_grid_descriptor() + .spatial_resolution(); + + let target_spatial_grid = if let Some(tsr) = target_spatial_resolution { + self.result_descriptor + .spatial_grid_descriptor() + .with_changed_resolution(tsr) + } else { + *self.result_descriptor.spatial_grid_descriptor() + }; + + let target_spatial_grid = if let Some(tor) = target_origin_reference { + // if the request is to move the origin of the query to a different point, we generate a new grid aligned to that point. + target_spatial_grid + .with_moved_origin_to_nearest_grid_edge(tor) + .as_derived() + .replace_origin(tor) + } else { + target_spatial_grid + }; + + let res = if self + .result_descriptor + .spatial_grid_descriptor() + .is_compatible_grid(&target_spatial_grid) + { + // TODO: resample if origin is not allgned to query? (maybe n + self + } + // Query resolution is smaller then workdlow + else if target_spatial_grid.spatial_resolution().x <= rd_resolution.x + && target_spatial_grid.spatial_resolution().y <= rd_resolution.y + //TODO: we should allow to use the "interpolation" as long as the fraction is > 0.5. This would require to keep 4 tiles which seems to be fine. The edge case of resampling with same resolution should also use the interpolation since bilieaner woudl make sense here? + { + log::debug!( + "Target res: {target_spatial_resolution:?}, workflow res: {rd_resolution:?} --> injecting interpolation" + ); + /* + let interpolation_method = if self + .result_descriptor + .bands + .bands() + .iter() + .all(|b| b.measurement.is_continuous()) + { + InterpolationMethod::BiLinear + } else { + InterpolationMethod::NearestNeighbor + }; + */ + + let interpolation_params = InterpolationParams { + interpolation: InterpolationMethod::NearestNeighbor, + output_resolution: InterpolationResolution::Resolution( + target_spatial_grid.spatial_resolution(), + ), + output_origin_reference: None, + }; + + let iop = Interpolation { + params: interpolation_params.clone(), + sources: self.operator.into(), + }; + + let iip = InitializedInterpolation::new_with_source_and_params( + CanonicOperatorName::from(&iop), + WorkflowOperatorPath::initialize_root(), // FIXME: this is not correct since the root is the child operator + self.initialized_operator, + &interpolation_params, + tiling_spec, + )?; + let rd = iip.result_descriptor().clone(); + Self::new(iop.boxed(), iip.boxed(), rd) + } else { + log::debug!( + "Query res: {target_spatial_resolution:?}, workflow res: {rd_resolution:?} --> injecting downsampling" + ); + + let downsample_params = DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_resolution: DownsamplingResolution::Resolution( + target_spatial_grid.spatial_resolution(), + ), + output_origin_reference: None, + }; + let dop = Downsampling { + params: downsample_params, + sources: self.operator.into(), + }; + + let ido = InitializedDownsampling::new_with_source_and_params( + CanonicOperatorName::from(&dop), + WorkflowOperatorPath::initialize_root(), // FIXME: this is not correct since the root is the child operator + self.initialized_operator, + downsample_params, + tiling_spec, + )?; + let rd = ido.result_descriptor().clone(); + Self::new(dop.boxed(), ido.boxed(), rd) + }; + Ok(res) + } + + pub fn wrap_with_projection_and_resample( + self, + target_origin_reference: Option, + target_spatial_resolution: Option, + target_sref: SpatialReference, + tiling_spec: TilingSpecification, + ) -> Result { + self.wrap_with_projection(target_sref, target_origin_reference, tiling_spec)? + .wrap_with_resample( + target_origin_reference, + target_spatial_resolution, + tiling_spec, + ) + } +} diff --git a/services/benches/quota_check.rs b/services/benches/quota_check.rs index f83b7f9a0..4aabc7b22 100644 --- a/services/benches/quota_check.rs +++ b/services/benches/quota_check.rs @@ -56,7 +56,7 @@ async fn bench() { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { data: dataset }, + params: GdalSourceParameters::new(dataset), } .boxed(), }, diff --git a/services/ne2_rgb_colorizer.png b/services/ne2_rgb_colorizer.png new file mode 100644 index 000000000..4ccfe01c4 Binary files /dev/null and b/services/ne2_rgb_colorizer.png differ diff --git a/services/src/api/apidoc.rs b/services/src/api/apidoc.rs index 3d0cf1e66..77c9974ce 100644 --- a/services/src/api/apidoc.rs +++ b/services/src/api/apidoc.rs @@ -3,24 +3,16 @@ use crate::api::handlers::datasets::VolumeFileLayersResponse; use crate::api::handlers::permissions::{ PermissionListOptions, PermissionListing, PermissionRequest, Resource, }; -use crate::api::handlers::plots::WrappedPlotOutput; -use crate::api::handlers::spatial_references::{AxisOrder, SpatialReferenceSpecification}; -use crate::api::handlers::tasks::{TaskAbortOptions, TaskResponse}; -use crate::api::handlers::upload::{UploadFileLayersResponse, UploadFilesResponse}; -use crate::api::handlers::users::AddRole; -use crate::api::handlers::users::{Quota, UpdateQuota, UsageSummaryGranularity}; -use crate::api::handlers::wfs::{CollectionType, GeoJson}; -use crate::api::handlers::workflows::{ProvenanceEntry, RasterStreamWebsocketResultType}; use crate::api::model::datatypes::{ AxisLabels, BandSelection, BoundingBox2D, Breakpoint, CacheTtlSeconds, ClassificationMeasurement, Colorizer, ContinuousMeasurement, Coordinate2D, DataId, DataProviderId, DatasetId, DateTimeParseFormat, DateTimeString, ExternalDataId, FeatureDataType, GdalConfigOption, LayerId, LinearGradient, LogarithmicGradient, Measurement, MultiLineString, MultiPoint, MultiPolygon, NamedData, NoGeometry, Palette, PlotOutputFormat, - PlotQueryRectangle, RasterColorizer, RasterDataType, RasterPropertiesEntryType, - RasterPropertiesKey, RasterQueryRectangle, RgbaColor, SpatialPartition2D, - SpatialReferenceAuthority, SpatialResolution, StringPair, TimeGranularity, TimeInstance, - TimeInterval, TimeStep, VectorDataType, VectorQueryRectangle, + RasterColorizer, RasterDataType, RasterPropertiesEntryType, RasterPropertiesKey, + RasterToDatasetQueryRectangle, RgbaColor, SpatialPartition2D, SpatialReferenceAuthority, + SpatialResolution, StringPair, TimeGranularity, TimeInstance, TimeInterval, TimeStep, + VectorDataType, }; use crate::api::model::operators::{ CsvHeader, FileNotFoundHandling, FormatSpecifics, GdalDatasetGeoTransform, @@ -40,13 +32,28 @@ use crate::api::model::responses::{ ZipResponse, }; use crate::api::model::services::{ - AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, MetaDataSuggestion, - Provenance, ProvenanceOutput, Provenances, UpdateDataset, Volume, + AddDataset, CreateDataset, DataPath, Dataset, DatasetDefinition, MetaDataDefinition, + MetaDataSuggestion, Provenance, ProvenanceOutput, Provenances, UpdateDataset, Volume, }; use crate::api::ogc::{util::OgcBoundingBox, wcs, wfs, wms}; +use crate::api::{ + handlers::{ + plots::WrappedPlotOutput, + spatial_references::{AxisOrder, SpatialReferenceSpecification}, + tasks::{TaskAbortOptions, TaskResponse}, + upload::{UploadFileLayersResponse, UploadFilesResponse}, + users::{AddRole, Quota, UpdateQuota, UsageSummaryGranularity}, + wfs::{CollectionType, GeoJson}, + workflows::{ProvenanceEntry, RasterStreamWebsocketResultType}, + }, + model::{ + datatypes::{GeoTransform, GridBoundingBox2D, GridIdx2D, SpatialGridDefinition}, + operators::{SpatialGridDescriptor, SpatialGridDescriptorState}, + }, +}; use crate::contexts::SessionId; use crate::datasets::listing::{DatasetListing, OrderBy}; -use crate::datasets::storage::{AutoCreateDataset, Dataset, SuggestMetaData}; +use crate::datasets::storage::{AutoCreateDataset, SuggestMetaData}; use crate::datasets::upload::{UploadId, VolumeName}; use crate::datasets::{DatasetName, RasterDatasetFromWorkflow, RasterDatasetFromWorkflowResult}; use crate::layers::layer::{ @@ -261,9 +268,7 @@ use utoipa::{Modify, OpenApi}; VectorColumnInfo, RasterDatasetFromWorkflow, RasterDatasetFromWorkflowResult, - RasterQueryRectangle, - VectorQueryRectangle, - PlotQueryRectangle, + RasterToDatasetQueryRectangle, BandSelection, TaskAbortOptions, @@ -408,6 +413,13 @@ use utoipa::{Modify, OpenApi}; RasterStreamWebsocketResultType, CacheTtlSeconds, + SpatialGridDefinition, + SpatialGridDescriptorState, + SpatialGridDescriptor, + GridBoundingBox2D, + GridIdx2D, + GeoTransform, + PermissionRequest, Resource, Permission, diff --git a/services/src/api/handlers/datasets.rs b/services/src/api/handlers/datasets.rs index 3f57aa361..489269519 100755 --- a/services/src/api/handlers/datasets.rs +++ b/services/src/api/handlers/datasets.rs @@ -6,7 +6,7 @@ use crate::{ datasets::{DatasetNameResponse, errors::*}, }, services::{ - AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, + AddDataset, CreateDataset, DataPath, Dataset, DatasetDefinition, MetaDataDefinition, MetaDataSuggestion, Provenances, UpdateDataset, Volume, }, }, @@ -15,7 +15,7 @@ use crate::{ datasets::{ DatasetName, listing::{DatasetListOptions, DatasetListing, DatasetProvider}, - storage::{AutoCreateDataset, Dataset, DatasetStore, SuggestMetaData}, + storage::{AutoCreateDataset, DatasetStore, SuggestMetaData}, upload::{AdjustFilePath, Upload, UploadDb, UploadId, UploadRootPath, VolumeName, Volumes}, }, error::{self, Error, Result}, @@ -242,6 +242,8 @@ pub async fn get_dataset_handler( .await .context(CannotLoadDataset)?; + let dataset: Dataset = dataset.into(); + Ok(web::Json(dataset)) } @@ -1369,47 +1371,52 @@ async fn create_system_dataset( #[cfg(test)] mod tests { + use super::*; - use crate::api::model::datatypes::NamedData; - use crate::api::model::responses::IdResponse; - use crate::api::model::responses::datasets::DatasetNameResponse; - use crate::api::model::services::{DatasetDefinition, Provenance}; - use crate::contexts::PostgresContext; - use crate::contexts::{Session, SessionId}; - use crate::datasets::DatasetIdAndName; - use crate::datasets::storage::DatasetStore; - use crate::datasets::upload::{UploadId, VolumeName}; - use crate::error::Result; - use crate::ge_context; - use crate::projects::{PointSymbology, RasterSymbology, Symbology}; - use crate::test_data; - use crate::users::UserAuth; - use crate::util::tests::admin_login; - use crate::util::tests::{ - MockQueryContext, SetMultipartBody, TestDataUploads, add_file_definition_to_datasets, - read_body_json, read_body_string, send_test_request, + use crate::{ + api::model::{ + datatypes::NamedData, + responses::{IdResponse, datasets::DatasetNameResponse}, + services::{DatasetDefinition, Provenance}, + }, + contexts::{PostgresContext, Session, SessionId}, + datasets::{ + DatasetIdAndName, + storage::DatasetStore, + upload::{UploadId, VolumeName}, + }, + error::Result, + ge_context, + projects::{PointSymbology, RasterSymbology, Symbology}, + test_data, + users::UserAuth, + util::tests::{ + MockQueryContext, SetMultipartBody, TestDataUploads, add_file_definition_to_datasets, + admin_login, read_body_json, read_body_string, send_test_request, + }, }; use actix_web; use actix_web::http::header; use actix_web_httpauth::headers::authorization::Bearer; use futures::TryStreamExt; - use geoengine_datatypes::collections::{ - GeometryCollection, MultiPointCollection, VectorDataType, + use geoengine_datatypes::{ + collections::{GeometryCollection, MultiPointCollection, VectorDataType}, + operations::image::{RasterColorizer, RgbaColor}, + primitives::{BoundingBox2D, ColumnSelection, SpatialQueryRectangle}, + raster::{GridShape2D, TilingSpecification}, + spatial_reference::SpatialReferenceOption, }; - use geoengine_datatypes::operations::image::{RasterColorizer, RgbaColor}; - use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection, SpatialResolution}; - use geoengine_datatypes::raster::{GridShape2D, TilingSpecification}; - use geoengine_datatypes::spatial_reference::SpatialReferenceOption; - use geoengine_operators::engine::{ - ExecutionContext, InitializedVectorOperator, QueryProcessor, StaticMetaData, - VectorOperator, VectorResultDescriptor, WorkflowOperatorPath, - }; - use geoengine_operators::source::{ - OgrSource, OgrSourceDataset, OgrSourceErrorSpec, OgrSourceParameters, + use geoengine_operators::{ + engine::{ + ExecutionContext, InitializedVectorOperator, QueryProcessor, StaticMetaData, + VectorOperator, VectorResultDescriptor, WorkflowOperatorPath, + }, + source::{OgrSource, OgrSourceDataset, OgrSourceErrorSpec, OgrSourceParameters}, + util::gdal::create_ndvi_meta_data, }; - use geoengine_operators::util::gdal::create_ndvi_meta_data; use serde_json::{Value, json}; use tokio_postgres::NoTls; + use uuid::Uuid; #[ge_context::test] #[allow(clippy::too_many_lines)] @@ -1700,6 +1707,79 @@ mod tests { .map_err(Into::into) } + fn ctx_tiling_spec_600x600() -> TilingSpecification { + TilingSpecification { + tile_size_in_pixels: GridShape2D::new([600, 600]), + } + } + + #[ge_context::test(tiling_spec = "ctx_tiling_spec_600x600")] + async fn create_dataset(app_ctx: PostgresContext) -> Result<()> { + let mut test_data = TestDataUploads::default(); // remember created folder and remove them on drop + + let session = app_ctx.create_anonymous_session().await.unwrap(); + let session_id = session.id(); + let session_context = app_ctx.session_context(session); + + let upload_id = upload_ne_10m_ports_files(app_ctx.clone(), session_id).await?; + test_data.uploads.push(upload_id); + + let dataset_name = + construct_dataset_from_upload(app_ctx.clone(), upload_id, session_id).await; + let exe_ctx = session_context.execution_context()?; + + let source = make_ogr_source( + &exe_ctx, + NamedData { + namespace: dataset_name.namespace, + provider: None, + name: dataset_name.name, + }, + ) + .await?; + + let query_processor = source.query_processor()?.multi_point().unwrap(); + let query_ctx = session_context.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let query = query_processor + .query( + VectorQueryRectangle { + spatial_query: SpatialQueryRectangle:: { + spatial_bounds: BoundingBox2D::new( + (1.85, 50.88).into(), + (4.82, 52.95).into(), + )?, + }, + time_interval: Default::default(), + attributes: ColumnSelection::all(), + }, + &query_ctx, + ) + .await?; + + let result: Vec = query.try_collect().await?; + + let coords = result[0].coordinates(); + assert_eq!(coords.len(), 10); + assert_eq!( + coords, + &[ + [2.933_686_69, 51.23].into(), + [3.204_593_64_f64, 51.336_388_89].into(), + [4.651_413_428, 51.805_833_33].into(), + [4.11, 51.95].into(), + [4.386_160_188, 50.886_111_11].into(), + [3.767_373_38, 51.114_444_44].into(), + [4.293_757_362, 51.297_777_78].into(), + [1.850_176_678, 50.965_833_33].into(), + [2.170_906_949, 51.021_666_67].into(), + [4.292_873_969, 51.927_222_22].into(), + ] + ); + + Ok(()) + } + #[ge_context::test] async fn it_creates_system_dataset(app_ctx: PostgresContext) -> Result<()> { let session = app_ctx.create_anonymous_session().await.unwrap(); @@ -2908,13 +2988,12 @@ mod tests { /// override the pixel size since this test was designed for 600 x 600 pixel tiles fn create_dataset_tiling_specification() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape2D::new([600, 600]), } } #[ge_context::test(tiling_spec = "create_dataset_tiling_specification")] - async fn create_dataset(app_ctx: PostgresContext) -> Result<()> { + async fn create_datasets(app_ctx: PostgresContext) -> Result<()> { let mut test_data = TestDataUploads::default(); // remember created folder and remove them on drop let session = app_ctx.create_anonymous_session().await.unwrap(); @@ -2938,12 +3017,11 @@ mod tests { let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::with_bounds( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &query_ctx, ) .await diff --git a/services/src/api/handlers/layers.rs b/services/src/api/handlers/layers.rs index 0043ba9e4..6b615170f 100644 --- a/services/src/api/handlers/layers.rs +++ b/services/src/api/handlers/layers.rs @@ -1,10 +1,16 @@ +use std::sync::Arc; + +use super::tasks::TaskResponse; + use crate::api::model::datatypes::{DataProviderId, LayerId}; use crate::api::model::responses::IdResponse; use crate::config::get_config_element; use crate::contexts::ApplicationContext; -use crate::datasets::{RasterDatasetFromWorkflow, schedule_raster_dataset_from_workflow_task}; -use crate::error::Error::NotImplemented; -use crate::error::{Error, Result}; +use crate::datasets::{ + RasterDatasetFromWorkflowParams, schedule_raster_dataset_from_workflow_task, +}; +use crate::error::Error::{LayerResultDescriptorMissingFields, NotImplemented}; +use crate::error::Result; use crate::layers::layer::{ AddLayer, AddLayerCollection, CollectionItem, Layer, LayerCollection, LayerCollectionListing, ProviderLayerCollectionId, UpdateLayer, UpdateLayerCollection, @@ -19,14 +25,12 @@ use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::WorkflowId; use crate::{contexts::SessionContext, layers::layer::LayerCollectionListOptions}; use actix_web::{FromRequest, HttpResponse, Responder, web}; -use geoengine_datatypes::primitives::{BandSelection, QueryRectangle}; -use geoengine_operators::engine::WorkflowOperatorPath; +use geoengine_datatypes::primitives::{BandSelection, SpatialGridQueryRectangle}; +use geoengine_operators::engine::{ExecutionContext, WorkflowOperatorPath}; + use serde::{Deserialize, Serialize}; -use std::sync::Arc; use utoipa::IntoParams; -use super::tasks::TaskResponse; - pub const ROOT_PROVIDER_ID: DataProviderId = DataProviderId::from_u128(0x1c3b_8042_300b_485c_95b5_0147_d9dc_068d); @@ -799,33 +803,30 @@ async fn layer_to_dataset( let result_descriptor = raster_operator.result_descriptor(); - let qr = QueryRectangle { - spatial_bounds: result_descriptor.bbox.ok_or( - Error::LayerResultDescriptorMissingFields { - field: "bbox".to_string(), - cause: "is None".to_string(), - }, - )?, - time_interval: result_descriptor - .time - .ok_or(Error::LayerResultDescriptorMissingFields { - field: "time".to_string(), - cause: "is None".to_string(), - })?, - spatial_resolution: result_descriptor.resolution.ok_or( - Error::LayerResultDescriptorMissingFields { - field: "spatial_resolution".to_string(), - cause: "is None".to_string(), - }, - )?, - attributes: BandSelection::first(), // TODO: add to API - }; + let sqr = SpatialGridQueryRectangle::new( + result_descriptor + .tiling_grid_definition(execution_context.tiling_specification()) + .tiling_grid_bounds(), + ); + + let qr_time = result_descriptor + .time + .ok_or(LayerResultDescriptorMissingFields { + field: "time".to_string(), + cause: "is None".to_string(), + })?; + + let qr = geoengine_datatypes::primitives::RasterQueryRectangle::new( + sqr, + qr_time, + BandSelection::first_n(result_descriptor.bands.len() as u32), + ); - let from_workflow = RasterDatasetFromWorkflow { + let from_workflow = RasterDatasetFromWorkflowParams { name: None, display_name: layer.name, description: Some(layer.description), - query: qr.into(), + query: qr, as_cog: true, }; @@ -834,8 +835,8 @@ async fn layer_to_dataset( let task_id = schedule_raster_dataset_from_workflow_task( format!("layer {item}"), + raster_operator, workflow_id, - layer.workflow, ctx, from_workflow, compression_num_threads, @@ -1185,48 +1186,46 @@ async fn remove_collection_from_collection( mod tests { use super::*; - use crate::api::model::responses::ErrorResponse; - use crate::config::get_config_element; - use crate::contexts::PostgresContext; - use crate::contexts::SessionId; - use crate::datasets::RasterDatasetFromWorkflowResult; - use crate::ge_context; - use crate::layers::layer::Layer; - use crate::layers::storage::INTERNAL_PROVIDER_ID; - use crate::tasks::util::test::wait_for_task_to_finish; - use crate::tasks::{TaskManager, TaskStatus}; - use crate::users::{UserAuth, UserSession}; - use crate::util::tests::admin_login; - use crate::util::tests::{ - MockQueryContext, TestDataUploads, read_body_string, send_test_request, - }; - use crate::{contexts::Session, workflows::workflow::Workflow}; - use actix_web::dev::ServiceResponse; - use actix_web::{http::header, test}; - use actix_web_httpauth::headers::authorization::Bearer; - use geoengine_datatypes::primitives::{CacheHint, Coordinate2D}; - use geoengine_datatypes::primitives::{ - RasterQueryRectangle, SpatialPartition2D, TimeGranularity, TimeInterval, - }; - use geoengine_datatypes::raster::{ - GeoTransform, Grid, GridShape, RasterDataType, RasterTile2D, TilingSpecification, + use crate::{ + api::model::responses::ErrorResponse, + contexts::{PostgresContext, Session, SessionId}, + datasets::RasterDatasetFromWorkflowResult, + ge_context, + layers::{layer::Layer, storage::INTERNAL_PROVIDER_ID}, + tasks::{TaskManager, TaskStatus, util::test::wait_for_task_to_finish}, + users::{UserAuth, UserSession}, + util::tests::{TestDataUploads, admin_login, read_body_string, send_test_request}, + workflows::workflow::Workflow, }; - use geoengine_datatypes::spatial_reference::SpatialReference; - use geoengine_datatypes::util::test::TestDefault; - use geoengine_operators::engine::{ - ExecutionContext, InitializedRasterOperator, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, SingleRasterOrVectorSource, TypedOperator, + use actix_web::{ + dev::ServiceResponse, + http::header, + test::{TestRequest, read_body_json}, }; - use geoengine_operators::mock::{MockRasterSource, MockRasterSourceParams}; - use geoengine_operators::processing::{TimeShift, TimeShiftParams}; - use geoengine_operators::source::{GdalSource, GdalSourceParameters}; - use geoengine_operators::util::raster_stream_to_geotiff::{ - GdalGeoTiffDatasetMetadata, GdalGeoTiffOptions, raster_stream_to_geotiff_bytes, + use actix_web_httpauth::headers::authorization::Bearer; + use geoengine_datatypes::{ + primitives::{ + CacheHint, Coordinate2D, RasterQueryRectangle, TimeGranularity, TimeInterval, + }, + raster::{ + GeoTransform, Grid, GridBoundingBox2D, GridShape, RasterDataType, RasterTile2D, + TilingSpecification, + }, + spatial_reference::SpatialReference, + util::test::TestDefault, }; use geoengine_operators::{ - engine::VectorOperator, - mock::{MockPointSource, MockPointSourceParams}, + engine::{ + RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + SingleRasterOrVectorSource, SpatialGridDescriptor, TypedOperator, VectorOperator, + }, + mock::{MockPointSource, MockPointSourceParams, MockRasterSource, MockRasterSourceParams}, + processing::{TimeShift, TimeShiftParams}, + source::{GdalSource, GdalSourceParameters}, + util::test::assert_eq_two_raster_operator_res_u8, }; + use uuid::Uuid; + use std::sync::Arc; use tokio_postgres::NoTls; @@ -1239,7 +1238,7 @@ mod tests { let collection_id = ctx.db().get_root_layer_collection_id().await.unwrap(); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!("/layerDb/collections/{collection_id}/layers")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(serde_json::json!({ @@ -1263,7 +1262,7 @@ mod tests { assert!(response.status().is_success(), "{response:?}"); - let result: IdResponse = test::read_body_json(response).await; + let result: IdResponse = read_body_json(response).await; ctx.db() .load_layer(&result.id.clone().into()) @@ -1299,9 +1298,10 @@ mod tests { description: "Layer Description".to_string(), workflow: Workflow { operator: MockPointSource { - params: MockPointSourceParams { - points: vec![(0.0, 0.1).into(), (1.0, 1.1).into()], - }, + params: MockPointSourceParams::new(vec![ + (0.0, 0.1).into(), + (1.0, 1.1).into(), + ]), } .boxed() .into(), @@ -1328,7 +1328,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!( "/layerDb/collections/{collection_id}/layers/{layer_id}" )) @@ -1354,7 +1354,7 @@ mod tests { let collection_id = ctx.db().get_root_layer_collection_id().await.unwrap(); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!("/layerDb/collections/{collection_id}/collections")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(serde_json::json!({ @@ -1365,7 +1365,7 @@ mod tests { assert!(response.status().is_success(), "{response:?}"); - let result: IdResponse = test::read_body_json(response).await; + let result: IdResponse = read_body_json(response).await; ctx.db() .load_layer_collection(&result.id, LayerCollectionListOptions::default()) @@ -1393,7 +1393,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::put() + let req = TestRequest::put() .uri(&format!("/layerDb/collections/{collection_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(serde_json::json!({ @@ -1430,6 +1430,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -1456,6 +1457,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(4., 5.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -1466,7 +1468,7 @@ mod tests { properties: Default::default(), }; - let req = test::TestRequest::put() + let req = TestRequest::put() .uri(&format!("/layerDb/layers/{layer_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(serde_json::json!(update_layer.clone())); @@ -1506,7 +1508,7 @@ mod tests { } } }); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!("/layerDb/collections/{collection_id}/layers")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(invalid_workflow_layer.clone()); @@ -1530,6 +1532,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -1548,7 +1551,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::put() + let req = TestRequest::put() .uri(&format!("/layerDb/layers/{layer_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(invalid_workflow_layer); @@ -1579,6 +1582,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -1597,7 +1601,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!("/layerDb/layers/{layer_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); @@ -1645,7 +1649,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!( "/layerDb/collections/{collection_a_id}/collections/{collection_b_id}" )) @@ -1693,9 +1697,10 @@ mod tests { description: "Layer Description".to_string(), workflow: Workflow { operator: MockPointSource { - params: MockPointSourceParams { - points: vec![(0.0, 0.1).into(), (1.0, 1.1).into()], - }, + params: MockPointSourceParams::new(vec![ + (0.0, 0.1).into(), + (1.0, 1.1).into(), + ]), } .boxed() .into(), @@ -1709,7 +1714,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!( "/layerDb/collections/{collection_id}/layers/{layer_id}" )) @@ -1749,7 +1754,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!("/layerDb/collections/{collection_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx.clone()).await; @@ -1763,7 +1768,7 @@ mod tests { // try removing root collection id --> should fail - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!("/layerDb/collections/{root_collection_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx.clone()).await; @@ -1793,7 +1798,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!( "/layerDb/collections/{root_collection_id}/collections/{collection_id}" )) @@ -1823,7 +1828,7 @@ mod tests { let session_id = session.id(); - let req = test::TestRequest::get() + let req = TestRequest::get() .uri(&format!("/layers/{INTERNAL_PROVIDER_ID}/capabilities",)) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx.clone()).await; @@ -1840,7 +1845,7 @@ mod tests { let root_collection_id = ctx.db().get_root_layer_collection_id().await.unwrap(); - let req = test::TestRequest::get() + let req = TestRequest::get() .uri(&format!( "/layers/collections/search/{INTERNAL_PROVIDER_ID}/{root_collection_id}?limit=5&offset=0&searchType=fulltext&searchString=x" )) @@ -1859,7 +1864,7 @@ mod tests { let root_collection_id = ctx.db().get_root_layer_collection_id().await.unwrap(); - let req = test::TestRequest::get() + let req = TestRequest::get() .uri(&format!( "/layers/collections/search/autocomplete/{INTERNAL_PROVIDER_ID}/{root_collection_id}?limit=5&offset=0&searchType=fulltext&searchString=x" )) @@ -1880,12 +1885,7 @@ mod tests { } impl MockRasterWorkflowLayerDescription { - fn new( - has_time: bool, - has_bbox: bool, - has_resolution: bool, - time_shift_millis: i32, - ) -> Self { + fn new(has_time: bool, time_shift_millis: i32) -> Self { let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(1_671_868_800_000, 1_671_955_200_000), @@ -1907,35 +1907,28 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: if has_time { + Some(TimeInterval::new_unchecked( + 1_671_868_800_000, + 1_672_041_600_000, + )) + } else { + None + }, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: if has_time { - Some(TimeInterval::new_unchecked( - 1_671_868_800_000, - 1_672_041_600_000, - )) - } else { - None - }, - bbox: if has_bbox { - Some(SpatialPartition2D::new_unchecked( - (0., 2.).into(), - (2., 0.).into(), - )) - } else { - None - }, - resolution: if has_resolution { - Some(GeoTransform::test_default().spatial_resolution()) - } else { - None - }, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1959,19 +1952,17 @@ mod tests { }; let tiling_specification = TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape::new([2, 2]), }; - let query_rectangle = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 2.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked( - 1_671_868_800_000 + i64::from(time_shift_millis), - 1_672_041_600_000 + i64::from(time_shift_millis), + let query_rectangle = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-2, -1, 0, 1).unwrap(), + TimeInterval::new_unchecked( + 1_671_868_800_000 - i64::from(time_shift_millis), + 1_672_041_600_000 - i64::from(time_shift_millis), ), - spatial_resolution: GeoTransform::test_default().spatial_resolution(), - attributes: BandSelection::first(), - }; + BandSelection::first(), + ); MockRasterWorkflowLayerDescription { workflow, @@ -2032,7 +2023,7 @@ mod tests { let provider_id = layer.id.provider_id; // create dataset from workflow - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!("/layers/{provider_id}/{layer_id}/dataset")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .append_header((header::CONTENT_TYPE, mime::APPLICATION_JSON)); @@ -2071,46 +2062,6 @@ mod tests { response } - async fn raster_operator_to_geotiff_bytes( - ctx: &C, - operator: Box, - query_rectangle: RasterQueryRectangle, - ) -> geoengine_operators::util::Result>> { - let exe_ctx = ctx.execution_context().unwrap(); - let query_ctx = ctx.mock_query_context().unwrap(); - - let initialized_operator = operator - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap(); - let query_processor = initialized_operator - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - raster_stream_to_geotiff_bytes( - query_processor, - query_rectangle, - query_ctx, - GdalGeoTiffDatasetMetadata { - no_data_value: Some(0.), - spatial_reference: SpatialReference::epsg_4326(), - }, - GdalGeoTiffOptions { - compression_num_threads: get_config_element::() - .unwrap() - .compression_num_threads, - as_cog: true, - force_big_tiff: false, - }, - None, - Box::pin(futures::future::pending()), - exe_ctx.tiling_specification(), - ) - .await - } - async fn raster_layer_to_dataset_success( app_ctx: PostgresContext, mock_source: MockRasterWorkflowLayerDescription, @@ -2129,45 +2080,37 @@ mod tests { // query the layer let workflow_operator = mock_source.workflow.operator.get_raster().unwrap(); - let workflow_result = raster_operator_to_geotiff_bytes( - &ctx, - workflow_operator, - mock_source.query_rectangle.clone(), - ) - .await - .unwrap(); // query the newly created dataset let dataset_operator = GdalSource { - params: GdalSourceParameters { - data: response.dataset.into(), - }, + params: GdalSourceParameters::new(response.dataset.into()), } .boxed(); - let dataset_result = raster_operator_to_geotiff_bytes( - &ctx, + + assert_eq_two_raster_operator_res_u8( + &ctx.execution_context().unwrap(), + &ctx.query_context(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + workflow_operator, dataset_operator, - mock_source.query_rectangle.clone(), + mock_source.query_rectangle, + false, ) - .await - .unwrap(); - - assert_eq!(workflow_result.as_slice(), dataset_result.as_slice()); + .await; } fn test_raster_layer_to_dataset_success_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, true, 0); + let mock_source = MockRasterWorkflowLayerDescription::new(true, 0); mock_source.tiling_specification } #[ge_context::test(tiling_spec = "test_raster_layer_to_dataset_success_tiling_spec")] async fn test_raster_layer_to_dataset_success(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, true, 0); + let mock_source = MockRasterWorkflowLayerDescription::new(true, 0); raster_layer_to_dataset_success(app_ctx, mock_source).await; } fn test_raster_layer_with_timeshift_to_dataset_success_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, true, 1_000); + let mock_source = MockRasterWorkflowLayerDescription::new(true, 1_000); mock_source.tiling_specification } @@ -2175,18 +2118,19 @@ mod tests { tiling_spec = "test_raster_layer_with_timeshift_to_dataset_success_tiling_spec" )] async fn test_raster_layer_with_timeshift_to_dataset_success(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, true, 1_000); + let mock_source: MockRasterWorkflowLayerDescription = + MockRasterWorkflowLayerDescription::new(true, 1_000); raster_layer_to_dataset_success(app_ctx, mock_source).await; } fn test_raster_layer_to_dataset_no_time_interval_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(false, true, true, 0); + let mock_source = MockRasterWorkflowLayerDescription::new(true, 0); mock_source.tiling_specification } #[ge_context::test(tiling_spec = "test_raster_layer_to_dataset_no_time_interval_tiling_spec")] async fn test_raster_layer_to_dataset_no_time_interval(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(false, true, true, 0); + let mock_source = MockRasterWorkflowLayerDescription::new(false, 0); let session = admin_login(&app_ctx).await; @@ -2204,58 +2148,4 @@ mod tests { ) .await; } - - fn test_raster_layer_to_dataset_no_bounding_box_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(true, false, true, 0); - mock_source.tiling_specification - } - - #[ge_context::test(tiling_spec = "test_raster_layer_to_dataset_no_bounding_box_tiling_spec")] - async fn test_raster_layer_to_dataset_no_bounding_box(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(true, false, true, 0); - - let session = admin_login(&app_ctx).await; - - let session_id = session.id(); - - let layer = mock_source.create_layer_in_context(&app_ctx).await; - - let res = send_dataset_creation_test_request(app_ctx, layer, session_id).await; - - ErrorResponse::assert( - res, - 400, - "LayerResultDescriptorMissingFields", - "Result Descriptor field 'bbox' is None", - ) - .await; - } - - fn test_raster_layer_to_dataset_no_spatial_resolution_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, false, 0); - mock_source.tiling_specification - } - - #[ge_context::test( - tiling_spec = "test_raster_layer_to_dataset_no_spatial_resolution_tiling_spec" - )] - async fn test_raster_layer_to_dataset_no_spatial_resolution(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, false, 0); - - let session = admin_login(&app_ctx).await; - - let session_id = session.id(); - - let layer = mock_source.create_layer_in_context(&app_ctx).await; - - let res = send_dataset_creation_test_request(app_ctx, layer, session_id).await; - - ErrorResponse::assert( - res, - 400, - "LayerResultDescriptorMissingFields", - "Result Descriptor field 'spatial_resolution' is None", - ) - .await; - } } diff --git a/services/src/api/handlers/permissions.rs b/services/src/api/handlers/permissions.rs index 173055109..df437475e 100644 --- a/services/src/api/handlers/permissions.rs +++ b/services/src/api/handlers/permissions.rs @@ -1,14 +1,16 @@ -use crate::api::model::datatypes::LayerId; -use crate::contexts::{ApplicationContext, GeoEngineDb, SessionContext}; -use crate::datasets::DatasetName; -use crate::datasets::storage::DatasetDb; -use crate::error::{self, Error, Result}; -use crate::layers::listing::LayerCollectionId; -use crate::machine_learning::MlModelDb; -use crate::permissions::{ - Permission, PermissionDb, PermissionListing as DbPermissionListing, ResourceId, Role, RoleId, +use crate::{ + api::model::datatypes::LayerId, + contexts::{ApplicationContext, GeoEngineDb, SessionContext}, + datasets::{DatasetName, storage::DatasetDb}, + error::{self, Error, Result}, + layers::listing::LayerCollectionId, + machine_learning::MlModelDb, + permissions::{ + Permission, PermissionDb, PermissionListing as DbPermissionListing, ResourceId, Role, + RoleId, + }, + projects::ProjectId, }; -use crate::projects::ProjectId; use actix_web::{FromRequest, HttpResponse, web}; use geoengine_datatypes::error::BoxedResultExt; use geoengine_datatypes::machine_learning::MlModelName; @@ -382,6 +384,7 @@ mod tests { let gdal = GdalSource { params: GdalSourceParameters { data: gdal_dataset_name, + overview_level: None, }, } .boxed(); @@ -652,6 +655,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), diff --git a/services/src/api/handlers/plots.rs b/services/src/api/handlers/plots.rs index 9c9f96426..6c0c612cb 100644 --- a/services/src/api/handlers/plots.rs +++ b/services/src/api/handlers/plots.rs @@ -10,11 +10,10 @@ use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::WorkflowId; use actix_web::{FromRequest, HttpRequest, Responder, web}; use base64::Engine; -use geoengine_datatypes::operations::reproject::reproject_query; +use geoengine_datatypes::operations::reproject::reproject_spatial_query; use geoengine_datatypes::plots::PlotOutputFormat; -use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, SpatialResolution, VectorQueryRectangle, -}; +use geoengine_datatypes::primitives::{BoundingBox2D, SpatialResolution}; +use geoengine_datatypes::primitives::{PlotQueryRectangle, PlotSeriesSelection}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_operators::engine::{ QueryContext, ResultDescriptor, TypedPlotQueryProcessor, WorkflowOperatorPath, @@ -131,17 +130,22 @@ async fn get_plot_handler( let request_spatial_ref: SpatialReference = params.crs.ok_or(error::Error::MissingSpatialReference)?; - let query_rect = VectorQueryRectangle { - spatial_bounds: params.bbox, - time_interval: params.time.into(), - spatial_resolution: params.spatial_resolution, - attributes: ColumnSelection::all(), - }; + let query_rect = PlotQueryRectangle::with_bounds( + params.bbox, + params.time.into(), + PlotSeriesSelection::all(), + ); let query_rect = if request_spatial_ref == workflow_spatial_ref { Some(query_rect) } else { - reproject_query(query_rect, workflow_spatial_ref, request_spatial_ref)? + let repr_spatial_query = reproject_spatial_query( + query_rect.spatial_query(), + workflow_spatial_ref, + request_spatial_ref, + )?; + repr_spatial_query + .map(|r| PlotQueryRectangle::new(r, query_rect.time_interval, query_rect.attributes)) }; let Some(query_rect) = query_rect else { @@ -162,18 +166,18 @@ async fn get_plot_handler( let data = match processor { TypedPlotQueryProcessor::JsonPlain(processor) => { - let json = processor.plot_query(query_rect.into(), &query_ctx); + let json = processor.plot_query(query_rect, &query_ctx); abortable_query_execution(json, conn_closed, query_abort_trigger).await? } TypedPlotQueryProcessor::JsonVega(processor) => { - let chart = processor.plot_query(query_rect.into(), &query_ctx); + let chart = processor.plot_query(query_rect, &query_ctx); let chart = abortable_query_execution(chart, conn_closed, query_abort_trigger).await; let chart = chart?; serde_json::to_value(chart).context(error::SerdeJson)? } TypedPlotQueryProcessor::ImagePng(processor) => { - let png_bytes = processor.plot_query(query_rect.into(), &query_ctx); + let png_bytes = processor.plot_query(query_rect, &query_ctx); let png_bytes = abortable_query_execution(png_bytes, conn_closed, query_abort_trigger).await; let png_bytes = png_bytes?; @@ -224,12 +228,14 @@ mod tests { use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::DateTime; use geoengine_datatypes::raster::{ - Grid2D, RasterDataType, RasterTile2D, TileInformation, TilingSpecification, + GeoTransform, Grid2D, GridBoundingBox2D, RasterDataType, RasterTile2D, TileInformation, + TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::{ PlotOperator, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + SpatialGridDescriptor, }; use geoengine_operators::mock::{MockRasterSource, MockRasterSourceParams}; use geoengine_operators::plot::{ @@ -239,6 +245,17 @@ mod tests { use tokio_postgres::NoTls; fn example_raster_source() -> Box { + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new_min_max(-3, 0, 0, 2).unwrap(), + ), + time: None, + bands: RasterBandDescriptors::new_single_band(), + }; + MockRasterSource { params: MockRasterSourceParams { data: vec![RasterTile2D::new_with_tile_info( @@ -254,21 +271,14 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() } fn json_tiling_spec() -> TilingSpecification { - TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()) + TilingSpecification::new([3, 2].into()) } #[ge_context::test(tiling_spec = "json_tiling_spec")] @@ -320,7 +330,7 @@ mod tests { "plotType": "Statistics", "data": { "Raster-1": { - "valueCount": 24, // Note: this is caused by the query being a BoundingBox where the right and lower bounds are inclusive. This requires that the tiles that inculde the right and lower bounds are also produced. + "valueCount": 6, // TODO: investigate why the bbox is satisfied with 6 pixels while the borders should in theory also be included ... "validCount": 6, "min": 1.0, "max": 6.0, @@ -334,7 +344,7 @@ mod tests { } fn json_vega_tiling_spec() -> TilingSpecification { - TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()) + TilingSpecification::new([3, 2].into()) } #[ge_context::test(tiling_spec = "json_vega_tiling_spec")] diff --git a/services/src/api/handlers/spatial_references.rs b/services/src/api/handlers/spatial_references.rs index 82a5617ec..92d39083f 100755 --- a/services/src/api/handlers/spatial_references.rs +++ b/services/src/api/handlers/spatial_references.rs @@ -203,15 +203,16 @@ pub(crate) async fn get_spatial_reference_specification_handler, _session: C::Session, ) -> Result { - spatial_reference_specification(&srs_string).map(web::Json) + let spatial_ref = SpatialReference::from_str(&srs_string)?; + spatial_reference_specification(spatial_ref).map(web::Json) } /// custom spatial references not known by proj or that shall be overriden fn custom_spatial_reference_specification( - srs_string: &str, + spatial_ref: SpatialReference, ) -> Option { // TODO: provide a generic storage for custom spatial reference specifications - match srs_string.to_uppercase().as_str() { + match spatial_ref.srs_string().to_uppercase().as_str() { "SR-ORG:81" => Some(SpatialReferenceSpecification { name: "GEOS - GEOstationary Satellite".to_owned(), spatial_reference: SpatialReference::new(SpatialReferenceAuthority::SrOrg, 81), @@ -234,18 +235,22 @@ fn custom_spatial_reference_specification( } } -pub fn spatial_reference_specification(srs_string: &str) -> Result { - if let Some(sref) = custom_spatial_reference_specification(srs_string) { +pub fn spatial_reference_specification( + spatial_reference: SpatialReference, +) -> Result { + if let Some(sref) = custom_spatial_reference_specification(spatial_reference) { return Ok(sref); } - let spatial_reference = - geoengine_datatypes::spatial_reference::SpatialReference::from_str(srs_string)?; - let json = proj_json(srs_string).ok_or_else(|| Error::UnknownSrsString { - srs_string: srs_string.to_owned(), + let spatial_reference: geoengine_datatypes::spatial_reference::SpatialReference = + spatial_reference.into(); + let srs_string = spatial_reference.srs_string(); + + let json = proj_json(&srs_string).ok_or_else(|| Error::UnknownSrsString { + srs_string: srs_string.clone(), })?; - let proj_string = proj_proj_string(srs_string).ok_or_else(|| Error::UnknownSrsString { - srs_string: srs_string.to_owned(), + let proj_string = proj_proj_string(&srs_string).ok_or_else(|| Error::UnknownSrsString { + srs_string: srs_string.clone(), })?; let extent: geoengine_datatypes::primitives::BoundingBox2D = @@ -323,7 +328,10 @@ mod tests { #[test] fn spec_webmercator() { - let spec = spatial_reference_specification("EPSG:3857").unwrap(); + let spec = spatial_reference_specification( + SpatialReference::from_str("EPSG:3857").unwrap().into(), + ) + .unwrap(); assert_eq!(spec.name, "WGS 84 / Pseudo-Mercator"); assert_eq!( spec.spatial_reference, @@ -351,7 +359,10 @@ mod tests { #[test] fn spec_wgs84() { - let spec = spatial_reference_specification("EPSG:4326").unwrap(); + let spec = spatial_reference_specification( + SpatialReference::from_str("EPSG:4326").unwrap().into(), + ) + .unwrap(); assert_eq!( SpatialReferenceSpecification { name: "WGS 84".to_owned(), @@ -374,7 +385,10 @@ mod tests { #[test] fn spec_utm32n() { - let spec = spatial_reference_specification("EPSG:32632").unwrap(); + let spec = spatial_reference_specification( + SpatialReference::from_str("EPSG:32632").unwrap().into(), + ) + .unwrap(); assert_eq!( SpatialReferenceSpecification { name: "WGS 84 / UTM zone 32N".to_owned(), @@ -395,7 +409,10 @@ mod tests { #[test] fn spec_geos() { - let spec = spatial_reference_specification("SR-ORG:81").unwrap(); + let spec = spatial_reference_specification( + SpatialReference::from_str("SR-ORG:81").unwrap().into(), + ) + .unwrap(); assert_eq!( SpatialReferenceSpecification { name: "GEOS - GEOstationary Satellite".to_owned(), diff --git a/services/src/api/handlers/wcs.rs b/services/src/api/handlers/wcs.rs index ba534dfe9..f45ca7913 100644 --- a/services/src/api/handlers/wcs.rs +++ b/services/src/api/handlers/wcs.rs @@ -1,5 +1,4 @@ use crate::api::handlers::spatial_references::{AxisOrder, spatial_reference_specification}; -use crate::api::model::datatypes::TimeInterval; use crate::api::ogc::util::{OgcProtocol, OgcRequestGuard, ogc_endpoint_url}; use crate::api::ogc::wcs::request::{DescribeCoverage, GetCapabilities, GetCoverage, WcsVersion}; use crate::config; @@ -12,15 +11,14 @@ use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::WorkflowId; use actix_web::{FromRequest, HttpRequest, HttpResponse, web}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, RasterQueryRectangle, SpatialPartition2D, + AxisAlignedRectangle, BandSelection, RasterQueryRectangle, SpatialResolution, TimeInterval, }; -use geoengine_datatypes::raster::GeoTransform; -use geoengine_datatypes::{primitives::SpatialResolution, spatial_reference::SpatialReference}; +use geoengine_datatypes::raster::GridShape2D; +use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_operators::call_on_generic_raster_processor_gdal_types; -use geoengine_operators::engine::{ExecutionContext, RasterOperator, WorkflowOperatorPath}; -use geoengine_operators::engine::{ResultDescriptor, SingleRasterOrVectorSource}; -use geoengine_operators::processing::{Reprojection, ReprojectionParams}; -use geoengine_operators::util::input::RasterOrVectorOperator; +use geoengine_operators::engine::{ + ExecutionContext, InitializedRasterOperator, WorkflowOperatorPath, +}; use geoengine_operators::util::raster_stream_to_geotiff::{ GdalGeoTiffDatasetMetadata, GdalGeoTiffOptions, raster_stream_to_multiband_geotiff_bytes, }; @@ -217,46 +215,42 @@ async fn wcs_describe_coverage_handler( let result_descriptor = operator.result_descriptor(); + let spatial_grid_descriptor = result_descriptor.spatial_grid_descriptor(); + let spatial_reference: Option = result_descriptor.spatial_reference.into(); let spatial_reference = spatial_reference.ok_or(error::Error::MissingSpatialReference)?; + let bounds = spatial_grid_descriptor.spatial_partition(); + + let (bbox_ll_0, bbox_ll_1, bbox_ur_0, bbox_ur_1) = + match spatial_reference_specification(spatial_reference.into())? + .axis_order + .ok_or(Error::AxisOrderingNotKnownForSrs { + srs_string: spatial_reference.srs_string(), + })? { + AxisOrder::EastNorth => ( + bounds.lower_left().x, + bounds.lower_left().y, + bounds.upper_right().x, + bounds.upper_right().y, + ), + AxisOrder::NorthEast => ( + bounds.lower_left().y, + bounds.lower_left().x, + bounds.upper_right().y, + bounds.upper_right().x, + ), + }; + + let GridShape2D { + shape_array: [raster_size_y, raster_size_x], + } = spatial_grid_descriptor.grid_shape(); - let resolution = result_descriptor - .resolution - .unwrap_or(SpatialResolution::zero_point_one()); - - let pixel_size_x = resolution.x; - let pixel_size_y = -resolution.y; - - let bbox = if let Some(bbox) = result_descriptor.bbox { - bbox - } else { - spatial_reference.area_of_use_projected()? - }; - - let axis_order = spatial_reference_specification(&spatial_reference.proj_string()?)? - .axis_order - .ok_or(Error::AxisOrderingNotKnownForSrs { - srs_string: spatial_reference.srs_string(), - })?; - let (bbox_ll_0, bbox_ll_1, bbox_ur_0, bbox_ur_1) = match axis_order { - AxisOrder::EastNorth => ( - bbox.lower_left().x, - bbox.lower_left().y, - bbox.upper_right().x, - bbox.upper_right().y, - ), - AxisOrder::NorthEast => ( - bbox.lower_left().y, - bbox.lower_left().x, - bbox.upper_right().y, - bbox.upper_right().x, - ), - }; - - let geo_transform = GeoTransform::new(bbox.upper_left(), pixel_size_x, pixel_size_y); - - let [raster_size_x, raster_size_y] = - *(geo_transform.lower_right_pixel_idx(&bbox) + [1, 1]).inner(); + let SpatialResolution { + x: pixel_size_x, + y: pixel_size_y, + } = spatial_grid_descriptor.spatial_resolution(); + + let band_0 = &result_descriptor.bands[0]; let mock = format!( r#" @@ -277,7 +271,7 @@ async fn wcs_describe_coverage_handler( 0 0 - {raster_size_y} {raster_size_x} + {raster_size_x} {raster_size_y} urn:ogc:def:crs:{srs_authority}::{srs_code} @@ -306,9 +300,15 @@ async fn wcs_describe_coverage_handler( workflow_id = identifiers, srs_authority = spatial_reference.authority(), srs_code = spatial_reference.code(), - origin_x = bbox.upper_left().x, - origin_y = bbox.upper_left().y, - band_name = result_descriptor.bands[0].name, + origin_x = bounds.upper_left().x, + origin_y = bounds.upper_left().y, + bbox_ll_0 = bbox_ll_0, + bbox_ll_1 = bbox_ll_1, + bbox_ur_0 = bbox_ur_0, + bbox_ur_1 = bbox_ur_1, + band_name = band_0.name, + pixel_size_y = -pixel_size_y, // TODO: use the "real" sign in the resolution? + pixel_size_x = pixel_size_x ); Ok(HttpResponse::Ok().content_type(mime::TEXT_XML).body(mock)) @@ -364,21 +364,13 @@ async fn wcs_get_coverage_handler( .map(Duration::from_secs), ); + let request_spatial_ref: SpatialReference = request.spatial_ref().map(Into::into)?; + let request_resolution = request.spatial_resolution().transpose()?; let request_partition = request.spatial_partition()?; - - if let Some(gridorigin) = request.gridorigin { - ensure!( - gridorigin.coordinate(request.gridbasecrs)? == request_partition.upper_left(), - error::WcsGridOriginMustEqualBoundingboxUpperLeft - ); - } - - if let Some(bbox_spatial_reference) = request.boundingbox.spatial_reference { - ensure!( - request.gridbasecrs == bbox_spatial_reference, - error::WcsBoundingboxCrsMustEqualGridBaseCrs - ); - } + let request_time: TimeInterval = request + .time + .map_or_else(default_time_from_config, Into::into); + let request_no_data_value = request.nodatavalue; let ctx = app_ctx.session_context(session); @@ -395,84 +387,35 @@ async fn wcs_get_coverage_handler( .initialize(workflow_operator_path_root, &execution_context) .await?; - // handle request and workflow crs matching - let workflow_spatial_ref: Option = - initialized.result_descriptor().spatial_reference().into(); - let workflow_spatial_ref = workflow_spatial_ref.ok_or(error::Error::InvalidSpatialReference)?; - - let request_spatial_ref: SpatialReference = request.gridbasecrs.into(); - let request_no_data_value = request.nodatavalue; - - // perform reprojection if necessary - let initialized = if request_spatial_ref == workflow_spatial_ref { - initialized - } else { - log::debug!( - "WCS query srs: {request_spatial_ref}, workflow srs: {workflow_spatial_ref} --> injecting reprojection" - ); - - let reprojection_params = ReprojectionParams { - target_spatial_reference: request_spatial_ref, - }; - - // create the reprojection operator in order to get the canonic operator name - let reprojected_workflow = Reprojection { - params: reprojection_params, - sources: SingleRasterOrVectorSource { - source: RasterOrVectorOperator::Raster(operator), - }, - } - .boxed(); - - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - - // TODO: avoid re-initialization and re-use unprojected workflow. However, this requires updating all operator paths - - // In order to check whether we need to inject a reprojection, we first need to initialize the - // original workflow. Then we can check the result projection. Previously, we then just wrapped - // the initialized workflow with an initialized reprojection. IMHO this is wrong because - // initialization propagates the workflow path down the children and appends a new segment for - // each level. So we can't re-use an already initialized workflow, because all the workflow path/ - // operator names will be wrong. That's why I now build a new workflow with a reprojection and - // perform a full initialization. I only added the TODO because we did some optimization here - // which broke at some point when the workflow operator paths were introduced but no one noticed. - - let irp = reprojected_workflow - .initialize(workflow_operator_path_root, &execution_context) - .await?; - - Box::new(irp) - }; - - let processor = initialized.query_processor()?; - - let spatial_resolution: SpatialResolution = - if let Some(spatial_resolution) = request.spatial_resolution() { - spatial_resolution? - } else { - // TODO: proper default resolution - SpatialResolution { - x: request_partition.size_x() / 256., - y: request_partition.size_y() / 256., - } - }; + let tiling_spec = execution_context.tiling_specification(); - // snap bbox to grid - let geo_transform = GeoTransform::new( - request_partition.upper_left(), - spatial_resolution.x, - -spatial_resolution.y, + // TODO: add resample push down! + let wrapped = + geoengine_operators::util::WrapWithProjectionAndResample::new_create_result_descriptor( + operator, + initialized, + ) + .wrap_with_projection_and_resample( + Some(request_partition.upper_left()), // TODO: set none if not changed? But how to handle mapping to grid? + request_resolution, + request_spatial_ref, + tiling_spec, + )?; + + let query_tiling_pixel_grid = wrapped + .result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(request_partition); + + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + query_tiling_pixel_grid.grid_bounds(), + request_time, + BandSelection::first(), ); - let idx = geo_transform.lower_right_pixel_idx(&request_partition) + [1, 1]; - let lower_right = geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(idx); - let snapped_partition = SpatialPartition2D::new(request_partition.upper_left(), lower_right)?; - let query_rect = RasterQueryRectangle { - spatial_bounds: snapped_partition, - time_interval: request.time.unwrap_or_else(default_time_from_config).into(), - spatial_resolution, - attributes: BandSelection::first(), // TODO: support multi bands in API and set the selection here - }; + let processor = wrapped.initialized_operator.query_processor()?; let query_ctx = ctx.query_context(identifier.0, Uuid::new_v4())?; @@ -514,7 +457,7 @@ fn default_time_from_config() -> TimeInterval { .and_then(|ogc| ogc.default_time) .map_or_else( || { - geoengine_datatypes::primitives::TimeInterval::new_instant( + TimeInterval::new_instant( geoengine_datatypes::primitives::TimeInstance::now(), ) .expect("is a valid time interval") @@ -524,7 +467,6 @@ fn default_time_from_config() -> TimeInterval { }, |time| time.time_interval(), ) - .into() } #[cfg(test)] @@ -538,12 +480,17 @@ mod tests { use actix_web::http::header; use actix_web::test; use actix_web_httpauth::headers::authorization::Bearer; - use geoengine_datatypes::raster::{GridShape2D, TilingSpecification}; + use geoengine_datatypes::raster::GridShape2D; + use geoengine_datatypes::raster::TilingSpecification; use geoengine_datatypes::test_data; use geoengine_datatypes::util::ImageFormat; use geoengine_datatypes::util::assert_image_equals_with_format; use tokio_postgres::NoTls; + fn tiling_spec() -> TilingSpecification { + TilingSpecification::new(GridShape2D::new([600, 600])) + } + #[ge_context::test] async fn get_capabilities(app_ctx: PostgresContext) { let session = app_ctx.create_anonymous_session().await.unwrap(); @@ -662,7 +609,7 @@ mod tests { xmlns:ogc="http://www.opengis.net/ogc" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:gml="http://www.opengis.net/gml" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wcs/1.1.1 http://127.0.0.1:3030/api/wcs/1625f9b9-7408-58ab-9482-4d5fb0e3c6e4/schemas/wcs/1.1.1/wcsDescribeCoverage.xsd"> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wcs/1.1.1 http://127.0.0.1:3030/api/wcs/{workflow_id}/schemas/wcs/1.1.1/wcsDescribeCoverage.xsd"> Workflow {workflow_id} {workflow_id} @@ -706,13 +653,6 @@ mod tests { // TODO: add get_coverage with masked band - fn tiling_spec() -> TilingSpecification { - TilingSpecification { - origin_coordinate: (0., 0.).into(), - tile_size_in_pixels: GridShape2D::new([600, 600]), - } - } - #[ge_context::test(tiling_spec = "tiling_spec")] async fn get_coverage_with_nodatavalue(app_ctx: PostgresContext) { // override the pixel size since this test was designed for 600 x 600 pixel tiles diff --git a/services/src/api/handlers/wfs.rs b/services/src/api/handlers/wfs.rs index 98b2239e8..e3e72aec6 100644 --- a/services/src/api/handlers/wfs.rs +++ b/services/src/api/handlers/wfs.rs @@ -13,12 +13,9 @@ use actix_web::{FromRequest, HttpRequest, HttpResponse, web}; use futures::future::BoxFuture; use futures_util::TryStreamExt; use geoengine_datatypes::collections::ToGeoJson; +use geoengine_datatypes::collections::{FeatureCollection, MultiPointCollection}; use geoengine_datatypes::primitives::VectorQueryRectangle; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; -use geoengine_datatypes::{ - collections::{FeatureCollection, MultiPointCollection}, - primitives::SpatialResolution, -}; use geoengine_datatypes::{ primitives::{FeatureData, Geometry, MultiPoint}, spatial_reference::SpatialReference, @@ -28,7 +25,9 @@ use geoengine_operators::engine::{ VectorOperator, VectorQueryProcessor, }; use geoengine_operators::engine::{QueryProcessor, WorkflowOperatorPath}; -use geoengine_operators::processing::{Reprojection, ReprojectionParams}; +use geoengine_operators::processing::{ + DeriveOutRasterSpecsSource, Reprojection, ReprojectionParams, +}; use geoengine_operators::util::abortable_query_execution; use geoengine_operators::util::input::RasterOrVectorOperator; use reqwest::Url; @@ -488,6 +487,7 @@ async fn wfs_feature_handler( let reprojection_params = ReprojectionParams { target_spatial_reference: request_spatial_ref, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }; // create the reprojection operator in order to get the canonic operator name @@ -519,15 +519,11 @@ async fn wfs_feature_handler( let processor = initialized.query_processor()?; - let query_rect = VectorQueryRectangle { - spatial_bounds: request.bbox.bounds_naive()?, - time_interval: request.time.unwrap_or_else(default_time_from_config).into(), - // TODO: find reasonable default - spatial_resolution: request - .query_resolution - .map_or_else(SpatialResolution::zero_point_one, |r| r.0), - attributes: ColumnSelection::all(), - }; + let query_rect = VectorQueryRectangle::with_bounds( + request.bbox.bounds_naive()?, + request.time.unwrap_or_else(default_time_from_config).into(), + ColumnSelection::all(), + ); let query_ctx = ctx.query_context(type_names.0, Uuid::new_v4())?; let (json, cache_hint) = match processor { @@ -1126,7 +1122,6 @@ x;y /// override the pixel size since this test was designed for 600 x 600 pixel tiles fn raster_vector_join_tiling_spec() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape2D::new([600, 600]), } } @@ -1179,7 +1174,10 @@ x;y "rasters": [{ "type": "GdalSource", "params": { + "data": ndvi_name, + "overviewLevel:": null + } }], } diff --git a/services/src/api/handlers/wms.rs b/services/src/api/handlers/wms.rs index 0c0c605ac..28e9e5f68 100644 --- a/services/src/api/handlers/wms.rs +++ b/services/src/api/handlers/wms.rs @@ -9,29 +9,25 @@ use crate::api::ogc::wms::request::{ use crate::config; use crate::config::get_config_element; use crate::contexts::{ApplicationContext, SessionContext}; -use crate::error::Result; -use crate::error::{self, Error}; +use crate::error::{self, Error, Result}; use crate::util::server::{CacheControlHeader, connection_closed, not_implemented_handler}; use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::WorkflowId; use actix_web::{FromRequest, HttpRequest, HttpResponse, web}; -use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, RasterQueryRectangle, SpatialPartition2D, + AxisAlignedRectangle, BandSelection, CacheHint, SpatialResolution, }; -use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_operators::engine::{ - RasterOperator, ResultDescriptor, SingleRasterOrVectorSource, WorkflowOperatorPath, -}; -use geoengine_operators::processing::{Reprojection, ReprojectionParams}; -use geoengine_operators::util::input::RasterOrVectorOperator; +use geoengine_datatypes::primitives::{RasterQueryRectangle, SpatialPartition2D}; use geoengine_operators::{ - call_on_generic_raster_processor, util::raster_stream_to_png::raster_stream_to_png_bytes, + call_on_generic_raster_processor, + engine::{ExecutionContext, WorkflowOperatorPath}, + util::raster_stream_to_png::raster_stream_to_png_bytes, }; use reqwest::Url; use snafu::ensure; use std::str::FromStr; use std::time::Duration; +use tracing::debug; use uuid::Uuid; pub(crate) fn init_wms_routes(cfg: &mut web::ServiceConfig) @@ -275,8 +271,21 @@ async fn wms_map_handler( .map(Duration::from_secs), ); + // TODO: use a default spatial reference if it is not set? + let request_spatial_ref: SpatialReference = + request.crs.ok_or(error::Error::MissingSpatialReference)?; + + let request_bounds: SpatialPartition2D = request.bbox.bounds(request_spatial_ref)?; + let x_request_res = request_bounds.size_x() / f64::from(request.width); + let y_request_res = request_bounds.size_y() / f64::from(request.height); + let request_resolution = SpatialResolution::new(x_request_res.abs(), y_request_res.abs())?; + + let raster_colorizer = raster_colorizer_from_style(&request.styles)?; + let ctx = app_ctx.session_context(session); + let tiling_spec = ctx.execution_context()?.tiling_specification(); + let workflow_id = WorkflowId::from_str(&request.layers)?; let workflow = ctx.db().load_workflow(&workflow_id).await?; @@ -291,65 +300,43 @@ async fn wms_map_handler( .initialize(workflow_operator_path_root, &execution_context) .await?; - // handle request and workflow crs matching - let workflow_spatial_ref: SpatialReferenceOption = - initialized.result_descriptor().spatial_reference().into(); - let workflow_spatial_ref: Option = workflow_spatial_ref.into(); - let workflow_spatial_ref = - workflow_spatial_ref.ok_or(error::Error::InvalidSpatialReference)?; + /* + let request_geo_transform = GeoTransform::new( + request_bounds.upper_left(), + request_resolution.x, + -request_resolution.y, + ); - // TODO: use a default spatial reference if it is not set? - let request_spatial_ref: SpatialReference = - request.crs.ok_or(error::Error::MissingSpatialReference)?; + let tiling_based_origin = request_geo_transform + .nearest_pixel_edge_coordinate(tiling_spec.tiling_origin_reference()); - // perform reprojection if necessary - let initialized = if request_spatial_ref == workflow_spatial_ref { - initialized - } else { - log::debug!( - "WMS query srs: {request_spatial_ref}, workflow srs: {workflow_spatial_ref} --> injecting reprojection" - ); + */ - let reprojection_params = ReprojectionParams { - target_spatial_reference: request_spatial_ref.into(), - }; - - // create the reprojection operator in order to get the canonic operator name - let reprojected_workflow = Reprojection { - params: reprojection_params, - sources: SingleRasterOrVectorSource { - source: RasterOrVectorOperator::Raster(operator), - }, - } - .boxed(); - - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - - // TODO: avoid re-initialization and re-use unprojected workflow. However, this requires updating all operator paths - - // In order to check whether we need to inject a reprojection, we first need to initialize the - // original workflow. Then we can check the result projection. Previously, we then just wrapped - // the initialized workflow with an initialized reprojection. IMHO this is wrong because - // initialization propagates the workflow path down the children and appends a new segment for - // each level. So we can't re-use an already initialized workflow, because all the workflow path/ - // operator names will be wrong. That's why I now build a new workflow with a reprojection and - // perform a full initialization. I only added the TODO because we did some optimization here - // which broke at some point when the workflow operator paths were introduced but no one noticed. - - let irp = reprojected_workflow - .initialize(workflow_operator_path_root, &execution_context) - .await?; - - Box::new(irp) - }; + let wrapped = + geoengine_operators::util::WrapWithProjectionAndResample::new_create_result_descriptor( + operator, + initialized, + ) + .wrap_with_projection_and_resample( + None, // Some(tiling_based_origin), + Some(request_resolution), + request_spatial_ref.into(), + tiling_spec, + )?; + // TODO: add a resammple operator for downsampling AND resample push down! + + let initialized = wrapped.initialized_operator; let processor = initialized.query_processor()?; - let query_bbox: SpatialPartition2D = request.bbox.bounds(request_spatial_ref)?; - let x_query_resolution = query_bbox.size_x() / f64::from(request.width); - let y_query_resolution = query_bbox.size_y() / f64::from(request.height); + let query_tiling_pixel_grid = wrapped + .result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(request_bounds); - let raster_colorizer = raster_colorizer_from_style(&request.styles)?; + debug!("WMS re-scale-project: {:?}", query_tiling_pixel_grid); let attributes = raster_colorizer.as_ref().map_or_else( || BandSelection::new_single(0), @@ -360,18 +347,17 @@ async fn wms_map_handler( }, ); - let query_rect = RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: request.time.unwrap_or_else(default_time_from_config).into(), - spatial_resolution: SpatialResolution::new_unchecked( - x_query_resolution, - y_query_resolution, - ), + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + query_tiling_pixel_grid.grid_bounds(), + request.time.unwrap_or_else(default_time_from_config).into(), attributes, - }; + ); + + debug!("WMS query rect: {:?}", query_rect); let query_ctx = ctx.query_context(workflow_id.0, Uuid::new_v4())?; + // The raster to png code already resamples when the tiles are filled. We should add resample for lower resolutions call_on_generic_raster_processor!( processor, p => @@ -490,23 +476,20 @@ mod tests { use crate::ge_context; use crate::users::UserAuth; use crate::util::tests::{ - MockQueryContext, check_allowed_http_methods, read_body_string, + admin_login, check_allowed_http_methods, read_body_string, register_ndvi_workflow_helper, register_ndvi_workflow_helper_with_cache_ttl, register_ne2_multiband_workflow, send_test_request, }; - use crate::util::tests::{admin_login, register_ndvi_workflow_helper}; use actix_http::header::{self, CONTENT_TYPE}; use actix_web::dev::ServiceResponse; use actix_web::http::Method; use actix_web_httpauth::headers::authorization::Bearer; use geoengine_datatypes::operations::image::{Colorizer, RgbaColor}; use geoengine_datatypes::primitives::CacheTtlSeconds; - use geoengine_datatypes::raster::{GridShape2D, RasterDataType, TilingSpecification}; + use geoengine_datatypes::raster::{GridBoundingBox2D, GridShape2D, TilingSpecification}; use geoengine_datatypes::test_data; use geoengine_datatypes::util::assert_image_equals; - use geoengine_operators::engine::{ - ExecutionContext, RasterQueryProcessor, RasterResultDescriptor, - }; + use geoengine_operators::engine::{ExecutionContext, RasterQueryProcessor}; use geoengine_operators::source::GdalSourceProcessor; use geoengine_operators::util::gdal::create_ndvi_meta_data; use std::convert::TryInto; @@ -638,32 +621,29 @@ mod tests { let ctx = app_ctx.session_context(session.clone()); let exe_ctx = ctx.execution_context().unwrap(); + let meta_data = create_ndvi_meta_data(); + let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + produced_result_descriptor: meta_data.result_descriptor.clone(), tiling_specification: exe_ctx.tiling_specification(), - meta_data: Box::new(create_ndvi_meta_data()), + overview_level: 0, + meta_data: Box::new(meta_data), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_partition = - SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); - let (image_bytes, _) = raster_stream_to_png_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_partition, - time_interval: geoengine_datatypes::primitives::TimeInterval::new( + RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-900, 899, -1800, 1799).unwrap(), + geoengine_datatypes::primitives::TimeInterval::new( 1_388_534_400_000, 1_388_534_400_000 + 1000, ) .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked(1.0, 1.0), - attributes: BandSelection::first(), - }, - ctx.mock_query_context().unwrap(), + BandSelection::first(), + ), + ctx.query_context(Uuid::new_v4(), Uuid::new_v4()).unwrap(), 360, 180, None, @@ -673,7 +653,7 @@ mod tests { .await .unwrap(); - // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, test_data!("wms/raster_small.png")); + // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, "raster_small_22.png"); assert_image_equals(test_data!("wms/raster_small.png"), &image_bytes); } @@ -681,7 +661,6 @@ mod tests { /// override the pixel size since this test was designed for 600 x 600 pixel tiles fn get_map_test_helper_tiling_spec() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape2D::new([600, 600]), } } @@ -751,7 +730,7 @@ mod tests { let image_bytes = actix_web::test::read_body(response).await; - // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, "get_map_ndvi.png"); + // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, "get_map_ndvi_2.png"); assert_image_equals(test_data!("wms/get_map_ndvi.png"), &image_bytes); } @@ -900,8 +879,8 @@ mod tests { ("version", "1.3.0"), ("layers", &id.to_string()), ("bbox", "-90,-180,90,180"), - ("width", "600"), - ("height", "300"), + ("width", "3600"), // TODO: use smaller area + ("height", "1800"), ("crs", "EPSG:4326"), ( "styles", @@ -966,8 +945,8 @@ mod tests { ("version", "1.3.0"), ("layers", &id.to_string()), ("bbox", "-90,-180,90,180"), - ("width", "600"), - ("height", "300"), + ("width", "3600"), + ("height", "1800"), ("crs", "EPSG:4326"), ( "styles", @@ -1130,7 +1109,7 @@ mod tests { - No CoordinateProjector available for: SpatialReference { authority: Epsg, code: 4326 } --> SpatialReference { authority: Epsg, code: 432 } + Spatial reference system 'EPSG:432' is unknown "# ); @@ -1194,7 +1173,13 @@ mod tests { .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let res = send_test_request(req, app_ctx).await; - ErrorResponse::assert(res, 200, "NoCoordinateProjector", "No CoordinateProjector available for: SpatialReference { authority: Epsg, code: 4326 } --> SpatialReference { authority: Epsg, code: 432 }").await; + ErrorResponse::assert( + res, + 200, + "UnknownSrsString", + "Spatial reference system 'EPSG:432' is unknown", + ) + .await; } #[ge_context::test] @@ -1205,7 +1190,7 @@ mod tests { let (_, id) = register_ndvi_workflow_helper(&app_ctx).await; - let req = actix_web::test::TestRequest::get().uri(&format!("/wms/{id}?service=WMS&version=1.3.0&request=GetMap&layers={id}&styles=&width=335&height=168&crs=EPSG:4326&bbox=-90.0,-180.0,90.0,180.0&format=image/png&transparent=FALSE&bgcolor=0xFFFFFF&exceptions=application/json&time=2014-04-01T12%3A00%3A00.000%2B00%3A00", id = id.to_string())).append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); + let req = actix_web::test::TestRequest::get().uri(&format!("/wms/{id}?service=WMS&version=1.3.0&request=GetMap&layers={id}&styles=&width=3600&height=1800&crs=EPSG:4326&bbox=-90.0,-180.0,90.0,180.0&format=image/png&transparent=FALSE&bgcolor=0xFFFFFF&exceptions=application/json&time=2014-04-01T12%3A00%3A00.000%2B00%3A00", id = id.to_string())).append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx).await; assert_eq!( @@ -1230,7 +1215,7 @@ mod tests { let (_, id) = register_ndvi_workflow_helper_with_cache_ttl(&app_ctx, CacheTtlSeconds::new(60)).await; - let req = actix_web::test::TestRequest::get().uri(&format!("/wms/{id}?service=WMS&version=1.3.0&request=GetMap&layers={id}&styles=&width=335&height=168&crs=EPSG:4326&bbox=-90.0,-180.0,90.0,180.0&format=image/png&transparent=FALSE&bgcolor=0xFFFFFF&exceptions=application/json&time=2014-04-01T12%3A00%3A00.000%2B00%3A00", id = id.to_string())).append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); + let req = actix_web::test::TestRequest::get().uri(&format!("/wms/{id}?service=WMS&version=1.3.0&request=GetMap&layers={id}&styles=&width=360&height=180&crs=EPSG:4326&bbox=-9.0,-18.0,9.0,18.0&format=image/png&transparent=FALSE&bgcolor=0xFFFFFF&exceptions=application/json&time=2014-04-01T12%3A00%3A00.000%2B00%3A00", id = id.to_string())).append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx).await; assert_eq!( @@ -1247,6 +1232,15 @@ mod tests { cache_header == "private, max-age=60" || cache_header == "private, max-age=59" || cache_header == "private, max-age=58" + || cache_header == "private, max-age=57" + || cache_header == "private, max-age=56" + || cache_header == "private, max-age=55" + || cache_header == "private, max-age=54" + || cache_header == "private, max-age=53" + || cache_header == "private, max-age=52" + || cache_header == "private, max-age=51" // TODO: find out what takes so long. Keep in mind we use a lot of tiles here... + || cache_header == "private, max-age=50", + "Cache header is {cache_header:?} and not one of the exprected 60, 59, 58" ); } } diff --git a/services/src/api/handlers/workflows.rs b/services/src/api/handlers/workflows.rs index 5c0cff2a6..4defba8a1 100755 --- a/services/src/api/handlers/workflows.rs +++ b/services/src/api/handlers/workflows.rs @@ -5,12 +5,13 @@ use crate::api::ogc::util::{parse_bbox, parse_time}; use crate::config::get_config_element; use crate::contexts::{ApplicationContext, SessionContext}; use crate::datasets::listing::{DatasetProvider, Provenance, ProvenanceOutput}; -use crate::datasets::{RasterDatasetFromWorkflow, schedule_raster_dataset_from_workflow_task}; +use crate::datasets::{ + RasterDatasetFromWorkflow, RasterDatasetFromWorkflowParams, + schedule_raster_dataset_from_workflow_task, +}; use crate::error::Result; use crate::layers::storage::LayerProviderDb; -use crate::util::parsing::{ - parse_band_selection, parse_spatial_partition, parse_spatial_resolution, -}; +use crate::util::parsing::{parse_band_selection, parse_spatial_partition}; use crate::util::workflows::validate_workflow; use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::{Workflow, WorkflowId}; @@ -19,13 +20,10 @@ use actix_web::{FromRequest, HttpRequest, HttpResponse, Responder, web}; use futures::future::join_all; use geoengine_datatypes::error::{BoxedResultExt, ErrorSource}; use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, - VectorQueryRectangle, + BoundingBox2D, ColumnSelection, RasterQueryRectangle, SpatialPartition2D, VectorQueryRectangle, }; use geoengine_operators::call_on_typed_operator; -use geoengine_operators::engine::{ - ExecutionContext, OperatorData, TypedResultDescriptor, WorkflowOperatorPath, -}; +use geoengine_operators::engine::{ExecutionContext, OperatorData, WorkflowOperatorPath}; use serde::{Deserialize, Serialize}; use snafu::Snafu; use std::collections::HashMap; @@ -200,11 +198,11 @@ async fn get_workflow_metadata_handler( async fn workflow_metadata( workflow: Workflow, execution_context: C::ExecutionContext, -) -> Result { +) -> Result { // TODO: use cache here let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - let result_descriptor: TypedResultDescriptor = call_on_typed_operator!( + let result_descriptor: geoengine_operators::engine::TypedResultDescriptor = call_on_typed_operator!( workflow.operator, operator => { let operator = operator @@ -216,7 +214,7 @@ async fn workflow_metadata( } ); - Ok(result_descriptor) + Ok(result_descriptor.into()) } /// Gets the provenance of all datasets used in a workflow. @@ -443,12 +441,32 @@ async fn dataset_from_workflow_handler( let compression_num_threads = get_config_element::()?.compression_num_threads; + // FIXME: dont initialize the workflow here, but in the task + let operator = workflow + .clone() + .operator + .get_raster() + .expect("must be raster here") + .initialize( + WorkflowOperatorPath::initialize_root(), + &ctx.execution_context()?, + ) + .await?; + + let result_descriptor = operator.result_descriptor(); + + let info = RasterDatasetFromWorkflowParams::from_request_and_result_descriptor( + info.into_inner(), + result_descriptor, + ctx.execution_context()?.tiling_specification(), + ); + let task_id = schedule_raster_dataset_from_workflow_task( format!("workflow {id}"), + operator, id, - workflow, ctx, - info.into_inner(), + info, compression_num_threads, ) .await?; @@ -466,9 +484,6 @@ pub struct RasterStreamWebsocketQuery { #[serde(deserialize_with = "parse_time")] #[param(value_type = String)] pub time_interval: TimeInterval, - #[serde(deserialize_with = "parse_spatial_resolution")] - #[param(value_type = crate::api::model::datatypes::SpatialResolution)] - pub spatial_resolution: SpatialResolution, #[serde(deserialize_with = "parse_band_selection")] #[param(value_type = String)] pub attributes: BandSelection, @@ -525,12 +540,26 @@ async fn raster_stream_websocket( .get_raster() .boxed_context(error::WorkflowMustBeOfTypeRaster)?; - let query_rectangle = RasterQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: query.time_interval.into(), - spatial_resolution: query.spatial_resolution, - attributes: query.attributes.clone().try_into()?, - }; + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized_operator = operator + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let query = query.into_inner(); + + let query_bounds = initialized_operator + .result_descriptor() + .tiling_grid_definition(execution_context.tiling_specification()) + .tiling_geo_transform() + .spatial_to_grid_bounds(&query.spatial_bounds); + let query_rectangle = RasterQueryRectangle::new_with_grid_bounds( + query_bounds, + query.time_interval.into(), + query.attributes.clone().try_into()?, + ); // this is the only result type for now debug_assert!(matches!( @@ -539,9 +568,8 @@ async fn raster_stream_websocket( )); let stream_handler = RasterWebsocketStreamHandler::new::( - operator, + initialized_operator, query_rectangle, - ctx.execution_context()?, ctx.query_context(workflow_id.0, Uuid::new_v4())?, ) .await?; @@ -562,9 +590,6 @@ pub struct VectorStreamWebsocketQuery { #[serde(deserialize_with = "parse_time")] #[param(value_type = String)] pub time_interval: TimeInterval, - #[serde(deserialize_with = "parse_spatial_resolution")] - #[param(value_type = crate::api::model::datatypes::SpatialResolution)] - pub spatial_resolution: SpatialResolution, pub result_type: RasterStreamWebsocketResultType, } @@ -611,12 +636,11 @@ async fn vector_stream_websocket( .get_vector() .boxed_context(error::WorkflowMustBeOfTypeVector)?; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: query.time_interval.into(), - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + query.spatial_bounds, + query.time_interval.into(), + ColumnSelection::all(), + ); // this is the only result type for now debug_assert!(matches!( @@ -660,7 +684,6 @@ mod tests { use super::*; use crate::api::model::responses::ErrorResponse; - use crate::config::get_config_element; use crate::contexts::PostgresContext; use crate::contexts::Session; use crate::datasets::storage::DatasetStore; @@ -683,16 +706,16 @@ mod tests { use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ ContinuousMeasurement, FeatureData, Measurement, MultiPoint, RasterQueryRectangle, - SpatialPartition2D, SpatialResolution, TimeInterval, + TimeInterval, + }; + use geoengine_datatypes::raster::{ + GeoTransform, GridBoundingBox2D, GridShape, RasterDataType, TilingSpecification, }; - use geoengine_datatypes::raster::{GridShape, RasterDataType, TilingSpecification}; use geoengine_datatypes::spatial_reference::SpatialReference; - use geoengine_datatypes::test_data; - use geoengine_datatypes::util::ImageFormat; - use geoengine_datatypes::util::assert_image_equals_with_format; + use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::{ - ExecutionContext, MultipleRasterOrSingleVectorSource, PlotOperator, RasterBandDescriptor, - RasterBandDescriptors, TypedOperator, + MultipleRasterOrSingleVectorSource, PlotOperator, RasterBandDescriptor, + RasterBandDescriptors, SpatialGridDescriptor, TypedOperator, }; use geoengine_operators::engine::{RasterOperator, RasterResultDescriptor, VectorOperator}; use geoengine_operators::mock::{ @@ -702,10 +725,7 @@ mod tests { use geoengine_operators::plot::{Statistics, StatisticsParams}; use geoengine_operators::source::{GdalSource, GdalSourceParameters}; use geoengine_operators::util::input::MultiRasterOrVectorOperator::Raster; - use geoengine_operators::util::raster_stream_to_geotiff::{ - GdalGeoTiffDatasetMetadata, GdalGeoTiffOptions, - single_timestep_raster_stream_to_geotiff_bytes, - }; + use geoengine_operators::util::test::assert_eq_two_raster_operator_res_u8; use serde_json::json; use std::io::Read; use std::sync::Arc; @@ -723,9 +743,7 @@ mod tests { let workflow = Workflow { operator: MockPointSource { - params: MockPointSourceParams { - points: vec![(0.0, 0.1).into(), (1.0, 1.1).into()], - }, + params: MockPointSourceParams::new(vec![(0.0, 0.1).into(), (1.0, 1.1).into()]), } .boxed() .into(), @@ -763,9 +781,7 @@ mod tests { async fn register_missing_header(app_ctx: PostgresContext) { let workflow = Workflow { operator: MockPointSource { - params: MockPointSourceParams { - points: vec![(0.0, 0.1).into(), (1.0, 1.1).into()], - }, + params: MockPointSourceParams::new(vec![(0.0, 0.1).into(), (1.0, 1.1).into()]), } .boxed() .into(), @@ -1006,8 +1022,11 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + geoengine_datatypes::raster::GridBoundingBox2D::new([0, 0], [199, 199]) + .unwrap(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), Measurement::Continuous(ContinuousMeasurement { @@ -1041,8 +1060,22 @@ mod tests { "dataType": "U8", "spatialReference": "EPSG:4326", "time": null, - "bbox": null, - "resolution": null, + "spatialGrid": { + "descriptor": "source", + "spatialGrid": { + "geoTransform": {"originCoordinate":{"x":0.0,"y":0.0}, "xPixelSize": 1., "yPixelSize": -1.}, + "gridBounds": { + "bottomRightIdx": { + "xIdx": 199, + "yIdx": 199 + }, + "topLeftIdx": { + "xIdx": 0, + "yIdx": 0 + } + } + } + }, "bands": [{ "name": "band", "measurement": { @@ -1157,7 +1190,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: dataset }, + params: GdalSourceParameters::new(dataset), } .boxed(), ), @@ -1236,7 +1269,6 @@ mod tests { fn test_download_all_metadata_zip_tiling_spec() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape::new([600, 600]), } } @@ -1261,9 +1293,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: dataset_name.clone(), - }, + params: GdalSourceParameters::new(dataset_name.clone()), } .boxed(), ), @@ -1292,7 +1322,8 @@ mod tests { "operator": { "type": "GdalSource", "params": { - "data": dataset_name + "data": dataset_name, + "overviewLevel": null } } }) @@ -1308,19 +1339,22 @@ mod tests { "start": 1_388_534_400_000_i64, "end": 1_404_172_800_000_i64, }, - "bbox": { - "upperLeftCoordinate": { - "x": -180.0, - "y": 90.0, - }, - "lowerRightCoordinate": { - "x": 180.0, - "y": -90.0 + "spatialGrid": { + "descriptor": "source", + "spatialGrid" : { + "geoTransform": {"originCoordinate":{"x":-180.0,"y":90.0}, "xPixelSize": 0.1, "yPixelSize": -0.1}, + "gridBounds": { + "bottomRightIdx": { + "xIdx": 3599, + "yIdx": 1799 + }, + "topLeftIdx": { + "xIdx": 0, + "yIdx": 0 + } + } } - }, - "resolution": { - "x": 0.1, - "y": 0.1 + }, "bands": [{ "name": "ndvi", @@ -1356,8 +1390,7 @@ mod tests { /// override the pixel size since this test was designed for 600 x 600 pixel tiles fn dataset_from_workflow_task_success_tiling_spec() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), - tile_size_in_pixels: GridShape::new([600, 600]), + tile_size_in_pixels: GridShape::new([512, 512]), } } @@ -1371,13 +1404,13 @@ mod tests { let (_, dataset) = add_ndvi_to_datasets(&app_ctx).await; + let operator_a = GdalSource { + params: GdalSourceParameters::new(dataset), + } + .boxed(); + let workflow = Workflow { - operator: TypedOperator::Raster( - GdalSource { - params: GdalSourceParameters { data: dataset }, - } - .boxed(), - ), + operator: TypedOperator::Raster(operator_a.clone()), }; let workflow_id = ctx.db().register_workflow(workflow).await.unwrap(); @@ -1394,12 +1427,12 @@ mod tests { "query": { "spatialBounds": { "upperLeftCoordinate": { - "x": -10.0, - "y": 80.0 + "x": 0.0, + "y": 52.0 }, "lowerRightCoordinate": { - "x": 50.0, - "y": 20.0 + "x": 52.0, + "y": 0.0 } }, "timeInterval": { @@ -1442,56 +1475,25 @@ mod tests { }; // query the newly created dataset - let op = GdalSource { - params: GdalSourceParameters { - data: response.dataset.into(), - }, + let operator_b = GdalSource { + params: GdalSourceParameters::new(response.dataset.into()), } .boxed(); - let exe_ctx = ctx.execution_context().unwrap(); - - let o = op - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap(); - - let query_ctx = ctx.query_context(workflow_id.0, Uuid::new_v4()).unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked(1_388_534_400_000, 1_388_534_400_000 + 1000), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: geoengine_datatypes::primitives::BandSelection::first(), - }; - - let processor = o.query_processor().unwrap().get_u8().unwrap(); + let query_rectangle = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-512, -1, 0, 511).unwrap(), + TimeInterval::new_unchecked(1_388_534_400_000, 1_388_534_400_000 + 1000), + geoengine_datatypes::primitives::BandSelection::first(), + ); - let result = single_timestep_raster_stream_to_geotiff_bytes( - processor, - query_rect, - query_ctx, - GdalGeoTiffDatasetMetadata { - no_data_value: Some(0.), - spatial_reference: SpatialReference::epsg_4326(), - }, - GdalGeoTiffOptions { - compression_num_threads: get_config_element::() - .unwrap() - .compression_num_threads, - as_cog: false, - force_big_tiff: false, - }, - None, - Box::pin(futures::future::pending()), - exe_ctx.tiling_specification(), + assert_eq_two_raster_operator_res_u8( + &ctx.execution_context().unwrap(), + &ctx.query_context(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + operator_a, + operator_b, + query_rectangle, + false, ) - .await - .unwrap(); - - assert_image_equals_with_format( - test_data!("raster/geotiff_from_stream_compressed.tiff"), - result.as_slice(), - ImageFormat::Tiff, - ); + .await; } } diff --git a/services/src/api/model/datatypes.rs b/services/src/api/model/datatypes.rs index 1ab5be794..92a8548c8 100644 --- a/services/src/api/model/datatypes.rs +++ b/services/src/api/model/datatypes.rs @@ -4,17 +4,19 @@ use geoengine_datatypes::operations::image::RgbParams; use geoengine_datatypes::primitives::{ AxisAlignedRectangle, MultiLineStringAccess, MultiPointAccess, MultiPolygonAccess, }; +use geoengine_datatypes::raster::GridBounds; use geoengine_macros::type_tag; use ordered_float::NotNan; use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Visitor}; use snafu::ResultExt; +use std::borrow::Cow; use std::{ collections::{BTreeMap, HashMap}, fmt::{Debug, Formatter}, str::FromStr, }; -use utoipa::{PartialSchema, ToSchema}; +use utoipa::{PartialSchema, ToSchema, openapi}; identifier!(DataProviderId); @@ -744,6 +746,109 @@ impl From for geoengine_datatypes::primitives::BoundingBox2D { } } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridDefinition { + pub geo_transform: GeoTransform, + pub grid_bounds: GridBoundingBox2D, +} + +impl From for SpatialGridDefinition { + fn from(value: geoengine_datatypes::raster::SpatialGridDefinition) -> Self { + Self { + geo_transform: value.geo_transform().into(), + grid_bounds: value.grid_bounds().into(), + } + } +} + +impl From for geoengine_datatypes::raster::SpatialGridDefinition { + fn from(value: SpatialGridDefinition) -> Self { + geoengine_datatypes::raster::SpatialGridDefinition::new( + value.geo_transform.into(), + value.grid_bounds.into(), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GeoTransform { + pub origin_coordinate: Coordinate2D, + x_pixel_size: f64, + y_pixel_size: f64, +} + +impl From for GeoTransform { + fn from(value: geoengine_datatypes::raster::GeoTransform) -> Self { + GeoTransform { + origin_coordinate: value.origin_coordinate().into(), + x_pixel_size: value.x_pixel_size(), + y_pixel_size: value.y_pixel_size(), + } + } +} + +impl From for geoengine_datatypes::raster::GeoTransform { + fn from(value: GeoTransform) -> Self { + geoengine_datatypes::raster::GeoTransform::new( + value.origin_coordinate.into(), + value.x_pixel_size, + value.y_pixel_size, + ) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GridIdx2D { + y_idx: isize, + x_idx: isize, +} + +impl From for GridIdx2D { + fn from(value: geoengine_datatypes::raster::GridIdx2D) -> Self { + Self { + y_idx: value.y(), + x_idx: value.x(), + } + } +} + +impl From for geoengine_datatypes::raster::GridIdx2D { + fn from(value: GridIdx2D) -> Self { + geoengine_datatypes::raster::GridIdx::new_y_x(value.y_idx, value.x_idx) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GridBoundingBox2D { + top_left_idx: GridIdx2D, + bottom_right_idx: GridIdx2D, +} + +impl From for GridBoundingBox2D { + fn from(value: geoengine_datatypes::raster::GridBoundingBox2D) -> Self { + Self { + top_left_idx: value.min_index().into(), + bottom_right_idx: value.max_index().into(), + } + } +} + +impl From for geoengine_datatypes::raster::GridBoundingBox2D { + fn from(value: GridBoundingBox2D) -> Self { + geoengine_datatypes::raster::GridBoundingBox2D::new_min_max( + value.top_left_idx.y_idx, + value.bottom_right_idx.y_idx, + value.top_left_idx.x_idx, + value.bottom_right_idx.x_idx, + ) + .expect("Bounds were correct before") // TODO: maybe try from? + } +} + /// An object that composes the date and a timestamp with time zone. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct DateTimeString { @@ -896,15 +1001,54 @@ impl From for geoengine_datatypes::primitives::Continuous } } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SerializableClasses(BTreeMap); + +impl PartialSchema for SerializableClasses { + fn schema() -> openapi::RefOr { + BTreeMap::::schema() + } +} + +impl ToSchema for SerializableClasses { + fn name() -> Cow<'static, str> { + as ToSchema>::name() // TODO: is this needed? + } +} + +impl Serialize for SerializableClasses { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + let classes: BTreeMap = + self.0.iter().map(|(k, v)| (k.to_string(), v)).collect(); + classes.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for SerializableClasses { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let tree: BTreeMap = Deserialize::deserialize(deserializer)?; + let classes: Result, _> = tree + .into_iter() + .map(|(k, v)| (k.parse::().map(|x| (x, v)))) + .collect(); + Ok(SerializableClasses( + classes.map_err(serde::de::Error::custom)?, + )) + } +} + #[type_tag(value = "classification")] #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] -#[serde( - try_from = "SerializableClassificationMeasurement", - into = "SerializableClassificationMeasurement" -)] pub struct ClassificationMeasurement { pub measurement: String, - pub classes: HashMap, + // use a BTreeMap to preserve the order of the keys + pub classes: SerializableClasses, } impl From @@ -914,7 +1058,7 @@ impl From Self { r#type: Default::default(), measurement: value.measurement, - classes: value.classes, + classes: SerializableClasses(value.classes), } } } @@ -922,52 +1066,16 @@ impl From impl From for geoengine_datatypes::primitives::ClassificationMeasurement { - fn from(value: ClassificationMeasurement) -> Self { - Self { + fn from( + value: ClassificationMeasurement, + ) -> geoengine_datatypes::primitives::ClassificationMeasurement { + geoengine_datatypes::primitives::ClassificationMeasurement { measurement: value.measurement, - classes: value.classes, - } - } -} - -/// A type that is solely for serde's serializability. -/// You cannot serialize floats as JSON map keys. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SerializableClassificationMeasurement { - pub measurement: String, - // use a BTreeMap to preserve the order of the keys - pub classes: BTreeMap, -} - -impl From for SerializableClassificationMeasurement { - fn from(measurement: ClassificationMeasurement) -> Self { - let mut classes = BTreeMap::new(); - for (k, v) in measurement.classes { - classes.insert(k.to_string(), v); - } - Self { - measurement: measurement.measurement, - classes, + classes: value.classes.0, } } } -impl TryFrom for ClassificationMeasurement { - type Error = ::Err; - - fn try_from(measurement: SerializableClassificationMeasurement) -> Result { - let mut classes = HashMap::with_capacity(measurement.classes.len()); - for (k, v) in measurement.classes { - classes.insert(k.parse::()?, v); - } - Ok(Self { - r#type: Default::default(), - measurement: measurement.measurement, - classes, - }) - } -} - /// A partition of space that include the upper left but excludes the lower right coordinate #[derive(Copy, Clone, Serialize, Deserialize, PartialEq, Debug, ToSchema, FromSql, ToSql)] #[serde(rename_all = "camelCase")] @@ -997,12 +1105,12 @@ impl From for geoengine_datatypes::primitives::SpatialPartit /// A spatio-temporal rectangle with a specified resolution #[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] -pub struct RasterQueryRectangle { +pub struct RasterToDatasetQueryRectangle { pub spatial_bounds: SpatialPartition2D, pub time_interval: TimeInterval, - pub spatial_resolution: SpatialResolution, } +/* /// A spatio-temporal rectangle with a specified resolution #[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] @@ -1019,39 +1127,7 @@ pub struct PlotQueryRectangle { pub time_interval: TimeInterval, pub spatial_resolution: SpatialResolution, } - -impl - From< - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::SpatialPartition2D, - geoengine_datatypes::primitives::BandSelection, - >, - > for RasterQueryRectangle -{ - fn from( - value: geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::SpatialPartition2D, - geoengine_datatypes::primitives::BandSelection, - >, - ) -> Self { - Self { - spatial_bounds: value.spatial_bounds.into(), - time_interval: value.time_interval.into(), - spatial_resolution: value.spatial_resolution.into(), - } - } -} - -impl From for geoengine_datatypes::primitives::RasterQueryRectangle { - fn from(value: RasterQueryRectangle) -> Self { - Self { - spatial_bounds: value.spatial_bounds.into(), - time_interval: value.time_interval.into(), - spatial_resolution: value.spatial_resolution.into(), - attributes: geoengine_datatypes::primitives::BandSelection::first(), // TODO: adjust once API supports attribute selection - } - } -} +*/ #[derive(Clone, Debug, PartialEq, Deserialize, Serialize, ToSchema)] pub struct BandSelection(pub Vec); diff --git a/services/src/api/model/operators.rs b/services/src/api/model/operators.rs index f999fb295..01734ee5f 100644 --- a/services/src/api/model/operators.rs +++ b/services/src/api/model/operators.rs @@ -1,14 +1,15 @@ +use super::datatypes::{ + BoundingBox2D, FeatureDataType, Measurement, RasterDataType, SpatialGridDefinition, + SpatialReferenceOption, TimeInterval, VectorDataType, +}; use crate::api::model::datatypes::{ - BoundingBox2D, CacheTtlSeconds, Coordinate2D, DateTimeParseFormat, FeatureDataType, - GdalConfigOption, Measurement, MultiLineString, MultiPoint, MultiPolygon, NoGeometry, - RasterDataType, RasterPropertiesEntryType, RasterPropertiesKey, SpatialPartition2D, - SpatialReferenceOption, SpatialResolution, TimeInstance, TimeInterval, TimeStep, - VectorDataType, + CacheTtlSeconds, Coordinate2D, DateTimeParseFormat, GdalConfigOption, MultiLineString, + MultiPoint, MultiPolygon, NoGeometry, RasterPropertiesEntryType, RasterPropertiesKey, + TimeInstance, TimeStep, }; use crate::error::{ RasterBandNameMustNotBeEmpty, RasterBandNameTooLong, RasterBandNamesMustBeUnique, Result, }; -use geoengine_datatypes::primitives::ColumnSelection; use geoengine_datatypes::util::ByteSize; use geoengine_macros::type_tag; use geoengine_operators::util::input::float_option_with_nan; @@ -27,11 +28,55 @@ pub struct RasterResultDescriptor { #[schema(value_type = String)] pub spatial_reference: SpatialReferenceOption, pub time: Option, - pub bbox: Option, - pub resolution: Option, + pub spatial_grid: SpatialGridDescriptor, pub bands: RasterBandDescriptors, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub enum SpatialGridDescriptorState { + Source, + Derived, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridDescriptor { + pub spatial_grid: SpatialGridDefinition, + pub descriptor: SpatialGridDescriptorState, +} + +impl From for geoengine_operators::engine::SpatialGridDescriptor { + fn from(value: SpatialGridDescriptor) -> Self { + let sp = geoengine_operators::engine::SpatialGridDescriptor::new_source( + value.spatial_grid.into(), + ); + match value.descriptor { + SpatialGridDescriptorState::Source => sp, + SpatialGridDescriptorState::Derived => sp.as_derived(), + } + } +} + +impl From for SpatialGridDescriptor { + fn from(value: geoengine_operators::engine::SpatialGridDescriptor) -> Self { + if value.is_source() { + let sp = value.source_spatial_grid_definition().expect("is source"); + return SpatialGridDescriptor { + spatial_grid: sp.into(), + descriptor: SpatialGridDescriptorState::Source, + }; + } + let sp = value + .derived_spatial_grid_definition() + .expect("if not source it must be derived"); + SpatialGridDescriptor { + spatial_grid: sp.into(), + descriptor: SpatialGridDescriptorState::Derived, + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, ToSchema)] pub struct RasterBandDescriptors(Vec); @@ -108,8 +153,7 @@ impl From for RasterResultD data_type: value.data_type.into(), spatial_reference: value.spatial_reference.into(), time: value.time.map(Into::into), - bbox: value.bbox.map(Into::into), - resolution: value.resolution.map(Into::into), + spatial_grid: value.spatial_grid.into(), bands: value.bands.into(), } } @@ -121,8 +165,7 @@ impl From for geoengine_operators::engine::RasterResultD data_type: value.data_type.into(), spatial_reference: value.spatial_reference.into(), time: value.time.map(Into::into), - bbox: value.bbox.map(Into::into), - resolution: value.resolution.map(Into::into), + spatial_grid: value.spatial_grid.into(), bands: value.bands.into(), } } @@ -332,6 +375,26 @@ impl From for TypedResultDes } } +impl From for geoengine_operators::engine::TypedResultDescriptor { + fn from(value: TypedResultDescriptor) -> Self { + match value { + TypedResultDescriptor::Plot(p) => { + geoengine_operators::engine::TypedResultDescriptor::Plot(p.result_descriptor.into()) + } + TypedResultDescriptor::Raster(r) => { + geoengine_operators::engine::TypedResultDescriptor::Raster( + r.result_descriptor.into(), + ) + } + TypedResultDescriptor::Vector(v) => { + geoengine_operators::engine::TypedResultDescriptor::Vector( + v.result_descriptor.into(), + ) + } + } + } +} + impl From for TypedResultDescriptor { fn from(value: geoengine_operators::engine::PlotResultDescriptor) -> Self { Self::Plot(TypedPlotResultDescriptor { @@ -431,10 +494,7 @@ impl geoengine_operators::engine::StaticMetaData< geoengine_operators::mock::MockDatasetDataSourceLoadingInfo, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, >, > for MockMetaData { @@ -442,10 +502,7 @@ impl value: geoengine_operators::engine::StaticMetaData< geoengine_operators::mock::MockDatasetDataSourceLoadingInfo, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, >, ) -> Self { Self { @@ -461,10 +518,7 @@ impl geoengine_operators::engine::StaticMetaData< geoengine_operators::source::OgrSourceDataset, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, >, > for OgrMetaData { @@ -472,10 +526,7 @@ impl value: geoengine_operators::engine::StaticMetaData< geoengine_operators::source::OgrSourceDataset, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, >, ) -> Self { Self { @@ -490,10 +541,7 @@ impl From for geoengine_operators::engine::StaticMetaData< geoengine_operators::mock::MockDatasetDataSourceLoadingInfo, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, > { fn from(value: MockMetaData) -> Self { @@ -509,10 +557,7 @@ impl From for geoengine_operators::engine::StaticMetaData< geoengine_operators::source::OgrSourceDataset, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, > { fn from(value: OgrMetaData) -> Self { diff --git a/services/src/api/model/services.rs b/services/src/api/model/services.rs index 043253b75..7e125bdc4 100644 --- a/services/src/api/model/services.rs +++ b/services/src/api/model/services.rs @@ -12,7 +12,8 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use validator::{Validate, ValidationErrors}; -use super::datatypes::DataId; +use super::datatypes::{DataId, DatasetId}; +use super::operators::TypedResultDescriptor; #[allow(clippy::large_enum_variant)] #[derive(Serialize, Deserialize, Debug, Clone, ToSchema, PartialEq)] @@ -221,3 +222,53 @@ impl From<&Volume> for crate::datasets::upload::Volume { } } } + +#[derive(Debug, Serialize, Deserialize, Clone, ToSchema, Validate)] +#[serde(rename_all = "camelCase")] +pub struct Dataset { + pub id: DatasetId, + pub name: DatasetName, + pub display_name: String, + pub description: String, + pub result_descriptor: TypedResultDescriptor, + pub source_operator: String, + pub symbology: Option, + pub provenance: Option>, + pub tags: Option>, +} + +impl From for crate::datasets::storage::Dataset { + fn from(value: Dataset) -> Self { + crate::datasets::storage::Dataset { + id: value.id.into(), + name: value.name, + display_name: value.display_name, + description: value.description, + result_descriptor: value.result_descriptor.into(), + source_operator: value.source_operator, + symbology: value.symbology, + provenance: value + .provenance + .map(|v| v.into_iter().map(Into::into).collect::>()), + tags: value.tags, + } + } +} + +impl From for Dataset { + fn from(value: crate::datasets::storage::Dataset) -> Self { + Dataset { + id: value.id.into(), + name: value.name, + display_name: value.display_name, + description: value.description, + result_descriptor: value.result_descriptor.into(), + source_operator: value.source_operator, + symbology: value.symbology, + provenance: value + .provenance + .map(|v| v.into_iter().map(Into::into).collect::>()), + tags: value.tags, + } + } +} diff --git a/services/src/api/ogc/util.rs b/services/src/api/ogc/util.rs index ecef3dcd8..fdb24a46d 100644 --- a/services/src/api/ogc/util.rs +++ b/services/src/api/ogc/util.rs @@ -151,6 +151,10 @@ where if s.is_empty() { return Ok(None); } + // don't fail on python queries where undefined resolution is not removed + if s == "None" { + return Ok(None); + } let split: Vec> = s.split(',').map(str::parse).collect(); @@ -249,11 +253,12 @@ pub fn rectangle_from_ogc_params( spatial_reference: SpatialReference, ) -> Result { let [a, b, c, d] = values; - match spatial_reference_specification(&spatial_reference.proj_string()?)? + let axis_order = spatial_reference_specification(spatial_reference)? .axis_order .ok_or(error::Error::AxisOrderingNotKnownForSrs { srs_string: spatial_reference.srs_string(), - })? { + })?; + match axis_order { AxisOrder::EastNorth => A::from_min_max((a, b).into(), (c, d).into()).map_err(Into::into), AxisOrder::NorthEast => A::from_min_max((b, a).into(), (d, c).into()).map_err(Into::into), } @@ -265,7 +270,7 @@ pub fn tuple_from_ogc_params( b: f64, spatial_reference: SpatialReference, ) -> Result<(f64, f64)> { - match spatial_reference_specification(&spatial_reference.proj_string()?)? + match spatial_reference_specification(spatial_reference)? .axis_order .ok_or(error::Error::AxisOrderingNotKnownForSrs { srs_string: spatial_reference.srs_string(), diff --git a/services/src/api/ogc/wcs/request.rs b/services/src/api/ogc/wcs/request.rs index 80e935aa3..f15d8e25b 100644 --- a/services/src/api/ogc/wcs/request.rs +++ b/services/src/api/ogc/wcs/request.rs @@ -150,6 +150,16 @@ impl GetCoverage { rectangle_from_ogc_params(self.boundingbox.bbox, spatial_reference) } + + pub fn spatial_ref(&self) -> Result { + let spatial_ref = self.gridbasecrs; // TODO: maybe this is something different. Lets investigate that later... + if let Some(bbx_sref) = self.boundingbox.spatial_reference { + if bbx_sref != spatial_ref { + return Err(error::Error::WcsBoundingboxCrsMustEqualGridBaseCrs); + } + } + Ok(spatial_ref) + } } #[derive(PartialEq, Debug, Deserialize, Serialize, ToSchema)] diff --git a/services/src/config.rs b/services/src/config.rs index 4f1d42913..65cda4af8 100644 --- a/services/src/config.rs +++ b/services/src/config.rs @@ -178,10 +178,6 @@ pub struct TilingSpecification { impl From for geoengine_datatypes::raster::TilingSpecification { fn from(ts: TilingSpecification) -> geoengine_datatypes::raster::TilingSpecification { geoengine_datatypes::raster::TilingSpecification { - origin_coordinate: geoengine_datatypes::primitives::Coordinate2D::new( - ts.origin_coordinate_x, - ts.origin_coordinate_y, - ), tile_size_in_pixels: geoengine_datatypes::raster::GridShape2D::from([ ts.tile_shape_pixels_y, ts.tile_shape_pixels_x, diff --git a/services/src/contexts/db_types.rs b/services/src/contexts/db_types.rs index 185387c6a..1b0d5480d 100644 --- a/services/src/contexts/db_types.rs +++ b/services/src/contexts/db_types.rs @@ -1312,9 +1312,7 @@ mod tests { use super::*; use crate::{ - datasets::external::{ - SentinelS2L2ACogsProviderDefinition, StacBand, StacQueryBuffer, StacZone, - }, + datasets::external::{SentinelS2L2ACogsProviderDefinition, StacQueryBuffer}, layers::external::TypedDataProviderDefinition, util::{postgres::assert_sql_type, tests::with_temp_context}, }; @@ -1345,27 +1343,6 @@ mod tests { ) .await; - assert_sql_type( - &pool, - "StacBand", - [StacBand { - name: "band".to_owned(), - no_data_value: Some(133.7), - data_type: geoengine_datatypes::raster::RasterDataType::F32, - }], - ) - .await; - - assert_sql_type( - &pool, - "StacZone", - [StacZone { - name: "zone".to_owned(), - epsg: 4326, - }], - ) - .await; - assert_sql_type( &pool, "SentinelS2L2ACogsProviderDefinition", @@ -1375,15 +1352,6 @@ mod tests { description: "A provider".to_owned(), priority: Some(1), api_url: "http://api.url".to_owned(), - bands: vec![StacBand { - name: "band".to_owned(), - no_data_value: Some(133.7), - data_type: geoengine_datatypes::raster::RasterDataType::F32, - }], - zones: vec![StacZone { - name: "zone".to_owned(), - epsg: 4326, - }], stac_api_retries: StacApiRetries { number_of_retries: 3, initial_delay_ms: 4, @@ -1442,15 +1410,6 @@ mod tests { priority: Some(3), id: DataProviderId::new(), api_url: "http://api.url".to_owned(), - bands: vec![StacBand { - name: "band".to_owned(), - no_data_value: Some(133.7), - data_type: geoengine_datatypes::raster::RasterDataType::F32, - }], - zones: vec![StacZone { - name: "zone".to_owned(), - epsg: 4326, - }], stac_api_retries: StacApiRetries { number_of_retries: 3, initial_delay_ms: 4, diff --git a/services/src/contexts/migrations/current_schema.sql b/services/src/contexts/migrations/current_schema.sql index 81f9e99f1..3a8cd50c5 100644 --- a/services/src/contexts/migrations/current_schema.sql +++ b/services/src/contexts/migrations/current_schema.sql @@ -2,40 +2,27 @@ CREATE TABLE geoengine ( clear_database_on_start boolean NOT NULL DEFAULT FALSE, database_version text NOT NULL ); - CREATE TYPE "SpatialReferenceAuthority" AS ENUM ( 'Epsg', 'SrOrg', 'Iau2000', 'Esri' ); - CREATE TYPE "SpatialReference" AS ( authority "SpatialReferenceAuthority", code OID ); - -CREATE TYPE "Coordinate2D" AS ( - x double precision, - y double precision -); - +CREATE TYPE "Coordinate2D" AS (x double precision, y double precision); CREATE TYPE "BoundingBox2D" AS ( lower_left_coordinate "Coordinate2D", upper_right_coordinate "Coordinate2D" ); - -CREATE TYPE "TimeInterval" AS ( - start bigint, - "end" bigint -); - +CREATE TYPE "TimeInterval" AS (start bigint, "end" bigint); CREATE TYPE "STRectangle" AS ( spatial_reference "SpatialReference", bounding_box "BoundingBox2D", time_interval "TimeInterval" ); - CREATE TYPE "TimeGranularity" AS ENUM ( 'Millis', 'Seconds', @@ -45,33 +32,29 @@ CREATE TYPE "TimeGranularity" AS ENUM ( 'Months', 'Years' ); - CREATE TYPE "TimeStep" AS ( granularity "TimeGranularity", step OID ); - CREATE TYPE "DatasetName" AS (namespace text, name text); - CREATE TYPE "Provenance" AS ( citation text, license text, uri text ); - CREATE DOMAIN "RgbaColor" AS smallint [4] CHECK ( - 0 <= ALL(value) AND 255 >= ALL(value) + 0 <= ALL(value) + AND 255 >= ALL(value) ); - CREATE TYPE "Breakpoint" AS ( "value" double precision, color "RgbaColor" ); - CREATE TYPE "ColorizerType" AS ENUM ( - 'LinearGradient', 'LogarithmicGradient', 'Palette' + 'LinearGradient', + 'LogarithmicGradient', + 'Palette' ); - CREATE TYPE "Colorizer" AS ( "type" "ColorizerType", -- linear/logarithmic gradient @@ -81,12 +64,10 @@ CREATE TYPE "Colorizer" AS ( under_color "RgbaColor", -- palette -- (colors --> breakpoints) - default_color "RgbaColor" - -- (no_data_color) + default_color "RgbaColor" -- (no_data_color) -- rgba -- (nothing) ); - CREATE TYPE "ColorParam" AS ( -- static color "RgbaColor", @@ -94,7 +75,6 @@ CREATE TYPE "ColorParam" AS ( attribute text, colorizer "Colorizer" ); - CREATE TYPE "NumberParam" AS ( -- static "value" bigint, @@ -103,43 +83,33 @@ CREATE TYPE "NumberParam" AS ( factor double precision, default_value double precision ); - CREATE TYPE "StrokeParam" AS ( width "NumberParam", color "ColorParam" ); - CREATE TYPE "TextSymbology" AS ( attribute text, fill_color "ColorParam", stroke "StrokeParam" ); - CREATE TYPE "PointSymbology" AS ( radius "NumberParam", fill_color "ColorParam", stroke "StrokeParam", text "TextSymbology" ); - CREATE TYPE "LineSymbology" AS ( stroke "StrokeParam", text "TextSymbology", auto_simplified boolean ); - CREATE TYPE "PolygonSymbology" AS ( fill_color "ColorParam", stroke "StrokeParam", text "TextSymbology", auto_simplified boolean ); - -CREATE TYPE "RasterColorizerType" AS ENUM ( - 'SingleBand', - 'MultiBand' -); - +CREATE TYPE "RasterColorizerType" AS ENUM ('SingleBand', 'MultiBand'); CREATE TYPE "RasterColorizer" AS ( "type" "RasterColorizerType", -- single band colorizer @@ -160,19 +130,16 @@ CREATE TYPE "RasterColorizer" AS ( blue_scale double precision, no_data_color "RgbaColor" ); - CREATE TYPE "RasterSymbology" AS ( opacity double precision, raster_colorizer "RasterColorizer" ); - CREATE TYPE "Symbology" AS ( "raster" "RasterSymbology", "point" "PointSymbology", "line" "LineSymbology", "polygon" "PolygonSymbology" ); - CREATE TYPE "RasterDataType" AS ENUM ( 'U8', 'U16', @@ -185,50 +152,29 @@ CREATE TYPE "RasterDataType" AS ENUM ( 'F32', 'F64' ); - -CREATE TYPE "ContinuousMeasurement" AS ( - measurement text, - unit text -); - -CREATE TYPE "SmallintTextKeyValue" AS ( - key smallint, - value text -); - -CREATE TYPE "TextTextKeyValue" AS ( - key text, - value text -); - +CREATE TYPE "ContinuousMeasurement" AS (measurement text, unit text); +CREATE TYPE "SmallintTextKeyValue" AS (key smallint, value text); +CREATE TYPE "TextTextKeyValue" AS (key text, value text); CREATE TYPE "ClassificationMeasurement" AS ( measurement text, classes "SmallintTextKeyValue" [] ); - CREATE TYPE "Measurement" AS ( -- "unitless" if all none continuous "ContinuousMeasurement", classification "ClassificationMeasurement" ); - CREATE TYPE "SpatialPartition2D" AS ( upper_left_coordinate "Coordinate2D", lower_right_coordinate "Coordinate2D" ); - -CREATE TYPE "SpatialResolution" AS ( - x double precision, - y double precision -); - +CREATE TYPE "SpatialResolution" AS (x double precision, y double precision); CREATE TYPE "VectorDataType" AS ENUM ( 'Data', 'MultiPoint', 'MultiLineString', 'MultiPolygon' ); - CREATE TYPE "FeatureDataType" AS ENUM ( 'Category', 'Int', @@ -237,28 +183,40 @@ CREATE TYPE "FeatureDataType" AS ENUM ( 'Bool', 'DateTime' ); - CREATE TYPE "VectorColumnInfo" AS ( "column" text, data_type "FeatureDataType", measurement "Measurement" ); - -CREATE TYPE "RasterBandDescriptor" AS ( - "name" text, - measurement "Measurement" +CREATE TYPE "RasterBandDescriptor" AS ("name" text, measurement "Measurement"); +CREATE TYPE "GridBoundingBox2D" AS ( + y_min bigint, + y_max bigint, + x_min bigint, + x_max bigint +); +CREATE TYPE "GeoTransform" AS ( + origin_coordinate "Coordinate2D", + x_pixel_size double precision, + y_pixel_size double precision +); +CREATE TYPE "SpatialGridDefinition" AS ( + geo_transform "GeoTransform", + grid_bounds "GridBoundingBox2D" +); +CREATE TYPE "SpatialGridDescriptorState" AS ENUM ('Source', 'Merged'); +CREATE TYPE "SpatialGridDescriptor" AS ( + "state" "SpatialGridDescriptorState", + spatial_grid "SpatialGridDefinition" ); - CREATE TYPE "RasterResultDescriptor" AS ( data_type "RasterDataType", -- SpatialReferenceOption spatial_reference "SpatialReference", "time" "TimeInterval", - bbox "SpatialPartition2D", - resolution "SpatialResolution", - bands "RasterBandDescriptor" [] + bands "RasterBandDescriptor" [], + spatial_grid "SpatialGridDescriptor" ); - CREATE TYPE "VectorResultDescriptor" AS ( data_type "VectorDataType", -- SpatialReferenceOption @@ -267,78 +225,62 @@ CREATE TYPE "VectorResultDescriptor" AS ( "time" "TimeInterval", bbox "BoundingBox2D" ); - CREATE TYPE "PlotResultDescriptor" AS ( -- SpatialReferenceOption spatial_reference "SpatialReference", "time" "TimeInterval", bbox "BoundingBox2D" ); - CREATE TYPE "ResultDescriptor" AS ( -- oneOf raster "RasterResultDescriptor", vector "VectorResultDescriptor", plot "PlotResultDescriptor" ); - -CREATE TYPE "MockDatasetDataSourceLoadingInfo" AS ( - points "Coordinate2D" [] -); - +CREATE TYPE "MockDatasetDataSourceLoadingInfo" AS (points "Coordinate2D" []); CREATE TYPE "DateTimeParseFormat" AS ( fmt text, has_tz boolean, has_time boolean ); - CREATE TYPE "OgrSourceTimeFormatCustom" AS ( custom_format "DateTimeParseFormat" ); - -CREATE TYPE "UnixTimeStampType" AS ENUM ( - 'EpochSeconds', - 'EpochMilliseconds' -); - +CREATE TYPE "UnixTimeStampType" AS ENUM ('EpochSeconds', 'EpochMilliseconds'); CREATE TYPE "OgrSourceTimeFormatUnixTimeStamp" AS ( timestamp_type "UnixTimeStampType", fmt "DateTimeParseFormat" ); - CREATE TYPE "OgrSourceTimeFormat" AS ( -- oneOf -- Auto custom "OgrSourceTimeFormatCustom", unix_time_stamp "OgrSourceTimeFormatUnixTimeStamp" ); - CREATE TYPE "OgrSourceDurationSpec" AS ( -- oneOf - infinite boolean, -- void - zero boolean, -- void + infinite boolean, + -- void + zero boolean, + -- void "value" "TimeStep" ); - CREATE TYPE "OgrSourceDatasetTimeTypeStart" AS ( start_field text, start_format "OgrSourceTimeFormat", duration "OgrSourceDurationSpec" ); - CREATE TYPE "OgrSourceDatasetTimeTypeStartEnd" AS ( start_field text, start_format "OgrSourceTimeFormat", end_field text, end_format "OgrSourceTimeFormat" ); - CREATE TYPE "OgrSourceDatasetTimeTypeStartDuration" AS ( start_field text, start_format "OgrSourceTimeFormat", duration_field text ); - CREATE TYPE "OgrSourceDatasetTimeType" AS ( -- oneOf -- None @@ -346,22 +288,12 @@ CREATE TYPE "OgrSourceDatasetTimeType" AS ( start_end "OgrSourceDatasetTimeTypeStartEnd", start_duration "OgrSourceDatasetTimeTypeStartDuration" ); - -CREATE TYPE "CsvHeader" AS ENUM ( - 'Yes', - 'No', - 'Auto' -); - -CREATE TYPE "FormatSpecificsCsv" AS ( - header "CsvHeader" -); - +CREATE TYPE "CsvHeader" AS ENUM ('Yes', 'No', 'Auto'); +CREATE TYPE "FormatSpecificsCsv" AS (header "CsvHeader"); CREATE TYPE "FormatSpecifics" AS ( -- oneOf csv "FormatSpecificsCsv" ); - CREATE TYPE "OgrSourceColumnSpec" AS ( format_specifics "FormatSpecifics", x text, @@ -373,25 +305,19 @@ CREATE TYPE "OgrSourceColumnSpec" AS ( "datetime" text [], rename "TextTextKeyValue" [] ); - -CREATE TYPE "OgrSourceErrorSpec" AS ENUM ( - 'Ignore', - 'Abort' -); - +CREATE TYPE "OgrSourceErrorSpec" AS ENUM ('Ignore', 'Abort'); -- We store `Polygon`s as an array of rings that are closed postgres `path`s. -- We do not use an array of `polygon`s as it is the same as storing a path -- plus a stored bbox that we don't want to compute and store (overhead). CREATE DOMAIN "Polygon" AS path []; - CREATE TYPE "TypedGeometry" AS ( -- oneOf - "data" boolean, -- void + "data" boolean, + -- void multi_point point [], multi_line_string path [], multi_polygon "Polygon" [] ); - CREATE TYPE "OgrSourceDataset" AS ( file_name text, layer_name text, @@ -406,50 +332,29 @@ CREATE TYPE "OgrSourceDataset" AS ( attribute_query text, cache_ttl int ); - CREATE TYPE "MockMetaData" AS ( loading_info "MockDatasetDataSourceLoadingInfo", result_descriptor "VectorResultDescriptor" ); - CREATE TYPE "OgrMetaData" AS ( loading_info "OgrSourceDataset", result_descriptor "VectorResultDescriptor" ); - CREATE TYPE "GdalDatasetGeoTransform" AS ( origin_coordinate "Coordinate2D", x_pixel_size double precision, y_pixel_size double precision ); - -CREATE TYPE "FileNotFoundHandling" AS ENUM ( - 'NoData', - 'Error' -); - -CREATE TYPE "RasterPropertiesKey" AS ( - domain text, - key text -); - -CREATE TYPE "RasterPropertiesEntryType" AS ENUM ( - 'Number', - 'String' -); - +CREATE TYPE "FileNotFoundHandling" AS ENUM ('NoData', 'Error'); +CREATE TYPE "RasterPropertiesKey" AS (domain text, key text); +CREATE TYPE "RasterPropertiesEntryType" AS ENUM ('Number', 'String'); CREATE TYPE "GdalMetadataMapping" AS ( source_key "RasterPropertiesKey", target_key "RasterPropertiesKey", target_type "RasterPropertiesEntryType" ); - CREATE DOMAIN "StringPair" AS text [2]; - -CREATE TYPE "GdalRetryOptions" AS ( - max_retries bigint -); - +CREATE TYPE "GdalRetryOptions" AS (max_retries bigint); CREATE TYPE "GdalDatasetParameters" AS ( file_path text, rasterband_channel bigint, @@ -464,22 +369,15 @@ CREATE TYPE "GdalDatasetParameters" AS ( allow_alphaband_as_mask boolean, retry "GdalRetryOptions" ); - -CREATE TYPE "TimeReference" AS ENUM ( - 'Start', - 'End' -); - +CREATE TYPE "TimeReference" AS ENUM ('Start', 'End'); CREATE TYPE "GdalSourceTimePlaceholder" AS ( "format" "DateTimeParseFormat", reference "TimeReference" ); - CREATE TYPE "TextGdalSourceTimePlaceholderKeyValue" AS ( "key" text, "value" "GdalSourceTimePlaceholder" ); - CREATE TYPE "GdalMetaDataRegular" AS ( result_descriptor "RasterResultDescriptor", params "GdalDatasetParameters", @@ -488,14 +386,12 @@ CREATE TYPE "GdalMetaDataRegular" AS ( step "TimeStep", cache_ttl int ); - CREATE TYPE "GdalMetaDataStatic" AS ( time "TimeInterval", params "GdalDatasetParameters", result_descriptor "RasterResultDescriptor", cache_ttl int ); - CREATE TYPE "GdalMetadataNetCdfCf" AS ( result_descriptor "RasterResultDescriptor", params "GdalDatasetParameters", @@ -505,18 +401,15 @@ CREATE TYPE "GdalMetadataNetCdfCf" AS ( band_offset bigint, cache_ttl int ); - CREATE TYPE "GdalLoadingInfoTemporalSlice" AS ( time "TimeInterval", params "GdalDatasetParameters", cache_ttl int ); - CREATE TYPE "GdalMetaDataList" AS ( result_descriptor "RasterResultDescriptor", params "GdalLoadingInfoTemporalSlice" [] ); - CREATE TYPE "MetaDataDefinition" AS ( -- oneOf mock_meta_data "MockMetaData", @@ -526,10 +419,8 @@ CREATE TYPE "MetaDataDefinition" AS ( gdal_metadata_net_cdf_cf "GdalMetadataNetCdfCf", gdal_meta_data_list "GdalMetaDataList" ); - -- seperate table for projects used in foreign key constraints CREATE TABLE projects (id uuid PRIMARY KEY); - CREATE TABLE project_versions ( id uuid PRIMARY KEY, project_id uuid REFERENCES projects (id) ON DELETE CASCADE NOT NULL, @@ -537,16 +428,11 @@ CREATE TABLE project_versions ( description text NOT NULL, bounds "STRectangle" NOT NULL, time_step "TimeStep" NOT NULL, - changed timestamp - with time zone NOT NULL + changed timestamp with time zone NOT NULL ); - CREATE INDEX project_version_idx ON project_versions (project_id, changed DESC); - CREATE TYPE "LayerType" AS ENUM ('Raster', 'Vector'); - CREATE TYPE "LayerVisibility" AS (data BOOLEAN, legend BOOLEAN); - CREATE TABLE project_version_layers ( layer_index integer NOT NULL, project_id uuid REFERENCES projects (id) ON DELETE CASCADE NOT NULL, @@ -564,7 +450,6 @@ CREATE TABLE project_version_layers ( layer_index ) ); - CREATE TABLE project_version_plots ( plot_index integer NOT NULL, project_id uuid REFERENCES projects (id) ON DELETE CASCADE NOT NULL, @@ -580,16 +465,12 @@ CREATE TABLE project_version_plots ( plot_index ) ); - CREATE TABLE workflows ( id uuid PRIMARY KEY, workflow json NOT NULL ); - -- TODO: add constraint not null - -- TODO: add length constraints - CREATE TABLE datasets ( id uuid PRIMARY KEY, name "DatasetName" UNIQUE NOT NULL, @@ -602,36 +483,27 @@ CREATE TABLE datasets ( symbology "Symbology", provenance "Provenance" [] ); - -- TODO: add constraint not null - -- TODO: add constaint byte_size >= 0 - CREATE TYPE "FileUpload" AS ( id UUID, name text, byte_size bigint ); - -- TODO: time of creation and last update - -- TODO: upload directory that is not directly derived from id - CREATE TABLE uploads ( id uuid PRIMARY KEY, -- user_id UUID REFERENCES users(id) ON DELETE CASCADE NOT NULL, files "FileUpload" [] NOT NULL ); - CREATE TYPE "PropertyType" AS (key text, value text); - CREATE TABLE layer_collections ( id uuid PRIMARY KEY, name text NOT NULL, description text NOT NULL, properties "PropertyType" [] NOT NULL ); - CREATE TABLE layers ( id uuid PRIMARY KEY, name text NOT NULL, @@ -641,7 +513,6 @@ CREATE TABLE layers ( properties "PropertyType" [] NOT NULL, metadata "TextTextKeyValue" [] NOT NULL ); - CREATE TABLE collection_layers ( collection uuid REFERENCES layer_collections ( id @@ -649,13 +520,11 @@ CREATE TABLE collection_layers ( layer uuid REFERENCES layers (id) ON DELETE CASCADE NOT NULL, PRIMARY KEY (collection, layer) ); - CREATE TABLE collection_children ( parent uuid REFERENCES layer_collections (id) ON DELETE CASCADE NOT NULL, child uuid REFERENCES layer_collections (id) ON DELETE CASCADE NOT NULL, PRIMARY KEY (parent, child) ); - CREATE TYPE "ArunaDataProviderDefinition" AS ( id uuid, "name" text, @@ -667,7 +536,6 @@ CREATE TYPE "ArunaDataProviderDefinition" AS ( description text, priority smallint ); - CREATE TYPE "DatabaseConnectionConfig" AS ( host text, port int, @@ -676,7 +544,6 @@ CREATE TYPE "DatabaseConnectionConfig" AS ( "user" text, "password" text ); - CREATE TYPE "GbifDataProviderDefinition" AS ( "name" text, db_config "DatabaseConnectionConfig", @@ -686,7 +553,6 @@ CREATE TYPE "GbifDataProviderDefinition" AS ( priority smallint, columns text [] ); - CREATE TYPE "GfbioAbcdDataProviderDefinition" AS ( "name" text, db_config "DatabaseConnectionConfig", @@ -694,7 +560,6 @@ CREATE TYPE "GfbioAbcdDataProviderDefinition" AS ( description text, priority smallint ); - CREATE TYPE "GfbioCollectionsDataProviderDefinition" AS ( "name" text, collection_api_url text, @@ -705,7 +570,6 @@ CREATE TYPE "GfbioCollectionsDataProviderDefinition" AS ( description text, priority smallint ); - CREATE TYPE "EbvPortalDataProviderDefinition" AS ( "name" text, "data" text, @@ -715,7 +579,6 @@ CREATE TYPE "EbvPortalDataProviderDefinition" AS ( description text, priority smallint ); - CREATE TYPE "NetCdfCfDataProviderDefinition" AS ( "name" text, "data" text, @@ -724,7 +587,6 @@ CREATE TYPE "NetCdfCfDataProviderDefinition" AS ( description text, priority smallint ); - CREATE TYPE "PangaeaDataProviderDefinition" AS ( "name" text, base_url text, @@ -732,13 +594,7 @@ CREATE TYPE "PangaeaDataProviderDefinition" AS ( description text, priority smallint ); - -CREATE TYPE "EdrVectorSpec" AS ( - x text, - y text, - "time" text -); - +CREATE TYPE "EdrVectorSpec" AS (x text, y text, "time" text); CREATE TYPE "EdrDataProviderDefinition" AS ( "name" text, id uuid, @@ -750,13 +606,11 @@ CREATE TYPE "EdrDataProviderDefinition" AS ( description text, priority smallint ); - CREATE TYPE "DatasetLayerListingCollection" AS ( "name" text, description text, tags text [] ); - CREATE TYPE "DatasetLayerListingProviderDefinition" AS ( id uuid, "name" text, @@ -764,39 +618,20 @@ CREATE TYPE "DatasetLayerListingProviderDefinition" AS ( collections "DatasetLayerListingCollection" [], priority smallint ); - -CREATE TYPE "StacBand" AS ( - "name" text, - no_data_value double precision, - data_type "RasterDataType" -); - -CREATE TYPE "StacZone" AS ( - "name" text, - epsg oid -); - CREATE TYPE "StacApiRetries" AS ( number_of_retries bigint, initial_delay_ms bigint, exponential_backoff_factor double precision ); - -CREATE TYPE "GdalRetries" AS ( - number_of_retries bigint -); - +CREATE TYPE "GdalRetries" AS (number_of_retries bigint); CREATE TYPE "StacQueryBuffer" AS ( start_seconds bigint, end_seconds bigint ); - CREATE TYPE "SentinelS2L2ACogsProviderDefinition" AS ( "name" text, id uuid, api_url text, - bands "StacBand" [], - zones "StacZone" [], stac_api_retries "StacApiRetries", gdal_retries "GdalRetries", cache_ttl int, @@ -804,7 +639,6 @@ CREATE TYPE "SentinelS2L2ACogsProviderDefinition" AS ( priority smallint, query_buffer "StacQueryBuffer" ); - CREATE TYPE "CopernicusDataspaceDataProviderDefinition" AS ( "name" text, id uuid, @@ -816,7 +650,6 @@ CREATE TYPE "CopernicusDataspaceDataProviderDefinition" AS ( priority smallint, gdal_config "StringPair" [] ); - CREATE TYPE "DataProviderDefinition" AS ( -- one of aruna_data_provider_definition "ArunaDataProviderDefinition", @@ -835,7 +668,6 @@ CREATE TYPE "DataProviderDefinition" AS ( copernicus_dataspace_provider_definition "CopernicusDataspaceDataProviderDefinition" ); - CREATE TABLE layer_providers ( id uuid PRIMARY KEY, type_name text NOT NULL, @@ -843,19 +675,14 @@ CREATE TABLE layer_providers ( definition "DataProviderDefinition" NOT NULL, priority smallint NOT NULL DEFAULT 0 ); - -- TODO: relationship between uploads and datasets? - -- EBV PROVIDER TABLE DEFINITIONS - CREATE TABLE ebv_provider_dataset_locks ( provider_id uuid NOT NULL, file_name text NOT NULL, - -- TODO: check if we need it PRIMARY KEY (provider_id, file_name) ); - CREATE TABLE ebv_provider_overviews ( provider_id uuid NOT NULL, file_name text NOT NULL, @@ -866,11 +693,9 @@ CREATE TABLE ebv_provider_overviews ( creator_name text, creator_email text, creator_institution text, - -- TODO: check if we need it PRIMARY KEY (provider_id, file_name) ); - CREATE TABLE ebv_provider_groups ( provider_id uuid NOT NULL, file_name text NOT NULL, @@ -880,71 +705,54 @@ CREATE TABLE ebv_provider_groups ( data_type "RasterDataType", data_range float [2], unit text NOT NULL, - -- TODO: check if we need it PRIMARY KEY (provider_id, file_name, name) DEFERRABLE, - FOREIGN KEY (provider_id, file_name) REFERENCES ebv_provider_overviews ( - provider_id, - file_name + provider_id, file_name ) ON DELETE CASCADE DEFERRABLE ); - CREATE TABLE ebv_provider_entities ( provider_id uuid NOT NULL, file_name text NOT NULL, id bigint NOT NULL, name text NOT NULL, - -- TODO: check if we need it PRIMARY KEY (provider_id, file_name, id) DEFERRABLE, - FOREIGN KEY (provider_id, file_name) REFERENCES ebv_provider_overviews ( - provider_id, - file_name + provider_id, file_name ) ON DELETE CASCADE DEFERRABLE ); - CREATE TABLE ebv_provider_timestamps ( provider_id uuid NOT NULL, file_name text NOT NULL, time bigint NOT NULL, - -- TODO: check if we need it PRIMARY KEY (provider_id, file_name, time) DEFERRABLE, - FOREIGN KEY (provider_id, file_name) REFERENCES ebv_provider_overviews ( - provider_id, - file_name + provider_id, file_name ) ON DELETE CASCADE DEFERRABLE ); - CREATE TABLE ebv_provider_loading_infos ( provider_id uuid NOT NULL, file_name text NOT NULL, group_names text [] NOT NULL, entity_id bigint NOT NULL, meta_data "GdalMetaDataList" NOT NULL, - -- TODO: check if we need it PRIMARY KEY (provider_id, file_name, group_names, entity_id) DEFERRABLE, - FOREIGN KEY (provider_id, file_name) REFERENCES ebv_provider_overviews ( - provider_id, - file_name + provider_id, file_name ) ON DELETE CASCADE DEFERRABLE ); - CREATE TYPE "MlModelMetadata" AS ( file_name text, input_type "RasterDataType", num_input_bands OID, output_type "RasterDataType" ); - CREATE TYPE "MlModelName" AS (namespace text, name text); - -CREATE TABLE ml_models ( -- noqa: +CREATE TABLE ml_models ( + -- noqa: id uuid PRIMARY KEY, name "MlModelName" UNIQUE NOT NULL, display_name text NOT NULL, @@ -952,18 +760,14 @@ CREATE TABLE ml_models ( -- noqa: upload uuid REFERENCES uploads (id) ON DELETE CASCADE NOT NULL, metadata "MlModelMetadata" ); - -- TODO: distinguish between roles that are (correspond to) users -- and roles that are not - -- TODO: integrity constraint for roles that correspond to users -- + DELETE CASCADE - CREATE TABLE roles ( id uuid PRIMARY KEY, name text UNIQUE NOT NULL ); - CREATE TABLE users ( id uuid PRIMARY KEY REFERENCES roles (id), email character varying(256) UNIQUE, @@ -973,30 +777,27 @@ CREATE TABLE users ( quota_available bigint NOT NULL DEFAULT 0, quota_used bigint NOT NULL DEFAULT 0, -- TODO: rename to total_quota_used? - CONSTRAINT users_anonymous_ck CHECK (( - email IS NULL - AND password_hash IS NULL - AND real_name IS NULL - ) - OR ( - email IS NOT NULL - AND password_hash IS NOT NULL - AND real_name IS NOT NULL - ) + CONSTRAINT users_anonymous_ck CHECK ( + ( + email IS NULL + AND password_hash IS NULL + AND real_name IS NULL + ) + OR ( + email IS NOT NULL + AND password_hash IS NOT NULL + AND real_name IS NOT NULL + ) ), CONSTRAINT users_quota_used_ck CHECK (quota_used >= 0) ); - -- relation between users and roles - -- all users have a default role where role_id = user_id - CREATE TABLE user_roles ( user_id uuid REFERENCES users (id) ON DELETE CASCADE NOT NULL, role_id uuid REFERENCES roles (id) ON DELETE CASCADE NOT NULL, PRIMARY KEY (user_id, role_id) ); - CREATE TABLE project_version_authors ( project_version_id uuid REFERENCES project_versions ( id @@ -1004,28 +805,23 @@ CREATE TABLE project_version_authors ( user_id uuid REFERENCES users (id) ON DELETE CASCADE NOT NULL, PRIMARY KEY (project_version_id, user_id) ); - CREATE TABLE user_uploads ( user_id uuid REFERENCES users (id) ON DELETE CASCADE NOT NULL, upload_id uuid REFERENCES uploads (id) ON DELETE CASCADE NOT NULL, PRIMARY KEY (user_id, upload_id) ); - CREATE TABLE sessions ( id uuid PRIMARY KEY, - project_id uuid REFERENCES projects (id) ON DELETE SET NULL, + project_id uuid REFERENCES projects (id) ON DELETE + SET NULL, view "STRectangle", user_id uuid REFERENCES users (id) ON DELETE CASCADE NOT NULL, created timestamp with time zone NOT NULL, valid_until timestamp with time zone NOT NULL ); - CREATE TYPE "Permission" AS ENUM ('Read', 'Owner'); - -- TODO: uploads, providers permissions - -- TODO: relationship between uploads and datasets? - CREATE TABLE external_users ( id uuid PRIMARY KEY REFERENCES users (id), external_id character varying(256) UNIQUE, @@ -1033,7 +829,6 @@ CREATE TABLE external_users ( real_name character varying(256), active boolean NOT NULL ); - CREATE TABLE permissions ( -- resource_type "ResourceType" NOT NULL, role_id uuid REFERENCES roles (id) ON DELETE CASCADE NOT NULL, @@ -1055,77 +850,55 @@ CREATE TABLE permissions ( ) = 1 ) ); - -CREATE UNIQUE INDEX ON permissions ( - role_id, - permission, - dataset_id -); - +CREATE UNIQUE INDEX ON permissions (role_id, permission, dataset_id); CREATE UNIQUE INDEX ON permissions (role_id, permission, layer_id); - CREATE UNIQUE INDEX ON permissions ( role_id, permission, layer_collection_id ); - -CREATE UNIQUE INDEX ON permissions ( - role_id, - permission, - project_id -); - -CREATE UNIQUE INDEX ON permissions ( - role_id, - permission, - ml_model_id -); - -CREATE VIEW user_permitted_datasets -AS +CREATE UNIQUE INDEX ON permissions (role_id, permission, project_id); +CREATE UNIQUE INDEX ON permissions (role_id, permission, ml_model_id); +CREATE VIEW user_permitted_datasets AS SELECT r.user_id, p.dataset_id, p.permission FROM user_roles AS r INNER JOIN permissions AS p ON ( - r.role_id = p.role_id AND p.dataset_id IS NOT NULL + r.role_id = p.role_id + AND p.dataset_id IS NOT NULL ); - -CREATE VIEW user_permitted_projects -AS +CREATE VIEW user_permitted_projects AS SELECT r.user_id, p.project_id, p.permission FROM user_roles AS r INNER JOIN permissions AS p ON ( - r.role_id = p.role_id AND p.project_id IS NOT NULL + r.role_id = p.role_id + AND p.project_id IS NOT NULL ); - -CREATE VIEW user_permitted_layer_collections -AS +CREATE VIEW user_permitted_layer_collections AS SELECT r.user_id, p.layer_collection_id, p.permission FROM user_roles AS r INNER JOIN permissions AS p ON ( - r.role_id = p.role_id AND p.layer_collection_id IS NOT NULL + r.role_id = p.role_id + AND p.layer_collection_id IS NOT NULL ); - -CREATE VIEW user_permitted_layers -AS +CREATE VIEW user_permitted_layers AS SELECT r.user_id, p.layer_id, p.permission FROM user_roles AS r INNER JOIN permissions AS p ON ( - r.role_id = p.role_id AND p.layer_id IS NOT NULL + r.role_id = p.role_id + AND p.layer_id IS NOT NULL ); - CREATE TABLE oidc_session_tokens ( session_id uuid PRIMARY KEY REFERENCES sessions ( id @@ -1136,18 +909,16 @@ CREATE TABLE oidc_session_tokens ( refresh_token bytea, refresh_token_encryption_nonce bytea ); - -CREATE VIEW user_permitted_ml_models -AS +CREATE VIEW user_permitted_ml_models AS SELECT r.user_id, p.ml_model_id, p.permission FROM user_roles AS r INNER JOIN permissions AS p ON ( - r.role_id = p.role_id AND p.ml_model_id IS NOT NULL + r.role_id = p.role_id + AND p.ml_model_id IS NOT NULL ); - CREATE TABLE quota_log ( timestamp timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, user_id uuid NOT NULL, @@ -1157,5 +928,4 @@ CREATE TABLE quota_log ( operator_path text NOT NULL, data text ); - CREATE INDEX ON quota_log (user_id, timestamp, computation_id); diff --git a/services/src/contexts/migrations/migration_0017_raster_result_desc.rs b/services/src/contexts/migrations/migration_0017_raster_result_desc.rs new file mode 100644 index 000000000..cb98a6492 --- /dev/null +++ b/services/src/contexts/migrations/migration_0017_raster_result_desc.rs @@ -0,0 +1,30 @@ +use super::{ + Migration0016MergeProviders, + database_migration::{DatabaseVersion, Migration}, +}; +use crate::error::Result; +use async_trait::async_trait; +use tokio_postgres::Transaction; + +/// This migration reworks the raster result descritptor and some other small changes from the rewrite branch +pub struct Migration0017RasterResultDesc; + +#[async_trait] +impl Migration for Migration0017RasterResultDesc { + fn prev_version(&self) -> Option { + Some(Migration0016MergeProviders.version()) + } + + fn version(&self) -> DatabaseVersion { + "0017_raster_result_desc".into() + } + + async fn migrate(&self, tx: &Transaction<'_>) -> Result<()> { + tx.batch_execute(include_str!("migration_0017_remove_stack_zone_band.sql")) + .await?; + + tx.batch_execute(include_str!("migration_0017_raster_result_desc.sql")) + .await?; + Ok(()) + } +} diff --git a/services/src/contexts/migrations/migration_0017_raster_result_desc.sql b/services/src/contexts/migrations/migration_0017_raster_result_desc.sql new file mode 100644 index 000000000..4c7ec51f9 --- /dev/null +++ b/services/src/contexts/migrations/migration_0017_raster_result_desc.sql @@ -0,0 +1,183 @@ +-- add the new types +CREATE TYPE "GridBoundingBox2D" AS ( + y_min bigint, + y_max bigint, + x_min bigint, + x_max bigint +); +CREATE TYPE "GeoTransform" AS ( + origin_coordinate "Coordinate2D", + x_pixel_size double precision, + y_pixel_size double precision +); +CREATE TYPE "SpatialGridDefinition" AS ( + geo_transform "GeoTransform", + grid_bounds "GridBoundingBox2D" +); +CREATE TYPE "SpatialGridDescriptorState" AS ENUM ('Source', 'Merged'); +CREATE TYPE "SpatialGridDescriptor" AS ( + "state" "SpatialGridDescriptorState", + spatial_grid "SpatialGridDefinition" +); +-- adapt the RasterResultDescriptor --> add the new attribute +ALTER TYPE "RasterResultDescriptor" +ADD ATTRIBUTE spatial_grid "SpatialGridDescriptor"; +-- migrate gdal_static metadata +WITH cte AS ( + SELECT + id, + (meta_data).gdal_static AS meta + FROM datasets + WHERE (meta_data).gdal_static IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.spatial_grid = ( + 'Source', + ( + ( + (cte).meta.params.geo_transform.origin_coordinate, + (cte).meta.params.geo_transform.x_pixel_size, + (cte).meta.params.geo_transform.x_pixel_size + )::"GeoTransform", + ( + 0, + (cte).meta.params.height - 1, + 0, + (cte).meta.params.width - 1 + )::"GridBoundingBox2D" + )::"SpatialGridDefinition" + )::"SpatialGridDescriptor" +FROM cte +WHERE datasets.id = cte.id; +-- migrate gdal_regular metadata +WITH cte AS ( + SELECT + id, + (meta_data).gdal_meta_data_regular AS meta + FROM datasets + WHERE (meta_data).gdal_meta_data_regular IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.spatial_grid = ( + 'Source', + ( + ( + (cte).meta.params.geo_transform.origin_coordinate, + (cte).meta.params.geo_transform.x_pixel_size, + (cte).meta.params.geo_transform.x_pixel_size + )::"GeoTransform", + ( + 0, + (cte).meta.params.height - 1, + 0, + (cte).meta.params.width - 1 + )::"GridBoundingBox2D" + )::"SpatialGridDefinition" + )::"SpatialGridDescriptor" +FROM cte +WHERE datasets.id = cte.id; +-- migrate gdal_metadata_net_cdf_cf +WITH cte AS ( + SELECT + id, + (meta_data).gdal_metadata_net_cdf_cf AS meta + FROM datasets + WHERE (meta_data).gdal_metadata_net_cdf_cf IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.spatial_grid = ( + 'Source', + ( + ( + (cte).meta.params.geo_transform.origin_coordinate, + (cte).meta.params.geo_transform.x_pixel_size, + (cte).meta.params.geo_transform.x_pixel_size + )::"GeoTransform", + ( + 0, + (cte).meta.params.height - 1, + 0, + (cte).meta.params.width - 1 + )::"GridBoundingBox2D" + )::"SpatialGridDefinition" + )::"SpatialGridDescriptor" +FROM cte +WHERE datasets.id = cte.id; +-- migrate gdal_metadata_lsit +CREATE FUNCTION pg_temp.spatial_grid_def_from_params_array( + t_slices "GdalLoadingInfoTemporalSlice" [] +) RETURNS "SpatialGridDefinition" AS $$ +DECLARE b_size_x double precision; +b_size_y double precision; +b_ul_x double precision; +b_ul_y double precision; +b_lr_x double precision; +b_lr_y double precision; +t_x double precision; +t_y double precision; +n "SpatialGridDefinition"; +t "GdalLoadingInfoTemporalSlice"; +BEGIN FOREACH t IN ARRAY t_slices LOOP IF t.params IS NULL THEN CONTINUE; +END IF; +t_x := (t).params.geo_transform.origin_coordinate.x + (t).params.geo_transform.x_pixel_size * (t).params.width; +t_y := (t).params.geo_transform.origin_coordinate.y + (t).params.geo_transform.y_pixel_size * (t).params.height; +IF b_size_x IS NULL THEN b_size_x := (t).params.geo_transform.x_pixel_size; +b_size_y := (t).params.geo_transform.y_pixel_size; +b_ul_x := (t).params.geo_transform.origin_coordinate.x; +b_ul_y := (t).params.geo_transform.origin_coordinate.y; +b_lr_x := t_x; +b_lr_y := t_y; +END IF; +b_ul_x := LEAST( + b_ul_x, + (t).params.geo_transform.origin_coordinate.x +); +b_ul_y := GREATEST( + b_ul_y, + (t).params.geo_transform.origin_coordinate.y +); +b_lr_x := GREATEST(b_lr_x, t_x); +b_lr_y := LEAST(b_lr_y, t_y); +END LOOP; +RETURN ( + ( + (b_ul_x, b_ul_y)::"Coordinate2D", + b_size_x, + b_size_y + )::"GeoTransform", + ( + 0, + ((b_ul_y - b_lr_y) / b_size_y) -1, + 0, + ((b_lr_x - b_ul_x) / b_size_x) -1 + )::"GridBoundingBox2D" +)::"SpatialGridDefinition"; +END; +$$ LANGUAGE plpgsql; +WITH cte AS ( + SELECT + id, + (meta_data).gdal_meta_data_list AS meta + FROM datasets + WHERE (meta_data).gdal_meta_data_list IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.spatial_grid = ( + 'Source', + pg_temp.spatial_grid_def_from_params_array(((cte).meta.params)) + )::"SpatialGridDescriptor" +FROM cte +WHERE datasets.id = cte.id; +-- remove the old attributes +ALTER TYPE "RasterResultDescriptor" DROP ATTRIBUTE bbox; +ALTER TYPE "RasterResultDescriptor" DROP ATTRIBUTE resolution; +-- mark the spatial_grid as NOT NULL +DROP FUNCTION IF EXISTS pg_temp.spatial_grid_def_from_params_array; diff --git a/services/src/contexts/migrations/migration_0017_remove_stack_zone_band.sql b/services/src/contexts/migrations/migration_0017_remove_stack_zone_band.sql new file mode 100644 index 000000000..097a85f67 --- /dev/null +++ b/services/src/contexts/migrations/migration_0017_remove_stack_zone_band.sql @@ -0,0 +1,4 @@ +ALTER TYPE "SentinelS2L2ACogsProviderDefinition" DROP ATTRIBUTE bands; +ALTER TYPE "SentinelS2L2ACogsProviderDefinition" DROP ATTRIBUTE zones; +DROP TYPE "StacBand"; +DROP TYPE "StacZone"; diff --git a/services/src/contexts/migrations/mod.rs b/services/src/contexts/migrations/mod.rs index ff42bad3f..cf34c0a14 100644 --- a/services/src/contexts/migrations/mod.rs +++ b/services/src/contexts/migrations/mod.rs @@ -10,10 +10,12 @@ mod current_schema; mod database_migration; mod migration_0015_log_quota; mod migration_0016_merge_providers; +mod migration_0017_raster_result_desc; #[cfg(test)] mod schema_info; +use migration_0017_raster_result_desc::Migration0017RasterResultDesc; #[cfg(test)] pub(crate) use schema_info::{AssertSchemaEqPopulationConfig, assert_migration_schema_eq}; @@ -25,6 +27,7 @@ pub fn all_migrations() -> Vec> { vec![ Box::new(Migration0015LogQuota), // cf. [`migration_0015_log_quota.rs`] why we start at `0015` Box::new(Migration0016MergeProviders), + Box::new(Migration0017RasterResultDesc), ] } diff --git a/services/src/contexts/mod.rs b/services/src/contexts/mod.rs index 113c68ec9..6e1d39343 100644 --- a/services/src/contexts/mod.rs +++ b/services/src/contexts/mod.rs @@ -112,6 +112,7 @@ pub trait GeoEngineDb: pub struct QueryContextImpl { chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, thread_pool: Arc, cache: Option>, quota_tracking: Option, @@ -121,10 +122,15 @@ pub struct QueryContextImpl { } impl QueryContextImpl { - pub fn new(chunk_byte_size: ChunkByteSize, thread_pool: Arc) -> Self { + pub fn new( + chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, + thread_pool: Arc, + ) -> Self { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); QueryContextImpl { chunk_byte_size, + tiling_specification, thread_pool, cache: None, quota_tracking: None, @@ -136,6 +142,7 @@ impl QueryContextImpl { pub fn new_with_extensions( chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, thread_pool: Arc, cache: Option>, quota_tracking: Option, @@ -144,6 +151,7 @@ impl QueryContextImpl { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); QueryContextImpl { chunk_byte_size, + tiling_specification, thread_pool, cache, quota_checker, @@ -173,6 +181,10 @@ impl QueryContext for QueryContextImpl { .ok_or(geoengine_operators::error::Error::AbortTriggerAlreadyUsed) } + fn tiling_specification(&self) -> TilingSpecification { + self.tiling_specification + } + fn quota_tracking(&self) -> Option<&geoengine_operators::meta::quota::QuotaTracking> { self.quota_tracking.as_ref() } diff --git a/services/src/contexts/postgres.rs b/services/src/contexts/postgres.rs index 037cab9c5..7ff5d086a 100644 --- a/services/src/contexts/postgres.rs +++ b/services/src/contexts/postgres.rs @@ -354,6 +354,7 @@ where Ok(QueryContextImpl::new_with_extensions( self.context.query_ctx_chunk_size, + self.context.exe_ctx_tiling_spec, self.context.thread_pool.clone(), Some(self.context.tile_cache.clone()), Some( @@ -462,69 +463,82 @@ where #[cfg(test)] mod tests { use super::*; - use crate::api::model::datatypes::RasterDataType as ApiRasterDataType; - use crate::config::QuotaTrackingMode; - use crate::datasets::external::netcdfcf::NetCdfCfDataProviderDefinition; - use crate::datasets::listing::{DatasetListOptions, DatasetListing, ProvenanceOutput}; - use crate::datasets::listing::{DatasetProvider, Provenance}; - use crate::datasets::storage::{DatasetStore, MetaDataDefinition}; - use crate::datasets::upload::{FileId, UploadId}; - use crate::datasets::upload::{FileUpload, Upload, UploadDb}; - use crate::datasets::{AddDataset, DatasetIdAndName}; - use crate::ge_context; - use crate::layers::add_from_directory::UNSORTED_COLLECTION_ID; - use crate::layers::layer::{ - AddLayer, AddLayerCollection, CollectionItem, LayerCollection, LayerCollectionListOptions, - LayerCollectionListing, LayerListing, ProviderLayerCollectionId, ProviderLayerId, - }; - use crate::layers::listing::{ - LayerCollectionId, LayerCollectionProvider, SearchParameters, SearchType, - }; - use crate::layers::storage::{ - INTERNAL_PROVIDER_ID, LayerDb, LayerProviderDb, LayerProviderListing, - LayerProviderListingOptions, - }; - use crate::machine_learning::{MlModel, MlModelDb, MlModelIdAndName, MlModelMetadata}; - use crate::permissions::{Permission, PermissionDb, Role, RoleDescription, RoleId}; - use crate::projects::{ - CreateProject, LayerUpdate, LoadVersion, OrderBy, Plot, PlotUpdate, PointSymbology, - ProjectDb, ProjectId, ProjectLayer, ProjectListOptions, ProjectListing, STRectangle, - UpdateProject, + use crate::{ + api::model::datatypes::RasterDataType as ApiRasterDataType, + config::QuotaTrackingMode, + datasets::{ + AddDataset, DatasetIdAndName, + external::netcdfcf::NetCdfCfDataProviderDefinition, + listing::{ + DatasetListOptions, DatasetListing, DatasetProvider, Provenance, ProvenanceOutput, + }, + storage::{DatasetStore, MetaDataDefinition}, + upload::{FileId, FileUpload, Upload, UploadDb, UploadId}, + }, + ge_context, + layers::{ + add_from_directory::UNSORTED_COLLECTION_ID, + layer::{ + AddLayer, AddLayerCollection, CollectionItem, LayerCollection, + LayerCollectionListOptions, LayerCollectionListing, LayerListing, + ProviderLayerCollectionId, ProviderLayerId, + }, + listing::{LayerCollectionId, LayerCollectionProvider, SearchParameters, SearchType}, + storage::{ + INTERNAL_PROVIDER_ID, LayerDb, LayerProviderDb, LayerProviderListing, + LayerProviderListingOptions, + }, + }, + machine_learning::{MlModel, MlModelDb, MlModelIdAndName, MlModelMetadata}, + permissions::{Permission, PermissionDb, Role, RoleDescription, RoleId}, + projects::{ + CreateProject, LayerUpdate, LoadVersion, OrderBy, Plot, PlotUpdate, PointSymbology, + ProjectDb, ProjectId, ProjectLayer, ProjectListOptions, ProjectListing, STRectangle, + UpdateProject, + }, + users::{ + OidcTokens, RoleDb, SessionTokenStore, UserClaims, UserCredentials, UserDb, UserId, + UserRegistration, + }, + util::tests::{ + MockQuotaTracking, admin_login, + mock_oidc::{MockRefreshServerConfig, mock_refresh_server}, + register_ndvi_workflow_helper, + }, + workflows::{registry::WorkflowRegistry, workflow::Workflow}, }; - use crate::users::{OidcTokens, SessionTokenStore}; - use crate::users::{RoleDb, UserClaims, UserCredentials, UserDb, UserId, UserRegistration}; - use crate::util::tests::mock_oidc::{MockRefreshServerConfig, mock_refresh_server}; - use crate::util::tests::{MockQuotaTracking, admin_login, register_ndvi_workflow_helper}; - use crate::workflows::registry::WorkflowRegistry; - use crate::workflows::workflow::Workflow; use bb8_postgres::tokio_postgres::NoTls; use futures::join; - use geoengine_datatypes::collections::VectorDataType; - use geoengine_datatypes::dataset::{DataProviderId, LayerId}; - use geoengine_datatypes::primitives::{ - BoundingBox2D, Coordinate2D, DateTime, Duration, FeatureDataType, Measurement, - RasterQueryRectangle, SpatialResolution, TimeGranularity, TimeInstance, TimeInterval, - TimeStep, VectorQueryRectangle, + use geoengine_datatypes::{ + collections::VectorDataType, + dataset::{DataProviderId, LayerId}, + primitives::{ + BoundingBox2D, CacheTtlSeconds, ColumnSelection, Coordinate2D, DateTime, Duration, + FeatureDataType, Measurement, RasterQueryRectangle, TimeGranularity, TimeInstance, + TimeInterval, TimeStep, VectorQueryRectangle, + }, + raster::{GeoTransform, GridBoundingBox2D, RasterDataType}, + spatial_reference::{SpatialReference, SpatialReferenceOption}, + test_data, + util::Identifier, }; - use geoengine_datatypes::primitives::{CacheTtlSeconds, ColumnSelection}; - use geoengine_datatypes::raster::RasterDataType; - use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceOption}; - use geoengine_datatypes::test_data; - use geoengine_datatypes::util::Identifier; - use geoengine_operators::engine::{ - MetaData, MetaDataProvider, MultipleRasterOrSingleVectorSource, PlotOperator, - RasterBandDescriptors, RasterResultDescriptor, StaticMetaData, TypedOperator, - TypedResultDescriptor, VectorColumnInfo, VectorOperator, VectorResultDescriptor, + use geoengine_operators::{ + engine::{ + MetaData, MetaDataProvider, MultipleRasterOrSingleVectorSource, PlotOperator, + RasterBandDescriptors, RasterResultDescriptor, StaticMetaData, TypedOperator, + TypedResultDescriptor, VectorColumnInfo, VectorOperator, VectorResultDescriptor, + }, + mock::{MockPointSource, MockPointSourceParams}, + plot::{Statistics, StatisticsParams}, + source::{ + CsvHeader, FileNotFoundHandling, FormatSpecifics, GdalDatasetGeoTransform, + GdalDatasetParameters, GdalLoadingInfo, GdalMetaDataList, GdalMetaDataRegular, + GdalMetaDataStatic, GdalMetadataNetCdfCf, OgrSourceColumnSpec, OgrSourceDataset, + OgrSourceDatasetTimeType, OgrSourceDurationSpec, OgrSourceErrorSpec, + OgrSourceTimeFormat, + }, + util::input::MultiRasterOrVectorOperator::Raster, }; - use geoengine_operators::mock::{MockPointSource, MockPointSourceParams}; - use geoengine_operators::plot::{Statistics, StatisticsParams}; - use geoengine_operators::source::{ - CsvHeader, FileNotFoundHandling, FormatSpecifics, GdalDatasetGeoTransform, - GdalDatasetParameters, GdalLoadingInfo, GdalMetaDataList, GdalMetaDataRegular, - GdalMetaDataStatic, GdalMetadataNetCdfCf, OgrSourceColumnSpec, OgrSourceDataset, - OgrSourceDatasetTimeType, OgrSourceDurationSpec, OgrSourceErrorSpec, OgrSourceTimeFormat, - }; - use geoengine_operators::util::input::MultiRasterOrVectorOperator::Raster; use httptest::Server; use oauth2::{AccessToken, RefreshToken}; use openidconnect::SubjectIdentifier; @@ -717,9 +731,7 @@ mod tests { .register_workflow(Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -963,9 +975,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -984,7 +994,7 @@ mod tests { let json = serde_json::to_string(&workflow).unwrap(); assert_eq!( json, - r#"{"type":"Vector","operator":{"type":"MockPointSource","params":{"points":[{"x":1.0,"y":2.0},{"x":1.0,"y":2.0},{"x":1.0,"y":2.0}]}}}"# + r#"{"type":"Vector","operator":{"type":"MockPointSource","params":{"points":[{"x":1.0,"y":2.0},{"x":1.0,"y":2.0},{"x":1.0,"y":2.0}],"spatialBounds":{"type":"none"}}}}"# ); } @@ -1097,6 +1107,7 @@ mod tests { source_operator: "OgrSource".to_owned(), symbology: None, tags: vec!["upload".to_owned(), "test".to_owned()], + // create a TypedResultDescriptor object then concert it to the API model result_descriptor: TypedResultDescriptor::Vector(VectorResultDescriptor { data_type: VectorDataType::MultiPoint, spatial_reference: SpatialReference::epsg_4326().into(), @@ -1112,6 +1123,7 @@ mod tests { time: None, bbox: None, }) + .into() }, ); @@ -1134,15 +1146,11 @@ mod tests { assert_eq!( meta_data - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into() - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all() - }) + .loading_info(VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all() + )) .await .unwrap(), loading_info @@ -1554,8 +1562,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReferenceOption::Unreferenced, time: None, - bbox: None, - resolution: None, + spatial_grid: geoengine_operators::engine::SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }; @@ -1747,9 +1757,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -1947,9 +1955,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -2307,6 +2313,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -2827,9 +2834,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -3011,6 +3016,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -3545,9 +3551,7 @@ mod tests { workflow: Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -3743,9 +3747,10 @@ mod tests { workflow: Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![ + Coordinate2D::new(1., 2.); + 3 + ]), } .boxed(), ), @@ -3812,9 +3817,10 @@ mod tests { workflow: Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![ + Coordinate2D::new(1., 2.); + 3 + ]), } .boxed(), ), @@ -3836,9 +3842,10 @@ mod tests { workflow: Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![ + Coordinate2D::new(1., 2.); + 3 + ]), } .boxed(), ), diff --git a/services/src/datasets/create_from_workflow.rs b/services/src/datasets/create_from_workflow.rs index 847e42a30..f912b9e0c 100644 --- a/services/src/datasets/create_from_workflow.rs +++ b/services/src/datasets/create_from_workflow.rs @@ -1,19 +1,20 @@ -use crate::api::model::datatypes::RasterQueryRectangle; +use crate::api::model::datatypes::RasterToDatasetQueryRectangle; +use crate::api::model::services::AddDataset; use crate::contexts::SessionContext; -use crate::datasets::AddDataset; use crate::datasets::listing::DatasetProvider; use crate::datasets::storage::{DatasetDefinition, DatasetStore, MetaDataDefinition}; use crate::datasets::upload::{UploadId, UploadRootPath}; use crate::error; use crate::tasks::{Task, TaskId, TaskManager, TaskStatusInfo}; -use crate::workflows::workflow::{Workflow, WorkflowId}; +use crate::workflows::workflow::WorkflowId; use geoengine_datatypes::error::ErrorSource; -use geoengine_datatypes::primitives::TimeInterval; +use geoengine_datatypes::primitives::{BandSelection, TimeInterval}; +use geoengine_datatypes::raster::TilingSpecification; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::Identifier; use geoengine_operators::call_on_generic_raster_processor_gdal_types; use geoengine_operators::engine::{ - ExecutionContext, InitializedRasterOperator, RasterResultDescriptor, WorkflowOperatorPath, + ExecutionContext, InitializedRasterOperator, RasterResultDescriptor, }; use geoengine_operators::source::{ GdalLoadingInfoTemporalSlice, GdalMetaDataList, GdalMetaDataStatic, @@ -34,18 +35,57 @@ use super::{DatasetIdAndName, DatasetName}; /// parameter for the dataset from workflow handler (body) #[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] -#[schema(example = json!({"name": "foo", "displayName": "a new dataset", "description": null, "query": {"spatialBounds": {"upperLeftCoordinate": {"x": -10.0, "y": 80.0}, "lowerRightCoordinate": {"x": 50.0, "y": 20.0}}, "timeInterval": {"start": 1_388_534_400_000_i64, "end": 1_388_534_401_000_i64}, "spatialResolution": {"x": 0.1, "y": 0.1}}}))] +#[schema(example = json!({"name": "foo", "displayName": "a new dataset", "description": null, "query": {"spatialBounds": {"upperLeftCoordinate": {"x": -10.0, "y": 80.0}, "lowerRightCoordinate": {"x": 50.0, "y": 20.0}}, "timeInterval": {"start": 1_388_534_400_000_i64, "end": 1_388_534_401_000_i64}}}))] #[serde(rename_all = "camelCase")] pub struct RasterDatasetFromWorkflow { pub name: Option, pub display_name: String, pub description: Option, - pub query: RasterQueryRectangle, + pub query: RasterToDatasetQueryRectangle, #[schema(default = default_as_cog)] #[serde(default = "default_as_cog")] pub as_cog: bool, } +pub struct RasterDatasetFromWorkflowParams { + pub name: Option, + pub display_name: String, + pub description: Option, + pub query: geoengine_datatypes::primitives::RasterQueryRectangle, + pub as_cog: bool, +} + +impl RasterDatasetFromWorkflowParams { + pub fn from_request_and_result_descriptor( + request: RasterDatasetFromWorkflow, + result_descriptor: &RasterResultDescriptor, + tiling_spec: TilingSpecification, + ) -> Self { + let query = request.query; + + let grid_bounds = result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_geo_transform() + .spatial_to_grid_bounds(&query.spatial_bounds.into()); // TODO: somehow clean up api and inner structs + + let raster_query = + geoengine_datatypes::primitives::RasterQueryRectangle::new_with_grid_bounds( + grid_bounds, + query.time_interval.into(), + BandSelection::first_n(result_descriptor.bands.len() as u32), + ); + + Self { + name: request.name, + display_name: request.display_name, + description: request.description, + query: raster_query, + as_cog: request.as_cog, + } + } +} + /// By default, we set [`RasterDatasetFromWorkflow::as_cog`] to true to produce cloud-optmized `GeoTiff`s. #[inline] const fn default_as_cog() -> bool { @@ -61,36 +101,28 @@ pub struct RasterDatasetFromWorkflowResult { impl TaskStatusInfo for RasterDatasetFromWorkflowResult {} -pub struct RasterDatasetFromWorkflowTask { +pub struct RasterDatasetFromWorkflowTask { pub source_name: String, + pub initialized_operator: R, pub workflow_id: WorkflowId, - pub workflow: Workflow, pub ctx: Arc, - pub info: RasterDatasetFromWorkflow, + pub info: RasterDatasetFromWorkflowParams, pub upload: UploadId, pub file_path: PathBuf, pub compression_num_threads: GdalCompressionNumThreads, } -impl RasterDatasetFromWorkflowTask { +impl RasterDatasetFromWorkflowTask { async fn process(&self) -> error::Result { - let operator = self.workflow.operator.clone(); + let result_descriptor = self.initialized_operator.result_descriptor(); - let operator = operator.get_raster()?; + let processor = self.initialized_operator.query_processor()?; - let execution_context = self.ctx.execution_context()?; + let query_rect: &geoengine_datatypes::primitives::QueryRectangle< + geoengine_datatypes::primitives::SpatialGridQueryRectangle, + BandSelection, + > = &self.info.query; - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - - let initialized = operator - .initialize(workflow_operator_path_root, &execution_context) - .await?; - - let result_descriptor = initialized.result_descriptor(); - - let processor = initialized.query_processor()?; - - let query_rect = self.info.query; let query_ctx = self.ctx.query_context(self.workflow_id.0, Uuid::new_v4())?; let request_spatial_ref = Option::::from(result_descriptor.spatial_reference) @@ -102,7 +134,7 @@ impl RasterDatasetFromWorkflowTask { call_on_generic_raster_processor_gdal_types!(processor, p => raster_stream_to_geotiff( &self.file_path, p, - query_rect.into(), + query_rect.clone(), query_ctx, GdalGeoTiffDatasetMetadata { no_data_value: Default::default(), // TODO: decide how to handle the no data here @@ -115,13 +147,11 @@ impl RasterDatasetFromWorkflowTask { }, tile_limit, Box::pin(futures::future::pending()), // datasets shall continue to be built in the background and not cancelled - execution_context.tiling_specification(), ).await)? .map_err(crate::error::Error::from)?; - // create the dataset let dataset = create_dataset( - self.info.clone(), + &self.info, res, result_descriptor, query_rect, @@ -137,7 +167,9 @@ impl RasterDatasetFromWorkflowTask { } #[async_trait::async_trait] -impl Task for RasterDatasetFromWorkflowTask { +impl Task + for RasterDatasetFromWorkflowTask +{ async fn run( &self, _ctx: C::TaskContext, @@ -179,12 +211,15 @@ impl Task for RasterDatasetFromWorkflowTask( +pub async fn schedule_raster_dataset_from_workflow_task< + C: SessionContext, + R: InitializedRasterOperator + 'static, +>( source_name: String, + initialized_operator: R, workflow_id: WorkflowId, - workflow: Workflow, ctx: Arc, - info: RasterDatasetFromWorkflow, + info: RasterDatasetFromWorkflowParams, compression_num_threads: GdalCompressionNumThreads, ) -> error::Result { if let Some(dataset_name) = &info.name { @@ -211,8 +246,8 @@ pub async fn schedule_raster_dataset_from_workflow_task( let task = RasterDatasetFromWorkflowTask { source_name, + initialized_operator, workflow_id, - workflow, ctx: ctx.clone(), info, upload, @@ -227,10 +262,10 @@ pub async fn schedule_raster_dataset_from_workflow_task( } async fn create_dataset( - info: RasterDatasetFromWorkflow, + info: &RasterDatasetFromWorkflowParams, mut slice_info: Vec, origin_result_descriptor: &RasterResultDescriptor, - query_rectangle: RasterQueryRectangle, + query_rectangle: &geoengine_datatypes::primitives::RasterQueryRectangle, ctx: &C, ) -> error::Result { ensure!(!slice_info.is_empty(), error::EmptyDatasetCannotBeImported); @@ -247,12 +282,29 @@ async fn create_dataset( .end(); let result_time_interval = TimeInterval::new(first_start, last_end)?; + let exe_ctx = ctx.execution_context()?; + + let source_tiling_spatial_grid = + origin_result_descriptor.tiling_grid_definition(exe_ctx.tiling_specification()); + let query_tiling_spatial_grid = + source_tiling_spatial_grid.with_other_bounds(query_rectangle.spatial_query.grid_bounds()); + let result_descriptor_bounds = origin_result_descriptor + .spatial_grid_descriptor() + .intersection_with_tiling_grid(&query_tiling_spatial_grid) + .ok_or(error::Error::EmptyDatasetCannotBeImported)?; // TODO: maybe allow empty datasets? + + // TODO: this is not how it is intended to work with the spatial grid descriptor. The source should propably not need that defined in its params since it can be derived from the dataset! + let (_state, dataset_source_descriptor_spatial_grid) = result_descriptor_bounds.as_parts(); + + let dataset_spatial_grid = geoengine_operators::engine::SpatialGridDescriptor::new_source( + dataset_source_descriptor_spatial_grid, + ); + let result_descriptor = RasterResultDescriptor { data_type: origin_result_descriptor.data_type, spatial_reference: origin_result_descriptor.spatial_reference, time: Some(result_time_interval), - bbox: Some(query_rectangle.spatial_bounds.into()), - resolution: Some(query_rectangle.spatial_resolution.into()), + spatial_grid: dataset_spatial_grid, bands: origin_result_descriptor.bands.clone(), }; //TODO: Recognize MetaDataDefinition::GdalMetaDataRegular @@ -278,14 +330,15 @@ async fn create_dataset( let dataset_definition = DatasetDefinition { properties: AddDataset { - name: info.name, - display_name: info.display_name, - description: info.description.unwrap_or_default(), + name: info.name.clone(), + display_name: info.display_name.clone(), + description: info.description.clone().unwrap_or_default(), source_operator: "GdalSource".to_owned(), symbology: None, // TODO add symbology? provenance: None, // TODO add provenance that references the workflow tags: Some(vec!["workflow".to_owned()]), - }, + } + .into(), meta_data, }; diff --git a/services/src/datasets/dataset_listing_provider.rs b/services/src/datasets/dataset_listing_provider.rs index 416f3a84c..9234280c5 100644 --- a/services/src/datasets/dataset_listing_provider.rs +++ b/services/src/datasets/dataset_listing_provider.rs @@ -463,11 +463,11 @@ mod tests { use geoengine_datatypes::{ collections::VectorDataType, primitives::{CacheTtlSeconds, TimeGranularity, TimeStep}, - raster::RasterDataType, + raster::{GeoTransform, GridBoundingBox, RasterDataType}, spatial_reference::SpatialReferenceOption, }; use geoengine_operators::{ - engine::{RasterBandDescriptors, StaticMetaData}, + engine::{RasterBandDescriptors, SpatialGridDescriptor, StaticMetaData}, source::{ FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, OgrSourceErrorSpec, @@ -711,8 +711,10 @@ mod tests { data_type: RasterDataType::U8, spatial_reference: SpatialReferenceOption::Unreferenced, time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., 0.).into(), 1., -1.), + GridBoundingBox::new([0, 0], [0, 0]).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }; @@ -744,8 +746,8 @@ mod tests { x_pixel_size: 0.0, y_pixel_size: 0.0, }, - width: 0, - height: 0, + width: 1, + height: 1, file_not_found_handling: FileNotFoundHandling::NoData, no_data_value: None, properties_mapping: None, diff --git a/services/src/datasets/external/aruna/mod.rs b/services/src/datasets/external/aruna/mod.rs index 3ac34e075..39217951c 100644 --- a/services/src/datasets/external/aruna/mod.rs +++ b/services/src/datasets/external/aruna/mod.rs @@ -1,9 +1,16 @@ -use std::collections::HashMap; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::path::PathBuf; -use std::str::FromStr; - +pub use self::error::ArunaProviderError; +use crate::contexts::GeoEngineDb; +use crate::datasets::external::aruna::metadata::{DataType, GEMetadata, RasterInfo, VectorInfo}; +use crate::datasets::listing::ProvenanceOutput; +use crate::layers::external::{DataProvider, DataProviderDefinition}; +use crate::layers::layer::{ + CollectionItem, Layer, LayerCollection, LayerCollectionListOptions, LayerListing, + ProviderLayerCollectionId, ProviderLayerId, +}; +use crate::layers::listing::{ + LayerCollectionId, LayerCollectionProvider, ProviderCapabilities, SearchCapabilities, +}; +use crate::workflows::workflow::Workflow; use aruna_rust_api::api::storage::models::v2::relation::Relation as ArunaRelationEnum; use aruna_rust_api::api::storage::models::v2::{ Dataset, InternalRelationVariant, KeyValue, KeyValueVariant, Object, ResourceVariant, @@ -18,26 +25,18 @@ use aruna_rust_api::api::storage::services::v2::{ GetDatasetRequest, GetDatasetsRequest, GetDownloadUrlRequest, GetObjectsRequest, GetProjectRequest, }; -use postgres_types::{FromSql, ToSql}; -use serde::{Deserialize, Serialize}; -use snafu::ensure; -use tonic::codegen::InterceptedService; -use tonic::metadata::{AsciiMetadataKey, AsciiMetadataValue}; -use tonic::service::Interceptor; -use tonic::transport::{Channel, Endpoint}; -use tonic::{Request, Status}; - use geoengine_datatypes::collections::VectorDataType; use geoengine_datatypes::dataset::{DataId, DataProviderId, LayerId}; use geoengine_datatypes::primitives::CacheTtlSeconds; use geoengine_datatypes::primitives::{ - FeatureDataType, Measurement, RasterQueryRectangle, SpatialResolution, VectorQueryRectangle, + FeatureDataType, Measurement, RasterQueryRectangle, VectorQueryRectangle, }; +use geoengine_datatypes::raster::{BoundedGrid, GeoTransform, GridShape2D}; use geoengine_datatypes::spatial_reference::SpatialReferenceOption; use geoengine_operators::engine::{ MetaData, MetaDataProvider, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, ResultDescriptor, TypedOperator, VectorColumnInfo, VectorOperator, - VectorResultDescriptor, + RasterResultDescriptor, ResultDescriptor, SpatialGridDescriptor, TypedOperator, + VectorColumnInfo, VectorOperator, VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{ @@ -46,24 +45,22 @@ use geoengine_operators::source::{ OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, OgrSourceDurationSpec, OgrSourceErrorSpec, OgrSourceParameters, OgrSourceTimeFormat, }; - -use crate::contexts::GeoEngineDb; -use crate::datasets::external::aruna::metadata::{DataType, GEMetadata, RasterInfo, VectorInfo}; -use crate::datasets::listing::ProvenanceOutput; -use crate::layers::external::{DataProvider, DataProviderDefinition}; -use crate::layers::layer::{ - CollectionItem, Layer, LayerCollection, LayerCollectionListOptions, LayerListing, - ProviderLayerCollectionId, ProviderLayerId, -}; -use crate::layers::listing::{ - LayerCollectionId, LayerCollectionProvider, ProviderCapabilities, SearchCapabilities, -}; -use crate::workflows::workflow::Workflow; - -pub use self::error::ArunaProviderError; - +use postgres_types::{FromSql, ToSql}; +use serde::{Deserialize, Serialize}; +use snafu::ensure; +use std::collections::HashMap; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::path::PathBuf; +use std::str::FromStr; +use tonic::codegen::InterceptedService; +use tonic::metadata::{AsciiMetadataKey, AsciiMetadataValue}; +use tonic::service::Interceptor; +use tonic::transport::{Channel, Endpoint}; +use tonic::{Request, Status}; pub mod error; pub mod metadata; + #[cfg(test)] #[macro_use] mod mock_grpc_server; @@ -512,19 +509,16 @@ impl ArunaDataProvider { crs: SpatialReferenceOption, info: &RasterInfo, ) -> geoengine_operators::util::Result { + let shape = GridShape2D::new_2d(info.width, info.height).bounding_box(); + + let geo_transform = GeoTransform::try_from(info.geo_transform) // TODO: convert into tiling based bounds? + .expect("GeoTransform should be valid"); // TODO: check if that can be false + Ok(RasterResultDescriptor { data_type: info.data_type, spatial_reference: crs, - time: Some(info.time_interval), - bbox: Some( - info.geo_transform - .spatial_partition(info.width, info.height), - ), - resolution: Some(SpatialResolution::try_from(( - info.geo_transform.x_pixel_size, - info.geo_transform.y_pixel_size, - ))?), + spatial_grid: SpatialGridDescriptor::source_from_parts(geo_transform, shape), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), info.measurement @@ -889,12 +883,12 @@ impl LayerCollectionProvider for ArunaDataProvider { ), DataType::SingleRasterFile(_) => TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: geoengine_datatypes::dataset::NamedData::with_system_provider( + params: GdalSourceParameters::new( + geoengine_datatypes::dataset::NamedData::with_system_provider( self.id.to_string(), id.to_string(), ), - }, + ), } .boxed(), ), @@ -1088,8 +1082,7 @@ mod tests { use geoengine_datatypes::collections::{FeatureCollectionInfos, MultiPointCollection}; use geoengine_datatypes::dataset::{DataId, DataProviderId, ExternalDataId, LayerId}; use geoengine_datatypes::primitives::{ - BoundingBox2D, CacheTtlSeconds, ColumnSelection, SpatialResolution, TimeInterval, - VectorQueryRectangle, + BoundingBox2D, CacheTtlSeconds, ColumnSelection, TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::{ @@ -2026,7 +2019,8 @@ mod tests { "operator": { "type": "GdalSource", "params": { - "data": "_:86a7f7ce-1bab-4ce9-a32b-172c0f958ee0:DATASET_ID" + "data": "_:86a7f7ce-1bab-4ce9-a32b-172c0f958ee0:DATASET_ID", + "overviewLevel": null } } }), @@ -2438,12 +2432,11 @@ mod tests { let ctx = MockQueryContext::test_default(); - let qr = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let qr = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let result: Vec = proc .query(qr, &ctx) diff --git a/services/src/datasets/external/copernicus_dataspace/ids.rs b/services/src/datasets/external/copernicus_dataspace/ids.rs index 5ac53c5e2..d9d41a558 100644 --- a/services/src/datasets/external/copernicus_dataspace/ids.rs +++ b/services/src/datasets/external/copernicus_dataspace/ids.rs @@ -1,8 +1,6 @@ use geoengine_datatypes::{ dataset::{DataId, DataProviderId, ExternalDataId, LayerId, NamedData}, - primitives::SpatialPartition2D, raster::RasterDataType, - spatial_reference::{SpatialReference, SpatialReferenceAuthority}, }; use std::str::FromStr; use strum::IntoEnumIterator; @@ -11,6 +9,7 @@ use strum_macros::{EnumIter, EnumString}; use crate::{ error::{Error, Result}, layers::listing::LayerCollectionId, + util::sentinel_2_utm_zones::UtmZone, }; #[derive(Debug, Clone)] @@ -91,6 +90,13 @@ impl Sentinel2ProductBand { Sentinel2ProductBand::L2A(band) => format!("{band}"), } } + + pub fn resolution_meters(self) -> usize { + match self { + Sentinel2ProductBand::L1C(l1c) => l1c.resolution_meters(), + Sentinel2ProductBand::L2A(l2a) => l2a.resolution_meters(), + } + } } #[derive(Debug, Clone, Copy, EnumString, strum::Display, EnumIter)] @@ -138,18 +144,6 @@ pub enum L2ABand { WVP_60M, } -#[derive(Debug, Clone, Copy)] -pub struct UtmZone { - pub zone: u8, - pub direction: UtmZoneDirection, -} - -#[derive(Debug, Clone, Copy)] -pub enum UtmZoneDirection { - North, - South, -} - impl Sentinel2ProductBand { // TODO: move to sentinel2 to separate concerns pub fn product_type(&self) -> &str { @@ -327,24 +321,6 @@ impl Sentinel2Band for Sentinel2ProductBand { } } -impl UtmZone { - pub fn epsg_code(self) -> u32 { - match self.direction { - UtmZoneDirection::North => 32600 + u32::from(self.zone), - UtmZoneDirection::South => 32700 + u32::from(self.zone), - } - } - - pub fn spatial_reference(self) -> SpatialReference { - SpatialReference::new(SpatialReferenceAuthority::Epsg, self.epsg_code()) - } - - pub fn extent(self) -> Option { - // TODO: as Sentinel uses enlarged grids, we could return a larger extent - self.spatial_reference().area_of_use().ok() - } -} - impl FromStr for CopernicusDataspaceLayerCollectionId { type Err = crate::error::Error; @@ -473,7 +449,7 @@ impl FromStr for Sentinel2LayerCollectionId { [product, zone] => Self::ProductZone { product: Sentinel2Product::from_str(product) .map_err(|_| Error::InvalidLayerCollectionId)?, - zone: UtmZone::from_str(zone)?, + zone: UtmZone::from_str(zone).map_err(|_| Error::InvalidLayerCollectionId)?, }, _ => return Err(Error::InvalidLayerCollectionId), }) @@ -522,7 +498,7 @@ impl FromStr for Sentinel2LayerId { [product, zone, band] => Self { product_band: Sentinel2ProductBand::with_product_and_band_as_str(product, band) .map_err(|_| Error::InvalidLayerId)?, - zone: UtmZone::from_str(zone)?, + zone: UtmZone::from_str(zone).map_err(|_| Error::InvalidLayerId)?, }, _ => return Err(Error::InvalidLayerId), }) @@ -555,62 +531,3 @@ impl From for DataId { }) } } - -impl FromStr for UtmZone { - type Err = crate::error::Error; - - fn from_str(s: &str) -> Result { - if s.len() < 5 || &s[..3] != "UTM" { - return Err(Error::InvalidLayerCollectionId); - } - - let (zone_str, dir_char) = s[3..].split_at(s.len() - 4); - let zone = zone_str - .parse::() - .map_err(|_| Error::InvalidLayerCollectionId)?; - - // TODO: check if zone is in valid range - - let north = match dir_char { - "N" => UtmZoneDirection::North, - "S" => UtmZoneDirection::South, - _ => return Err(Error::InvalidLayerCollectionId), - }; - - Ok(Self { - zone, - direction: north, - }) - } -} - -impl std::fmt::Display for UtmZone { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "UTM{}{}", - self.zone, - match self.direction { - UtmZoneDirection::North => "N", - UtmZoneDirection::South => "S", - } - ) - } -} - -impl UtmZone { - pub fn zones() -> impl Iterator { - (1..=60).flat_map(|zone| { - vec![ - UtmZone { - zone, - direction: UtmZoneDirection::North, - }, - UtmZone { - zone, - direction: UtmZoneDirection::South, - }, - ] - }) - } -} diff --git a/services/src/datasets/external/copernicus_dataspace/provider.rs b/services/src/datasets/external/copernicus_dataspace/provider.rs index e29f3fdb8..67c6c0a65 100644 --- a/services/src/datasets/external/copernicus_dataspace/provider.rs +++ b/services/src/datasets/external/copernicus_dataspace/provider.rs @@ -12,6 +12,7 @@ use crate::{ listing::LayerCollectionId, }, projects::RasterSymbology, + util::sentinel_2_utm_zones::UtmZone, workflows::workflow::Workflow, }; use async_trait::async_trait; @@ -44,7 +45,6 @@ use super::{ ids::{ CopernicusDataId, CopernicusDataspaceLayerCollectionId, CopernicusDataspaceLayerId, Sentinel2LayerCollectionId, Sentinel2LayerId, Sentinel2Product, Sentinel2ProductBand, - UtmZone, }, sentinel2::Sentinel2Metadata, }; @@ -292,6 +292,7 @@ impl CopernicusDataspaceDataProvider { self.id, ) .into(), + overview_level: None, }, } .boxed(), diff --git a/services/src/datasets/external/copernicus_dataspace/sentinel2.rs b/services/src/datasets/external/copernicus_dataspace/sentinel2.rs index c0120018c..7f9fe4e13 100644 --- a/services/src/datasets/external/copernicus_dataspace/sentinel2.rs +++ b/services/src/datasets/external/copernicus_dataspace/sentinel2.rs @@ -1,18 +1,22 @@ use std::path::PathBuf; -use crate::datasets::external::copernicus_dataspace::stac::{ - load_stac_items, resolve_datetime_duplicates, +use crate::{ + datasets::external::copernicus_dataspace::stac::{ + load_stac_items, resolve_datetime_duplicates, + }, + util::sentinel_2_utm_zones::UtmZone, }; use gdal::{DatasetOptions, GdalOpenFlags}; use geoengine_datatypes::{ primitives::{ - CacheTtlSeconds, DateTime, RasterQueryRectangle, SpatialResolution, TimeInstance, - TimeInterval, + AxisAlignedRectangle, CacheTtlSeconds, ColumnSelection, DateTime, RasterQueryRectangle, + TimeInstance, TimeInterval, VectorQueryRectangle, }, + raster::{GeoTransform, GridShape2D, SpatialGridDefinition, TilingSpecification}, spatial_reference::{SpatialReference, SpatialReferenceAuthority}, }; use geoengine_operators::{ - engine::{MetaData, RasterBandDescriptor, RasterResultDescriptor}, + engine::{MetaData, RasterBandDescriptor, RasterResultDescriptor, SpatialGridDescriptor}, source::{ GdalDatasetParameters, GdalLoadingInfo, GdalLoadingInfoTemporalSlice, GdalLoadingInfoTemporalSliceIterator, @@ -28,7 +32,7 @@ use snafu::{ResultExt, Snafu}; use url::Url; use super::{ - ids::{Sentinel2Band, Sentinel2ProductBand, UtmZone}, + ids::{Sentinel2Band, Sentinel2ProductBand}, stac::{CopernicusStacError, StacItemExt}, }; @@ -84,7 +88,7 @@ pub struct Sentinel2Metadata { impl Sentinel2Metadata { async fn crate_loading_info( &self, - query: RasterQueryRectangle, + query: VectorQueryRectangle, // TODO: here the name is misleading :( ) -> Result { let mut stac_items = load_stac_items( Url::parse(&self.stac_url).context(CannotParseStacUrl)?, @@ -241,14 +245,42 @@ impl MetaData for &self, query: RasterQueryRectangle, ) -> geoengine_operators::util::Result { - self.crate_loading_info(query).await.map_err(|e| { - geoengine_operators::error::Error::LoadingInfo { + let utm_extent = self.zone.native_extent(); + let px_size = self.product_band.resolution_meters() as f64; + let geo_transform = GeoTransform::new(utm_extent.upper_left(), px_size, -px_size); + let grid_bounds = geo_transform.spatial_to_grid_bounds(&utm_extent); + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); + + // FIXME: get tiling_spec! + let tiling_specification = TilingSpecification::new(GridShape2D::new_2d(512, 512)); + + let spatial_bounds = SpatialGridDescriptor::new_source(spatial_grid) + .tiling_grid_definition(tiling_specification) + .tiling_geo_transform() + .grid_to_spatial_bounds(&query.spatial_query.grid_bounds()); + + let spatial_bounds_query = VectorQueryRectangle::with_bounds( + spatial_bounds.as_bbox(), + query.time_interval, + ColumnSelection::all(), + ); + + self.crate_loading_info(spatial_bounds_query) + .await + .map_err(|e| geoengine_operators::error::Error::LoadingInfo { source: Box::new(e), - } - }) + }) } async fn result_descriptor(&self) -> geoengine_operators::util::Result { + let utm_extent = self.zone.native_extent(); + let px_size = self.product_band.resolution_meters() as f64; + let geo_transform = GeoTransform::new(utm_extent.upper_left(), px_size, -px_size); + let grid_bounds = geo_transform.spatial_to_grid_bounds(&utm_extent); + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); + + let spatial_grid_desc = SpatialGridDescriptor::new_source(spatial_grid); + Ok(RasterResultDescriptor { data_type: self.product_band.data_type(), spatial_reference: SpatialReference::new( @@ -264,11 +296,7 @@ impl MetaData for DateTime::new_utc(2015, 7, 4, 10, 10, 6), DateTime::now(), )), - bbox: self.zone.extent(), - resolution: Some(SpatialResolution::new( - self.product_band.resolution_meters() as f64, - self.product_band.resolution_meters() as f64, - )?), + spatial_grid: spatial_grid_desc, bands: vec![RasterBandDescriptor::new_unitless( self.product_band.band_name(), )] @@ -285,10 +313,13 @@ impl MetaData for #[cfg(test)] mod tests { - use std::env; - + use super::*; + use crate::{ + datasets::external::copernicus_dataspace::ids::L2ABand, + util::sentinel_2_utm_zones::UtmZoneDirection, + }; use geoengine_datatypes::{ - primitives::{BandSelection, Coordinate2D, DateTime, SpatialPartition2D}, + primitives::{Coordinate2D, DateTime, SpatialPartition2D}, test_data, }; use geoengine_operators::source::{FileNotFoundHandling, GdalDatasetGeoTransform}; @@ -297,10 +328,7 @@ mod tests { matchers::{contains, request, url_decoded}, responders::status_code, }; - - use crate::datasets::external::copernicus_dataspace::ids::{L2ABand, UtmZoneDirection}; - - use super::*; + use std::env; fn add_partial_responses( server: &Server, @@ -479,18 +507,18 @@ mod tests { // time=2020-07-01T12%3A00%3A00.000Z/2020-07-03T12%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=256&HEIGHT=256&CRS=EPSG%3A32632&BBOX=482500%2C5627500%2C483500%2C5628500 let loading_info = metadata - .crate_loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( + .crate_loading_info(VectorQueryRectangle::with_bounds( + SpatialPartition2D::new_unchecked( (482_500., 5_627_500.).into(), (483_500., 5_628_500.).into(), - ), - time_interval: TimeInterval::new_unchecked( + ) + .as_bbox(), + TimeInterval::new_unchecked( DateTime::parse_from_rfc3339("2020-07-01T12:00:00.000Z").unwrap(), DateTime::parse_from_rfc3339("2020-07-03T12:00:00.000Z").unwrap(), ), - spatial_resolution: SpatialResolution::new(10., 10.).unwrap(), - attributes: BandSelection::first(), - }) + ColumnSelection::all(), + )) .await .unwrap(); diff --git a/services/src/datasets/external/copernicus_dataspace/stac.rs b/services/src/datasets/external/copernicus_dataspace/stac.rs index 261bc7260..b68315b5b 100644 --- a/services/src/datasets/external/copernicus_dataspace/stac.rs +++ b/services/src/datasets/external/copernicus_dataspace/stac.rs @@ -1,9 +1,6 @@ use geoengine_datatypes::{ operations::reproject::{CoordinateProjection, CoordinateProjector, ReprojectClipped}, - primitives::{ - AxisAlignedRectangle, BoundingBox2D, DateTime, Duration, RasterQueryRectangle, - SpatialPartitioned, - }, + primitives::{AxisAlignedRectangle, DateTime, Duration, VectorQueryRectangle}, spatial_reference::SpatialReference, }; use snafu::{ResultExt, Snafu}; @@ -41,7 +38,7 @@ pub enum CopernicusStacError { } fn bbox_time_query( - query: &RasterQueryRectangle, + query: &VectorQueryRectangle, query_projection: SpatialReference, ) -> Result<[(&'static str, String); 2], CopernicusStacError> { // TODO: add query buffer like in Element84 provider? @@ -52,11 +49,7 @@ fn bbox_time_query( CoordinateProjector::from_known_srs(query_projection, SpatialReference::epsg_4326()) .context(CannotReprojectBbox)?; - let spatial_partition = query.spatial_partition(); // TODO: use SpatialPartition2D directly - let bbox = BoundingBox2D::new_upper_left_lower_right_unchecked( - spatial_partition.upper_left(), - spatial_partition.lower_right(), - ); + let bbox = query.spatial_query.spatial_bounds; // TODO: use SpatialPartition2D directly // TODO: query the whole zone instead? (for Sentinel-2) let bbox = bbox @@ -97,7 +90,7 @@ fn bbox_time_query( pub async fn load_stac_items( stac_url: Url, collection: &str, - query: RasterQueryRectangle, + query: VectorQueryRectangle, query_projection: SpatialReference, product_type: &str, ) -> Result, CopernicusStacError> { diff --git a/services/src/datasets/external/edr.rs b/services/src/datasets/external/edr.rs index 5a84bf490..3ce34d25f 100644 --- a/services/src/datasets/external/edr.rs +++ b/services/src/datasets/external/edr.rs @@ -16,15 +16,18 @@ use gdal::Dataset; use geoengine_datatypes::collections::VectorDataType; use geoengine_datatypes::dataset::{DataId, DataProviderId, LayerId}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BoundingBox2D, CacheTtlSeconds, ContinuousMeasurement, Coordinate2D, - FeatureDataType, Measurement, RasterQueryRectangle, SpatialPartition2D, TimeInstance, - TimeInterval, VectorQueryRectangle, + BoundingBox2D, CacheTtlSeconds, ContinuousMeasurement, Coordinate2D, FeatureDataType, + Measurement, RasterQueryRectangle, TimeInstance, TimeInterval, VectorQueryRectangle, +}; +use geoengine_datatypes::raster::{ + BoundedGrid, GeoTransform, GridIdx2D, GridShape2D, GridSize, RasterDataType, + SpatialGridDefinition, }; -use geoengine_datatypes::raster::RasterDataType; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_operators::engine::{ MetaData, MetaDataProvider, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, - StaticMetaData, TypedOperator, VectorColumnInfo, VectorOperator, VectorResultDescriptor, + SpatialGridDescriptor, StaticMetaData, TypedOperator, VectorColumnInfo, VectorOperator, + VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{ @@ -681,16 +684,30 @@ impl EdrCollectionMetaData { fn get_raster_result_descriptor( &self, + geo_transform: GeoTransform, + grid_shape: GridShape2D, ) -> Result { - let bbox = self.get_bounding_box()?; - let bbox = SpatialPartition2D::new_unchecked(bbox.upper_left(), bbox.lower_right()); + // IF the dataset has a fliped y-axis and we want to use it up-up we need to flip the grid! + + let spatial_grid = if geo_transform.y_axis_is_neg() { + SpatialGridDefinition::new(geo_transform, grid_shape.bounding_box()) + } else { + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_shape.bounding_box()); + spatial_grid + .flip_axis_y() + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + spatial_grid.grid_bounds.axis_size_y() as isize, + 0, + )) + }; + + let spatial_grid_def = SpatialGridDescriptor::new_source(spatial_grid); Ok(RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), time: Some(self.get_time_interval()?), - bbox: Some(bbox), - resolution: None, + spatial_grid: spatial_grid_def, bands: RasterBandDescriptors::new_single_band(), }) } @@ -985,12 +1002,12 @@ impl LayerCollectionProvider for EdrDataProvider { let operator = if collection.is_raster_file()? { TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: geoengine_datatypes::dataset::NamedData::with_system_provider( + params: GdalSourceParameters::new( + geoengine_datatypes::dataset::NamedData::with_system_provider( self.id.to_string(), id.to_string(), ), - }, + ), } .boxed(), ) @@ -1188,8 +1205,23 @@ impl MetaDataProvider = + GridShape2D::new_2d(first_params.height, first_params.width); + Ok(Box::new(GdalMetaDataList { - result_descriptor: collection.get_raster_result_descriptor()?, + result_descriptor: collection + .get_raster_result_descriptor(geo_transform, grid_shape)?, params, })) } @@ -1228,7 +1260,8 @@ mod tests { use geoengine_datatypes::{ dataset::ExternalDataId, hashmap, - primitives::{BandSelection, ColumnSelection, SpatialResolution}, + primitives::{BandSelection, ColumnSelection}, + raster::GridBoundingBox2D, util::gdal::hide_gdal_errors, }; use geoengine_operators::{engine::ResultDescriptor, source::GdalDatasetGeoTransform}; @@ -1630,15 +1663,11 @@ mod tests { .await .unwrap(); let loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .unwrap(); assert_eq!( @@ -1672,7 +1701,6 @@ mod tests { cache_ttl: Default::default(), } ); - let result_descriptor = meta.result_descriptor().await.unwrap(); assert_eq!( result_descriptor, @@ -1759,23 +1787,15 @@ mod tests { if meta_result.is_err() { server.verify_and_clear(); } - meta_result.unwrap() }; let loading_info_parts = meta - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 90.).into(), - (360., -90.).into(), - ), - time_interval: TimeInterval::new_unchecked( - 1_692_144_000_000, - 1_692_500_400_000, - ), - spatial_resolution: SpatialResolution::new_unchecked(1., 1.), - attributes: BandSelection::first(), - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [361, 720]).unwrap(), + TimeInterval::new_unchecked(1_692_144_000_000, 1_692_500_400_000), + BandSelection::first(), + )) .await .unwrap() .info @@ -1857,11 +1877,14 @@ mod tests { 1_692_144_000_000, 1_692_500_400_000 )), - bbox: Some(SpatialPartition2D::new_unchecked( - (0., 90.).into(), - (359.500_000_000_000_06, -90.).into() - )), - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new( + (0., 90.).into(), + 0.499_305_555_555_555_6, + -0.498_614_958_448_753_5 + ), + GridBoundingBox2D::new_min_max(0, 360, 0, 719).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), } ); diff --git a/services/src/datasets/external/gbif.rs b/services/src/datasets/external/gbif.rs index 3ab455c54..4a62aeaf3 100644 --- a/services/src/datasets/external/gbif.rs +++ b/services/src/datasets/external/gbif.rs @@ -1549,7 +1549,7 @@ mod tests { use geoengine_datatypes::collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection}; use geoengine_datatypes::dataset::ExternalDataId; use geoengine_datatypes::primitives::{ - BoundingBox2D, CacheHint, FeatureData, MultiPoint, SpatialResolution, TimeInterval, + BoundingBox2D, CacheHint, FeatureData, MultiPoint, TimeInterval, }; use geoengine_datatypes::primitives::{ColumnSelection, TimeInstance}; use geoengine_datatypes::util::test::TestDefault; @@ -2213,15 +2213,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; @@ -2412,15 +2408,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; @@ -2550,15 +2542,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; @@ -2657,16 +2645,16 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new( (-61.065_22, 14.775_33).into(), (-61.065_22, 14.775_33).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = MockQueryContext::test_default(); let result: Vec<_> = processor @@ -3026,16 +3014,16 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new( (-61.065_22, 14.775_33).into(), (-61.065_22, 14.775_33).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = MockQueryContext::test_default(); let result: Vec<_> = processor @@ -3145,16 +3133,16 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new( (-61.065_22, 14.775_33).into(), (-61.065_22, 14.775_33).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = MockQueryContext::test_default(); let result: Vec<_> = processor @@ -3256,13 +3244,12 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(1_517_011_200_000).unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(1_517_011_200_000).unwrap(), + ColumnSelection::all(), + ); + let ctx = MockQueryContext::test_default(); let result: Vec<_> = processor @@ -3300,13 +3287,12 @@ mod tests { return Err(format!("{result:?} != {expected:?}")); } - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(1_517_443_200_000).unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(1_517_443_200_000).unwrap(), + ColumnSelection::all(), + ); + let ctx = MockQueryContext::test_default(); let result: Vec<_> = processor @@ -3350,7 +3336,6 @@ mod tests { add_test_data(&db_config).await; let result = test(ctx, db_config).await; - assert!(result.is_ok()); } @@ -3399,17 +3384,16 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new( + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new( TimeInstance::from_millis_unchecked(1_517_011_200_000), TimeInstance::from_millis_unchecked(1_517_443_200_000), ) .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + ColumnSelection::all(), + ); + let ctx = MockQueryContext::test_default(); let result: Vec<_> = processor @@ -3447,17 +3431,16 @@ mod tests { return Err(format!("{result:?} != {expected:?}")); } - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new( + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new( TimeInstance::from_millis_unchecked(1_517_011_200_000), TimeInstance::from_millis_unchecked(1_517_443_200_001), ) .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + ColumnSelection::all(), + ); + let ctx = MockQueryContext::test_default(); let result: Vec<_> = processor diff --git a/services/src/datasets/external/gfbio_abcd.rs b/services/src/datasets/external/gfbio_abcd.rs index dc492242a..9295ba20b 100644 --- a/services/src/datasets/external/gfbio_abcd.rs +++ b/services/src/datasets/external/gfbio_abcd.rs @@ -645,17 +645,14 @@ impl mod tests { use super::*; use crate::config; - use crate::contexts::SessionContext; - use crate::contexts::{PostgresContext, PostgresSessionContext}; + use crate::contexts::{PostgresContext, PostgresSessionContext, SessionContext}; use crate::layers::layer::ProviderLayerCollectionId; use crate::{ge_context, test_data}; use bb8_postgres::bb8::ManageConnection; use futures::StreamExt; use geoengine_datatypes::collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection}; use geoengine_datatypes::dataset::ExternalDataId; - use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, MultiPoint, SpatialResolution, TimeInterval, - }; + use geoengine_datatypes::primitives::{BoundingBox2D, FeatureData, MultiPoint, TimeInterval}; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::QueryProcessor; @@ -1281,15 +1278,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; @@ -1461,12 +1454,11 @@ mod tests { bbox: None, },meta, vec![]); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((0., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let ctx = MockQueryContext::test_default(); let result: Vec<_> = processor diff --git a/services/src/datasets/external/gfbio_collections.rs b/services/src/datasets/external/gfbio_collections.rs index e91f3f251..075881349 100644 --- a/services/src/datasets/external/gfbio_collections.rs +++ b/services/src/datasets/external/gfbio_collections.rs @@ -821,7 +821,7 @@ mod tests { use bb8_postgres::bb8::ManageConnection; use geoengine_datatypes::{ dataset::ExternalDataId, - primitives::{BoundingBox2D, ColumnSelection, SpatialResolution, TimeInterval}, + primitives::{BoundingBox2D, ColumnSelection, TimeInterval}, test_data, }; use httptest::{ @@ -1120,15 +1120,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::with_bounds( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; diff --git a/services/src/datasets/external/mod.rs b/services/src/datasets/external/mod.rs index 72f62bc7f..629772e40 100644 --- a/services/src/datasets/external/mod.rs +++ b/services/src/datasets/external/mod.rs @@ -10,6 +10,5 @@ mod sentinel_s2_l2a_cogs; pub use copernicus_dataspace::CopernicusDataspaceDataProviderDefinition; pub use sentinel_s2_l2a_cogs::{ - GdalRetries, SentinelS2L2ACogsProviderDefinition, StacApiRetries, StacBand, StacQueryBuffer, - StacZone, + GdalRetries, SentinelS2L2ACogsProviderDefinition, StacApiRetries, StacQueryBuffer, }; diff --git a/services/src/datasets/external/netcdfcf/loading.rs b/services/src/datasets/external/netcdfcf/loading.rs index 9d5cfe78d..3af2de915 100644 --- a/services/src/datasets/external/netcdfcf/loading.rs +++ b/services/src/datasets/external/netcdfcf/loading.rs @@ -16,7 +16,7 @@ use crate::{ use geoengine_datatypes::{ dataset::{DataProviderId, LayerId, NamedData}, operations::image::{Colorizer, RasterColorizer}, - primitives::{CacheTtlSeconds, TimeInstance}, + primitives::{CacheTtlSeconds, Duration, TimeInstance, TimeInterval}, }; use geoengine_operators::{ engine::{RasterOperator, RasterResultDescriptor, TypedOperator}, @@ -148,7 +148,7 @@ pub fn create_layer( workflow: Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: data_id }, + params: GdalSourceParameters::new(data_id), } .boxed(), ), @@ -217,7 +217,9 @@ fn create_loading_info_part( params.file_path = file_path.with_file_name(time_instance.as_datetime_string_with_millis() + ".tiff"); - time_instance.into() + // Note: was a TimeInstance before which is not valid so we add 1 millisecond just to get an interval. + TimeInterval::new(time_instance, time_instance + Duration::milliseconds(1)) + .expect("increasing one millisecond must work") } ParamModification::Channel { channel, @@ -225,7 +227,8 @@ fn create_loading_info_part( } => { params.rasterband_channel = channel; - time_instance.into() + TimeInterval::new(time_instance, time_instance + Duration::milliseconds(1)) + .expect("increasing one millisecond must work") } }; diff --git a/services/src/datasets/external/netcdfcf/mod.rs b/services/src/datasets/external/netcdfcf/mod.rs index c04d9d36c..7d4075a82 100644 --- a/services/src/datasets/external/netcdfcf/mod.rs +++ b/services/src/datasets/external/netcdfcf/mod.rs @@ -29,12 +29,14 @@ use geoengine_datatypes::primitives::{ CacheTtlSeconds, DateTime, Measurement, RasterQueryRectangle, TimeInstance, VectorQueryRectangle, }; -use geoengine_datatypes::raster::{GdalGeoTransform, RasterDataType}; +use geoengine_datatypes::raster::{ + BoundedGrid, GdalGeoTransform, GeoTransform, GridShape2D, RasterDataType, +}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::canonicalize_subpath; use geoengine_datatypes::util::gdal::ResamplingMethod; -use geoengine_operators::engine::RasterBandDescriptor; use geoengine_operators::engine::RasterBandDescriptors; +use geoengine_operators::engine::{RasterBandDescriptor, SpatialGridDescriptor}; use geoengine_operators::source::{ FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, }; @@ -453,6 +455,7 @@ impl NetCdfCfDataProvider { .boxed_context(error::UnexpectedExecution)? } + #[allow(clippy::too_many_lines)] fn meta_data_from_netcdf( base_path: &Path, dataset_id: &NetCdfCf4DDatasetId, @@ -464,9 +467,7 @@ impl NetCdfCfDataProvider { const TIME_DIMENSION_INDEX: usize = 1; let dataset = gdal_netcdf_open(Some(base_path), Path::new(&dataset_id.file_name))?; - let root_group = dataset.root_group().context(error::GdalMd)?; - let time_coverage = TimeCoverage::from_dimension(&root_group)?; let geo_transform = { @@ -502,30 +503,6 @@ impl NetCdfCfDataProvider { let dimensions = data_array.dimensions().context(error::GdalMd)?; - let result_descriptor = RasterResultDescriptor { - data_type: RasterDataType::from_gdal_data_type( - data_array - .datatype() - .numeric_datatype() - .try_into() - .unwrap_or(GdalDataType::Float64), - ) - .unwrap_or(RasterDataType::F64), - spatial_reference: SpatialReference::try_from( - data_array.spatial_reference().context(error::GdalMd)?, - ) - .context(error::CannotParseCrs)? - .into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( - "band".into(), - derive_measurement(data_array.unit()), - )]) - .context(error::GeneratingResultDescriptorFromDataset)?, - }; - let params = GdalDatasetParameters { file_path: netcfg_gdal_path( Some(base_path), @@ -551,6 +528,36 @@ impl NetCdfCfDataProvider { retry: None, }; + let pixel_shape = GridShape2D::new_2d(params.height as usize, params.width as usize); + let geo_transform = + GeoTransform::try_from(params.geo_transform).expect("GeoTransform must be valid"); // TODO: check how the axis in netcfd are stored; + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::from_gdal_data_type( + data_array + .datatype() + .numeric_datatype() + .try_into() + .unwrap_or(GdalDataType::Float64), + ) + .unwrap_or(RasterDataType::F64), + spatial_reference: SpatialReference::try_from( + data_array.spatial_reference().context(error::GdalMd)?, + ) + .context(error::CannotParseCrs)? + .into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + geo_transform, + pixel_shape.bounding_box(), + ), + bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "band".into(), + derive_measurement(data_array.unit()), + )]) + .expect("must work since derive_measurement can't fail"), + }; + let dimensions_time = dimensions .get(TIME_DIMENSION_INDEX) .map(Dimension::size) @@ -1578,24 +1585,30 @@ mod tests { use crate::ge_context; use crate::layers::layer::LayerListing; use crate::layers::storage::LayerProviderDb; - use crate::{tasks::util::NopTaskContext, util::tests::add_land_cover_to_datasets}; + use crate::tasks::util::NopTaskContext; + use crate::util::tests::add_land_cover_to_datasets; use geoengine_datatypes::dataset::ExternalDataId; use geoengine_datatypes::plots::{PlotData, PlotMetaData}; - use geoengine_datatypes::primitives::{BandSelection, PlotSeriesSelection}; + use geoengine_datatypes::primitives::{ + BandSelection, BoundingBox2D, Coordinate2D, PlotQueryRectangle, PlotSeriesSelection, + SpatialResolution, + }; use geoengine_datatypes::raster::RenameBands; + use geoengine_datatypes::raster::{GeoTransform, GridBoundingBox2D}; use geoengine_datatypes::{ - primitives::{ - BoundingBox2D, PlotQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - spatial_reference::SpatialReferenceAuthority, - test_data, + primitives::TimeInterval, spatial_reference::SpatialReferenceAuthority, test_data, util::gdal::hide_gdal_errors, }; use geoengine_operators::engine::{ MultipleRasterSources, RasterBandDescriptors, RasterOperator, SingleRasterSource, }; use geoengine_operators::processing::{ - RasterStacker, RasterStackerParams, RasterTypeConversion, RasterTypeConversionParams, + Interpolation, InterpolationMethod, InterpolationParams, RasterStacker, + RasterStackerParams, RasterTypeConversion, RasterTypeConversionParams, + }; + use geoengine_operators::source::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, + GdalLoadingInfoTemporalSlice, }; use geoengine_operators::source::{GdalSource, GdalSourceParameters}; use geoengine_operators::{ @@ -1605,10 +1618,6 @@ mod tests { MeanRasterPixelValuesOverTimePosition, }, processing::{Expression, ExpressionParams}, - source::{ - FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, - GdalLoadingInfoTemporalSlice, - }, }; use tokio_postgres::NoTls; @@ -1866,7 +1875,7 @@ mod tests { .await .unwrap(); - pretty_assertions::assert_eq!( + assert_eq!( collection, LayerCollection { id: ProviderLayerCollectionId { @@ -1896,7 +1905,7 @@ mod tests { provider_id: NETCDF_CF_PROVIDER_ID, collection_id: LayerCollectionId("dataset_sm.nc/scenario_3".to_string()) }, - name: "Regional Rivalry".to_string(), + name: "Regional Rivalry".to_string(), description: "SSP3-RCP6.0".to_string(), properties: Default::default(), }), CollectionItem::Collection(LayerCollectionListing { r#type: Default::default(), @@ -1956,26 +1965,24 @@ mod tests { spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3035) .into(), time: None, - bbox: None, - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((3_580_000.0, 2_370_000.0).into(), 1000.0, -1000.0), // FIXME: move to tiling bounds + GridBoundingBox2D::new( + [0, 0], // 0 + [9, 9] // 10 + ) + .unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), } ); let loading_info = metadata - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new( - (43.945_312_5, 0.791_015_625_25).into(), - (44.033_203_125, 0.703_125_25).into(), - ) - .unwrap(), - time_interval: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), - spatial_resolution: SpatialResolution::new_unchecked( - 0.000_343_322_7, // 256 pixel - 0.000_343_322_7, // 256 pixel - ), - attributes: BandSelection::first(), - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [9, 9]).unwrap(), + TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + )) .await .unwrap(); @@ -1995,10 +2002,11 @@ mod tests { ) .into(); + let expected_time = TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)); assert_eq!( loading_info_parts[0], GdalLoadingInfoTemporalSlice { - time: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), + time: TimeInterval::new_unchecked(expected_time, expected_time + 1), params: Some(GdalDatasetParameters { file_path, rasterband_channel: 4, @@ -2092,26 +2100,24 @@ mod tests { spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3035) .into(), time: None, - bbox: None, - resolution: Some(SpatialResolution::new_unchecked(1000.0, 1000.0)), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((3_580_000.0, 2_370_000.0).into(), 1000.0, -1000.0), + GridBoundingBox2D::new( + [0, 0], // 0 + [9, 9] // 10 + ) + .unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), } ); let loading_info = metadata - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new( - (43.945_312_5, 0.791_015_625_25).into(), - (44.033_203_125, 0.703_125_25).into(), - ) - .unwrap(), - time_interval: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), - spatial_resolution: SpatialResolution::new_unchecked( - 0.000_343_322_7, // 256 pixel - 0.000_343_322_7, // 256 pixel - ), - attributes: BandSelection::first(), - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new([0, 0], [9, 9]).unwrap(), // Fixme: adapt to tiling bounds + TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + )) .await .unwrap(); @@ -2126,10 +2132,11 @@ mod tests { .path() .join("dataset_sm.nc/scenario_5/metric_2/1/2000-01-01T00:00:00.000Z.tiff"); + let expected_time = TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)); assert_eq!( loading_info_parts[0], GdalLoadingInfoTemporalSlice { - time: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), + time: TimeInterval::new_unchecked(expected_time, expected_time + 1), params: Some(GdalDatasetParameters { file_path, rasterband_channel: 1, @@ -2275,10 +2282,10 @@ mod tests { }, sources: Expression { params: ExpressionParams { - expression: "A".to_string(), + expression: "if A is NODATA {NODATA} else {A}".to_string(), // FIXME: was "A" because nodata pixels would be skipped. --> The landcover pixels overlapping are NODATA, but why? output_type: RasterDataType::F64, output_band: None, - map_no_data: false, + map_no_data: true, }, sources: SingleRasterSource { raster: RasterStacker { @@ -2287,29 +2294,41 @@ mod tests { }, sources: MultipleRasterSources { rasters: vec![ - GdalSource { - params: GdalSourceParameters { - data: geoengine_datatypes::dataset::NamedData::with_system_provider( - EBV_PROVIDER_ID.to_string(), - serde_json::json!({ - "fileName": "dataset_irr_ts.nc", - "groupNames": ["metric_1"], - "entity": 0 - }) - .to_string(), - ), + Interpolation{ + params: InterpolationParams { + interpolation: InterpolationMethod::NearestNeighbor, + output_resolution: geoengine_operators::processing::InterpolationResolution::Resolution(SpatialResolution::new_unchecked(0.1, 0.1)), // The test data has a resolution of 1.0! + output_origin_reference: Some(Coordinate2D::new(0.0, 0.0)), }, - } - .boxed(), + sources: SingleRasterSource { + raster: GdalSource { + params: GdalSourceParameters { + data: geoengine_datatypes::dataset::NamedData::with_system_provider( + EBV_PROVIDER_ID.to_string(), + serde_json::json!({ + "fileName": "dataset_irr_ts.nc", + "groupNames": ["metric_1"], + "entity": 0 + }) + .to_string(), + ), + overview_level: None, + }, + } + .boxed(), + } + }.boxed(), RasterTypeConversion { params: RasterTypeConversionParams { output_data_type: RasterDataType::I16, }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: land_cover_dataset_name.into(), - }, + params: GdalSourceParameters::new( + geoengine_datatypes::dataset::NamedData::with_system_name( + land_cover_dataset_name.to_string(), + ) + ), }.boxed(), } }.boxed(), @@ -2317,9 +2336,7 @@ mod tests { } }.boxed() } - } - .boxed() - .into(), + }.boxed().into(), } .boxed(); @@ -2343,27 +2360,26 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new( + PlotQueryRectangle::with_bounds( + BoundingBox2D::new( (46.478_278_849, 40.584_655_660_000_1).into(), (87.323_796_021_000_1, 55.434_550_273).into(), ) .unwrap(), - time_interval: TimeInterval::new( + TimeInterval::new( DateTime::new_utc(1900, 4, 1, 0, 0, 0), DateTime::new_utc_with_millis(2055, 4, 1, 0, 0, 0, 1), ) .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked(0.1, 0.1), - attributes: PlotSeriesSelection::all(), - }, + PlotSeriesSelection::all(), + ), &query_context, ) .await .unwrap(); assert_eq!(result, PlotData { - vega_string: "{\"$schema\":\"https://vega.github.io/schema/vega-lite/v4.17.0.json\",\"data\":{\"values\":[{\"x\":\"2015-01-01T00:00:00+00:00\",\"y\":46.342800000000004},{\"x\":\"2055-01-01T00:00:00+00:00\",\"y\":43.54399999999997}]},\"description\":\"Area Plot\",\"encoding\":{\"x\":{\"field\":\"x\",\"title\":\"Time\",\"type\":\"temporal\"},\"y\":{\"field\":\"y\",\"title\":\"\",\"type\":\"quantitative\"}},\"mark\":{\"line\":true,\"point\":true,\"type\":\"line\"}}".to_string(), + vega_string: "{\"$schema\":\"https://vega.github.io/schema/vega-lite/v4.17.0.json\",\"data\":{\"values\":[{\"x\":\"2015-01-01T00:00:00+00:00\",\"y\":46.68000000000007},{\"x\":\"2055-01-01T00:00:00+00:00\",\"y\":43.72000000000009}]},\"description\":\"Area Plot\",\"encoding\":{\"x\":{\"field\":\"x\",\"title\":\"Time\",\"type\":\"temporal\"},\"y\":{\"field\":\"y\",\"title\":\"\",\"type\":\"quantitative\"}},\"mark\":{\"line\":true,\"point\":true,\"type\":\"line\"}}".to_string(), metadata: PlotMetaData::None, }); } diff --git a/services/src/datasets/external/netcdfcf/overviews.rs b/services/src/datasets/external/netcdfcf/overviews.rs index 678b72345..7e3e86a5a 100644 --- a/services/src/datasets/external/netcdfcf/overviews.rs +++ b/services/src/datasets/external/netcdfcf/overviews.rs @@ -664,7 +664,6 @@ impl CogRasterCreationOptionss { fn inner(this: &CogRasterCreationOptionss) -> Result { let mut options = RasterCreationOptions::new(); options.add_name_value("COMPRESS", &this.compression_format)?; - options.add_name_value("TILED", "YES")?; options.add_name_value("LEVEL", &this.compression_level)?; options.add_name_value("NUM_THREADS", &this.num_threads)?; options.add_name_value("BLOCKSIZE", COG_BLOCK_SIZE)?; @@ -769,12 +768,13 @@ mod tests { use crate::{contexts::SessionContext, ge_context, tasks::util::NopTaskContext}; use gdal::{DatasetOptions, GdalOpenFlags}; use geoengine_datatypes::{ - primitives::{DateTime, SpatialResolution, TimeInterval}, - raster::RasterDataType, + primitives::{DateTime, TimeInterval}, + raster::{GeoTransform, GridBoundingBox2D, RasterDataType}, spatial_reference::SpatialReference, test_data, util::gdal::hide_gdal_errors, }; + use geoengine_operators::engine::SpatialGridDescriptor; use geoengine_operators::{ engine::{RasterBandDescriptors, RasterResultDescriptor}, source::{ @@ -819,6 +819,8 @@ mod tests { ) .unwrap(); + let expected_time_1: TimeInstance = DateTime::new_utc(2020, 1, 1, 0, 0, 0).into(); + let expected_time_2: TimeInstance = DateTime::new_utc(2020, 2, 1, 0, 0, 0).into(); assert_eq!( loading_info, GdalMetaDataList { @@ -826,17 +828,15 @@ mod tests { data_type: RasterDataType::I16, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: Some(SpatialResolution::new_unchecked(1.0, 1.0)), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((50., 55.).into(), 1., -1.), + GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: vec![ GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(2020, 1, 1, 0, 0, 0), - DateTime::new_utc(2020, 1, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_1, expected_time_1 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: Path::new("foo/2020-01-01T00:00:00.000Z.tiff").into(), rasterband_channel: 1, @@ -858,11 +858,7 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }, GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(2020, 2, 1, 0, 0, 0), - DateTime::new_utc(2020, 2, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_2, expected_time_2 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: Path::new("foo/2020-02-01T00:00:00.000Z.tiff").into(), rasterband_channel: 1, @@ -994,6 +990,11 @@ mod tests { .await .unwrap() .unwrap(); + + let expected_time_1: TimeInstance = DateTime::new_utc(1900, 1, 1, 0, 0, 0).into(); + let expected_time_2: TimeInstance = DateTime::new_utc(2015, 1, 1, 0, 0, 0).into(); + let expected_time_3: TimeInstance = DateTime::new_utc(2055, 1, 1, 0, 0, 0).into(); + pretty_assertions::assert_eq!( sample_loading_info, GdalMetaDataList { @@ -1001,17 +1002,15 @@ mod tests { data_type: RasterDataType::I16, spatial_reference: SpatialReference::epsg_4326().into(), time: None, - bbox: None, - resolution: Some(SpatialResolution::new_unchecked(1.0, 1.0)), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((50., 55.).into(), 1., -1.), + GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: vec![ GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(1900, 1, 1, 0, 0, 0), - DateTime::new_utc(1900, 1, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_1, expected_time_1 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: dataset_folder .join("metric_2/0/1900-01-01T00:00:00.000Z.tiff"), @@ -1034,11 +1033,7 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }, GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(2015, 1, 1, 0, 0, 0), - DateTime::new_utc(2015, 1, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_2, expected_time_2 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: dataset_folder .join("metric_2/0/2015-01-01T00:00:00.000Z.tiff"), @@ -1061,11 +1056,7 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }, GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(2055, 1, 1, 0, 0, 0), - DateTime::new_utc(2055, 1, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_3, expected_time_3 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: dataset_folder .join("metric_2/0/2055-01-01T00:00:00.000Z.tiff"), diff --git a/services/src/datasets/external/pangaea/mod.rs b/services/src/datasets/external/pangaea/mod.rs index 27e2dc624..6c0b9025e 100644 --- a/services/src/datasets/external/pangaea/mod.rs +++ b/services/src/datasets/external/pangaea/mod.rs @@ -273,8 +273,8 @@ mod tests { }; use geoengine_datatypes::dataset::{DataId, ExternalDataId, LayerId}; use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, Coordinate2D, MultiPointAccess, SpatialResolution, - TimeInterval, VectorQueryRectangle, + BoundingBox2D, ColumnSelection, Coordinate2D, MultiPointAccess, TimeInterval, + VectorQueryRectangle, }; use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::{ @@ -515,12 +515,11 @@ mod tests { panic!("Expected Data QueryProcessor"); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let ctx = MockQueryContext::test_default(); let result = proc.query(query_rectangle, &ctx).await; @@ -580,12 +579,11 @@ mod tests { panic!("Expected MultiPoint QueryProcessor"); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let ctx = MockQueryContext::test_default(); let result: Vec = proc @@ -656,12 +654,11 @@ mod tests { panic!("Expected MultiPolygon QueryProcessor"); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let ctx = MockQueryContext::test_default(); let result: Vec = proc @@ -727,12 +724,11 @@ mod tests { panic!("Expected MultiPoint QueryProcessor"); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let ctx = MockQueryContext::test_default(); let result: Vec = proc diff --git a/services/src/datasets/external/sentinel_s2_l2a_cogs.rs b/services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs similarity index 85% rename from services/src/datasets/external/sentinel_s2_l2a_cogs.rs rename to services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs index 80ec7009c..f0ce4f777 100644 --- a/services/src/datasets/external/sentinel_s2_l2a_cogs.rs +++ b/services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs @@ -12,23 +12,23 @@ use crate::layers::listing::{ use crate::projects::{RasterSymbology, Symbology}; use crate::stac::{Feature as StacFeature, FeatureCollection as StacCollection, StacAsset}; use crate::util::operators::source_operator_from_dataset; +use crate::util::sentinel_2_utm_zones::UtmZone; use crate::workflows::workflow::Workflow; use async_trait::async_trait; use geoengine_datatypes::dataset::{DataId, DataProviderId, LayerId, NamedData}; use geoengine_datatypes::operations::image::{RasterColorizer, RgbaColor}; use geoengine_datatypes::operations::reproject::{ - CoordinateProjection, CoordinateProjector, ReprojectClipped, + CoordinateProjection, CoordinateProjector, Reproject, }; -use geoengine_datatypes::primitives::CacheTtlSeconds; +use geoengine_datatypes::primitives::{AxisAlignedRectangle, CacheTtlSeconds}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BoundingBox2D, DateTime, Duration, RasterQueryRectangle, - SpatialPartitioned, TimeInstance, TimeInterval, VectorQueryRectangle, + DateTime, Duration, RasterQueryRectangle, TimeInstance, TimeInterval, VectorQueryRectangle, }; -use geoengine_datatypes::raster::RasterDataType; +use geoengine_datatypes::raster::{GeoTransform, SpatialGridDefinition}; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority}; use geoengine_operators::engine::{ MetaData, MetaDataProvider, OperatorName, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, TypedOperator, VectorResultDescriptor, + RasterResultDescriptor, SpatialGridDescriptor, TypedOperator, VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{ @@ -40,14 +40,36 @@ use geoengine_operators::util::retry::retry; use log::debug; use postgres_types::{FromSql, ToSql}; use reqwest::Client; +use sentinel_2_l2a_bands::{ImageProduct, ImageProductpec}; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, ensure}; use std::collections::HashMap; use std::convert::TryInto; use std::fmt::Debug; +use std::ops::Neg; use std::path::PathBuf; +mod sentinel_2_l2a_bands; + static STAC_RETRY_MAX_BACKOFF_MS: u64 = 60 * 60 * 1000; +static ELEMENT_84_STAC_SENTINEL2_L2A_PRODUCTS: &[ImageProduct] = &[ + ImageProduct::B01, + ImageProduct::B02, + ImageProduct::B03, + ImageProduct::B04, + ImageProduct::B05, + ImageProduct::B06, + ImageProduct::B07, + ImageProduct::B08, + ImageProduct::B8A, + ImageProduct::B09, + ImageProduct::B10, + ImageProduct::B11, + ImageProduct::B12, + ImageProduct::Aot, + ImageProduct::Wvp, + ImageProduct::Scl, +]; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, FromSql, ToSql)] #[serde(rename_all = "camelCase")] @@ -57,8 +79,6 @@ pub struct SentinelS2L2ACogsProviderDefinition { pub description: String, pub priority: Option, pub api_url: String, - pub bands: Vec, - pub zones: Vec, #[serde(default)] pub stac_api_retries: StacApiRetries, #[serde(default)] @@ -129,8 +149,6 @@ impl DataProviderDefinition for SentinelS2L2ACogsProviderDefi self.name, self.description, self.api_url, - &self.bands, - &self.zones, self.stac_api_retries, self.gdal_retries, self.cache_ttl, @@ -155,24 +173,10 @@ impl DataProviderDefinition for SentinelS2L2ACogsProviderDefi } } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, FromSql, ToSql)] -#[serde(rename_all = "camelCase")] -pub struct StacBand { - pub name: String, - pub no_data_value: Option, - pub data_type: RasterDataType, -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, FromSql, ToSql)] -pub struct StacZone { - pub name: String, - pub epsg: u32, -} - #[derive(Debug, Clone, PartialEq)] pub struct SentinelDataset { - band: StacBand, - zone: StacZone, + band: ImageProduct, + zone: UtmZone, listing: Layer, } @@ -202,8 +206,6 @@ impl SentinelS2L2aCogsDataProvider { name: String, description: String, api_url: String, - bands: &[StacBand], - zones: &[StacZone], stac_api_retries: StacApiRetries, gdal_retries: GdalRetries, cache_ttl: CacheTtlSeconds, @@ -214,7 +216,7 @@ impl SentinelS2L2aCogsDataProvider { name, description, api_url, - datasets: Self::create_datasets(&id, bands, zones), + datasets: Self::create_datasets(&id), stac_api_retries, gdal_retries, cache_ttl, @@ -222,22 +224,19 @@ impl SentinelS2L2aCogsDataProvider { } } - fn create_datasets( - id: &DataProviderId, - bands: &[StacBand], - zones: &[StacZone], - ) -> HashMap { - zones - .iter() + fn create_datasets(id: &DataProviderId) -> HashMap { + UtmZone::zones() .flat_map(|zone| { - bands.iter().map(move |band| { - let layer_id = LayerId(format!("{}:{}", zone.name, band.name)); - let listing = Layer { + ELEMENT_84_STAC_SENTINEL2_L2A_PRODUCTS + .iter() + .map(move |band| { + let layer_id = LayerId(format!("{}:{}", zone, band.name())); + let listing = Layer { id: ProviderLayerId { provider_id: *id, layer_id: layer_id.clone(), }, - name: format!("Sentinel S2 L2A COGS {}:{}", zone.name, band.name), + name: format!("Sentinel S2 L2A COGS {}:{} ({})", zone, band.long_name(), band.name()), description: String::new(), workflow: Workflow { operator: source_operator_from_dataset( @@ -274,14 +273,14 @@ impl SentinelS2L2aCogsDataProvider { metadata: HashMap::new(), }; - let dataset = SentinelDataset { - zone: zone.clone(), - band: band.clone(), - listing, - }; + let dataset = SentinelDataset { + zone, + band: *band, + listing, + }; - (layer_id, dataset) - }) + (layer_id, dataset) + }) }) .collect() } @@ -377,13 +376,11 @@ impl LayerCollectionProvider for SentinelS2L2aCogsDataProvider { workflow: Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: NamedData { - namespace: None, - provider: Some(self.id.to_string()), - name: id.to_string(), - }, - }, + params: GdalSourceParameters::new(NamedData { + namespace: None, + provider: Some(self.id.to_string()), + name: id.to_string(), + }), } .boxed(), ), @@ -398,8 +395,8 @@ impl LayerCollectionProvider for SentinelS2L2aCogsDataProvider { #[derive(Debug, Clone)] pub struct SentinelS2L2aCogsMetaData { api_url: String, - zone: StacZone, - band: StacBand, + zone: UtmZone, + band: ImageProduct, stac_api_retries: StacApiRetries, gdal_retries: GdalRetries, cache_ttl: CacheTtlSeconds, @@ -438,7 +435,7 @@ impl SentinelS2L2aCogsMetaData { .filter(|f| { f.properties .proj_epsg - .is_some_and(|epsg| epsg == self.zone.epsg) + .is_some_and(|epsg| epsg == self.zone.epsg_code()) }) .collect(); @@ -465,7 +462,7 @@ impl SentinelS2L2aCogsMetaData { start_times[i + 1] } else { // (or end of query?) - query.time_interval.end() + query_end_buffer + start + query_end_buffer }; /* @@ -512,22 +509,27 @@ impl SentinelS2L2aCogsMetaData { let time_interval = TimeInterval::new(start, end)?; - if time_interval.start() <= query.time_interval.start() { - let t = if time_interval.end() > query.time_interval.start() { - time_interval.start() - } else { - time_interval.end() - }; - known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); - } + if time_interval.contains(&query.time_interval) { + let t1 = time_interval.start(); + let t2 = time_interval.end(); + known_time_start = Some(t1); + known_time_end = Some(t2); + } else { + if time_interval.end() <= query.time_interval.start() { + let t1 = time_interval.end(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } else if time_interval.start() <= query.time_interval.start() { + let t1 = time_interval.start(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } - if time_interval.end() >= query.time_interval.end() { - let t = if time_interval.start() < query.time_interval.end() { - time_interval.end() - } else { - time_interval.start() - }; - known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + if time_interval.start() >= query.time_interval.end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() >= query.time_interval.end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } } if time_interval.intersects(&query.time_interval) { @@ -536,16 +538,16 @@ impl SentinelS2L2aCogsMetaData { time_interval, feature .assets - .get(&self.band.name) + .get(self.band.name()) .map_or(&"n/a".to_string(), |a| &a.href) ); let asset = feature .assets - .get(&self.band.name) + .get(self.band.name()) .ok_or(error::Error::StacNoSuchBand { - band_name: self.band.name.clone(), + band_name: self.band.name().to_owned(), })?; parts.push(self.create_loading_info_part(time_interval, asset, self.cache_ttl)?); @@ -619,7 +621,7 @@ impl SentinelS2L2aCogsMetaData { width: stac_shape_x as usize, height: stac_shape_y as usize, file_not_found_handling: geoengine_operators::source::FileNotFoundHandling::NoData, - no_data_value: self.band.no_data_value, + no_data_value: self.band.no_data_value(), properties_mapping: None, gdal_open_options: None, gdal_config_options: Some(vec![ @@ -656,18 +658,21 @@ impl SentinelS2L2aCogsMetaData { let t_start = t_start - Duration::seconds(self.stac_query_buffer.start_seconds); let t_end = t_end + Duration::seconds(self.stac_query_buffer.end_seconds); + let native_spatial_ref = + SpatialReference::new(SpatialReferenceAuthority::Epsg, self.zone.epsg_code()); + let epsg_4326_ref = SpatialReference::epsg_4326(); + let projector = CoordinateProjector::from_known_srs(native_spatial_ref, epsg_4326_ref)?; + let native_bounds = self.zone.native_extent(); + // request all features in zone in order to be able to determine the temporal validity of individual tile - let projector = CoordinateProjector::from_known_srs( - SpatialReference::new(SpatialReferenceAuthority::Epsg, self.zone.epsg), - SpatialReference::epsg_4326(), - )?; - - let spatial_partition = query.spatial_partition(); // TODO: use SpatialPartition2D directly - let bbox = BoundingBox2D::new_upper_left_lower_right_unchecked( - spatial_partition.upper_left(), - spatial_partition.lower_right(), - ); - let bbox = bbox.reproject_clipped(&projector)?; // TODO: use reproject_clipped on SpatialPartition2D + let bbox = native_bounds + .reproject(&projector) + .inspect_err(|e| { + debug!( + "could not project zone bounds to EPSG:4326. Was: {native_bounds:?}. Source: {e}" + ); + }) + .ok(); Ok(bbox.map(|bbox| { vec![ @@ -795,16 +800,23 @@ impl MetaData } async fn result_descriptor(&self) -> geoengine_operators::util::Result { + let geo_transform = GeoTransform::new( + self.zone.native_extent().upper_left(), + self.band.resolution_m(), + self.band.resolution_m().neg(), + ); + let grid_bounds = geo_transform.spatial_to_grid_bounds(&self.zone.native_extent()); + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); + Ok(RasterResultDescriptor { - data_type: self.band.data_type, + data_type: self.band.data_type(), spatial_reference: SpatialReference::new( SpatialReferenceAuthority::Epsg, - self.zone.epsg, + self.zone.epsg_code(), ) .into(), time: None, - bbox: None, - resolution: None, // TODO: determine from STAC or data or hardcode it + spatial_grid: SpatialGridDescriptor::new_source(spatial_grid), bands: RasterBandDescriptors::new_single_band(), }) } @@ -842,8 +854,8 @@ impl MetaDataProvider> = provider @@ -952,20 +961,29 @@ mod tests { .await .unwrap(); + let data_bounds = + SpatialPartition2D::new((166_021.44, 9_329_005.18).into(), (534_994.66, 0.00).into()) + .unwrap(); + + let raster_result_descriptor = meta.result_descriptor().await.unwrap(); + let tiling_grid_definition = raster_result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(exe_ctx.tiling_specification()); + + let data_bounds_in_pixel_grid = tiling_grid_definition + .tiling_geo_transform() + .spatial_to_grid_bounds(&data_bounds); + let loading_info = meta - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new( - (166_021.44, 9_329_005.18).into(), - (534_994.66, 0.00).into(), - ) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2021, 1, 2, 10, 2, 26))?, - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }) + .loading_info(RasterQueryRectangle::new_with_grid_bounds( + data_bounds_in_pixel_grid, + TimeInterval::new_instant(DateTime::new_utc(2021, 1, 2, 10, 2, 26))?, + BandSelection::first(), + )) .await .unwrap(); + // we expect only one tile because there is only this one at the queried time let expected = vec![GdalLoadingInfoTemporalSlice { time: TimeInterval::new_unchecked(1_609_581_746_000, 1_609_581_758_000), params: Some(GdalDatasetParameters { @@ -1055,31 +1073,34 @@ mod tests { ); let op = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), } .boxed() .initialize(WorkflowOperatorPath::initialize_root(), &exe) .await .unwrap(); - let processor = op.query_processor()?.get_u16().unwrap(); - - let spatial_bounds = - SpatialPartition2D::new((166_021.44, 9_329_005.18).into(), (534_994.66, 0.00).into()) - .unwrap(); + let sp = SpatialPartition2D::new( + (600_000.000, 5_500_020.000).into(), // 1830 px + (709_800.000, 5_390_220.000).into(), // 1830 px + ) + .unwrap(); - let spatial_resolution = SpatialResolution::new_unchecked( - spatial_bounds.size_x() / 256., - spatial_bounds.size_y() / 256., + let processor = op.query_processor()?.get_u16().unwrap(); + let sp = processor + .raster_result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(exe.tiling_specification) + .tiling_geo_transform() + .spatial_to_grid_bounds(&sp); + + let query = RasterQueryRectangle::new_with_grid_bounds( + sp, + TimeInterval::new_instant(DateTime::new_utc(2018, 10, 19, 13, 23, 25))?, + BandSelection::first(), ); - let query = RasterQueryRectangle { - spatial_bounds, - time_interval: TimeInterval::new_instant(DateTime::new_utc(2021, 1, 2, 10, 2, 26))?, - spatial_resolution, - attributes: BandSelection::first(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let ctx = exe.mock_query_context(ChunkByteSize::MAX); let result = processor .raster_query(query, &ctx) @@ -1087,8 +1108,8 @@ mod tests { .collect::>() .await; - // TODO: check actual data - assert_eq!(result.len(), 1); + // This are 5 x 5 tiles since the sentinel tile intersects 5 x5 geo engine tiles + assert_eq!(result.len(), 25); // Ok(()) } @@ -1116,7 +1137,7 @@ mod tests { request::query(url_decoded(contains(("limit", "500")))), request::query(url_decoded(contains(( "bbox", - "[33.899332958586406,-2.261536424319933,33.900232774450984,-2.2606312588790414]" + "[9.396566748392315,-83.82852972938498,63.83756656611425,0]" )))), request::query(url_decoded(contains(( "datetime", @@ -1184,7 +1205,8 @@ mod tests { responders::status_code(206) .append_header("Content-Type", "application/json") .body( - include_bytes!("../../../../test_data/stac_responses/cog-header.bin").to_vec(), + include_bytes!("../../../../../test_data/stac_responses/cog-header.bin") + .to_vec(), ) .append_header( "x-amz-id-2", @@ -1252,7 +1274,7 @@ mod tests { .append_header("Content-Type", "application/json") .body( include_bytes!( - "../../../../test_data/stac_responses/cog-tile.bin" + "../../../../../test_data/stac_responses/cog-tile.bin" )[0..2] .to_vec() ).append_header( @@ -1277,7 +1299,7 @@ mod tests { .append_header("Content-Type", "application/json") .body( include_bytes!( - "../../../../test_data/stac_responses/cog-tile.bin" + "../../../../../test_data/stac_responses/cog-tile.bin" ) .to_vec() ).append_header( @@ -1309,15 +1331,6 @@ mod tests { description: "Access to Sentinel 2 L2A COGs on AWS".into(), priority: Some(22), api_url: server.url_str("/v0/collections/sentinel-s2-l2a-cogs/items"), - bands: vec![StacBand { - name: "B04".into(), - no_data_value: Some(0.), - data_type: RasterDataType::U16, - }], - zones: vec![StacZone { - name: "UTM36S".into(), - epsg: 32736, - }], stac_api_retries: Default::default(), gdal_retries: GdalRetries { number_of_retries: 999, @@ -1326,14 +1339,10 @@ mod tests { query_buffer: Default::default(), }); - let provider = provider_def - .initialize( - app_ctx - .session_context(app_ctx.create_anonymous_session().await.unwrap()) - .db(), - ) - .await - .unwrap(); + let session_ctx = + app_ctx.session_context(app_ctx.create_anonymous_session().await.unwrap()); + + let provider = provider_def.initialize(session_ctx.db()).await.unwrap(); let meta: Box> = provider @@ -1347,17 +1356,27 @@ mod tests { .await .unwrap(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (600_000.00, 9_750_100.).into(), - (600_100.0, 9_750_000.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2021, 9, 23, 8, 10, 44)) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked(10., 10.), - attributes: BandSelection::first(), - }; + let exe_ctx = session_ctx.execution_context().unwrap(); + let result_descriptor = meta.result_descriptor().await.unwrap(); + + let data_bounds = SpatialPartition2D::new_unchecked( + (600_000.00, 9_750_100.).into(), + (600_100.0, 9_750_000.).into(), + ); + + let tiling_geo_transform = result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(exe_ctx.tiling_specification()) + .tiling_geo_transform(); + + let sp: geoengine_datatypes::raster::GridBoundingBox<[isize; 2]> = + tiling_geo_transform.spatial_to_grid_bounds(&data_bounds); + let query = RasterQueryRectangle::new_with_grid_bounds( + sp, + TimeInterval::new_instant(DateTime::new_utc(2021, 9, 23, 8, 10, 44)).unwrap(), + BandSelection::first(), + ); let loading_info = meta.loading_info(query).await.unwrap(); let parts = if let GdalLoadingInfoTemporalSliceIterator::Static { parts } = loading_info.info { @@ -1425,21 +1444,14 @@ mod tests { name.clone(), Box::new(GdalMetaDataStatic { time: None, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U16, - spatial_reference: SpatialReference::from_str("EPSG:32736").unwrap().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, params, cache_ttl: CacheTtlSeconds::default(), }), ); let gdal_source = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), } .boxed() .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1452,20 +1464,21 @@ mod tests { let query_context = MockQueryContext::test_default(); + let data_bounds = SpatialPartition2D::new_unchecked( + (499_980., 9_804_800.).into(), + (499_990., 9_804_810.).into(), + ); + + let sp: geoengine_datatypes::raster::GridBoundingBox<[isize; 2]> = + tiling_geo_transform.spatial_to_grid_bounds(&data_bounds); + let stream = gdal_source .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (499_980., 9_804_800.).into(), - (499_990., 9_804_810.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2014, 3, 1, 0, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::new(10., 10.).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new_with_grid_bounds( + sp, + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ), &query_context, ) .await diff --git a/services/src/datasets/external/sentinel_s2_l2a_cogs/sentinel_2_l2a_bands.rs b/services/src/datasets/external/sentinel_s2_l2a_cogs/sentinel_2_l2a_bands.rs new file mode 100644 index 000000000..20ceba5c0 --- /dev/null +++ b/services/src/datasets/external/sentinel_s2_l2a_cogs/sentinel_2_l2a_bands.rs @@ -0,0 +1,95 @@ +use geoengine_datatypes::raster::RasterDataType; + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum ImageProduct { + B01, + B02, + B03, + B04, + B05, + B06, + B07, + B08, + B8A, + B09, + B10, + B11, + B12, + Scl, + Wvp, + Aot, + _Tci, +} + +pub trait ImageProductpec { + fn resolution_m(&self) -> f64; + fn name(&self) -> &str; + fn long_name(&self) -> &str; + fn no_data_value(&self) -> Option; + fn data_type(&self) -> RasterDataType; +} + +impl ImageProductpec for ImageProduct { + fn no_data_value(&self) -> Option { + Some(0.) + } + + fn resolution_m(&self) -> f64 { + match self { + ImageProduct::B02 + | ImageProduct::B03 + | ImageProduct::B04 + | ImageProduct::B08 + | ImageProduct::_Tci => 10., + ImageProduct::B05 + | ImageProduct::B06 + | ImageProduct::B07 + | ImageProduct::B8A + | ImageProduct::B11 + | ImageProduct::B12 + | ImageProduct::Scl + | ImageProduct::Wvp + | ImageProduct::Aot => 20., + ImageProduct::B01 | ImageProduct::B09 | ImageProduct::B10 => 60., + } + } + + fn name(&self) -> &str { + match self { + ImageProduct::B01 => "B01", + ImageProduct::B02 => "B02", + ImageProduct::B03 => "B03", + ImageProduct::B04 => "B04", + ImageProduct::B05 => "B05", + ImageProduct::B06 => "B06", + ImageProduct::B07 => "B07", + ImageProduct::B08 => "B08", + ImageProduct::B8A => "B8A", + ImageProduct::B09 => "B09", + ImageProduct::B10 => "B10", + ImageProduct::B11 => "B11", + ImageProduct::B12 => "B12", + ImageProduct::Scl => "SCL", + ImageProduct::Wvp => "WVP", + ImageProduct::Aot => "AOT", + ImageProduct::_Tci => "TCI", + } + } + + fn long_name(&self) -> &str { + match self { + ImageProduct::Scl => "Scene Classification", + ImageProduct::Wvp => "Water Vapour", + ImageProduct::Aot => "Aerosol Optical Thickness", + ImageProduct::_Tci => "True Colour Image", + _ => self.name(), + } + } + + fn data_type(&self) -> RasterDataType { + match self { + ImageProduct::Scl => RasterDataType::U8, + _ => RasterDataType::U16, + } + } +} diff --git a/services/src/datasets/listing.rs b/services/src/datasets/listing.rs index 4af6e3d3e..f86dbe67e 100644 --- a/services/src/datasets/listing.rs +++ b/services/src/datasets/listing.rs @@ -1,5 +1,6 @@ use super::DatasetName; use super::storage::MetaDataDefinition; +use crate::api::model::operators::TypedResultDescriptor; use crate::config::{DatasetService, get_config_element}; use crate::datasets::storage::{Dataset, validate_tags}; use crate::error::Result; @@ -8,7 +9,7 @@ use async_trait::async_trait; use geoengine_datatypes::dataset::{DataId, DatasetId}; use geoengine_datatypes::primitives::{RasterQueryRectangle, VectorQueryRectangle}; use geoengine_operators::engine::{ - MetaDataProvider, RasterResultDescriptor, TypedResultDescriptor, VectorResultDescriptor, + MetaDataProvider, RasterResultDescriptor, VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{GdalLoadingInfo, OgrSourceDataset}; diff --git a/services/src/datasets/mod.rs b/services/src/datasets/mod.rs index 2da973a78..b9c3ee3e4 100644 --- a/services/src/datasets/mod.rs +++ b/services/src/datasets/mod.rs @@ -8,7 +8,7 @@ pub mod storage; pub mod upload; pub(crate) use create_from_workflow::{ - RasterDatasetFromWorkflow, RasterDatasetFromWorkflowResult, + RasterDatasetFromWorkflow, RasterDatasetFromWorkflowParams, RasterDatasetFromWorkflowResult, schedule_raster_dataset_from_workflow_task, }; pub use name::{DatasetIdAndName, DatasetName, DatasetNameError}; diff --git a/services/src/datasets/postgres.rs b/services/src/datasets/postgres.rs index 834fd0faf..170750c51 100644 --- a/services/src/datasets/postgres.rs +++ b/services/src/datasets/postgres.rs @@ -228,6 +228,10 @@ where Ok(rows .iter() .map(|row| { + // get the real TypedResultDescriptor and convert it to the API one + let result_desc: TypedResultDescriptor = row.get(6); + let result_desc = result_desc.into(); + Result::::Ok(DatasetListing { id: row.get(0), name: row.get(1), @@ -235,7 +239,7 @@ where description: row.get(3), tags: row.get::<_, Option>>(4).unwrap_or_default(), source_operator: row.get(5), - result_descriptor: row.get(6), + result_descriptor: result_desc, symbology: row.get(7), }) }) diff --git a/services/src/datasets/storage.rs b/services/src/datasets/storage.rs index afa812444..4aa41a0e7 100755 --- a/services/src/datasets/storage.rs +++ b/services/src/datasets/storage.rs @@ -24,15 +24,13 @@ use validator::{Validate, ValidationError}; pub const DATASET_DB_ROOT_COLLECTION_ID: Uuid = Uuid::from_u128(0x5460_73b6_d535_4205_b601_9967_5c9f_6dd7); -#[derive(Debug, Serialize, Deserialize, Clone, ToSchema, Validate)] +#[derive(Debug, Serialize, Deserialize, Clone, Validate)] #[serde(rename_all = "camelCase")] pub struct Dataset { - #[schema(value_type = crate::api::model::datatypes::DatasetId)] pub id: DatasetId, pub name: DatasetName, pub display_name: String, pub description: String, - #[schema(value_type = crate::api::model::operators::TypedResultDescriptor)] pub result_descriptor: TypedResultDescriptor, pub source_operator: String, pub symbology: Option, @@ -49,7 +47,8 @@ impl Dataset { description: self.description.clone(), tags: self.tags.clone().unwrap_or_default(), // TODO: figure out if we want to use Option> everywhere or if Vec is fine source_operator: self.source_operator.clone(), - result_descriptor: self.result_descriptor.clone(), + // convert the TypedResultDescriptor to the API one + result_descriptor: self.result_descriptor.clone().into(), symbology: self.symbology.clone(), } } diff --git a/services/src/error.rs b/services/src/error.rs index 04ca664f8..8a783cb5e 100644 --- a/services/src/error.rs +++ b/services/src/error.rs @@ -499,6 +499,8 @@ pub enum Error { #[snafu(display("Raster band names must not be longer than 256 bytes"))] RasterBandNameTooLong, + ResolutionMissmatch, // FIXME: added this to mark sections where we need to do something about resolutions later + #[snafu(display("Resource id is invalid: type: {}, id: {}", resource_type, resource_id))] InvalidResourceId { resource_type: String, diff --git a/services/src/util/mod.rs b/services/src/util/mod.rs index 35f885c38..3d6b91a1e 100644 --- a/services/src/util/mod.rs +++ b/services/src/util/mod.rs @@ -20,6 +20,7 @@ pub mod parsing; pub mod postgres; pub mod server; // TODO: refactor to be gated by `#[cfg(test)]` +pub mod sentinel_2_utm_zones; pub mod tests; pub mod workflows; diff --git a/services/src/util/operators.rs b/services/src/util/operators.rs index 0f995de0f..b9418c772 100644 --- a/services/src/util/operators.rs +++ b/services/src/util/operators.rs @@ -23,7 +23,7 @@ pub fn source_operator_from_dataset( ), GdalSource::TYPE_NAME => TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), } .boxed(), ), diff --git a/services/src/util/sentinel_2_utm_zones.rs b/services/src/util/sentinel_2_utm_zones.rs new file mode 100644 index 000000000..26471b9c9 --- /dev/null +++ b/services/src/util/sentinel_2_utm_zones.rs @@ -0,0 +1,133 @@ +use std::str::FromStr; + +use geoengine_datatypes::{ + primitives::{Coordinate2D, SpatialPartition2D}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority}, +}; +use snafu::Snafu; +use strum::IntoStaticStr; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct UtmZone { + pub zone: u8, + pub direction: UtmZoneDirection, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum UtmZoneDirection { + North, + South, +} + +#[derive(Debug, Snafu, IntoStaticStr)] +pub enum UtmZoneError { + IdMustStartWithUtm, + DirectionNotNorthOrSouth, + ZoneSuffixNotANumber, +} + +impl UtmZone { + pub fn epsg_code(self) -> u32 { + match self.direction { + UtmZoneDirection::North => 32600 + u32::from(self.zone), + UtmZoneDirection::South => 32700 + u32::from(self.zone), + } + } + + pub fn spatial_reference(self) -> SpatialReference { + SpatialReference::new(SpatialReferenceAuthority::Epsg, self.epsg_code()) + } + + pub fn extent(self) -> Option { + // TODO: as Sentinel uses enlarged grids, we could return a larger extent + self.spatial_reference().area_of_use().ok() + } + + pub fn native_extent(self) -> SpatialPartition2D { + match (self.zone, self.direction) { + (32 | 34 | 36, UtmZoneDirection::North) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 8_000_040.0), + Coordinate2D::new(909_780.0, -9_780.0), + ), + (60, UtmZoneDirection::North) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 9_100_020.0), + Coordinate2D::new(809_760.0, -9_780.0), + ), + (_, UtmZoneDirection::North) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 9_400_020.0), + Coordinate2D::new(909_780.0, -9_780.0), + ), + (1, UtmZoneDirection::South) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(99_960.0, 10_000_000.0), + Coordinate2D::new(909_780.0, 690_220.0), + ), + (60, UtmZoneDirection::South) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 10_000_000.0), + Coordinate2D::new(809_760.0, 890_200.0), + ), + (_, UtmZoneDirection::South) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 10_000_000.0), + Coordinate2D::new(909_780.0, 690_220.0), + ), + } + } +} + +impl FromStr for UtmZone { + type Err = UtmZoneError; + + fn from_str(s: &str) -> Result { + if s.len() < 5 || &s[..3] != "UTM" { + return Err(UtmZoneError::IdMustStartWithUtm); + } + + let (zone_str, dir_char) = s[3..].split_at(s.len() - 4); + let zone = zone_str + .parse::() + .map_err(|_| UtmZoneError::ZoneSuffixNotANumber)?; + + // TODO: check if zone is in valid range + + let north = match dir_char { + "N" => UtmZoneDirection::North, + "S" => UtmZoneDirection::South, + _ => return Err(UtmZoneError::DirectionNotNorthOrSouth), + }; + + Ok(Self { + zone, + direction: north, + }) + } +} + +impl std::fmt::Display for UtmZone { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "UTM{}{}", + self.zone, + match self.direction { + UtmZoneDirection::North => "N", + UtmZoneDirection::South => "S", + } + ) + } +} + +impl UtmZone { + pub fn zones() -> impl Iterator { + (1..=60).flat_map(|zone| { + vec![ + UtmZone { + zone, + direction: UtmZoneDirection::North, + }, + UtmZone { + zone, + direction: UtmZoneDirection::South, + }, + ] + }) + } +} diff --git a/services/src/util/tests.rs b/services/src/util/tests.rs index 7614887fd..ca0434c88 100644 --- a/services/src/util/tests.rs +++ b/services/src/util/tests.rs @@ -51,7 +51,8 @@ use geoengine_datatypes::operations::image::RasterColorizer; use geoengine_datatypes::operations::image::RgbaColor; use geoengine_datatypes::primitives::CacheTtlSeconds; use geoengine_datatypes::primitives::Coordinate2D; -use geoengine_datatypes::primitives::SpatialResolution; +use geoengine_datatypes::raster::GeoTransform; +use geoengine_datatypes::raster::GridBoundingBox2D; use geoengine_datatypes::raster::RasterDataType; use geoengine_datatypes::raster::RenameBands; use geoengine_datatypes::spatial_reference::SpatialReference; @@ -63,6 +64,7 @@ use geoengine_operators::engine::QueryContext; use geoengine_operators::engine::RasterBandDescriptor; use geoengine_operators::engine::RasterBandDescriptors; use geoengine_operators::engine::RasterResultDescriptor; +use geoengine_operators::engine::SpatialGridDescriptor; use geoengine_operators::engine::WorkflowOperatorPath; use geoengine_operators::engine::{ChunkByteSize, MultipleRasterSources}; use geoengine_operators::engine::{RasterOperator, TypedOperator}; @@ -148,7 +150,7 @@ pub async fn register_ndvi_workflow_helper_with_cache_ttl( let workflow = Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: dataset }, + params: GdalSourceParameters::new(dataset), } .boxed(), ), @@ -239,7 +241,7 @@ pub async fn add_ndvi_to_datasets_with_cache_ttl( pub async fn add_land_cover_to_datasets(db: &D) -> DatasetName { let ndvi = DatasetDefinition { properties: AddDataset { - name: None, + name: Some(DatasetName::new(None, "land_cover_raster_test".to_string())), display_name: "Land Cover".to_string(), description: "Land Cover derived from MODIS/Terra+Aqua Land Cover".to_string(), source_operator: "GdalSource".to_string(), @@ -305,11 +307,10 @@ pub async fn add_land_cover_to_datasets(db: &D) -> DatasetName { data_type: RasterDataType::U8, spatial_reference: SpatialReferenceOption::SpatialReference(SpatialReference::epsg_4326()), time: Some(geoengine_datatypes::primitives::TimeInterval::default()), - bbox: Some(geoengine_datatypes::primitives::SpatialPartition2D::new((-180., 90.).into(), - (180., -90.).into()).unwrap()), - resolution: Some(SpatialResolution { - x: 0.1, y: 0.1, - }), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(-180., 90.), 0.1, -0.1), + GridBoundingBox2D::new_min_max(0,1799, 0, 1599).unwrap(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new("band".into(), geoengine_datatypes::primitives::Measurement::classification("Land Cover".to_string(), [ (0_u8, "Water Bodies".to_string()), @@ -379,18 +380,21 @@ pub async fn register_ne2_multiband_workflow( GdalSource { params: GdalSourceParameters { data: blue.name.into(), + overview_level: None, }, } .boxed(), GdalSource { params: GdalSourceParameters { data: green.name.into(), + overview_level: None, }, } .boxed(), GdalSource { params: GdalSourceParameters { data: red.name.into(), + overview_level: None, }, } .boxed(), diff --git a/services/src/workflows/raster_stream.rs b/services/src/workflows/raster_stream.rs index d86ecef37..e9d8de359 100644 --- a/services/src/workflows/raster_stream.rs +++ b/services/src/workflows/raster_stream.rs @@ -10,10 +10,9 @@ use geoengine_datatypes::{ }; use geoengine_operators::{ call_on_generic_raster_processor, - engine::{ - QueryAbortTrigger, QueryContext, QueryProcessorExt, RasterOperator, WorkflowOperatorPath, - }, + engine::{InitializedRasterOperator, QueryAbortTrigger, QueryContext, QueryProcessorExt}, }; +use tracing::debug; pub struct RasterWebsocketStreamHandler { state: RasterWebsocketStreamHandlerState, @@ -29,6 +28,16 @@ enum RasterWebsocketStreamHandlerState { Processing { _fut: SpawnHandle }, } +impl std::fmt::Debug for RasterWebsocketStreamHandlerState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Closed => write!(f, "Closed"), + Self::Idle { stream: _ } => write!(f, "Idle"), + Self::Processing { _fut: _ } => write!(f, "Processing"), + } + } +} + impl Default for RasterWebsocketStreamHandlerState { fn default() -> Self { Self::Closed @@ -44,7 +53,7 @@ impl StreamHandler> for RasterWebsocketSt fn finished(&mut self, ctx: &mut Self::Context) { ctx.stop(); - + debug!("Stream finished."); self.abort_processing(); } @@ -53,6 +62,7 @@ impl StreamHandler> for RasterWebsocketSt Ok(ws::Message::Ping(msg)) => ctx.pong(&msg), Ok(ws::Message::Text(text)) if &text == "NEXT" => self.next_tile(ctx), Ok(ws::Message::Close(reason)) => { + debug!("Stream was closed. Reason: {:?}", reason); ctx.close(reason); self.finished(ctx); @@ -65,20 +75,15 @@ impl StreamHandler> for RasterWebsocketSt impl RasterWebsocketStreamHandler { pub async fn new( - raster_operator: Box, + initialized_raster_operator: Box, query_rectangle: RasterQueryRectangle, - execution_ctx: C::ExecutionContext, mut query_ctx: C::QueryContext, ) -> Result { - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + let spatial_reference = initialized_raster_operator + .result_descriptor() + .spatial_reference; - let initialized_operator = raster_operator - .initialize(workflow_operator_path_root, &execution_ctx) - .await?; - - let spatial_reference = initialized_operator.result_descriptor().spatial_reference; - - let query_processor = initialized_operator.query_processor()?; + let query_processor = initialized_raster_operator.query_processor()?; let abort_handle = query_ctx.abort_trigger().ok(); @@ -147,6 +152,7 @@ fn send_result( } Some(Err(e)) => { // on error, send the error and close the connection + debug!("Tile error in stream: {e}"); actor.state = RasterWebsocketStreamHandlerState::Closed; ctx.close(Some(CloseReason { code: CloseCode::Error, @@ -157,6 +163,7 @@ fn send_result( None => { // stream ended actor.state = RasterWebsocketStreamHandlerState::Closed; + debug!("Sttream is empty --> ended."); ctx.close(Some(CloseReason { code: CloseCode::Normal, description: None, @@ -181,11 +188,12 @@ mod tests { use bytes::{Bytes, BytesMut}; use futures::channel::mpsc::UnboundedSender; use geoengine_datatypes::{ - primitives::{ - BandSelection, DateTime, SpatialPartition2D, SpatialResolution, TimeInterval, - }, + primitives::{BandSelection, DateTime, TimeInterval}, + raster::GridBoundingBox2D, util::arrow::arrow_ipc_file_to_record_batches, }; + + use geoengine_operators::engine::WorkflowOperatorPath; use tokio_postgres::NoTls; use uuid::Uuid; @@ -209,19 +217,24 @@ mod tests { let (workflow, workflow_id) = register_ndvi_workflow_helper(&app_ctx).await; - let query_rectangle = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let workflow_root_path = WorkflowOperatorPath::initialize_root(); + let initialized_operator = workflow + .operator + .get_raster() + .unwrap() + .initialize(workflow_root_path, &ctx.execution_context().unwrap()) + .await + .unwrap(); + + let query_rectangle = RasterQueryRectangle::new_with_grid_bounds( + GridBoundingBox2D::new_min_max(-90, 89, -180, 179).unwrap(), // This is just a part of the raster but the original test used a resolution of 1.0 instead of the 0.1 the data actually has + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ); let handler = RasterWebsocketStreamHandler::new::>( - workflow.operator.get_raster().unwrap(), + initialized_operator, query_rectangle, - ctx.execution_context().unwrap(), ctx.query_context(workflow_id.0, Uuid::new_v4()).unwrap(), ) .await diff --git a/services/src/workflows/vector_stream.rs b/services/src/workflows/vector_stream.rs index df59676b9..2f46bafb7 100644 --- a/services/src/workflows/vector_stream.rs +++ b/services/src/workflows/vector_stream.rs @@ -205,10 +205,7 @@ mod tests { use geoengine_datatypes::primitives::ColumnSelection; use geoengine_datatypes::{ collections::MultiPointCollection, - primitives::{ - BoundingBox2D, CacheHint, DateTime, FeatureData, MultiPoint, SpatialResolution, - TimeInterval, - }, + primitives::{BoundingBox2D, CacheHint, DateTime, FeatureData, MultiPoint, TimeInterval}, util::arrow::arrow_ipc_file_to_record_batches, }; use geoengine_operators::engine::ChunkByteSize; @@ -273,17 +270,12 @@ mod tests { ), }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_upper_left_lower_right( - (-180., 90.).into(), - (180., -90.).into(), - ) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)) + let query_rectangle = VectorQueryRectangle::with_bounds( + BoundingBox2D::new_upper_left_lower_right((-180., 90.).into(), (180., -90.).into()) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + ColumnSelection::all(), + ); let handler = VectorWebsocketStreamHandler::new::>( workflow.operator.get_vector().unwrap(), diff --git a/services/src/workflows/workflow.rs b/services/src/workflows/workflow.rs index 2cbab5f5b..7f1d09b24 100644 --- a/services/src/workflows/workflow.rs +++ b/services/src/workflows/workflow.rs @@ -47,9 +47,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -73,7 +71,10 @@ mod tests { }, { "x": 1.0, "y": 2.0 - }] + }], + "spatialBounds": { + "type": "none" + } } } }) diff --git a/test_data/dataset_defs/landcover.json b/test_data/dataset_defs/landcover.json index ed33a27a8..4acb47e96 100644 --- a/test_data/dataset_defs/landcover.json +++ b/test_data/dataset_defs/landcover.json @@ -49,18 +49,26 @@ "resultDescriptor": { "dataType": "U8", "spatialReference": "EPSG:4326", - "noDataValue": 255.0, "time": { "start": "-262143-01-01T00:00:00+00:00", "end": "+262142-12-31T23:59:59.999+00:00" }, - "bbox": { - "upperLeftCoordinate": [-180.0, 90.0], - "lowerRightCoordinate": [180.0, -90.0] - }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/natural_earth_2_blue.json b/test_data/dataset_defs/natural_earth_2_blue.json index 13193840b..bb1fa030e 100644 --- a/test_data/dataset_defs/natural_earth_2_blue.json +++ b/test_data/dataset_defs/natural_earth_2_blue.json @@ -49,13 +49,22 @@ "start": "-262143-01-01T00:00:00+00:00", "end": "+262142-12-31T23:59:59.999+00:00" }, - "bbox": { - "upperLeftCoordinate": [-180, 90.0], - "lowerRightCoordinate": [180.0, -90.0] - }, - "resolution": { - "x": 0.09999999999999, - "y": 0.09999999999999 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/natural_earth_2_green.json b/test_data/dataset_defs/natural_earth_2_green.json index ffbe3f03b..f3d9c6340 100644 --- a/test_data/dataset_defs/natural_earth_2_green.json +++ b/test_data/dataset_defs/natural_earth_2_green.json @@ -49,13 +49,22 @@ "start": "-262143-01-01T00:00:00+00:00", "end": "+262142-12-31T23:59:59.999+00:00" }, - "bbox": { - "upperLeftCoordinate": [-180, 90.0], - "lowerRightCoordinate": [180.0, -90.0] - }, - "resolution": { - "x": 0.09999999999999, - "y": 0.09999999999999 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/natural_earth_2_red.json b/test_data/dataset_defs/natural_earth_2_red.json index c4510eaf3..ca994894c 100644 --- a/test_data/dataset_defs/natural_earth_2_red.json +++ b/test_data/dataset_defs/natural_earth_2_red.json @@ -49,13 +49,22 @@ "start": "-262143-01-01T00:00:00+00:00", "end": "+262142-12-31T23:59:59.999+00:00" }, - "bbox": { - "upperLeftCoordinate": [-180, 90.0], - "lowerRightCoordinate": [180.0, -90.0] - }, - "resolution": { - "x": 0.09999999999999, - "y": 0.09999999999999 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/ndvi (3587).json b/test_data/dataset_defs/ndvi (3587).json index fbc2ec419..49e4f2cc6 100644 --- a/test_data/dataset_defs/ndvi (3587).json +++ b/test_data/dataset_defs/ndvi (3587).json @@ -21,17 +21,22 @@ "start": "2014-01-01T00:00:00.000Z", "end": "2014-07-01T00:00:00.000Z" }, - "bbox": { - "upperLeftCoordinate": [ - -20037508.3427892439067364, 19971868.8804085627198219 - ], - "lowerRightCoordinate": [ - 20027452.8429077081382275, -19966571.3752283006906509 - ] - }, - "resolution": { - "x": 14052.95025804873876, - "y": 14057.88111778840539 + "spatialGrid": { + "state": "source", + "spatialGrid":{ + "geoTransform": { + "originCoordinate": { + "x": -20037508.3427892439067364, + "y": 19971868.8804085627198219 + }, + "xPixelSize":14052.95025804873876, + "yPixelSize":-14057.88111778840539 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/ndvi.json b/test_data/dataset_defs/ndvi.json index cdbc2d893..dd47d2b45 100644 --- a/test_data/dataset_defs/ndvi.json +++ b/test_data/dataset_defs/ndvi.json @@ -296,13 +296,22 @@ "start": "2014-01-01T00:00:00.000Z", "end": "2014-07-01T00:00:00.000Z" }, - "bbox": { - "upperLeftCoordinate": [-180.0, 90.0], - "lowerRightCoordinate": [180.0, -90.0] - }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "state": "source", + "spatialGrid":{ + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/ndvi_flipped_y_axis.json b/test_data/dataset_defs/ndvi_flipped_y_axis.json index 75b10a32b..5766e3a56 100644 --- a/test_data/dataset_defs/ndvi_flipped_y_axis.json +++ b/test_data/dataset_defs/ndvi_flipped_y_axis.json @@ -292,13 +292,22 @@ "start": "2014-01-01T00:00:00.000Z", "end": "2014-07-01T00:00:00.000Z" }, - "bbox": { - "upperLeftCoordinate": [-180.0, 90.0], - "lowerRightCoordinate": [180.0, -90.0] - }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": -90.0 + }, + "xPixelSize":0.1, + "yPixelSize":0.1 + }, + "gridBounds": { + "min": [-1800, 0], + "max": [-1, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/ndvi_list.json b/test_data/dataset_defs/ndvi_list.json index deb161b5a..23577513b 100644 --- a/test_data/dataset_defs/ndvi_list.json +++ b/test_data/dataset_defs/ndvi_list.json @@ -293,13 +293,22 @@ "start": "2014-01-01T00:00:00.000Z", "end": "2014-07-01T00:00:00.000Z" }, - "bbox": { - "upperLeftCoordinate": [-180.0, 90.0], - "lowerRightCoordinate": [180.0, -90.0] - }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "state": "source", + "spatialGrid":{ + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/layer_collection_defs/test_collection.json b/test_data/layer_collection_defs/test_collection.json index 955625776..90bd653d8 100644 --- a/test_data/layer_collection_defs/test_collection.json +++ b/test_data/layer_collection_defs/test_collection.json @@ -6,6 +6,7 @@ "layers": [ "b75db46e-2b9a-4a86-b33f-bc06a73cd711", "c078db52-2dc6-4838-ad75-340cefeab476", - "83866f7b-dcee-47b8-9242-e5636ceaf402" + "83866f7b-dcee-47b8-9242-e5636ceaf402", + "52ef9e16-acd1-4c61-9a80-7d5b335d0d5a" ] } diff --git a/test_data/layer_defs/natural_earth_r.json b/test_data/layer_defs/natural_earth_r.json new file mode 100644 index 000000000..42bafea32 --- /dev/null +++ b/test_data/layer_defs/natural_earth_r.json @@ -0,0 +1,42 @@ +{ + "id": "52ef9e16-acd1-4c61-9a80-7d5b335d0d5a", + "name": "Natural Earth II – R", + "description": "A raster with one band (R from RGB)", + "workflow": { + "type": "Raster", + "operator": { + "type": "GdalSource", + "params": { + "data": "ne2_raster_red" + } + } + }, + "symbology": { + "type": "raster", + "opacity": 1, + "rasterColorizer": { + "type": "singleBand", + "band": 0, + "bandColorizer": { + "type": "linearGradient", + "breakpoints": [ + { + "value": 0, + "color": [255, 245, 240, 255] + }, + { + "value": 127, + "color": [250, 105, 73, 255] + }, + { + "value": 255, + "color": [254, 244, 239, 255] + } + ], + "noDataColor": [0, 0, 0, 0], + "underColor": [255, 245, 240, 255], + "overColor": [254, 244, 239, 255] + } + } + } +} diff --git a/test_data/layer_defs/rgb.json b/test_data/layer_defs/natural_earth_rgb.json similarity index 100% rename from test_data/layer_defs/rgb.json rename to test_data/layer_defs/natural_earth_rgb.json diff --git a/test_data/provider_defs/sentinel_s2_l2a_cogs.json b/test_data/provider_defs/sentinel_s2_l2a_cogs.json index 37f2d16c2..f998e8233 100644 --- a/test_data/provider_defs/sentinel_s2_l2a_cogs.json +++ b/test_data/provider_defs/sentinel_s2_l2a_cogs.json @@ -6,60 +6,6 @@ "priority": 50, "apiUrl": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items", "cacheTtl": 86400, - "bands": [ - { - "name": "B01", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "B02", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "B03", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "B04", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "B08", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "SCL", - "noDataValue": 0, - "dataType": "U8" - } - ], - "zones": [ - { - "name": "UTM32N", - "epsg": 32632 - }, - { - "name": "UTM36N", - "epsg": 32636 - }, - { - "name": "UTM36S", - "epsg": 32736 - }, - { - "name": "UTM37N", - "epsg": 32637 - }, - { - "name": "UTM37S", - "epsg": 32737 - } - ], "queryBuffer": { "startSeconds": 60, "endSeconds": 60 diff --git a/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif b/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif new file mode 100644 index 000000000..5c0b80823 Binary files /dev/null and b/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif differ diff --git a/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30_bytes.txt b/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30_bytes.txt new file mode 100644 index 000000000..d5efa4a9c --- /dev/null +++ b/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30_bytes.txt @@ -0,0 +1,30 @@ +255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 180 193 202 200 201 195 191 192 174 75 255 255 255 124 +255 255 255 255 255 255 255 255 255 255 255 255 255 255 131 172 185 188 192 196 200 191 189 197 187 106 255 255 169 155 +255 255 255 255 255 255 127 107 152 179 147 177 174 175 180 191 187 185 192 198 196 193 189 181 112 42 255 255 255 255 +255 255 255 255 255 164 185 182 173 159 125 138 180 176 160 121 128 147 166 167 176 181 183 135 45 255 255 255 255 255 +255 255 255 175 186 190 167 140 123 73 46 70 151 196 190 163 185 193 191 197 207 197 142 76 255 255 255 255 255 255 +255 255 161 175 184 173 170 188 123 91 76 102 176 195 195 192 194 193 194 199 200 185 129 255 255 255 255 255 255 255 +255 255 128 177 165 145 191 174 154 179 179 120 167 191 194 195 192 187 190 185 186 191 153 84 255 255 255 255 255 255 +255 117 100 174 159 147 99 135 179 187 161 112 111 164 190 193 195 191 194 188 188 193 177 111 255 255 255 255 255 255 +255 170 166 162 163 112 79 144 189 183 172 182 164 173 185 188 191 195 197 203 202 200 193 146 170 172 191 185 132 255 +255 157 187 192 191 174 154 174 182 184 188 184 182 181 186 191 192 195 197 190 185 196 200 194 189 187 189 194 190 129 +118 101 140 180 179 175 176 181 183 176 167 171 182 186 188 189 192 199 200 204 205 207 204 201 198 196 199 199 169 77 +255 187 178 178 178 181 182 185 182 167 163 173 183 189 189 190 191 196 198 196 194 195 186 145 167 189 183 162 123 255 +255 148 168 181 183 184 184 185 184 173 169 166 172 177 182 180 188 191 198 199 188 159 104 160 164 151 148 102 255 255 +255 51 74 164 185 188 182 183 188 183 171 166 175 185 186 186 189 177 189 192 196 195 158 255 255 255 255 255 255 255 +255 37 27 102 173 183 179 178 181 181 174 168 171 180 186 192 190 191 199 200 203 203 143 128 149 128 85 110 255 255 +255 136 145 148 174 180 175 173 176 172 176 174 179 185 194 200 198 190 180 170 152 123 51 255 255 173 113 34 255 255 +255 131 179 187 184 182 187 181 177 175 172 177 180 192 197 188 192 200 212 197 115 255 138 100 255 177 87 255 197 196 +255 163 177 185 181 186 181 185 181 182 183 176 186 195 196 191 183 176 145 134 119 121 101 134 255 124 117 255 255 198 +134 158 135 160 180 184 181 181 181 183 182 184 184 189 190 193 185 147 136 181 188 190 198 195 158 140 163 89 255 255 +255 255 107 103 122 160 183 187 185 180 179 178 182 191 191 192 159 161 190 196 193 197 199 183 159 172 182 125 255 255 +255 255 255 255 92 57 164 179 185 185 189 188 190 198 203 193 105 122 174 191 197 195 191 183 186 191 200 171 101 103 +255 255 255 255 52 75 144 186 184 182 183 180 186 192 193 203 182 121 166 201 195 196 200 200 194 197 200 175 82 255 +255 255 255 255 104 115 118 185 188 191 189 178 176 183 179 148 152 86 255 255 149 177 197 200 199 192 203 164 188 160 +255 255 255 66 67 54 144 182 174 183 184 179 179 183 175 177 185 190 185 138 166 149 149 159 189 186 161 103 175 115 +255 255 255 118 255 255 174 185 185 180 179 180 182 181 190 198 198 189 197 193 136 255 164 105 80 116 134 168 154 255 +255 255 255 105 97 109 179 198 202 199 196 187 184 179 173 169 179 141 149 143 123 255 255 194 146 91 154 172 255 66 +255 255 51 77 206 135 124 189 207 207 200 198 198 192 199 215 215 209 193 161 255 255 255 255 255 255 255 137 255 174 +255 255 45 101 115 123 79 108 185 201 208 212 205 198 202 210 213 213 209 193 115 255 255 255 255 255 255 255 255 255 +255 255 255 255 44 92 143 58 106 176 204 210 211 202 197 193 189 196 210 196 118 255 255 255 255 255 255 255 255 255 +255 255 255 255 43 47 91 109 147 167 199 206 207 210 194 196 199 198 178 180 204 175 187 189 151 255 255 255 255 166 diff --git a/test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst b/test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst new file mode 100644 index 000000000..b7eb4105f --- /dev/null +++ b/test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst @@ -0,0 +1,68 @@ +����������������������������������������������������������������a�oF���������������������s�}}!��������L�����������������������������y����������w=a���������Kormou}vw~}�~��w�vppqox|����������������~w����������������}�����������qct}������yz~{vv���wy|vomm���vutrmo�������������x����}���|zwy{vvg\^ZSWYNQLHHCADDU`RWark�xsvYBAQU]cRIC?>AEIjjm^JINLFIFEKJ@7>9307,#4$)25:Th#"/* $$(=aB#&<#' # )('89=8%0? �����������������������������������������������������������������4Lxs�����������������������:���������r�{�|�������������������������������������\���������M{o�{}~|���|���twysxwur{}���������������������������������������������|�x������|}usu{��~ooqzzxqnkkvwx}hfjfksv�������rkk�}orv{kdn|snisj`YYYSSMMRKMIJGFT[\UEQPVRUNEB=>jifUFAAK[hMl�va_PAA??QLPZ=983#!+1-!$//:LN3!$+# )B'+!" "(-/;IXvcUZV[V`g=@L>!!( &%  �����������������������������������������������������������������39V������������������Ɯ8�����������vf����^��������������������������������������Y������q~�������~|{z���~�|z�����y���}�������������������~���������z�������������}�������zzxx}y��|}xw||wpqmqqjl|����twsu����~z~��~��pkdjon}�zqmfc^[]WOIISECCNMILGDEABGCDECD@@?A?JUOPG>8:GWV^>5/0)-.4/%## AB4*)721,!%0+$!!)) &" #$%%2LZgclU_[JKD48/&!-! �����������������������������������������������������������������39V������������������Ɯ8�����������vf����^��������������������������������������Y������q~�������~|{z���~�|z�����y���}�������������������~���������z�������������}�������zzxx}y��|}xw||wpqmqqjl|����twsu����~z~��~��pkdjon}�zqmfc^[]WOIISECCNMILGDEABGCDECD@@?A?JUOPG>8:GWV^>5/0)-.4/%## AB4*)721,!%0+$!!)) &" #$%%2LZgclU_[JKD48/&!-! ����������������������������������������������������������������������������������j���C;N�����������͕�������������������������������������������u�����L���������������������������������~����������������������������������������������������}��xuvnp|rnvmsy}mq�������{{|qqs}��~jjgfhll���krhahhcNMFFONA>DEPSYXG??@BCCAB@@=?DP`XI@<;?JJPXVei`WKC23>IGNZ<"",,!' ,?SL<6-'77*"'0:5-) ##"$++*(&,,3Hcp�n@@UR?767-!!*%!)  �������������������������������������������������������������������Z�������������������������������²��V�s������������������|��������������������������������������������������������~�xz������������������~|}x}���������������������z��zv{��xztolt}x}uxqvqrz����������oqtr��ymffgfigk~solokcfb`_X[ZJLB?GLVPOPI?ADECBCLJJFERaWC?==?D]libVspaWK3#'5TMVU+$!)OgXWC:6@?/'?;8649+)0-0/$#$ $#'$+')5Ee�sm=\qLKP//2.! �������������������������������������������������������������������:����������������î���T��~�����������v��q;�����������������������Ž����������������������������������^c������������������������������������������������������������|�w�����~��wswuutwx~z|�txl�������v����mgefol��qrcg_aefd`\XVTOOE>BECCDIC?BHCDDEdspYJ^`FFHF>;7,(]G80!*'19GOR[dG;:JI616:9"-95!(  #,+((2KYafYct\leZL2P,C<& �������������������������������������������������������������������:����������������î���T��~�����������v��q;�����������������������Ž����������������������������������^c������������������������������������������������������������|�w�����~��wswuutwx~z|�txl�������v����mgefol��qrcg_aefd`\XVTOOE>BECCDIC?BHCDDEdspYJ^`FFHF>;7,(]G80!*'19GOR[dG;:JI616:9"-95!(  #,+((2KYafYct\leZL2P,C<& ������������������������������������������������������������������%)�����������������/�����������������ů���vx����u�����������������������������������������������������wt~�������������������������������������x{��~�|�����������������yz{x��}�����~~���xw}������ysz���������~}szfdin���qrthbb]gngPT[WKBACA?;BABACA>AGGDZ]YRK^[HJKC;:FbbIItt[>1"K2 +.@HQNHHX_IFEEB>&&49=4/?4)4&/,('5SKFGPs�piN,%/# ����������������������������������������������������������������13&�m���´������������������������ƽ����������}�������������������������ï��ź����������������������������������������������������������������������sr������������������~srz�������}}����}zuni����u���|�����qz��qnw{�si}}|gfmoie_OMNPPBAC@AB@@AA?>?CABJWRI@AO]cNB;=AYarwoteP=<8.%3(($%MR\G@T^^TCM_fN;78;>?@?BAA@ANPFBI>=FWG5,:KgaiqpbPQ6." !7/2;BZLMPY\K[_YikX@70==;=,A7) "+34*&*7;AWUKL?2FH=. ����������������������������������������������������������������(���������������������������������������»�~p��jn�����������������������ö�������������������������������������������Ĺ�����������������������Û�����ubl��������������������nz{����������������}pz�xsw{�������w��������zjd~zqnjg`h`[]YWHGGJHKD@>>?@?BAA@ANPFBI>=FWG5,:KgaiqpbPQ6." !7/2;BZLMPY\K[_YikX@70==;=,A7) "+34*&*7;AWUKL?2FH=. �������������������������������������������������������rZ_RL`Z#������������������ɸ���������������������������9������������������������������Ƿ������������������������[?s�������������~���������������������������~y{|�}���������������������������~����������rnttu�������������up�tnlprqjiggg_QMMSSGACRFDC==>>>@GEEHD?>=>=>>=77;AO`msslK:3$+5+'#,5;6<>2/9N<;]j�paBCC7J?(& (*%'.>GGTC:KIBT@2"���������������������������������������������������>I=7m�����ƌ�P�������Ş���������������������������������������{�������������������������������������������������������������������������������������������������������������������������������������������������zrwz����������������}qmq}wig_UZ[^QSPOHB>AA?B==<<>??JE>???<49>=@:7=LLUeuud:+#)+#"!2&#'!1B8@G49''3C3[qrd`WH<5A=!#0+$$"2 '(*++8HA/:BV^Y[B*�����������������������������������������������2'+�Q~������θg��x����������bs�����������������������Ÿ��������������������������������������������������������������������������������������������������|z~������������}z���������������y�������������}����������{{�����}����~����wnqqsuhe^ZMV[XLKDB@CEA=>=>=>@AD><>E?>>AA?=><>@?EUq`B3@%.1#)#$"%#I>BDKO>&+0&4S_fnvuPC@;=>=>@AD><>E?>>AA?=><>@?EUq`B3@%.1#)#$"%#I>BDKO>&+0&4S_fnvuPC@;>=?@ADGB@>>===?=AXb8(+32+%"&'2R>&/7;//!/=@PRZmsCF=AYI1*&!+>;-%$"$&#%;4LSWVWzX4$))�a�������������������������������������]�p�s������Ÿ����������������������������������������������������������ĺ������������������̴��������������������������������������������������������������������������~�������������������������������|{~�����{���������������wy�������������xnntnjuujdikQJFISYQTPPIB@>@DC??AC@?>CB@B?A>=<=>>ERaG+!,3 :/)"@FY_fbe`^]|;@O;06&&)- ()*(!!#$$*FGURfpbL8 ��Z����������������������������������0J7P��������Խþ�������˾��տ�����������������������������������������������������������������ĝ���������������������������������������������������������������������|������~��������������������~���������������|}z������������vory�vgs�������yqlhhhswokhgeRUNNNRSGBIJ@??CDDBC@AGBAD@@?@??=EGMLPjyA1 ('>&$5YOUiOOVFEI^.1B)'(-04# #+%'# "'3&$5YOUiOOVFEI^.1B)'(-04# #+%'# "'3=>>AHJHHE?>>=LdTJ�c=<8)  '4% ))4K/$/:-#.HHGNFDJFGFD!(('1+# (46$'"$##'6F*;>YQK:(&,6'!|^-�����������������������������������"����������ļ�������������������������������������������������İ����������������������¨�������������������������������������������������������������������������ʷ�����~��u���}���������������{x������}�����{���{|x���xuwytvs�rm����{~trpmuojoqqnsyfgomi[NFHQVTRIODBIIDFH>>@CUu�hJB@?>;SQE^�xL:98$"& '*ABWJHCD<39<@B?(;<>A@7)-'7J_T9"!!#',:GPKBD!# * ?������x��T�������������������������fU���H������ȶ��������������������ʵ�����������������������������������������������������������������������ʷ�������������������������������������������������������Ƨ�����������������������������}������������z|������{��|ws{��|�����ykronqqppsrcmmhcdkldRFEEFGVHAA?FIU]\HDA?EJgyLBD@?=NPUA" ?������x��T�������������������������fU���H������ȶ��������������������ʵ�����������������������������������������������������������������������ʷ�������������������������������������������������������Ƨ�����������������������������}������������z|������{��|ws{��|�����ykronqqppsrcmmhcdkldRFEEFGVHAA?FIU]\HDA?EJgyLBD@?=NPUA" ju���������Q���������������������������p�ٙ������¾��������������������������������������������������Ƽ���������������������������������������������������������������������������������������������������������������pv������������������s����������no|���������������������tolqkkimpmmnkggkjfYOMKLEHLA?CGDCJ\OBCCAWaSOLCBC><>BDJgkLH;3&&'//'%/.*)$ $4&%)22.9BC:/&&@>ChURT;7##+ 9!*$64;963>>9%;FG>/2# &!#$%.7B@3+!" �������������L������������������������Цd��xr�����ü����������������������ǿ������������������������������������������������������������������������������������������������������������������������������x��������������������������������������|v���������}�z{z~|w���v�����otyyqppvyyyocgkkria]aaJKHFCMXPZacmmUJYXNPHGFEECEA@?AAEOiYA7%$9;>"(>B<>=?D=?BLMMED?),!A9!$(*;@?>?B;=>?A?:*  "02.0+" �������������b�����������������������������}����̺������������������ü���������������������������������v���������������������������������������������ĸ�����������������������������������������������������������������������������������������}�������������~||��qs�����vv���{wqsxyqjlljmjgooorRMLPJPYan`[jmXMXUNRLEFIKFC?>?BLMMED?),!A9!$(*;@?>?B;=>?A?:*  "02.0+" ����ů�������f�������������������������J�t��������Ĺ�Ű�������������������������������������������������~������������������������������������������������������������������������������������������p��������������������v�����������������������nm��������~t�����}y����������pyuvutskmnoujksrstrnoa\UN_ggf^\[j^NS`fgeSKFDHHB?>=>@HDB@>:<-<<+%(.49138>>A>?#5@>><73.  (0/85 & !2K7%����������Ƚ�8��������������������p��������������Ӽ��ɿ����������ļ�ø�����������������������������������������������������������������������������������������������������������������������������������������������������������������������s����������������}w��~}�������tqqvvyvjXYrtutmjklrpsqtxnosvhhdiX^fyqb`jdbRQOCIDAC==@?@>?@>;7;5,?=9 !,:ADCC?@@;0=A;9=9?7**12./"  #,&����������ɫM��������������������d����ʫ�����������ʶ������������������������������������������������������������������������������������������������������������|ti����������������������������������������������������������������������������������������������zx���}yxqvzuvyvvsquwspkjkjlouptu~uqrmgjemiekvvcxq`JLQFBEAB@>>@==<=<:9>@==<=<:9?<><;?>BDCC?::5'/2"$+-.*   �����������h��������������������������ѳ��ʲ�����������������������������������������ı���������������������������������������������������������������}������������������������������������������q��������������������}{y}����������������������}y��������������}wu~���zzrtzxqkpr}qhhsslkmvwstzmtjqijffdie__fiylc[d^NBACCBA@>=?>@?;=ED>3$%07B=56>@EEIK;:8420+EBBB9:7=A@@B65/(# ���������vw��������������������`���������˾�ì�Ŀ�����������������������������������������ÿ���������������������������������������������������������������������������������������������������������������������������������������������������������������������~r������wrrtxx���qkkwpmktvpmoklqsvjiosleejiikfgjk��w]]ZX>BPCA@B>??A=@==@FF/*4( !"!!3>B@97AED@16CA;AACAFHD>AA@@?>61:+#   ���������j�����������������������������Ǽ��������ĵ�����������������������ø�����Ƿ���Ǽ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������{{��wrqqvxy|{zxomotpmpqsoiilpjpsmnrsikqmkhqlirt��sa``cOXZMD==@ACC>A>>@AE<AGEA?HB3@HF@GFCBDGFAC><:9:9?5     ���������j�����������������������������Ǽ��������ĵ�����������������������ø�����Ƿ���Ǽ���������������������������������������������������������������������������������������������������������������������������������������������������������������������������{{��wrqqvxy|{zxomotpmpqsoiilpjpsmnrsikqmkhqlirt��sa``cOXZMD==@ACC>A>>@AE<AGEA?HB3@HF@GFCBDGFAC><:9:9?5     ��þ����f��������������������~fzv������ú���������������������������������Ƕ������ƶ���������������������¹��������������������������������������������mu������������������������������������������������������������������������������z����������z~��������������������y|yyqlpvvvtrqvvtupkpjpmgnqmousyvqonmnklopr\~mjaZ]`_dcD??A?AV]@@>@@FF=?I6+*30)586'%BJH4"%57=CBEFGIKDHE?@=;;37>AC?7    �����������������������������y�sx������������������������������ñ������¼����������������������������������������������������������������������������������������������������������������������������������������������������������������y�����}~�y�����|������{����wuw}xyotyzqtxvmswvswuppmrptwrnjidmkohkmlgpZPggfi^][]ffAABFBS[_LDACBDD@CB99:<7-  ������y�������������������������������������������������������������û�ÿ���������������������IJ���������������������������������������������������������������������������������������������������¨�����������������������������������������y��vt|~{w{�}{��{xt|���x{zwkqwxxxrpttvtrsltwsvusslookmmikjnhkkgjrigfdgfb`\_gifX[LMMQ]RJ@ILOFRCEF?5*#*&,BA=6>>>@<502A=GBBCADB<:@HFCA>0#4A@929=>.(3(!  ����xn��������������������Wmg�lv���������������������������ƿ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������r}u{�������~~�}~|}��ux|zxprrpx�zwqntuyyxuppfpntponkdWjfqulknngjqodhgeecfhgij]cdd^f}MEDHC@?J<5-""(3=C?41=>AAA0-DB@EFBCBDC@?6::9;9,/8;;A>>6'!'  ����xn��������������������Wmg�lv���������������������������ƿ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������r}u{�������~~�}~|}��ux|zxprrpx�zwqntuyyxuppfpntponkdWjfqulknngjqodhgeecfhgij]cdd^f}MEDHC@?J<5-""(3=C?41=>AAA0-DB@EFBCBDC@?6::9;9,/8;;A>>6'!'  ������������������������}���pi~ms����������������������������������ò���������������������������������ǻ��î���������ð������������������������������������������������������~����������������������������������������������������������������������vsz~vv����~��{{|��zx��{z}vt{{~~~vvsxyx|rlmvsqnomlc\hxut{nnjdjlkibikqyqnoliccgjdiz[ZRH?AIF;2&#!#6@FF><@@>?, :HGFCADHC??<9#(#&+1+,7676;;<2(7?<%!¿������������������������������}g������������������������������������������������������������������������Ⱥ��ÿ�����������ľ��������������������������������������������������sy������������������������������������������������t|�����������w����~t}�~|�}pzz��wv{w��|zzrq�}~smq~}{|�uwzxsrzvlnntpsZCMa�tgggfmhkp~momqrkr}}mjflqqosvvihHJKKC<5!".:C;BA<@CCA3IDEHCFCBC>?95#1.-3?:5'5-))5=6+&%-)ź��Ĺ��������������������ļ������������������������������������������������������������������������Ǻ�������������ï�������įþ�������������������������������������������������������~����������������������������������������������������eJ����qvx����wxx���}w|ww�{smiuyxuo}�xpyx~{z��vuoqloonmmjikq��ulllkosnsl`gxol�kfdkoljpps�xuGDAACC@/).-)(9:9AC?=0?D>%8RGECAAAD@>FC5,&%5$#/:<3 .5-'+' ��������s7����Pj����������«����������������������������������µ���������������������Ű����������������Ÿ���������¯����������ŷ�������������������������������������������������������������������������������������������}��|�������|����~$3��}�vy�wuyzsy~y���}{}~uxxokmumv|���~|z}�}}�~�mskfetwkosv{kqjgglpspv�nkoukr�pqkmggmnmur}rrraFBCEA;1);51.0',:?:>>=836>75+0FCC?>FC<9?A@=<)%=43.06$ 6/.>=91&*.,!)��������s7����Pj����������«����������������������������������µ���������������������Ű����������������Ÿ���������¯����������ŷ�������������������������������������������������������������������������������������������}��|�������|����~$3��}�vy�wuyzsy~y���}{}~uxxokmumv|���~|z}�}}�~�mskfetwkosv{kqjgglpspv�nkoukr�pqkmggmnmur}rrraFBCEA;1);51.0',:?:>>=836>75+0FCC?>FC<9?A@=<)%=43.06$ 6/.>=91&*.,!)��������������������������������������������������������½������������������������������������¿������ÿ���������������������������������������������������������������������������������������������������������������������xyy�������������Q8����t�vyw|}}��uu}��zvlsuwsil{}x{���x�u��y}rkniltupqmrrtomkheiinqsihgu~uwwnpgklhjiprruvxuRBBFA@DA9(88;@,2<4@A4(>CC>:%GEB@?>@>>?B?>=--=C-(@FA-""50+37CB>=B;>>?52%'$ ����}4���������������������������u�������������������������������������������������������������������������������������������������������������������}��������������������������������������~�����������������������������������������������);��so��rzz}���uxrovzy��}}ynyw|zy{��y|�wuxorkllnrojlrpp{u{tmkllptooegfjoimmkjbkkmlhjjivw{v`C>BE@?@?C74>@>A?A@DC<)#! 5>/!EHD?AE@<;1=<2/8?EDDCCAACEBDBBCC@&!��>�������������������������������������������˳�������������������������������������������������������������������������x��������������������������}����������������������������������������{����|u������������������������������������hb��{rzq{rz}}v��vvssnkkuyvpqkn{vtzxt�z}�yptv|srozsrwrpnntqprqrlv{mrmhfim}wujjkikopvongjnmv{`DF??CDA?@DA:<:=>>=AEB:,!066%6IFD@BAD@=BC=B?B?>B1"(;BEBFFFFDGEGGFGDC?AAB@�����������z�������������������������������������������������������������������������������������������������������������������������������������������������������������~������������������������������~{z{uw~z|}���������������������~����O��y~yp����tw�x{yx}upvvvrx~wy��wy}q|�zwwyzvokhqtxjnm}�vpmtsnjlponrqkkh{{volmikiqsqkmvny�vSTX@EG>?AD;:>;=?=8EFED:!)/%(@FJJHGEIFEB>CF=CB>BCAAGFDCA?EFCCDIIJJJLC:A@=@<(.,-�����������z�������������������������������������������������������������������������������������������������������������������������������������������������������������~������������������������������~{z{uw~z|}���������������������~����O��y~yp����tw�x{yx}upvvvrx~wy��wy}q|�zwwyzvokhqtxjnm}�vpmtsnjlponrqkkh{{volmikiqsqkmvny�vSTX@EG>?AD;:>;=?=8EFED:!)/%(@FJJHGEIFEB>CF=CB>BCAAGFDCA?EFCCDIIJJJLC:A@=@<(.,-����������������������������������������������¢���������������������������������������¼����������������uy�������������������������������������������������������������{����������������������������������vvzw|~~~������������}������������p�{rp|�wrxwrotuwwz{t|}~uwzpzr��u}w~|z~�}�toytqkrvyyrlwwxujnomspimruvllsq{~murlnromprwudYn_MT\SWKC@C@CA=?<=>CIGA<.10'/6INHEHFFFICCBF@EDD?>EFAIHHFCBECEIJLEEKMFFFC>>37@?:7(&%&�����������m���������������������������������½��ø��������������������������������������������������������������������������������������������������������������������������������������������������{�������z��|������������}�����������w\]�|tlw|��z}lnjvur~qx�ywqw}�z�|uu����{}}u}�xwtw|xquqwsvqlruuuqroqpjgn{zkpnmprhqqmuzxuffccpplkzXK@@>@A=;9=BHNLI?@93=72):OFFJKJJJEDECGFIHELJKMONFGHHJIHKJKKMMKFGFDBA>/@=@=&&/"#�������������������������������������������������ƻ�����������������������������������������������������������������������������������������������~zvy����|�����������������������������������z~����������������}������������������ox�tv��{xr�vlr{}{tsslkswijnq{�wrmp��{|x��ws}wovzvsurrrs{�xonnpnru~y{t{tqupoqqtrnxmonlowx~l^WITnoknfeaQNECAA?>?DKGIHF?>@@.BD==/K@@DGKOMH?CEDCEMRONLJHNIHGFFHGJLJJLLGHGDDBC?3:AA@3+'1,�����������u������������������������������������������������������������������������������������������}��������������������������������������������~vv����������������������������������������������������������������������������rw{u}������wuwxz�{qywvlgpnmtpturirw��r}{���xyr|}}pssoruy��xvpkjmstwwpzos|nouyovprvzqqlwyzvvkQ@Haoszts_xuMEFJCGGIIE?C@?DDDC?;?(:A>53JHLNLBFDDEGHLMOMMMHGEEGHIGFFHMNMMLIGFFCB:6<>>ABD0 -*"$'$$'���������������������������������ȼ����������ƿ����������������������������������������������������������������������������������������ǣ��������������������pz���������������������������������������������������~���������|����wxxr����z|di����~��rhnnmljkrpownnzx���{t}�z|���|tvwt}{qs��pjorrxqqlprnsnx|vttqtuwqyytxkvpkieSQXn}yuqz|�yBCBILHMQOGBGCAEDGF?: 1IUOIDHIJMKGJGGHS\Z^XMNOLJIIKKIIKKLNOQOLGFFAEGHCGBAD?A>:1)" *+!##111##&-+-���������������������������������ȼ����������ƿ����������������������������������������������������������������������������������������ǣ��������������������pz���������������������������������������������������~���������|����wxxr����z|di����~��rhnnmljkrpownnzx���{t}�z|���|tvwt}{qs��pjorrxqqlprnsnx|vttqtuwqyytxkvpkieSQXn}yuqz|�yBCBILHMQOGBGCAEDGF?: 1IUOIDHIJMKGJGGHS\Z^XMNOLJIIKKIIKKLNOQOLGFFAEGHCGBAD?A>:1)" *+!##111##&-+-���������������������������������ƻ���·����ÿ�����������������������������������������������������������������q����������������������������������������|lr��������������������������������������������}�����~{{w|����r}��zwz��{suy��|zzwz}�yk�zmwzw|sroinsklwtnmsur��xqq��x����~~}~lmrw�w~uwuuzuovtvupnjsxz~xt{sqtnsnkzvtknmw�i{wzxesxt\GOEGACLINGEKD@>@DB<?@DEBHDHKJHFKHFEJN`ebd]RQPLFIKHIMMDFLQOLFEBFDEJHBDEEIE@A<85-,*%$"" #035,)0.*32��������������þ������������������ž�������¼�����������������������������������������������������������������������������������������������������~���������������������������������������������������������������|�ywzzu���}}{z{{qw{��{}�~�~v�lIlB�zyrhjovmoonfkmqlj���|ru�������swxvu`dor|uasvuwozymossumusx{~xwuvpmooni|~}�sswtruxvrqwmjLO[SKEKH>AACAA=??=9,8BHFIHHDDCHHDBDNHTX`dgj_NPIDLKGQPPPGJPKLJHFEGGFKKHKJFB=@;8:4//1%  ''*',..(,/&!!"),(14..���������������������~������������¹������º��������������������������������������������������������������������������������������������������|�������������������������������������������vt���������|������������vutu~z}xxz|zxrv{tu���{~���uvyw}T�swprmswqsrmltlfm�v�xuy�mpsux�|x|vtpz~vuuvuz�xnmqqzww�wou}umspuruopqt~�tmwqkppyqnmthU`_aTC@?ABB@@=@C@?>)=21/0100* &/ ).+,&&(3.0/0.-/-/-,,//.2,'&����������Ļ�Ƹ�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������{~�����~~x�}}}vy�}����~��u~�xz{~�}�����y~����iv�xuz|qpqmjpsqmjzyuntt�t{uuux{rw}|�{y�z{tzwuz�~wnkkjlqpwsr|vtsutpptuqrux~vu|qrnuqkgv}xZJIJHFECC<=FMKACECAGEMLGIGILMJLHGQXfutgk_VPTQL8?KTRRRNOOMIIGHHHIKLMC@BB76;8<>?<:910177:7'((10-"!452221389665613/,+)-����������Ļ�Ƹ�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������{~�����~~x�}}}vy�}����~��u~�xz{~�}�����y~����iv�xuz|qpqmjpsqmjzyuntt�t{uuux{rw}|�{y�z{tzwuz�~wnkkjlqpwsr|vtsutpptuqrux~vu|qrnuqkgv}xZJIJHFECC<=FMKACECAGEMLGIGILMJLHGQXfutgk_VPTQL8?KTRRRNOOMIIGHHHIKLMC@BB76;8<>?<:910177:7'((10-"!452221389665613/,+)-�������������Ľ��������������������Ľ���������������������������������������������������������������������������������������������������������������������������������������������������������������{zx���|u{x��z|vsvw}~�~z�{|{z�{����y��������|{zt|notpnnssqlujlsmiu�vuqkows|wu|y��q|twvyy{�tqqpknqqxyplr|usynkqoowxxzr}~�wyupvluuoe_KBBBA?CD=ASLFCID70?CHNNKJIHKJLLHAEHfbYujQKJJOUJEHKDNTNLLJHJHCEIKONMKCCC64:><;:CA@52666"!%.&+2032-,,/1465899:24500/./-*.,��������������þ������������������������ÿ����������������������v�������������������������������������������������������������������������������ʻ����������������������������������������������������x�{}�z{qv|�~~u{��|������y{��|����������������uWvyryq`oosprxormomsrq�wqsuq�trw}x{��uuy�nxusvvophlomqyz|ywrurknpouw{syqnrsonmgjmijgfa]RIJCFBCFGKEGCC6(IFKIHHEJEFIgdhYRXbkjkkk\_HDFGONLJKMMSPQFGFHEDEJKJOMDCB<:>B?=78773())$*.,+.(.42&*49474243//1,020/..�ſ�����������ǽ���������������������¾�������������������������������������������������������������������������������������������������������������������������������������������������������������������~||�qrvv����������������}�����������|�~|�������tY<-A,(mnpnot|wst��xjkoo{u}�z�ttry{{zw}vx|rxtrvrprmsyqvstpzzsnitvtnxqrkeirwqnjlnfifjmhkkQVSEWYLBEDG<(CQ\f[LG4HFWqwvuplqw{zt|{pjaOQKFLLMPQRSQJ>FNJJJMJFDKPLOKK<;BDAE@A?:7=<86&!-111168*../-%+0654(110-0.01/10.�����������������������������������������������������������������������������������������������������������������������������������������������������������iv}��������������������������������������z}�������s��|~���~����vwv~�������������}��������������?-_lnpolmwx�{vosq{u|vswsru}{qt}���{Zptuotsoyyzsvy{tusunnn|�vx}uuz~tn{�|numlonpnnosfgfdh`VICBC2;LRP\VOPKJKRbwqvrsuvvwt_]p@:PQNJNPQRQQOLE@DFHOOMMMMOQSLONLOMKIC@D?9:>842$&(!!!/1+-347//*--../450/.,-23552-230�����������������������������������������������������������������������������������������������������������������������������������������������������������iv}��������������������������������������z}�������s��|~���~����vwv~�������������}��������������?-_lnpolmwx�{vosq{u|vswsru}{qt}���{Zptuotsoyyzsvy{tusunnn|�vx}uuz~tn{�|numlonpnnosfgfdh`VICBC2;LRP\VOPKJKRbwqvrsuvvwt_]p@:PQNJNPQRQQOLE@DFHOOMMMMOQSLONLOMKIC@D?9:>842$&(!!!/1+-347//*--../450/.,-23552-230���ƺ�����������������������������������������������������������������������������������������������þ���������������������������������������������������wv}Pt~�������������������������������������~�����z��y�z}~�����y|�������������|��������~}|�|�{���n/ $Spyvwx�wxvxtxyyvtvturyrtkvts�����wpouvv|z�xu~~vzuyxr{~~��xw{~~snu|�ukfo�omsp]UDCJHXhOLIE<=XT]w_FGIMPXT"k{tuy{�xsjc^eaZTOFHKNRQOMQSMJKKAGIRRSRNKMIMH;LKKHFFB@>>>:8;<@<4 ,$-,%*/2350/4.0/4554531/.,.0/0/1141,�������÷��ø�������������������������������������������������������������������������������������������������������������������������������������������������{������������������������������������������������y���~}����{}�����|����������}���~zwu~~ux�|q|}y_kPdjp[pzux{�tlpjk}yuwt{ywpyzuvmnt��xqt~����|xz{zqzx�~}zzy��zxzx~sqt{x}wx~q{y[UPGF_i_MLH#+RirvnWHMauy{}���s~~�~�wL2eIW]_ILLMIIIFDFTUPJJGC?BKVLJKJMD>HEFD@;769;;8575171//) $()!.' )191*-..256540)$000.1-).-.011.�ʿ�Ľ�����ĺ����w{������������������������������������������{w�����������������������������������������������������������������������������������������s����������������������������������������������r|�~|���y���~yy~��{{y��vv�~���������~�����v�xzx}~y{vrwwu�yw`_urv~�|iroksu���|v{yzno��y��uuw���zv���ssw�zy}��~�uty�zyt�wwopt���~�xr|��kNJE[g_VJIA)L[jphf_drrsw{}wwmqyutmmmupourikf_bdSENE@@DFHGGIHB@AJMPKECCBEILH:8:<>@???:7124-%"%'!" $.*,5=4/-+*.0,,, .141101'*,--./12����ǵ���Һ��Ƿ��ot��������������������������������������������������������������������������������������������������������������~������������������������������������������������������Ǻ��������������wy�}�v��}}{}}����v��zx��w�~|�������������xwyunrvyz�vimxzqvvzyS3X~�yquz�|rszxuyuvmq|mmt{nqpsuloq}���{sw�yz�~���wz�u��|yvv������y�}z��b[rYOVTGE2^mUuw\TSqxrpiv~��pqvkX[e}|yv{wnc\_bbfheYRGNEEEFGHDGHHMME?>BADFE<9?<<74,!#$%'01+&#&(!"(0-1:40(,#!'&-.0(+--,*0/*,,..11��Ŀ��������Ʒ�����������������������������������������������������ġ��������������������������������������������������������}��������������������������Ķ�������������������������������ƽ���������{��y����|�sz�����|������}~x���}z��}�}~���~�}stw|rptq��w{~~�~zuoziQu�zwpx����zusvmsvnnrxtwpmjqmjtyrvs|�{zvv�������}�}z��|�~�{rr�����|{{{��u}u[i_SB>@`ed{}rms����}�t�xm[RORUWY^d_e`ZVQPQTYdf[Y\ccZIFC?>=DIJIC??=>CC=;4/..264;8:;<750781)*10)./**-((0./0'*124.(%"$&%*'%!"*,*)12,-,,/0/������������ȵ������������������������������������������˓pk�����̽�����������������������������������������������������������|���������������������������������������������������������ȵ����������������|���t}���xz}|s{���y��|������zys�����w{sqyortuz�tuxy{��|xzx{vvGX{w~~���~�{oinpppvrrsostwvqmporwpt~��y�uw�{|�����������~�}�������z��TTd�mfW@AMZcc]bj�������{uz}ud58[UWWWW\[XUUSMJMOOS[][Z^fc\ZUIA?CMNKC@>?9===;00/0175788<;92)12-"+..0.'',+("*+*+.-022.,5-#&" !%0/,0,*---00������������ȵ������������������������������������������˓pk�����̽�����������������������������������������������������������|���������������������������������������������������������ȵ����������������|���t}���xz}|s{���y��|������zys�����w{sqyortuz�tuxy{��|xzx{vvGX{w~~���~�{oinpppvrrsostwvqmporwpt~��y�uw�{|�����������~�}�������z��TTd�mfW@AMZcc]bj�������{uz}ud58[UWWWW\[XUUSMJMOOS[][Z^fc\ZUIA?CMNKC@>?9===;00/0175788<;92)12-"+..0.'',+("*+*+.-022.,5-#&" !%0/,0,*---00���­��������͸�����������������������������������������ǘ}x�����ξ�����������������������������������������������������������������������������������������������������������������������������������������|qny~yw}}����}|u}�����������}~~yy�|��nsv~}umkmtts|����v}}vpy�fWq}vr\xs{swtkslkkjmollotpqtplpwqjx}tpkuxyzx�{���|u|����y{z�������|�xtiy�tj]LHC@bejg`[\b�����{pwvodb73SUTRVWTTRQONLKLNLOXWX\YUQ^[R<=FM@ABJB@@AAA@010-/.2455:80)/0#" !(,-.//-***#'-,.-.0/1020//*+!$(,+/2(.,-/../2��������������º��������������������������������������ʿ���}����л����������������������������������������������������������|����������������������������������������������������������������ů��������������w}y|{}zx{{�|�����������������zyy�|suy}vxwx{~~{v~|���~}�}�vy��zzseSd|v}qwuwxsws{wymlorrrwsdkytpittvwwz�z�ytwvztw}�|�����������xeA/=��z^fbX^cc[cdYYe`mnw�xytojhgke>?N\ZWV[SPOMJIJIIJJMSWfb\[bb\OJJGG7`b^]LKJHKHGHIKKPSS_^[[`\\XS^PGKNJGC@<=D>41/.244<;<:4.%*'&,...///01$*530--413(  (+/!"043222/����������ĵ�������������������������������������������ý����������������������������������������������������������������}~|���w�����������������������������������������������������������f���ȡ�������qw}x|z��swxx~}|z��������}������~�������~yy�~�����yr{|twxuw��{{���}���|x~{uu|twxpv|x{yw|trttzrxpspxstrvwtjtx|�y{twx{tqxv�����{�������NG���{{�{wy��y��������pVsx�|qQOvqtq{vmmnjh`SKHLGFKLNNOQPVWYS]]aUUK?@@CEGGB777;@:19529;?=::<8385)'+""+--,01.+# ' %-(+-)*++)+"#$ '/02451/����������ã���������������½��������������ý��������ƺ�������Ǿ�������������������������n|������������������������������������z�����|�������������������������������������}{�������������������������������������y{{�zy�����~��������������~����|���{�}�{{}ww|�|Zp|w��}{xws{wyyvtu{�rw|w}�|wsvrrpsvuoorqkqxtr{x~zvvrrrr�|��������{�ak�����z~w{{�|p���������whsuwvYQhknudl}|umiopjZKLKIJMONOQQMVZW^d\LBJOMV\DD=0.16:B?:648:=:7526544601/1,  %&&$+.-.//1,'!'))-*26.*)))**&*$!%$!$334410����������ã���������������½��������������ý��������ƺ�������Ǿ�������������������������n|������������������������������������z�����|�������������������������������������}{�������������������������������������y{{�zy�����~��������������~����|���{�}�{{}ww|�|Zp|w��}{xws{wyyvtu{�rw|w}�|wsvrrpsvuoorqkqxtr{x~zvvrrrr�|��������{�ak�����z~w{{�|p���������whsuwvYQhknudl}|umiopjZKLKIJMONOQQMVZW^d\LBJOMV\DD=0.16:B?:648:=:7526544601/1,  %&&$+.-.//1,'!'))-*26.*)))**&*$!%$!$334410����������������������������������������������������û���tz�����Ĥ��������������������������������������ƿ����������������������������z��~����������������������������������������������������Ģ������������������~w~���yz����}�������������������������xvx��}����n|x|x{�xruz�sxxzsq|}|tx}�vxsrwnhcdmuxxnpr�yssppkqroqr|xvz���{z�������w�Z)e����|}�y��{����������qlp}�xvxnmpyz{~�zkddie]SQRMNLHLPQSUYYQV_gZW]cXV\LFC<5446=C@A>==<9112487434235-& ,$#*,..-./,,)$#&"**),.//.440+(*((+0//+,0*+),,'*('*,30��������Ŀ�����������������������������������������������njv����Ħ������������ʽ�д������������ɪ����������������Ŀÿ�������������������������������������������������������}������������������������ͼ����~zx�����v|~�~|����~z������������������������~��y{���w�{je|wty~w���~ycruywy��~z��{urnpjge~{{}x{y~x{yz|{x~~�~{���{�����������^tyGK������w{�~|������z���~ohcmoouxypojgqnhstk_e]Y[PTWTSQNNNOWVWSMTgYYTPUTTSMBE?AJL747=?B?:95421120122323300-+0%'.-...,0-).$#(+)).*-..-0---++++,--* %-6/((,-0,*#!-,-222������˺��������������������������������������������������~������������������������ʿ�������ƶ��|��Ǻ�����������������������������4t�����������������������������������������������������������������Ż����~~�����yl�~y�����~������������������������qr|�~��~mi}u{|~~���}n_uuqrru�{}q}�|uuljptvo~zwx�y�ywq��|zyw{�wx��}|���������N=eK*N��~}|�lw���������i{�|rzdPbflsrifellgkrohd_YYRSUSOKMRSQXRSRJQ_WPMLLORQ@C>AKMNF<38;<==62112220/001/001100/'++,,./+)**,+--,-/./0/2/--10&()+,+.*-/10*'*-.,0.(*,/222������ǰ���������������������������������������������������������������������������������������������������l~t�������������������wQ�������������������������������������|~����������������������������¸�����wxv��������{x�����~y~����������������}��wvvwvu��~�����zz��������sxyztw{���{tx|vrwtgcrh{�|w}��{v����sy�sy��������������W/,Yfj�{���v�|�����w���nz{wwyui]P\jv|{|vskkopkfhjf_QPOQONQSUPTTVQPVSVQJHJLLKFCAEDCKIE<4003752234310../.-.0330+%#"+,++-12-+++,,/+,+,01.,-,0.,..-+))*,,))(,(&)*..30++13232������Я���������������������������������������������������������������mw��������Ƶ��������{����z�����}i��`=;X�|����po�����������v���������������������������������������������������������������v���������}up}�����������~��zz{�������������������{|s~|����zy���y�{�wywt��wz��t1p}}yxy�z|~zyy}�yorrvu~~{w}����w}�����z�|������������N1(3x���~����}~�����}����~{|twwwtuh[\cr{�~trtiib^ec]`SNQNNJIOQSTXVUNNMMIEGEGGGKLI94=8316:47876530.---/12211+,*#''),.-.242.+/+-**(**,-+--,**(()*-,))*+-)(())((((,22.%,0211�������²��������������������������������������������������������������ͧYK��̬�����o����~y|������ozP<^a�gY~�������sUv��������»��������������x�����v�������������������������������������������X\������������}����������������|�������������z{�����{z������������z{dx]-H30"02>_{��|pqx{�|}yyuspnvuzzorz�����||��v���~z}������{�����VS|����������������������}yyxz||vtyW`blsvupkkfe_\_a`\VQPROMLKOSVVTQGHFEDBB@>BDHJJBB@:97=>77589:>>8663//.-/43121/31010/( .---0010+.2*%'(&'(+,+,.+*+)((()(*++)++*)(((&&&*264*,2311��������������������������ʿ���ɽ���������������������������������������˾��˜I�Z>�����CGNu���vYWj~@=���gN���xEnr�w��������������������������������oz������������������������������������������������������{����������~��~�������z����{������������������������������T$U��z<,@����z}tw��{tqtyyy��}�u{���������||�sz}�~�������|w�������������������������|{z{|wsvz{pXT`i~�strfea`^dcV[[VKHIJRSTYTNPDABAAAADB?@FGFEB=<30129:78777;852,...332531-/0/0,0.-(%0//0.(++01/$****'(***,1-2*(())**+,*))'()*,,()(,122.//0--��������������������������ʿ���ɽ���������������������������������������˾��˜I�Z>�����CGNu���vYWj~@=���gN���xEnr�w��������������������������������oz������������������������������������������������������{����������~��~�������z����{������������������������������T$U��z<,@����z}tw��{tqtyyy��}�u{���������||�sz}�~�������|w�������������������������|{z{|wsvz{pXT`i~�strfea`^dcV[[VKHIJRSTYTNPDABAAAADB?@FGFEB=<30129:78777;852,...332531-/0/0,0.-(%0//0.(++01/$****'(***,1-2*(())**+,*))'()*,,()(,122.//0--�������������������������������ü���������������������������������������ɿ�{`h28$CIOtc�nPW���1@2D7V�g���}q`8QF=`tG_�������������������������������������������������������v������������������������������Ĺ��}}v{~{z���uw|��������xo�����������{}�}������������������j^u����������~|zyqxyzuu~s���x�ov������������}}r����}��������������������������������}y��|yvmnv{xshfY`gwokhcc_]WRTWWOMKLIJRTXXPOK=>CB@?AEFC@DFB@:572333;==7/1663161-.212345522,+-./++* ! +011/,2-+0,)+-32210*,,+766*)))(****)+'(*)/./+*'*2363210-,������������������������������¼�ļ������������������������������Ų��¦\L��}O8&/BM��yr���c0i-;_f\YWX^VR''%&+2Qn��������³������������������������������������������������������������������������������������{sqx����~�}z�}|}z�{w����z|���������}��������������������Ms������������yqtrswxy~��vy�����~�}z�hSmndhqcgqzz��������������������p{�����������������{vorqvw~��zoW^fuqlj][YXYUUUTLKOMPQQRRNMMI?CEGBDBJFB@DB><351476:=DH>52251382.1/.4202510-+--.*'&((*)+)+/.,,00--20,-.1540+,,,.70.++)))()*))*(+*')()(''+.3//51+/-��������������ð��������������ö����������������������~���������ΰ�����h[��y�P7"3A{fGP"-j+3a.:P0*6T~}��������������������������������������������������ü������������������������������������u���������ouz����tszy{|�utuzw�����������������������������������rt�������������|qrvtwwrx}xw�v�z��uhcXp��X`_`r������paqu���������������z������������������~xqruz{~�~~wZW[enpmea`aZTRPUPPMMRPQTPIIHFGLI>@FFG>@AJHEABCB>?@AADF>;74623344433455532/.,+.,*++&%(,1//*),)-/.*1,))*+.6.*5..2728,+,*+++))-/-(((02.**/8B:5.24/20����������˹��ʳ������������������ž��������������������������ϵ���61h2m�e;M&+,j8.&#+"" %0Dd|p���j�v������������������������u������������������������������������������������������������������toru������|vs��uku{}�������������}����~v������������lE��������������yxpo|�sjq{z��|?>f%3�������lur{��������s���������������������������������~wutzuvz���yhdZUddiccad[TTSTPKMNRRPRPKGKND5,!%/FAFHKGFIIJJGDEA=ACBFA/-1454777652;;5/32/../,-.'%&*22123/,1--1-+)'*+092*2-62124))**)*+.574)(,5777595q������������������������������������������������������������������xx��������uyvzurt}�yv�����������ye������b��}���������Dv~�������������}okrx|y{y��haa���������������{|�}������������������������������������������yr{}|zx{~���sppo_efa_^\VWRMOLJJNUVOOPNQT?!A@D=JK")#%C?DACB=8<9/-))5:801487<78422-.00(()*/220.052/--//0++(+.01572<:9=9:1*--#++,692-,059;;82..333993478:������������žŲ�����˼�ɺ�������º�����������«��{d~�����ªT<�X.4q7`iv &0 &4AVFj_K1$AIA1?zmFQ^����������������������������b=���������������������������������������������������������������������x}��|�����xy|�xszzvv}�����|{���v���vvzm���������`Pd������������w~}olrtkoq~zh����������u�������~�~���������������������������������������~{xxz}}w{��}ywtttm_Qb`[\]URKJLMST[ZXMEIH( ")>,  -FF?<69?=>7610-32031595312240220.--10..,*,011/.31,,,+/.,.955.+6;4513'#).202204985;>;640214300/523������������žŲ�����˼�ɺ�������º�����������«��{d~�����ªT<�X.4q7`iv &0 &4AVFj_K1$AIA1?zmFQ^����������������������������b=���������������������������������������������������������������������x}��|�����xy|�xszzvv}�����|{���v���vvzm���������`Pd������������w~}olrtkoq~zh����������u�������~�~���������������������������������������~{xxz}}w{��}ywtttm_Qb`[\]URKJLMST[ZXMEIH( ")>,  -FF?<69?=>7610-32031595312240220.--10..,*,011/.31,,,+/.,.955.+6;4513'#).202204985;>;640214300/523�����������������¶��������ſ������̮�����������������������oT[g?(1QOr#" JI1L�{saH=>@3-OCHSp��m��������������������������oST}������������������������������������������������������������y�����Ž�}�}���|�|w��w}}{z|�{�������zq����yu|�gc���������wluz��������������{wtvsu\Sc<����������? ��������|uz�����w�������������������}��������������}}�|�~}|��xusmppoj^SVWYYROOLNPPUUUSPIB   + %EF>;:?674>>:71/,;94422223/--.+*((+.,++)((*,/-.0.-,-,,/-.2&'0-+.-+++((*'*23.-699=B@<;:@?AD<4255539BD877012201+.---*)%)///1/450.2..))%%.+*%+)((&'(/'.6?B?AA=;>?>>>=:01./+-������������������������̵���������̺����������_f�����u?-1/I)!#*13���}A%!'1qPCchkbOouz������������������������������������������������������������������������������������������������������s|�������x��������~��������EYb^���fs[JgInjnsxYx�{������������~�H{�C���1��������������������������������������������~��xd�����z|{�����{{|}z|y~|�{|yxxw{uqpnf[[\Z`Z_]^[[[Z[VUUS+ $D>=?C<=:><8>>:614207=@<;8(+-035������������������������״���������ή���������}w�rFVltDOE1"BP##8(#>���t34":BOsQ?Y���kCLBXv�t�������������������������������������������������������������������������������������������������wy}������}������������sw���U������bOVPpfeogmvt�����������~����lJH`�������������������������������������������������Zmu�����~��{��}}}|sqr�������zyzurkgfb]Z\]Ya`a[]XWZPNRVOF " $=9:<<2/-+@A>ABAA>;;//14<;������������������м����׬���������ң���������N�v��Qo?#.@KGFE&97:f8K=F6(7>'2Ol���MX14BC?>:ED239AAD@5/+)+))***)(((*-23?=722/01(&&31!## + + &++84/,-34)')5>>>>C?>>@ABFGE96;B?>934-)*.-*-0-.232.3<@<9:5000$ + /25.2.4 //--%#,*.)()).88;;;=?:BDB78@A@@>:9����ǿ����������������������������з��������������hO.}hG0'mKYey�X]{�x�|ymPi-[1J��Z3J�nx��gg�������������������������������������������������������������Ż��������������������������������������������������zw~�����||���������������yg]�����������ve�|xtsgct�p}ZHBC?T)�\I�����������������������c;kpf�������������x����y������|�y��||Xdt�������������}{|z��zow{ytqiaa[X]]^_WedYUUYEM?  + !" @ED=BEB7;4/3/0/5;>:<838=?@;<=;6397(8981/67)-78/#+*+,''.47278:<;:8:>?59@BAA>;;���������ð������������ǹ�����ɳ��ѳ��������{������mPKe/CU&$  '-! 1ED?<=@==>=<<=;79::;=BC8=<:@@BB@CB<=@@>918;>?8655@?3)15.4;A7"-.2())('267899:;;:79938=?>===;���������ð������������ǹ�����ɳ��ѳ��������{������mPKe/CU&$  '-! 1ED?<=@==>=<<=;79::;=BC8=<:@@BB@CB<=@@>918;>?8655@?3)15.4;A7"-.2())('267899:;;:79938=?>===;��������Ƕ��������ÿ��������������ư�������������f8wD).;>37@==EGB?:04<:/%*=6D2:KE)2>>7>BD2)+69(-1-226978:895;<<:46:88;<<<��������ͭ��ü��������������Ŀ������������������}n;B5-$Ac~��������������ZAS��v�^e�}�|��������o{�{����������������������������������������������������������������������������������������������������������uvzr}z��|j~��������������S������������������������}ny�����UOL5����������������������^A'=n��y���������~���}~����������������r��~yqyv��}����z|���zljjjkhdksmqon`aic]\\UA3B  0@GE?==CCCCGE?54:68>;<;82786B>?@>>=6CB@=:9/ )<432><67!03:9:8D?=<;:4766689;6:;9;;943827=<>>������������ʾ�������Ļ�����������Ž�p������������RD9\oun;"8N�������������������vRn\�������������jmHOoX��������������������������������������������������������������������������������������������������������|�~o�v{��xy����������l4������������������������������������K4:���������������������_bKZnpw���������������������������~��������v�~t���y�����~~����gdbgc_altnihrhZ`bWYUTMHB)     7F?=B;;@>?CDB931:>ABB><<>;8=:7;9=B82:?AB:A2),03.02931.48<9275?CB@=?9831,113876721233360/0;;;8;=�����������������Ž���������Ō����ʽ����������}yj�$0,A; d_4h�������������������o���r�����������socfB�G����������������������������������������������������������ĸ�ö������������������������������������������||ngu����vz����������TG+��������������������������������������t/2������������������Bbl�{�������������������������������||x}������~�qy��y������|nswlcffcb^gjqksuoZRD457DG=   + )>81'363=??@@;5/7DEA@>=@=:=;6#)ABAADD><0*,,+$(3?B<4;9A4@=225274463423<6159<;2./,-/1:>�������������Ǵ��ɸ��u������ğ����ų�������������]T\D%L���z����������������h�������������aVic�����~���������������������������������������������������������Ƽ�ü�����������������d����kNUpg���������������z��tj{��gcx�u�������8����������������������q~����������������y#������������������X]b|xv���������������������������������{w��~s���z|��x|����ywnszxnddekjm|rrvtK*B1'     :;1<-%##&2<@:C@6$:A88;=;93+499;;87@>>@ABB<3$../(#*/*)*26>D7.-03597-.634777923;:993777353365124=����������������������������ȼ������������������9fCAac����jlly��������������{������������dSU3�����������������������������������������������������}z����������ʻ�����������������v���������a���y������������~�|��Zu{{zZ�xv��ohX&����������������������������������������a&$���^!��slI�vM����BJo|ly}�������������������������w}�����y|��x{|mqzvvz}�zi����xplpttslnrwkjkuxq_)$%(97 +   *# (1$!#$,<;>@=>;;;;<<=;;===@><=>>?AB8&/20-*'%%)'',06C=0,0//46.26558679::7593-.*127422//0/1�������������˽���ü��_�����˷����ý���������y�<_Y'Mz����sagpp�������������������������{ZJ���������������y���������������������������������������������������������������j�^����UA^������������v�������~yxx����uux}���c~=}���j]ec�����������������������{�����������������we(*�p����x;!oxBVSKJWgwzzur�����������������������x|��~���~twwwz}ytnoxx}��qy�ryzwwsrjrjidjnxqfR&: "=6/311/  #.147BB>;==<>===<<;;;><=>>=<<;<=?-,/210.&)('(**64//3.14/.726566578865601AA6=<8578;;4,�������������¹���Ī���������������������������2^5/E3$~����ubb_kq����������������~�������ykN�����������������{�������������������������������������������������������������dAO������������������������������s���k�������zeDiy�rbhnd�������������������������l��nu��������������r�������Cc���MYn\uoo{w��������������������������sw{���|�zqx�ohktpsqqjjm~y�yq���~ttuakjlltv}td?6G=& + +(+:7-/7@:>=<6/-**.33,$'+2426<<8<<:<=>>==<<;;<>>=;;<<98;6!+0/09;# ('&'()01452+(+++3886/.03475658@HH?3=D4-/<=;7�������������¹���Ī���������������������������2^5/E3$~����ubb_kq����������������~�������ykN�����������������{�������������������������������������������������������������dAO������������������������������s���k�������zeDiy�rbhnd�������������������������l��nu��������������r�������Cc���MYn\uoo{w��������������������������sw{���|�zqx�ohktpsqqjjm~y�yq���~ttuakjlltv}td?6G=& + +(+:7-/7@:>=<6/-**.33,$'+2426<<8<<:<=>>==<<;;<>>=;;<<98;6!+0/09;# ('&'()01452+(+++3886/.03475658@HH?3=D4-/<=;7�������������Ź��ɾ�Wp������������������������C"+psm��������uuvx}~����������������x}������v���������������iW�������������������������������������������������~������������k���������������������������||���~��q{�������������o_fiU���������������������������� _���������������������|Pt�m3Ctx��}xzx�~������wV����������������|vp����zrrqrkgjdkkhmjmjm|���z{zzxumihhnrtsjP$ ,PXQ9*754.-(/8=;876?>74#**:@>A@=:9?@@;;;=<;=;988;:&&--2?= "&&&(((*,,--+(#'156764426:<7565;JIG;<>>??@<;1$!!.6=>94:<=><:<>><;::89:.!--/:<: &%%(((((().,(-.13899765759:76;EHECD;;@EGECEA������������������õ�������·�������������z��fL-;m����������������������´����������u}���h+����������������[���������������������������������������������qm����������������������������������������������������y�������5KnoheL����������������������������������~�������vKC�������������k���������h��ui;,�����������������z{vtwvy{moidhpojshmoesnnv�����xuvokmqpmjbbA (1,% & #!'3;>>>HLF><>;=?AA?>"##.57>ADB??8>AA?>:69;<;:::;<<;;:98:;&&)+0/-+  %$%''''''(/66//239<=888:416>8@GA@CIJIEBFCBEK������������������ƾ����������������������U���f#53&U���������������������������������������|8�����������������kq�������������������������������������������t���x��w����������������������z|z~}���~���x}����������������������hNB_ZS3�������������������������������������yydcgD����������������}������������~}|��������������������rt|{��zonorzslpkmljoxtn�������tvopntlfiYV0 2) % !(*(+%!,9=>@?=:=:8>=9:>?=0CGBBBB@@?=>?@=89<;;<:9::;;<:999998:=5(+,*+%$%%%&&&&''(012/1458DFHHHGHHH=;�����Ǿþ���������̼����������������������}xvZi#K.3'!6�������������������������ǵ�����������}T&�����������wO���C�nx�����������������������������������������������|����������������������yz|w{��}z{��||����������������������-#@���������������������������������������|VG��(��������������������v{������������������������}yu���|u{|����tyrw�owtqvu�rgo��������uysqsushbO5*$%&&!"$5=:99:?=;:=@>.%?CDB@=?>;;>?><4/5;;;:;<>:9:::9897669/&'(,*%&&$#&&&'''(+..1246;==<<>@41654/6==65:;>DAEEE;��������������������Žs���������������������k9.p7VU1%d��������������}����������������������]K�������������������pj���������������������������������������������������w�������������������}����������������������������������*�����������������������������������������|=�������������������������zy��������������������{x��~}��uuj{v{��xuqxzzwqu�rvq�kkt}}���z~}{{}wqjP9#".''&))''())5<;=;9874305<@97GFFBA?><;::;:8:;;<:89::<99::999;:9878%*+*,(#! &'&$$%&)+-/112356:@?=;:<>:676548:<:789=;=<<<;:::::;:::;9787876/ '+,*)$#""# %)*+-10975667;><;:9;978898767:;:789::88t���������}y�����������������������������������������������}f����������̲������������������������������������������������������������������������`��������������������{�����������������������������������Ev����f����������������������������������~��������������}}�uw�zqssoipzy~qy}xmr~���~uznz||��}e\Yd`fcS/ /'$"#&"%AADBDD>?FGGCBA>311(+,++!(D@>=;9:;<=@;>=<<;;<;:;::;967899998987,!&),-'&$#$%$"'),/33;8:=<;<<;;9:9769:98557865876654335����������������������ø����������µ����������}nw:<")@����������_9h��������u���������������������������������������`qu���|����ʸ������������������������������������������������������������������������������������������ğh���������������������������������������dZ������������������������������������������������������qu|rjpujfjuinqrwtmu{�{}ngjupup����h`\`hccid\%%c<)%%#08<<53.))/:?>=;88:;;DH;=><:<:9988:8:<74467578997;-&&'*'&$##$$"()0358;;:;:<;;9:77879:777884455765533445���������������������ſ���������������u��������bj�"!@>>6W���n����� v������e]��������������L������������������������}���������������������������������������������������������¯�����������ë��������������Ɵ�������������U������������������������������������������������������������������������������s�����������������ulrtwoqznR_pwpkqq��yzz���oqryivpipz��xme_`daU^]/Z0&,3$&+%%(6EH7-15CDAA?>=88<9AA>@;40/;?=;::::<>BEE=??;::9:78999:;:633445667686.**-($$#$$%#$%'#*+/56798::::;997777668778956554454323355���������������������ſ���������������u��������bj�"!@>>6W���n����� v������e]��������������L������������������������}���������������������������������������������������������¯�����������ë��������������Ɵ�������������U������������������������������������������������������������������������������s�����������������ulrtwoqznR_pwpkqq��yzz���oqryivpipz��xme_`daU^]/Z0&,3$&+%%(6EH7-15CDAA?>=88<9AA>@;40/;?=;::::<>BEE=??;::9:78999:;:633445667686.**-($$#$$%#$%'#*+/56798::::;997777668778956554454323355���������������������ò�������Į�����w�����������J5g���\����������������chw�����������{P����������������������������������������������������������������������������������ƾ������������������������������{����������i����������������������������������������������������������������������������������������������~�}}wwvqmjkggummhvqt~|~��|xwuqzphblpy{slnb_eflef]dF#F05-'((("%!9A?G4DC=?@A<8989:BADD@;<;;:9<;:9999;;<9862333212336<72/'&&%&+*%$*4&+,4567;<=;;;9776787988785426543555432233��������������������;����������������������������s\Qyv�~4syh���������������|��������������������~H�����������������Ͷ���������vZw�����������������������������������������������������̿���������ɳ�����ķ��y�����������������u������������������������������������������������������������������������������������x�\0z�������worqlgkgmmnmbdfkepv}{{���}t�tvqknjeb`nqocouptrk`W/"*'&'/43)&"($"!)A6:>=87;;;<:@0(,.-+@7%###?>+;8896=DB@A>@@=<;:59:=>::;;:9999::;><9<89::<>@<96511//..124F764SF82>A664.../7565788:87764313777::::4345444454323345�������������������ʿ����������������������������������������ot��������������������������������������l������������������������~}u��{��������������������������������������������������������������Ĺ�����}������������������ͫj���������������������������������������������������������������������������������������r\EEE[_j���~ytnjf?AKR_e`^bdallppzkgxpvwppqqnkgggigmuxxvv_TQKY!QVY7% %+'%&)0%!/&*>358556<==;<9<>;?><<;9;@AA?<:85100.15549SEC?HB@:HMO451-5863..-15566545347667<=;:6545444555343434���������������������������������������������������������}t�X������������������ȯ���������������������tQ��������������~sw�����������|�u�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������tZWF.!06u^asz�}ulrWB@BA><:8::8896788::;;;<977:=>=:;;30.3444556;3?NA:@B>:EG>218651/00/3565564465679;9996554444554433444���������������������������������{���������������������u_�����������������������������������������������m\��������������x����������zp~��dj���gt�����������������������������������������������������q��������������������������L������������������������������������������������������������������������������������������XD(0d?:M[jjmvmc71]PUcgiosqr�tjpwolsjgnedhkjfkqx���z}peUJSy~<+3!$"('&#+& + 8S/244;77559;;><<KLILEG21:@;77765543333456655598:;<;;997544444544444443������������������Ǿ���ij�~��sUQb{m=������������������zU��������������������������������������������������^���������������������������|~_l�N�xo��������������������������������������������������������������������������������3������������������������������������������������������������������������������������������N1&'7*@A.7Qk\�i?*DBOZnt�qnj^^adlq{z|yrdgkfnpmy}������j0AFDtf228*0#,+  7,+95169678:68:;@BA>CD?:;;<=<;87689:;9:86899:;:99:76435232:KFF<.7G432<;73<;98764333368686::<=<;;986655444555555555�����������������ļ�����~{}�w%��2Cxpr4{�������������8����������������������[����������������������������|����������������������������zs�}��q�������������������������������������������������������������������������������-������������������������������������������������������������������������������������������)j�??#""L/$fE,&,3DYQal�}~Ydi^{�qmhnpqmkjowy|���������e]N7hhQO(?H; + :(<7),8777674247;=9==5;99>>@>988:9999877:878655762 $45'��r����������n��������������������������������������������������������������������������������������pwypUT�������������������������������zq����������������������������������������w@Cw�����������������������������������������������������������������������������������������u��Zi`LoK*)&-BO@REQ�}{tjnrkeetqlmpppmkotz��������{dTO@hpQG23:J + +#,2!.F@6);=979704568:9:;C4-459;<:88:::9797888698646216616:CH;@CE>984557768986424323799;;:<;::8966545544665555566��������������������}�{{�ppA�������������s�������ϧ��������������������������������ͺ������������������������������������������������������z��y|bz����������������������������������������������������������������������c�������������������������������������������������������������������������������������������������wh��w" '%;54MBN{��zqsi^fgo�}|wvqnqvw���}�y���zjMO:T]H8+ABF     #">8-=6=:;@?:43489:=><;8/4;84689979:;::::9998865987 31,9A?G@CE@=?9:?;815731../03.389;;;::9987776656777666555566�����������������������V�����������������r����������������������������������������͸��������^l������������t������������������������©���������w�p|SM~����������������������������}z��������������}z�����������������������������������������������������������������������������������������������������������������������������jXC2G=#()<1Mf�}lecbhlt���xvvnx���{w�u�wm{e)\?FV6'%6>2 +//84;75<:>=?<;:<<;;<=<<DG=CC<>DCCB>=D<5210201058<:::<;98965898777776766666666�������������������IJ�x9�����������������������|���������������������������u��������ƴ�ª���������������������������������������ʡ��ù��������w��VKYs������������w�����������������x����������������������������������������������������������������������������������������������������������������������������������������������~cI:HNQL<6%1.4H���vkph��������������sntzs�d ;Q7M1!  + &5:99>=;?==BA@;;989;>://37:98:::<9889899;77898831#*33459FHB@EJFEBEBDBF@6213333449;9:9<999:89;;8999988776776666=l������������������ɶ�I������������������������������������������������Ԫ�����������ſ��������������pZ������������������������?�������������zyoztt0S�����������������������������������������������������������������|�����������������������������������������������������������������������������������������������������������+ )�t5!9.!4Z=Thlo~{���pgv���|{�ykXbaGjqvGHX-A!''   %14@BB?=83*2ED>8877689,68:999879:99879889754787223332338HKLHGLFCBDDBBE@7863344269:;99::;<::;;;:;;;:98777887756=l������������������ɶ�I������������������������������������������������Ԫ�����������ſ��������������pZ������������������������?�������������zyoztt0S�����������������������������������������������������������������|�����������������������������������������������������������������������������������������������������������+ )�t5!9.!4Z=Thlo~{���pgv���|{�ykXbaGjqvGHX-A!''   %14@BB?=83*2ED>8877689,68:999879:99879889754787223332338HKLHGLFCBDDBBE@7863344269:;99::;<::;;;:;;;:98777887756=�W}}@_��|��jĿ������^.������������������������������������������������ɽ�����̢������ü�������������pP����f�������������������������������~xyz�o=`l������wy{���������������������������������������������������������������������������������������������������������������������������������������������������������������������_>:9(*/8)0&*DhbPSQ^PcQETSk|nMFEC:6+AD;:999:<7+88:78;:999:7885589<93565122333356GJHI?4FCFBD<;<;8976300289:9989<;=<;9765:;;:98779988876#<6-Z[+'CG0dtz]�������v9��������������������������������������������������G������u��Ⱥ����������������������y������������������i���������������|x~��a������e�������������������������������{���jb_�����������������������������������������������������������������������������������������������������������������������������|�����og+4!?^93 $";671$+'PQ`H9`bfb[SLOPZZX^h~x��6(  +  +   ( ""9&$)6@9::76;9848.49::889987998887798;9743343234457AGFGIGIFFEC;9887868741228:;;99<<>><5455336;:;<989::9888NZ-'.2KID)'+Qj�����yd������������������������������������������������ʒD������������ú���������������l����h���������������������������������~|{u}�yw�lc��\~���������������~�������{����������~����������������������@)����������������������������������������������������������������������������������������������������������������_NLGIQCIWdgeZZUkvyzR #    &%*($$;2( 7(88776897762 89:;8788778888:4676655765445541;>FGFGGHHJHC=:88:8745556637::9;:<<=:555795339=@@;99:98888�~bl7FDE��\r,-Tu���ǥ�t`������������������������������������������տ���չ|�����������Ƚ���ɼ�Ŷ��������������X���������������������������������Txyz���l2YnzI������������������������o������������������������������Ÿw]�������������������������������������������������������������������������������������������������������������������XZm}��saeI%,:$")+!3(1#.#*/0B^VGFS\gZ[Zb~g  +!!"! + +  "5351. ;1-3886657556456;;87778667535664996334688457756=FEDCIILHJD>:9::;7646667646899;:;;94455699424:@A=:;:88878������OT���^KX�p>Fx�����l��������������������������������������������[��̼s������������������¼��ŵ�|���lCN����(����������������������������������{|���~m03RW������������������������������������������������ҿ�����Π��c����������������������������������������������������������������������������������������������������������������������`o_PD0(OK+ #:I>YE7&5.3KU60OXY`fYVV\|�~ $!   +!(%&,"54"+987876557736798683576322123433566333795554006C?A?ABGEB;:98::96567766538:99::9;633334498248>@?=9999988��������������ynr~���Ѣ��A������������������������������������������y\���Ú�����@����_�N�����©���Ʃ������{�����T���������������������������������}�����oZSWy���������������������������������������������������ȸ������o!���������������������������������������������������������������������������������������������������������������������xl~��qV}gGCKJR`c64$"?<==?BAA=:6869:8877776;:;:99<=820344275146:????=<;<;;���������������������˻�h���������������������������������������������l��Ŏ�����������������Ĺ����������]tz��ol���s�����������������������������������7p�x�yz�����yM�����������a�����l:=<>?>4-37:877753:;;:9;<9952243340136788:<<<>?@>���������������������ζ���������������������������������������������ʿ��̥v����������������q����ù�������~���`b������d�����������������������������������������}JUx�����������������R���������������Ǽ������������������k�������������������������������������������������������������������������������������������������������������������������pbfkizzu|{��~��l,('@J-,15QRW]o����}1 +  + + /;:95 655322232232223321221.13/1K5'++,*+,6878:<:==;;<=?8/.044364149:;::;9:9:711330//453378877;<;=�����������������������ɹ��������������������������������������������ʡ��p���������������������О������y���nuu����������m���������������������������������wxvy�g]-V�����������������������������������²������������������e�������������������������������������������������������U��������������������������������������������������¼�����Nk�������xt|nichi{}~�zrwu���G%(=LWYf��wsc& +   ;;9656432343232212222220/31*,.$+-,+-,,3899;56<;;;==>>=<46=9945::<;;;8>=<:41//3562334;:836639>�������������������̾�ϵq������������������������������������������H�����H��������������������w�����������mCCk��������ɵ���������������������������������������w12|�~��������������������ü�������������������������������Q����������������������������������ſ�������������������������������������������������������������������������t2&8%Sa\d���uz���iht~��~w��|v��{Z9+HQTiuo�n0+   =;&155333532343222232100100! " ---./0/59=:54378;>??>=<=<<96=;9:;;<==:@@@<43338988;=>A@@9=>7;?�������������������Ÿ����������������������������������������������������Q����������������������Ŀ����º���id��������ȼ�Ȩ�����������������������������������k�o>b����Po��������������G���DZ���������������������¨�������s��������������������������������Ż����������������������z�����5����������������������������������������������4?(+BW����tkA.ANdzr}|^ywlv{��sxt����@$*8/PT`�~���8!   "=+'54553421143001122///412&$)(.-00.259:8568:;:>>>?>9?A>9568;<<<<76@AB>8235878:533237<@DC@I��������������¸���з��������������������������������������������������ˡ_����������������������������������c��������ɸ���ɻ����������������������������h����o��[Tp���rz���}����������mo�������������������������Ƴ��������\�������������������������������������������~|��}�������������s�����������������������������������������������vf8Mfppllf)^]hdz}~neg^kdkt�������r;@AAA@@>>@;9778;;;;<;9B@B@;8589<=5458768:AFIENq������������������wX��������������������������������������������������Í�������������������������������Ž����������������ԲE�����������������������������{��_���\_�}�������������������������������������������½���������h��������������������������������������������ph����t�������������������������������������������������������p=YdH"[~jacXK ]ar~{umg]dbc]ez�������V'$AYOPXv{{�����   "%+  5=*2776445322233111210/00144*,2651144267:=;:9:9:;;?>?@@;=A<8858:=?=>=AAA?9989<>=313899:;DIONR���������������]Y���������������������������������������������������ʡ�������������������������������ű�¶���������������Ӊ��������������������������������`r��e�������{|��������������������������������������Ⱥ����Ϳ����v����������������������������������������lrqt���������|�����3����������������������������������������Ϳ�vl+BI22;;'8ZdN:[kyzvozznp]ZSS\f{�������A"&/\UOf|������. !,&4P3  #3.9986454123211122132222656;;973-/38>?>;:::::;:==AABB>;<:98:;;;<@@@@@><:;;?@;1332(16>DJPVU������������lz�rK��������������������������������������������������������������������������������������ʶ����������������ɸ��}qM����������������������������otqy�������~��������������w��|�����������������������������Ę����s����������������������������������������������g����������rK���ga���������������������������������̮��Y'T�dE- K]g]9& Ijsz~}uqyYW[QSWW������{2!7OCJSfx�����s")]?- 6." -57634444430,+2523014449;9534-128=:<;:::;;;<<;<;?@BB@;97:==98>=@A>;><>879:6525;>ABGIQY]u~z������{S<��������������������������������������������������������������������������������������������������½����������ɮ������]�������������������³������|w~���������kz������������������������������������������Ʊ��������������������vL����ź������������������x���|p|�����y����������������Н��������������������������z7M�}vy5,JReUWH%/$?Zdnqqv�}z�{jZgVJSPOWcciw��{\:*"9MVy}���a   +5LL1-/+88655656:60373223489;5949:;74565::;<<;;<<<<7688:::;+(0889<<;@@;79655;<877679:9BHKMdev�}����we�������������������������������������������������������k��������ř����������������������������������ƶ���������������¶����{o������������������������n��Xz���������������������������������}��������������������ʹ������Xl����������;�����´���������������~��th��zwkgq����������������ʨʹ��������¿æ��U�����Я��ð..,yrX' NCGQ%4-(AH.:Ubjjty��tu{zq]WUDDH9XX_pxq���u:2'6imab�p-   B41.$ 587889768972.!176589:79898456898<====<<<<==8689:<;:'(5227=@@BB=9:66679767679:8INWX[Z������r������������������������������������������������������������������ħS��������������������������������������������Ʒ����������}������������������O������|��ZlIie���rg������ø������������w}��IZ��������������½�o������_m`TZu�������������������������yzz�vkmtw|�gk�����Xd�}����������������ļ������ò��������������ˤ��iS�pfj9#331: (FC(1G96:Epv|sryy�|tzzpOHKD/&>Letu������QBQkns�n  + + &+" :985746772/87'.887::9:@=9:=;7888=<<<<====?>;689:;<<;5597;=@DD><=65998788867==Letu������QBQkns�n  + + &+" :985746772/87'.887::9:@=9:=;7888=<<<<====?>;689:;<<;5597;=@DD><=65998788867==<::789;:<<======?@@=789:<==;:34:99./26>=><::?BCDSZ\bfs����_="����������������������������������������������������������¯��������x���������������������������������q����������������ı����ͼ������������������������e����x���yf~����������������������7�������������������r���������������ic~������������������|rx���nphepcgoc}l�oiq�q�{bYWdqx��������������������������r������mMy+=Q]_PU[`XW](&LSSTXTYW@)=DJWqszjv}{|wtn}��YJ`fnoz��������zgaUcdX+2@ +6288 .!%  .67;:3354:449;97)08:8:=>=?<878779:<=>>??>@AB@?989;>==<;:89:ABBA99741014339;9:?EKRUT`hp����E>&����������������������������������������������������������ü���ӹ��Ʉ��������������������������������.�����t������������ñ��Ļ�ľ����T������������������}�����~��������w|u���������������������������������_�����������������������¾��������������u{��ebjeXMP\h[^UQWb[nh`i_jkv|�����������w�oyg���]Yu�qQF>LW J<AI^]WUVbgLNYe /BYWW^ZQL[A.7)246LZ]`fpwoy}vf[f`gdyom�wu������zwLGDP9$/  %%&0+++-($:9761-*.+4;<;:;96336:999;:9/)4:67?><:7889999:==>?@??AAA@>:8:;=>><;=<=:CCCA99845410//23.-)=NRU^dff����1���������������������������������������������������������������������ѱx�����������������������������������1�������������������������������������Tx�����zqS}����~����������z^|�������_�����u����������wQ\���S����{������b�������������������������a�����vxqqnhnUGMRWV[PLYWQ_k\j`hx|�|������s}ww~}U-"7aT`9)Qqwf?%#)&+0 )RI6FTGVOKWL"MVXWVHFVW`D;*8XQ?:PRrwy~y�|hXTtw�}k{���~�����qnXUOGKSKTS%  + /'$' -@99455557547899=:>:7696<::<<<7."/78>78889:999:<>>?@@ABBBA@?;9:;>>=<<>?B;@CD=9873773--,**(+7?EJU[T[a���N����������������������������������������������������������������������ϯ��������������������������������������������������������^��������������������������}|k�������yr�����m�����������������������U�����ì�f���z�d��w����������UL_�������������������x_�zrurmemUKEFNRQZkaRQXorrdwysv����{����~z��{^6$ ;?@)!=\_]L 1.>8;LRRA%>?@BA?@C<9:70-2:0+***+.8NP #!@?OU>.QRUYWTIM]]X=@A@A;9>;:<82114D;,,+,/9<?=;63*/8:469614788<;70647689898789:??@C@??A@@<<=>@@@A@?=<;861/---/.*,.3CAFFMS\nzz���������������������������������������������������������������������ۯ����������������������������������������������º������������������������}������{�����kiv��������������������������x��{�������������ª����������Ҷ����qm�������������{�����}qcV]ZU\diV[\_h_bQLNYYKL\TWZIEIS]filrwtpij{olRNchYZnYacVSZU^a^b`NQRSQUNCI1/ "KLJKPN4INKJKLORWR_UOY%%KRPRB4Tr|��������|}�yyyo�~mhZWS8  '&<93443334788?CB@;7332158704888=><<=????@@?>?=:=;74../.-.-+8KQYPJHLO[ed\�������������������������������T�������������������������������������ϟ���}�����������������������������������������������������`�����������������������������t���x����������U�b����������t��������LcY�uD����¾�ȴĽ�������ɾ��������������||������nT8AJXLUaY_]Yc\aOTYX]Z\[e_ZOEN[s]Vttjfddfib]]S[cT\f5JYqhhgg=;VdXWYPG@='('%5;87#3>I^]w��������\knhhs����omWJ6# #  "#0;103000138<=8::543.54158878=;8137669:=<:=??<69;<>@@@@A@BCA=<<=@@@>>?>>=;;:73//-,1/,/>=;:8223-.46554685558:<=??@@>94:67<=AAB@@@AB@===>A@@???>==<;874.---/-+,>QY[WPES[^pr����������������������������������������������������������������sp�������¾F�����������������������������������������������������������������������������`�z��������u������������������������������������������������ʿ����������������������|q�qfZgru�eLNLPTUU[Y`bYU[efSNMT\U[bSQQ[cc`hdbgdh_VWWUWW`ce`_^Y468ZF7"07(>FFBAB>=,":.!9>7" *".?JD? 6KQOS_f?U]MI<1C*Qeg`������������������]F8&"&4:.%10-*-02?B@::;;;83045305777767568:?>::;;67??ACACBAA?>>=??@A@?@>==>;851/./,-+*3JRXVB''UJP]e��������������������������������������������������������������������������������������������������������������������������o�����y�������������������������©�����������������������������������������|�����������ĺ���ý���¾��������������jjldc^YYLSQNGGV[RQ\ZOUTQWfab_PRMKR`]\bhe`dodkm{wofTNUPR[^ZZZPPXRQNI3WN13K@8%=NOQM% " )! JQTLJ:.D3NU2MCEIMQTQSL4Sdna~������������zt����jT(+$09.'#! !*04311/4?GKJ<7543024677:;=:<<=:>?@A@@@??>==6640.-.-.-.>QRVXB6>EGNN_�����������������������������������������������������������������������������������������������������������������������������������W���������������������дv��������S�������������������������������������������Ǿ���������ĸ��������t�����]jrlh\]`JUcHHIU]SUZUQUPTWbdc[\acST\]`]O]cln����tk]OSSSTUYWRW@MGGMONANFIKIS\F7=KTNQ!)$&' 2@>C=PNOL<7HPIC8KSGOMJK6LL(($@dqxf�����������s^`lr}kjQ%/0*#&#$((%!(&#%-305FCFEE@/./159989:=>:<>@=BBCBAA<<;;;:::;=?@:9==<=BBA??>ABA@@@@?==<5362-----20EC46:24:ADGS[��������������������f����x��������������������������������������������������������������������������������������������������������������������������������“9��ƹ���qu�����������������������������}���������Ĺ�������������������������gpp���xVW\QXYRHEGPJNWSYPTRRTo{h^SOOQNSURMV^ZZg���y]WWPSTPRTO<8:?@AC@@@A@@A@@>==><6316.-,-JYQ1)*)126;>@@R\������������������vu�����]�����������������������������������������������������������������������������������������������������������w��������������������źy��ü���Ymy���������Ʋ�����������������t�������������������������������������ph���wSPKMQKMUJFMUSYSUPLONTasg^]QOKLLJOY[bZX^�}yLd\ZTTOHSRESYSPOJQTKGG#!QA%@JK# H:.NPGMSRL &0">;&@NHHGP\KBECG!CLet���tz���}lhmw����wd��i^#- (-*)  ,%&,.))** !4772/25664312678=<6334456789=<;<;;6312.+,.DXY.*/2>PPDDGK�������������������������������������������������������������������������͸�����������������������������������������������������������������������������������������������������������������t����������������ì���������������������������u~�}aiXFGMFEQ^RVOTUPQYTP_ZdnncXOMFDCIPVS_YW]wq^[][^TPPSQOIOPQXRNJPG7,2NZD5@+87EB%$$QPNMLMMKXV/@QJHIGGCJQFG)")U~��uot�w{gqv�������dH# '+$(* "%**)""'(+.964(.RVC<:;8?EJIGFB?@@BAB9:;1/011122457;;<<;:;974077:403BE1,..%16;BDKLK������������������������=�������������������������������������������Ϲ���ެ������������������������������������������������������������������������������������ƺ�¸ŭ����������ӿ���������{�����������������������ÿ�����������������wpljopz�rv^NEDEDINPPLRROU]aMZXdLZc_\V?AAFZYjVLSXQUXXhhZSRRVOLHMONDSSLTTO:'EULR?:L89^[_O-3CK/F=3MJLOK_c0!38GKIHNC?YWR5!"P~}��}�t{}zz}w������|}ll; + !"*!$'(''%(+).2325BX\7AKKeYFNQPMKGDEDA?:::1...0112323689=:=><=@@AB@?:;<:;77420MMJ@61/-.//+.6>HIOLI����������������������w�����������������������������������������c�����Q��������������������������������������������������������������{��������������������������ʽ��Ŵ�����������ĥ�����ĵ�I�����������������ƻ�����������������������y{shditu��obRGEHKKPRLNPQMN\PTWRAGIVVRWIMMPOMMRZX\TOSTSWUYYPRFKQSUQ6DZXPVPH\YM]W[_bhiVUK?U)IWa>*;RNNPh`eJ$7F&%%?>GFA?D5TWSE-FW���q����kuy�������x)H.&   "#()**-+'%&+::9==H;<9RaTJRPLABFGFC?<;920001112322467778:=?@@AA>@<98;=:3220/@OPB2,'+/00/05=QRPU������������O���������������������������������������������������������w���v���������������������������������������������������������������������������������������H�����������Ĩ��������Ɣ���������������������x�¥����������������������igimp�xYSOOJEKRTU^YZRTPKJU@X^_QT[Y[aYNLCJVZ]RBQ\QMRVWPNMONQSUP=L^YY^MX_ZP^^benn`^]0K8'A_`P-UTKNVD@ (QX1366(#EFBEA?ANKKED?%DLy���s���yswj{�����Ϋ!&'1# %$ *$"*++*'&&+9=8>A?FHOL?YlGKONDEDHDA?:8733//0//012355545447]bnaYcfhphcPIKKJ^\ZRN7.LSROQMMMJKNVXSaUUWQhghY]j^`psihb[k_mic[ZWALTI:4C:4-0NO')3*6HCCCGHD;CD>ilqs�{wmid`^cfp��º��j 5!(6,('$%%5 ())'()(8@>:EMM<>S^YeoJLMGBCDDC@>:8401/..,-/234555431116:;>?==<==><80/.--//0BLA/.141*-48IU`cr���������t��������������������������������������������������������uyF������������������������������������������������@�����������f�'�������������������������������¿�¨������������¥h��������z�����������������������������������������pcjrpaifmda[]YTUVQTNLIJ_A4RZdoll]WUWbUTMe]h__VZNMNUNOMFIMKHJQOZd\Y^MSOMMRUYcrfW1UreZ[`qo`]G;A+$"5NGBN8440F@./C9DJTKJJCGF??QYhrrbdW_h`hfib������Y!)+;?4#   #  )++,.-/049HQOQPW[\afb3*@@<<=??=<552-,-,--./244323355201458::=;>><<8//.-,-../7J1.,14035BFRl\���������������������������������������������������������������������������������������������������������������������k�������øťY������������������������������Ș������Ⱦ����������Ʋ����������������������������xi�����������������~pw��|�uqldcaR_rdVSXUWUUNHIOUKJD1Ead`[ONWQQMTQaHHOMNT8IUHKOOSMMHLMNTZrpEKMY^Z^Yb\kgbfjgmswtpVWL>CI4-+G+9!"%D!+(4/849OB<5:3#!$((:KFCCH?/0HFD@;)19JVKI=RR\o[`^bt���������yx��k`TKMD2HT)"#.0+)!9&ACLG>58HD;BB?;=;:9;BC8(.$!nkUUbq��"$(00&&*) + +  -.-./.8DDASYY\cr��|bXVagiN@AOP:LJ720/1510000003231//047545378211205///01365631//12==CGPY]cj��������������������������������������������������������������������������������������������������������B���������������Þ����{�����������������������������������V���������������|�������������7�����������������±�����ƺ�������~zlfh`t��zmbdl^mlotl�ljton]XSUVRVPQTUEFSTRVUNANMTVWZcdTq`LTlofYVWPOJ?XG@;:OckG[gm{����������е�����trpV>Ih)98.GP\O7+%EKJ>8A?:98@V\s}w���{VRNNRSPPBEMNQG=<33133100011245300123447649::5000.//1158799300113=CGEGYWY����������������������������������������������������������������������������������������������������������v~����������¾�������������������������������������������������Ǽ�����������pxd�c���������������������������������������kthZl���}qlNZPP]myr}ja^XNQYYUKTTIFFEHMTYJ0NQVTKK\[[^okkIqojJSZSL=HNX;9BAY\Zey���������������ҫ���xsh^YY,8& 8<"1AFDFI<%.:Ga=:<<8925;/0124645<750/0014@NMHRhg����������������������������������������������������������������������������������������������7�b��������û�����������ȪW����Y������������������������������������������ͽ�������������xWf�~���{�}_����������������������������������nh������|p=SVVRJtrgh`UHKYaeKUZNNNIOTOY[RQVU[:;W\x~��y�w]mETJSWWYE@IFR[qnW����������������Һ������eghdZJ4)(@15;:>CKDAGOKOPQTR#8��!+*,'/%," + +&,.//.//;DCb_[]XZUL^O:CKJOIEHEEMQOPCIGFZ7643112:?BH@952357A@@<HIJO<0EMXO2k��)03$**'! (./21136CJNX[Y[d_SWCBCB8:AAEE<==AMY`cRZYXSE<223:HNFEHN@<875436743:887;856@F=@D?GK�����������������������������������������������������������������������������������������������������������������Ǽ�b^i��������������������������������������������������������������n��y��������������������������������Ƽ������������|~MqllQz��V~GF}pne\PaastUPQUREZILT`RKRUSE s�����z����x\ZMWjgdgy�������������]���������klt��xi_^ZD.O($21 0OGDPU&.(;LE!BJF?H@'!,GFOSI67~�-&0&#&."&21004@>GRS^k}}�{gL;?B?=@>GE>?<963FVlf`bXTMH635<;@BB?<5985;FQTWRJE@;7445545::789987<>FFBCG9C������������������������������������������������������������������������������������������������mo�������������ƶ�����W������������������������������������������Ǡ�������®���������������z�������z����������~��������������������}���xxxYkgu��}Cp^Jnqk_aVVX`jZ[[_JEMOXOOKMNRSMZ������������}dK[^wcab�����u|�������\v��������phehitpp]CXbcZM)%8'%8NUdW?I@(CIHMNMMSURFSTOBRy�{1%,'%"##85389;FLYi�����zZ@>CEEHQQNRD?@>;41CMNKHLE=:<98=885533226768899?ABFIDDC@���������������������������������������������������������������������g����������������������������yd�������ǿ�����������������������������������������������������������Ŀ¹�����������{��������������������|�������me�������������rsy{khkmqt�����|tjyztmp^j_RYYXRMRJMJKieXQTTWcvv��������������{�u|lmnx��s}��ȴ�­�x}����zysnlnohnrvshsm`D#4D4)31M%&QJCSBHI97& MNNLNNQPKFMPSPGAg���O!!$.)(! #( 9:=ABDFHUt����~_G>ONHALTPLCGDBAA>;?GHFCAA?I:?A>>>DEFCHDBC9:>VQPOLG@:87675456669?B?DFFI?2=<9�����������������������������������������������������������������������������������������������������V��������������Ǘ�����������������������������������������������������������������k�����������p��n����������������������������u��}p]af�w�����~�e}qetyjh_RVTUNZPJKHEPZZXWI@n�v�����������������u����p��gj{��������������troosrhnv~��~~wrqlW-J)B2*-%%#+%GJL`!-';SKIGJEMIKNKOOOEGKMNP-3C|���Za[P[)&% +!)+)&.(FADIHNENo����|^ODMICY]NN??DJIEAA=>@AJHJJBJJ?FKG;BFHEGDFB<=>HMLHFG;;88888889:978;AHHNO>,!<:��������������������������������������������������������������������������������������������������������������������ȷ����������������������������������������������������ŭ�����������xS����������������������Z�������Ǻ���������wo�}|x`_h�������o�qnyjinh[c]`]_UZ[JBCFFD6*)7�������ğ|���������x~ugbcsnbeYy����������ͅ�xnknpvw|}������unpcZiR-I;)1W2:7&;SQf,'.GPQJJJDAHKNNNMEFHHL8;?S���t���xso)  &/#/.4SOWB;E?DIGER\RMKJNI;BCLRFKI>BE;;;=DD;<:99<;<<=@A:7;CKKNJG3.5:�����������������������������������������������������������������������~���È��{������������������������Z�������������k���������������������������������������������������ȿ������������������v�������������������������Ǿ��������~[}|�uaYTy��������dt��Xt_W\fdmgXPQIEGLGC2'GK����������mx�ʶ�����osvhfdkzqXVU^������������¶���{���|���������yo|`Ug^LU]QDYH;BbQS;$/4GHKKIECBABGIFA;AIJIHG@<���mly}~}z9- +� +)/6++%SPIEPaa|��Ƕ��F>@@;:6?998BA:?===>AEMNOQmldLBGMBIPNKIO(<@><:6:C?=<>AA@@BBEICCDGPLOPG@@>A����������������������������������������������������������������������̸�������N�������������������������������������Ņo���������������������������������������������������������������������������������q��������������ȼ����¬����nr{~MT`bx�������~����Na_Z[\]SfVMPRONO5$I~����������yY_�������lb\U^bY]s}bV]Zt�~v�����������ɲ���������������~wljbndup_SM/!5JZO: (GS[HJKHA=AAB?<6:7@CDJKVI=���z�����E:$ �������&"10%"ife{�������o?><;9::8548=>@6;<@@D?<>BCDNUX]nnfSHEKMJPFFI?�������������������������������������������������ʁ�����З����������v�������ռ�*����������������������������Fy������ş���������������������������������������������������è����������������������������x���������������û����������|�tZT\m������������w*DG;34KYSU[ZUC>IKJHIB>BBA@>=99;=CIMJLQRORPQY]`ZVMFNJEEI�����������������������������~[Q������������������¢��������u��̟i�η����������=K�����w�����������������������������Ƽq���������������������������������������������������������������������������������������w���}����������û�����v{ijHPOp���y��������r*2L`jgftd`XQOLFCW����}��k�������������ptmfSOVXVr��x�osaUYNd����²��������������������{wojt��qr���t,%W]]caWZ]hX]XVA44/@HD88>>A@@#CVYJJi�����m���S�$��  +b���������cUJD759:661356558;>?=;;<3;89@FA3'FHIEDRU@GGDFC><:FD=@?CF@16GY`OJ��sY\t6;t�T!#&%7U2AO@15([¿������zNKF;78:921676563458BENNONPNPW]\[W\`driGNMWFGG�����������������������em��������������B��������������¾���������ý����������ǫt]���ơ������������������������������������������������������������������������������������������r���n�����������k�������������������������������������jWaZS?i��������������~FOTtZ]x}n_XQbr}��a*�lht���y{������xd^e����wm|�������^Ti�������������������������oh|������������t&"-RZSSVZRNRQSR<$ !-?==9BHD>65Pg[NffgmOPQU3Tq��[ '*9'6Ya�������������{^V?8::97776137>DEIOK96667:98>B@:KB@DEFKD?>GHJTRRNPRTU_dartvu|mN[XLLTF����������������X�g@�������ʿ�����Ȯ��ΰ������ź�������������Ϲ�˶�����«���ɻħ�����l������������������������������Y��������������������������������������������������������������������������u���w����������p����������������m������bHugQF���������������X[_fthrmkshqev���V����Wq|�v��������jd^}���}rt��|���{���u�_]qrqpxjz���������������uU������������u|UT9VS?W\MOSRROR@7.E23BA:8CDCDGCIW<76887661=CEMQDCDFIECESRKMIOS>:>BBDFUOOUPPRLTZ]iki�����}emqrk^Y���������s�����������������������ļ������������¿��ſ����������Ȼ�������÷�ɹ�������p�����������������������������������������������������������������������������������������������������������������������{�r�����s�����������������(RUm\�������������s����|~�osxybcuh�����������yO>�������fbuw����{mr��gs|�~�xeeXNSPMMVf|��yqs�}pv��������������������}qXf.*^BPX\\WQRRQRK@@=$6?B>BBB@ACGB:9Pdh\YQ[^;(UFs�����t{tm\hr~�������y\JF0.0/7;51342>4671//32-2=;8;845978896EOKN`XE=;DV=<JXbVahy~���f������������������nn}�����|x|��Yoomcil`ZMPSOJLPQVUXuZWYOY_\m{x`n��������������{qRWM7cSNWba\UQVTMHDA- 5@>==?D@?ACF=5M]_YXGHKWNC2/*��xol�����y����������p98:.29:983247A6253211433./.;?748?959><(EF;7GE=9ETOC@@:9>N:<@NRZZNQQHUY^_]aex���������i����s��h������������������������������������{����������������è������������Ƽ���������b������������������������������������������������������������������������������������������������������������������������������������{�������������������u�����������������������������4"��������������������vsqWZsskiktSNQ_g[aXGBAEH?AAAJGDWLSPJR^m����������y~������{t�s������VjwWY\]TSRUFCEE@CEDCABBB@EFF>HQLNRTSef_ODA1C.!)7.,5^���������ygND9844722/0444402/-./1144<:44370.-/7:AGNM.%6BADPB<;>@AF<;<>FPT]b]]SRNPY]Umyr����������vls}�hw���������������������������������t���zxex����������������������������ü��������{]��������������������������������+�������������������������������������������������������������������������������������������������������x�������������x��������������������������p�{a��������������������|wwxngk�z�w=GLJT[YuaAMG><7h`Y^dhb\PRPPZY}�}|��������dwf^hrbi������������������������{{�������w��m~�cy�rt�����������������qlku}����ȵ����sn���X����������������������������������������������������������������������������������������������������������������������������B���������{�����������������������������������������K��������������������������t���whn/BSeKEJNS[qlh^VFAGNH@:ADGMDIZcaf��������xoh���������������x��|_fkdM=SaVR=BEFD@>EB?>>=@CDC?DDQNOPVdbdv}]S:*!>5/,?h}��^QA<97632/.-000-.01442/0/.1110;D59;11774>C@>C<71?E:=A;;>==D=;F]Z_gejoWURUMR^�����������o^eewwbUT�����ƶ�����������������xsj``eafimpjnxyvusne]i�w�����������q^S_e���͢��������~}vM�����������������������������zR���������������������������������������������������������������������������������������������������������������������������������������������������������������������������v���jWXB!=HkSABM_ixo�jBLMfbZE@BFHKEDVFKNj|�lwsrea_��|������oq~����|ypdi_ZJURRIDGEDB?>CA@@>8;??BBB@HPQTYZVQ7Qj`J@8(8/$-;XW0-<;87443/-*)./.++-.0320///4445;;47942832:>>C=>63777:;7;D>;Xfa[hxxrqgY\W`ckk����������ybXXZZPPKC������������������nlhlvrn\]UTYONR^YTz���pmjusu�}h�ois���xiSZXg�����������z��}[qvQ�����������������1������������q�����������������������������������������������������������������������������������������������d��|���u��������������������������������������������������������������������lz��tAB@6=<<==;=DEDMOOXW]T/4MQKA<5.-73#"'#'26D781;:85310-,-+.0.+,--//0/110356855628657:85;760338<:6;BCEFFC:;KV�������aOSj|||����{u{�tdS[SYOSOA/�������{����yu}sila`W\yzsm_YYPIFH]OORX^����hlp�|�f�vWXNu|l\dqi�z�����wrnp�������rOG�������������������������������������������������������������������������������������������������������������������������������}������������������������������������������������������������������������uu��NF??=<;JWRJHFH@BDGZ^WA9<:9KPRR@?9::9DJBLKNM@FKfri`aUr�s���������~��ux}^C9&EOLEBB??A><::><=<:>CHIGEGPXS@566CL<4(=<:5/19:88640/,-/2331.1/-.-/-/243675::5677755889<812//6=C::EGEFID535_{~ldYgXflp����|}|pdadd]dQOHMKJLG/����������|}qplfa^`]Xkjqleb]VUKAK\cJGGR[]]^_w������vMIz�uu^ex}�{��z�ke_bswz�wf`lxg]��������������������������������������������������������������������������������������������������R���������������������������\��������������������������������������������������������y?������������ɏqs��]J?AD@=GKJJDCB5578<:6X>>;EKH\C==:;96PC;JHCC;ABZZc\UX]{v|����xu��}tpn}ioc]RNG=DLGCB>?B=;;;?BD>0?@E?>?IGNFE?=@PSP???B>>AC8?>:41774201310/,+/2/03,,++*,//.395785663333587<<<>>388646=:GE:G8>>?:4TVc\\uwoka������mpuld^VWRE7/=ACE6$���z��������~�{tpp`c\^dfoumk^ZRLA4CH;=JGKKFXjs�fq|jn�����x|obcr��ntz{nfa]^k}��[/qvxW�����������|f������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������v���^OHEKGCJMDCBB:79<9698Qg@<=<>H@<;:>95E.6H@998;;HFYOTipj�������ux��bv{�xjkfYYFOTMHHGCC@>=>A@:>9>@@FKI:AFG;=AJVff88;<6:<3-.43/64-..10.....*,1.+,.-011/438778631/015515;9;6C:=@HBOL:=7PLF9?Skz��qsi~�����|hqos[GE8,6-,"0540���������������zslcb_^dmsqj^PROHD601127SHA?S\]U���v{���vtbggde��peoxpnqua`kio��jbFlt�w������������M��������������������������������������������������������������������������������������ɸ���������j������������upt}�����������������������������������������������������������������z������}t��`SK@DC?FGDPD799;97<==8XqL<:<;997<886?QNKX�����Zr��~}t������gqbYgQRSMNIIGB==<==BDEEG@925;=?A@JC;=>53C8UmF:344766344221.-,-10-0123,-.-+)*0320133267430-,/32367@@;>;?>8887DMGNA30>DB=;EH_k[Zno�������ng^ML1+ $"����������~���xoegebd[\lqliVLQLLT[G?D<1PNSAFKHIqiu��~�kbmUZeysmslclexmnnytcv�rmd_npvyo����������������������������������������������������������������������������������������������������~X����°����L����}������������������������������������������������������������}pH������������������~t}paTG@BCBEEFFC9<8:;>>:977@^@8G457/=:=8::86434533<@CQV����������{zk�������^bK\XTOSQVJA@?;;:;HKFEMKD;0-/?DAUSC;9845:9_nd8812353220.-...-+--/1/-,.----)-11000044212-,.-..1488668>8@ABE<>>FCS�����ydmfjQ\_`\F*!?SY*�����������}{o]V[]ffcbhd[N*7QNFS^PKFEC>JJHTO]e]iek}snDI`XWRj\Y^fafkilijpt^]kfYffc^iq>����������������������������������������������������������������������������������������������������f�����}��Ʒ�������X���������������������������������������������������������oN������������������������^OLBCCBJEGBB?:<85:=>=;9;>eJG755+;><8:<6123151179ALn������w��{ttz������DTb]=RZHOXOIAF==:;AIIFVJEDC96@@?B=98?>611NrV4152AY91../..---+*++)++-/21/./.--++*+.21000.247;7?NL>0458<@><<=:<48F53DSYq_ZsqaRDLQIGIQZYQZdgaem_\fgca[h_jeOTT^m�����������������������������������������������������������������������������������������������������mf�������ƶ�������x��������������������������������������������������}tzs���sQ���������������������z��f]WMFEDDCDMB@C>869>>=>988kv>43342<97796033273198@JUky�����q{twzxvu�����lada6O]JTWRNEFC@>>HGBENG??=@6;?95BA>??;9?KPOY_a`NHJLKMMLKJJJLPQT_d_q�uhhbX[TRPRTWe����������������������������������������������������������������������������������������������������������|�����������������������������������������������������������������������e���������������������}l`^KEB@BABECEAE?=<>>=;;;;@o_964253125560244543F;AOQD?C=:1*--./463/-,10..00////00//0//./000/0//.()))(*..-.-,,+,+.13116;;:2147D>>;;CE:<8;5138?A9=AA8Cb|migHEL^�licVIB9&)/�������xohf_caUOEETUJ:NUPhbWRONLTHFA=?@?><:;;=@URLLF?DHHJROGMQKJPQOKVacmbda_[SGbRKWZY-�����������������������������������������������������������������������������������������������������X��������������a����������������������������������������������������������czmj$��������������������xxkcYGA>=B@BDABCA@?==?97679BXv966432224102244477==?G]POmy����|�~monsq�~ynR`[=[N^]YVTQPIVAILVDBCB@A:?DHZN?@3)*+-,-.*+-..,./0//..000/1000../111000/+*)*+//+-/--/--+,//-,49:512117>>==BH9;9342323998:=@ELadigLNHRYhi�j]N7 #(-/����zkfhgeh__UKGFITTKH`b`eURMJGDJF@@?=@EFEA?<=???<=>;;>>ACEIMRHGMLMNUZ_bc`[[`QKYXUX[M�(��������������������������������������������������������������������������������������������������pk�����������r�������������������������������������������������������������h����������������b�������vm[ZWB>=?ECGJ?@EJOLD@:<::989;[J:5411013333422976A<;Hb_iL]{���|�{zluy�}ghSgw]]h_TUVVRPNRFLJE=99EB@;88=>@Chwiq@2//+,-.-/0/..,/0/.../0//011.-.00000/00,+*,-/0.263011')*)--.0////12124;=8;J=4D6851025D?;?>FDHHHKOEHGLXe�cOE)#"/0"���}mmhc_^^VSQKJHHSPUZVSTRSTQJEBNDA>=;9;DZL==CE@A>9867999AELZZRNMNWUOSXZZ\Z\\QRRNVfG0�F7������������������������������������������������������������������������������������������������������������������������������������������������������������������������vtkkQ������������������������xmmXUQHAFEECBECDLKPOKCJJBDGB89BIZfzxifh���u|���YS_cx�echTNQWOSQv^E2BB=:L@>>>;9;;=HOaE25532/25:A>=@AGDIIDHINMNTnQ=6,-rysgZYWTUUZVTVUTKKSW\_YbkYMKHDGEF><====GFQJ:>KG<=ABC?E567?>BCNNRNTYUWMNSSYSWYONKKWG+�L-������������������������������������������������������������������������������������������������������������������������������������������������������������������������ryR����������������������������z^UJCDD@FFFJGFHNVJMMJFNDCJGG=A?XzZ=5335F=769779<68;@IS\z�jjk_m{��llx����aZacl�sglk[_^DAQod]T#8<=XFC;;99761AC=98PE++)))'&+,20010/101/./00110/.10/./01/,,+,)+.0,-.1-,+,.4125230/02>>1248@75D:>5523/0506769?FFOD>=-40;@I*-%- #"ZVSNPNC=CKGTa]TWMKU\_redVOHECDEE@><=;:;9<@999;=637:8;K984677LJE9=>=98112<972BCA5223311/113=@>>==?8;:/%'2653)(OMMJME;=ROQY\ZTRQMRWZ[dUKDA?@EG@>;:<999;777568:401832=9ID469;@BI@EECSLLSIIL@@LQG>3�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������t��mhroXVP88:9CAGAF?:==@ITX^ah\HEBABA>:>dQRSPPRQFFD>95787JORZqy�qGR��d������}���v`aaj}{�eADh`\Wp\fTHPC?8323265/125??@B?;,@A?=#(*)-144/( JRVOC>8CTVO[Z[QVRQLOZYEDJ>@@AB??=9::78:666555681-.00365>:467:99=;:99BEFGGC=8??BGMPQFAASUYROVH@>>CHA@BFSSLFBBIAB+-(<8==4EVXnw{��OW���w{�o�{~k|�|cebavuqZgegYVRfgUNOTOD;BI;=IB568;:85;7982,*%''(///0///0112455.-.-.//.../-,,,+)+-.0.4=650-././269.7;.43205;9343/./2>;73185=4A\WMG@8-6MTNF=;8=72/*'GCB;AJMJUWPV[YSTUTUVYX@IMBB@@?@A=;9777666665778. !004557=55=F457676767999::;?E<���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������}9XbI?:67899?A?CJKMNKMMRVN\FGD@?DGIGGGOt`D@>BD==;5=>@<3BWXb���ol��xe?Fnrz�l���nheagtkr}tg[QKXplVJEGJ@?I9=P@>:83148457>86111.----.../--22454--.-...-.0-,,,+)**.//-/5453-2./13444,,-/,/5/39021/--0433442147;QkbXCCJMFD:0:85+*!@:<=?GIXZQQVYRTWWPC@DKAFBEC?>??>;988766666669=351-1342544334332676679:97768:=A+��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������|RMbP=<88:798>?BGOTNJMKGHFTfPMEFKTTPPMNOLUDA?B?<=>;@CD><8GRXg}�������_uT/~kowo��|ngjfg`kcjgkUVUPHZ^DEEGG?<8:32/.,--0.,,+,,233410/0.//114+(*++(,+./21061..,0/2212342104/2123=000/,+,./20643669?><:?BBC;;:81,.,+,)**'OLEFIHJRPNGHQOTHDG;>@ACB@?>=>==;;:9955544547<9I+011123443121277B.-,-31232239@<:7544689;CBBDF>=?>>@RLJJPPXNSMEDBDAAFA@@@>>ADVOBF3tKTZu����zp����zgrpbarnodb_lj`ja^^URTZVMB6`ZPCCDU@?7;5-488954/6=JC<712020..313.-221111188;8;;9)%$$)**,,0439<98A933/,-.127:9009<3564573,*/1/04545>@C@?AB>2#"/242541$"!!',51"MOMIIIJQOOGNSKDDED;>=BELO;:==<;;;;::8653556666;55323335422322763-+-06521133=AAH#���HLI�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������n�jxsKEH?>84555889:>A@B@<>B?=@NQRRKJMW`KCAAE>CD>B@??BFCSROC5M,ZP_t��w������|tkhV_`uessjljhf^jPPPVp�a"7qcHFD62-001-3511/3332158ACDC84/$&$#)**.-025==<9:H310---79357428:78;98841-/3//2224?DGGC@@:;:9=:65)%$.$!$*+,,KIIJHHMLXLDG?AA@BJU@;:;<;;;;;;::876767564244423465301493--,+,..G@754457<@C7+��CHD������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������}��`BIMD976569:>;<>=<<>>>CCCEVZWOTX``WTJDAEJDAB;;<>?CCFNGA>;;XXXr~q���q����Tk[M_cinj~seo�olkWZXV[bULJcgOLJD=<9>?=6738885-+8632>9,+///313330234137>>>;<96&'&$++,0-/13:::=<<111--,1<0381269659C::93200-*('.4;ABECAB<<=68-)"'&$),*(!%$HHIHHLHLMCDGA==<==A@DB@?><;;<;;:;;;::98777765254443474720044420/+,./5A966558:>?MG;@&������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������}�e^HNN9;742469;==>>?@D@BGHJNKNQLRXaYXUQOLHHGB>:>?=<>@::D:;BHVLRjx|}����z��]I[]RWjq`T^��rnhabc`sONkj\ZEUUNJCEA=FIG<6556562-2044;M7,+&)--//1.0227767;;<788*,*%*,.2///,8:8=>93110-,.01653378;:6D:;;3160,'$)/38:>B@?=A;;A@@?===<;;;<<<:;;::987776741933459;88100122451,,-.EJ:65559=CIIGB==�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������|vffL]�D6347:7=GLDFFAEPKUW`[UQWVURA><>=>B>;;QQPl\^Zy�z|qgggl��uucY[PZYFFH@;<>9TQ9-315774.68<7C97'!!',-..-.2.25347;;93%+++,-0122/,3;5;<52201./.1//11342588;>:51///-*(.1234<@=<=;??<**38443..044% #%'12FEC@A>@?>@ADA=<:;<>==>=<<<;;;;;;;::;;:96665:40723335:9310///1323.-.09G955458:>FHHHE9>;�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������~WSXRdsF887832ECD?<@A=>=>?@?B@EHSX_caWRONJH=::<=<@@<;<<9;NI]h}gi{���UeeT\bTE33gHMmcTTRr��ywkl���zmaOIJ\UIG@::53_G1,3624353./866B?20-*+,00/00.02238:750$%+.,,/11532,35;<93213/../-.012567548C@63/,2-./012248?9<=>:9::8762-*321/+'+%1/(,>?@>>>=@@CL@A@gdea[QPOJFB;:;<<=>?><=9;:8:9)>_^Tx�k:URLWLEXD9:>@==?@???=;:::;<;<;;<=====>===;:::;853331./522233443110/0211//.//02577679:;GGKEECB%)5������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������~KH@3348:419:::<;;=??>ACKPA?@jcd\UUUPNF>::;<==>??@?:999994@_���g[cpfZOFK::<2JLU[\Y_aRs�zvxo^YTHQA^j:'-CB8528/.001213334666578970/.+)%',.20-$#-/,&#'(-10010./04.(-3484001./011211135323268>52/00001010386E469:8988975421123##&29?B@>>;<<<<<=:8::;;;;;<==<====;;;:99:876422./6132332432112111111002345677778;FFD?BCCGH@0����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������QOF31136363547;;ABKEC@@ZRXXSWRTKC=?=<=<<>CBDC:9:?;:::=G]{����f��|dB7003PbjZUWh\W[b��~|xb^UA\^d_<+&2B>-;7%)-/0.-,33245653874622-,+''*/066,(*+(%%)+-,.0/,.//751-4473..0/012256206620/221422///0/1003642665:=@?>==624841*.)118==>=;:;;<=<;:;:;;:;<=<<>=<<;::;:99766531.021222224431122222212245656687789>?><=A@AEA9998�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������ɻ��cOK72159@O4456;;>BGEGF=;<<@<;58BFO}�������k51.0KJCeLLQ^`a`acdtmhaVWK_`PGNC7B0$:967//..-,8736671642/11++**)+.442,/,,/&',11.,.--12165101133/-0/./147320251../1//00/,.110.-7843;<<>BB?@B@;92<,#'%-19CMH;RZB@?BCFFDC@@IJMHFDBCABFA??A>@CEBHH?;;=?AF94;?Kfb��}��txB104KbERKNJXT_TQgu]lqlidW[db_[WH$#2,"(>6(*301,,434465757320-*(),,*.1++(()('$%-230),.-23232332/0/0./.--13512010....-.2372./01/..1348?=9>@;=?@=>:46!3:8,7CHGB>==<;989::96555113411111223433222343344456667888788:9979:;@@A7?DMMJ@[:��\ULYcV�.�����������������������������������������������������������c�������:������������������������������������������������������������������������������������������������gb���k����mVQPI66;:9;8:99;;;=@?DFFFA>?ORKPDB?BCBA@CEA=AFCEGG@;;=@C;-/5;@Zs������vZa616R\>FRMLMPUOV^sgiacjhii^[ce_BAT9?<;6<:4/-+2CD93156575/0)*+*#*1.,+*,)('',,/1),-/23353203./10--.-,213221..-/0..-/3530001//028?>::7;:8<;99997659668>AHJHE]]?=>===<<:9<<9:965532233200222234333223444445566777778898677665>>:7=6EZbkvtuv�~~{ms��vsc���������������������������������������������������������������}cO[����������������������������������������������������������������������������������������������|����{����^SZ\P;7;;;8474988;??@@A@FIKIB<:;=>?&1KL^o�����xsgV?2.19JA8HFOMLERWr�pTZk�}cbZ\WG?IK8,?86475133?YL;6./21464-*)**').+.),*,*'*-**0),-/23454/14200.,,.-,*-23110///2..03102////0587:88:4776=AERTLLVX><<;;;;:;;<:::;;;::::9<=>@@?==<<;::::9967773233321222233323122344455556667888886555468=@<<<4;NUp�����}�{q��zqouK�������������������������������������������������~��Ĭ������fUUWZbPL;�������������������������������������������������������������������������������������������������zzz[UM]SB8:997674:9;==>><=ACA@GQ_ZPMKI@@BBB?@BCB?CIGLB@><;=?$%Dh��}z����xuYJD9?Z\D6;LGEeh_YWqtWQVZdbeja`^( # 6?<9<536CBDP637;=?><:98<;6;CHLYORUab9;<====<<<<=<<;;;;:978:=>A@AA;;;===<::;99965311344333333444433335688655677:::::7654458:<@:88@>FOaipmdqlhc]\\UQLNfVOTF>����������������������������������������D_�ұ��te_VE>;::889=AC0������������������������������������������������������������������������������������������s�������zkO>98JD=532:275469<>=>>9:<;@DFBDD@@B?=??@=??=<>CQ@BHIGD>AFBB>=AFY{ep}���z~���^]^�oYZF77ZRShYw|TUu�i_iyrvi]i^=-(-47@C725:G75713346,+../2100.0143.2-)+.+*-5-*,/3345332010-.//-,,.0105330.12.0303143017335-,5J=276?ABBEDB9;?87>FYTQniiUQN;;>=====<<===<=<;<;:879>??@B@:::<<;<<;;::9454333344343343444433367887656769:;::8865568:;;::;;=FJLN`qXccZba^NF@<>@H]nkU4��������������������������������������Zf�����YIBA=99887768=>=������������������������������������������������������������������������������������������w����w|�siZ9:<><72217214549===>>::=>>?G=>>>@EG?>>=<;;<;;ARHEHFDA?CB>BA;@AMf��v�z�|Lz��Zf]�zSL[93KRUYeZ���Ƙ�ujnvxwsjskS)-4AD9:>=DH><7343421.,0..871-4,-./,*(*)*/,//,,-3636411011+*//,+++---023/,.-001/./00.320982/EH-15;@BACA?>;:779=BZ[a��[@RI<<<>=<<======<<;;:99889>?=@B@;:;:;;;=;;;96454444444433343435433467787765568<<;;;;96679:::::<>@HHRLL\U^VUZYP?<;:;;;?FSc7!�������������������������������������_x���zA=<<:>8:875589=AD5����������������������������������������������������������������������������������������������msz}p\9698410334334569<=>??>>@A@ACA=>BCDA?@===?@A>=>@BDFCD@=?@?BC>?ADW|zxxnvi_XlqY]Ynm\QV@8DPIJo_Qi�������hmoptuo;D%"!HIQD977B;7;?7686/2#&100563-8.00)))'''(0102//-43384//030-+,.+-+)*),.--.+-.,132001,,11078-.171338;BGU@:;A;<97;CWcjeaVZPJ=<==========<<:99988899>?CCE@;::;:::<<<::5445554444444444444434466777765566:=<<;;:889:;:;BBCCGMIIOQLa`SOLCF?<9976679?@AB?><==?CDD??B?@?DFD@>=@CD@?=?@G�zkwkx{lb82[Z[YTSaYMCBKDg^Tid����o}�jue��lv)?>0E?IE<68::798789758+"/1015621/-030))')-.10301/158;82/.//.,+-,++)'+)++,..++&*1410/((*-/42/11110078>B]WB;:98:==HT^\dNSV?>===<====>==<;;<;:98777:>@@BA>:;;;:::<=;9:88565666545554444444445666776745568;<<:;<;9:<>>AHB@FLEDLYUQlRLQ>8:<;9866676695.�����������������������������������8HdjhcTG<<7787986698677=CECBBDG?74$���������������������������������������������������������������������������������s���rY\ueVA332///0545855659>@AAAACDA>@AA>>>>=><<<=ADED?<=?ADBCBA@=?D@=><83*���`bn{ycZfcc_UXW^�sQ@ZHKc]`al�}�yqfiletqszX7C6>NILA:::;9:>;8:793'#'-.0033116242/-)(.422./0-./79::62//0-,,--+*(&()+,-,0/--,081//-*),---0000-.03248Q<68988;;9;AJGNHTUBD=>>><=>>=<;;;:=<<:88789<=AA@?:;<::::<;;89<755666555555554445545556677765568;::;;:;>?@AACDHKHK\NQWQMMXLCB7489;;:88676660(�����������������������������������CI[[VNG<96767?:64557778;@?@ECCDGCACFFI@?<��������������������������������������������������������������������������?��~|dMWpZEA3568125:9787999=>>@@A?BCBAAAB@??@>>>@??ACCE?=-YHJGHDD=>>>>=<;<:9888;;;:88889=>>@@?:;<;:::<<;9834556655455555555555555556777744469889;<;=>BCKLPQOMNIFE@A???<;9679;<=<8877765-/#����������������������������������KHY?DG<:847778755567778:::;>=<:==>@A@:=@A@A@?CEFECAGGCBBCB@?>>EEEDBCD@ACCEEC?;75{��ccnrrypYf^jtt`�ugC?>@9HWJQ^���~�iYhn^n��iL:!>>>>><;:95658:;99888:<;AA@><;<<:::<<<97546656554555555555555555556778544577789;=;=>>ACECB@=>;:=?=?<:<=9;;:<;8;=;666602+����������������������������������D>KC;B?:;778999877777899998:::9=><=;;<>B?��������������������������������������������������������������������������{����K;,baB44765456666:?<93;=>>=8;??>>>?A@B?ACKRKI@CCBA@?BAABECCDBBC@CD?><:��zhdf]�u~vTf\mt���x\bhuu>>=>?>=<;877688868888:98>???<;=<:;;=<9635455553444555556555555556556775445:::;<;;<<<<<=>=<<;;;:<;<<::=><=<;;:<987:766332����������������������������������=:7?;><;989::997:::88999::99999:8889:<<@F�����B:������������������������������������������������������������������x����jE$ruM;458:;<764.7:<94688:89:;==<=><<>A?A@HHGF@BC@?A?@??@FDFECEDAAA?><:S�l{dgjrnRHHWbnp~��ae�mmzw��}aFE����lSPggZiuQAjbc]YA8MF@?:8;37632332.-,02585;73123*)-13174686259;93210042/-+*)()(),,30+)(*+*.740/-+)-.3/0/0010/.2;/024454Id|xUTKG_J79H>>=<=>=><;888888768889:;>==<==<;;<;8345455544444555556665655555566665434:;::<::::::9:<<<<<<;;;::;;<==>===;99>:6887766<�������������������������������I�69459@;:?89::;98999:::9:99879:899988889;A4/0=MRNMIG;��������������������������������3.7km:��/&������������������T����n^9$fU@6247<==;3251345698479999;;;::;9<><=>@CBC@ABA>>>=>@AEDCDHEDACC?@>>:8=77641452687/+,.156618;.052,)01059567428;9850000110+++)()*)+/.++(-1-.,+0//.''./,///0//097./026643TrmW:<;IGGXZI==<<<>?>=;99988877988999;>==<<=><;;<:::5433444444545555666666666665656666549999::999:;::::::;<;;:::;;:<<>>==?<:9999888796 ��������������������������������4485:=>:;999::998878:999988899899988889:G?=>@?ACDBEDCABCDBAA@=:=TON[_WY4^Yfy�ks�or�zp���erxx�aTn�MTSZUTenc?+KXYNDJ<3;?084065445460-.-123564=44:61)*+05=723539;;:72/0-020,++)&&***(.),6*/.--*/430-'/---.././0/././35255_oUs==>8==GFM;::;=??>==;99988888888989=;:;<<<=;;;;::5423434555555556666666666666656666556788:89899:;::999;;;;::;:;<<===<<;9899999:9<8773-�������������������������������39678;9:9999;;;8889:;;<:9;:;999898888999;;?ADCFHNNIGBPA:44��������������������(���̫�������ƔF ���������������z���meeW6/[?21237>?@=753/.03122035778899<:9;<;A@?AAACB=<=@>==>?@BCCBBBBCFGEDCA>><@=GSSWb>BUqr��}�~�‘~g_DZlj�oFSfFOSgm`^deTB:HOA*4N==D?;95;;7552260,-//.444875:5-*--.074126877777310/000,**++)+))**)*,+.-)//.3132*0-23,+---,,/00343210?J[Y=;:===>;;9:;<>=<<:99:99888878888;<<;;:<<;;::9998544333335656666666666666666665556665444588889:9::9:::8:;:9::877999:;:988;<;9::<9:87755:>><=9778898888889:<;;;<=<;<=@>==;:=:��������������F�������Ϳ�������ň�<���!-A=>?=<:732/22./07:;78:<<===?BCC?@><+-;P93457:>?=>??>>><2041/.-3765689<;D@;;<9:@GDC@CCCEC@A@BBACB@;?<=:978974018((,*,0/0/.353-&*21530/1127432301,-.--+,110,,-+*$*,,*)-++,,*))8@A;971*+,-,-.//./.-..BX8:;:68:==>68:;;<<::998878777777898::;:;<<;;:;;9953343333345566666666666666666666666654357789:::::::::::;:98789::::8899989:999999999987888899779::<=87������������������>=::;;87899::;<<<==;=@@A?@@<<<;9:;<;889888799888898779999:;;;899;763����#BN����������Ȼ��»��ksY?6976543444246=D=8;:::2?=<71388:>@?>@@>:77410,.+,1687766:;@JIDBA@<97>>DB==<;=GF@BA??AABBECBBBDB@=<=?@O:*AQL`W^N]YWbeYO611:9G_JF88<>?nMF\U_a�okLYK\Q*-??@?;;67733107('*,),,,--0.)'+*/796/-..08885313.0..,,,00/--,*+&,/-,,-,,.,*+,9DQ<:.**,-./0/.../--.0Q;7;<:68::CD57:::;;;:9987777777778;9:99:;<<:;:8976534433333444556666666666666666667666544447999:;;:::::::;::9:;:;;:9988:8689::;99:999999899889889988::::+��������������:>=;9;=<8778::;=====<;;<<:99998777776777667777788899998899+.=Ba`}������¾��������żn]L755643332443345778788637C@76489:=?A@@>>?=87,-*-/.17:8558:=B=<:;<<<=B@?=>;:?ABF>?@BBBEFCCCFCC@=>??=?J?.1LD=W|k\FfF8=0/)6DIRL@Z>FTkJJJW_jp���Z@;Z]N'B:EH==@737922,*//1*(')*,,'&)7**154606;68875113/100///0/..10*.333-,,.036**)-3?790**+,--..../.--..3P66;:78:<>=>?BA@@@A?>=<;;<<<;;;:998998756665665667776677666569CF@cw�r���������¹���½�msD66554446673576458769;:67EC8849<;>@AA@@?@B@873+10147865789<=??=;<>=<:=>??>?>=:;AFEDEFDDB?ADEDGGA@B@;;<>AL3+494F+'324D@1$6@U[YNQYbD?DJUYwz~��hL8J_^N(6IM>6B:26<:50+.2//)(*+,,**)15,*;22613;57::2272210..-/./00..2:87.,/./630,++,/#&-+*,-,--..-,-.///7H5557889<;?@8987998987676655556677=;99999:;;;;:976546533333444455566666666666677777766544579:<;;;::;;;;<<:;;;;<<=<;987976899::;:99989::9999:999999888888887.�����������52@?::=<;;:9:<=======>?A@@?>>=<;<<==<<=<;:99:97766555554566666555654334456432;ix�~���������Ÿ��``H65433456754674468956::47C<8;89:;?@@AB?<@AA>ET614598766689<<==;:;?><:==>A@<;:9;>FFHDACBB@A>BCHFBB@A><>>>@A9476749=>976BF42BkVPIDF=CA;LX_�~z��h^K?FSW.;JIIOJ>>==;;8353-.0+,,-00/,//41/-.482/98686452211//-..//--04=;8/.---421-)))*++,*+---....-...///I;5678;8:<>AB8878888776655555556676<=8::99:;:7:997655654333445445566666666666777788876653358:;<;;;;;<<<=<;<<<;<=<==<;9898899:::;;99988::98::9977999988899887.#��������5457<:7:;:6599:;<=<<<<>??@?=<===;;<<>>>>>>>;:::988765545444445555212334433333233<68B56;;:<<;::;=@>;;<<@@=<=:;<=@CH?;;?@=779=EBC<@B3/7;##'HSG;,*4AD\sbYc�qvzqwkSNAB=@FGKNM;@<>;:77101-1/),/4541-//9327.4744>>457636410//-/-.-036=;8+))),142.*))*++-++,--.//...3?>BC955747;<>??CC8778887655544445556777688999:::;668776767654445555455666666666677888998765434478::=<<;==<<=;<<=<;<>>>>=<998:999:;:::;;897;:879:;99778898888999885&������/;;:8::9986424899:;;<;<<==<;<>=<;;<==>>>?==>=<==9877775333233344223244333343353334=RcVLs�û��ʘ�zbDCf85445677872158668:<::66=H579>=<<@?@@@?@=<>@GXKE=LT>;888:9:99889===;<=>>>@<9@A@==CGA<>BIKHJMIIHHLEDDCA>=;8::9785.4668+.+$ ,EOlfr{[sltvkd�txiD;96IHMIEDE<=@C443/,/6&+.07754212><<2/5667>7:94169110/-..-.047=:4))''*.20-)(**+,.,,--..///.0<84446554468;>@@BD7778776555444455567775<76;99::;;989978877765556655556666666666778999:98765421558::<==:>=<;==>==<>>>>>>=;;99:::;::::;;<977;<9899<998798888788889877-���%02;<:88:98652/056899:::9998::;;;;:;<=<===<<<=>=<>;988785223332233555773556555785445AKGJJg������F:M=69#>6457878983346878;;:868@F789;946;<=?@@@<<=>FnV@Q;D?8879::888545;<<<:99::;;;?>===??B>=;9:;:88==@C@>==B8/+1<21*$$(+ORMjPp�_^fkw����~k43(8RVQJHIGHDG>774.-.((+.62458411.5108369A@712/31.10/-..--257;82++*&),..+))**++.--.//0000171222234212357:?@BD667665554443445666777:876::;;<<<:79;9887777777666655667666666778:;;;:986544524379:<<;:;;===>>>=>?>>>>>=;<::::;;::::;;<=:8:=<;;:;=;98:;9888888898774233:;767888999750..1477787776769999:::;<;;;;<<<<;?==><;::87412354339;9834799:977887766?R]GLgX\��B5654326065677778756856777<:656AC7:;:348;<==???@?>@T��d<6;C688:;@8884:;;;<;<::;:9::;;<;==>@AAFELLPLKJQNNEEEFF?>=;<=;>;<;:>A?B@KA5)*,:$/,' ($;JPc;DN[Y`_lyw����P=CHNBeVKHLHKDD??90+/-))+1658<82/.,+-3878;98--26/,100/-,,,038851--,%)).-++++++,,01/./02433011010231122357:;A@555434434445677777:=@=<9:;:::<;;:;;::988888877766666666677667::;;;::87644666542479;::;<<=>>>>>>>=>==<<<=<<<<=<;9:;;:;<=<:;:<=:;<9:<:::87654778:9::5544346765567787/02234566777777888888788999:::;;<;;<<==<<<<:65766777778;;;;;98966777665456788998IB:74224679E75666825457764357:896?C?89;<<;==<::999=<<;Lw��|>349;9ADD668::;@:::978997979<9:<<>>=?<8;<;;:<:8:<<;9889989887766666666777789;;;;;:97653445664455799;=;><@?????>====><;==<====;:;;;;<<<<;86:<<;<:8::::868<8789:8:::85333345655678730-033665455566778887788789:::::;;:9::==;==:666677785688999;978876677887667997955g551138=?A,857658454369<57<7;89187<;==><@@<;84538::;;<99:;8885578889<===>DIJGOSWQHKOURNIEECE8659<@?A@@@4237CF"!"./$����� ����7WVXW_lpqttQR\[LWan{rkSNNDEGD:9340+-3413449158735622252850.0////01001521/-**),+&)--,**-,)*+++)'(.0012322100.-./334579::943334545677786669B??@@9:>?>>;:;;;;<;999999988777665666676789:;::99887444455664466699:>>?>>=>==<>=<=====><;<<<<<=<<<=878;===9:8989749:86889878:986431/13689:9641///245455555567777677766789;;9:;:;;;===<:84563787899;;8777787:97778888887987644s<67447;97632568<<<;>=<=??CHLRSNNGIPNMKAA@C:76:;9;?@?;112245%%'&�������������HM`W[]eqpnq[?GVJJPgrul^Z]JIH@F;63.,//211126<:::946;626/570.212/.000..10.-+..+-+)().-*(,,+*)))&')00001331.--,,-,0136999::3334576565678777?B@@>;@>::<<<:;::::99998877666666666789:::99878956876776556568:;<=?@@>?>?>>>>=======>>====<;<<<==<=<=9998<<=;<<:98769:857889778998554./2689:9556110264444445555555456565434689;=;;=>??>:73474279;<=;:?8877797997888668988886444xI6::;6743�/69;95.12::==59>:98/587868;;344344589::::<<@XZLFg;68988;8888;<<=9888;997679::<;>>>=>?;;>KNQOIJLJQOI=@AB=87<866=BC2/00/22*%)"��������������HNTP\]`ibn�qMFJLGK^`zuN]D"?=BD=0...+++-./4:<<>2/59//27662/33//./0//.--,-)*/-+,,&'/0,(,+(&()))+,/0002332-.-.///223577788345566655688:9;=A?BC=@<;;;;;:::998877666666666789::8887999679:8668667659;:<>>?@@??>>>>>=<<=>>=>>====:;;<<===<==:::9<<<=<<:88779;;768:98667889640015::978774423345333455677775544553122337;;=@@@?<842560.477:<:7<;<=7789888778877767865432�Y8:;<575-� 65;83-.6@>>=9:;:66.45565445233344677897677:>L?YF999988879;;:88889::<<===<;:9===D0//014:1"'���������������IMKSZUiro�lhRNQTOS]hv\L<>B81::>/022*+,)/35A=>>;1445434683453...-----,-+)((.1,-+)*+./*&(*('((*+-01124453-/01///25635786434666667666:?>@C@@A;:?BAAB@=<>BC><<<<;;:9:98887766666677799887778886798;987779978:;==??AA?>>>>======<===>><<;:;::;<<<<=>=::<:<<;<<;:6775888:6899865666887556899999885653332333455566666555543233334:;<>><9764571/./3158976<=?9::979966789876542(BiS�L7<=<:=>5��43993/18=@=:9:985//3487;654200455677674448789898988899989;<:9;:;;;<>?@BA@>@@>?@?>=<;=<=??HMPLKKIHE?CEE>96:==A8:A1/033:9%$�����������������>@e^Y]xji�t]IYWWdU\fVPW76@B46?4593,.0(1=<8@>>9()8555-/10262--.-.,++++)*)(031.+..)*+*&$(+)())*,243334312321134<<8-./31-466666688778:@AA?@=>AADAC@?>?>AB><<=<<;:99:988776676667778776678455478::<:999:;::;<=?@AAA??>>>=>>>===>>>==<;;<<:;<<<<=;<;::98;<;;;;;6897887:9687664444788758::;;:999767732222334556666555554444444578:::888;:780./3543689;=:>>>=;:8876687764@>o����F6<==>>?A$�13:985539@?98:::)138;:86782/046799:74589;;996798877:99878::;<;<;;CDCEDDF?@DCDCA>=<==<;;<>EONKHEHG?CDC97;ED=79?800/131��%'�����������������5HW\fTTQx�pTU^^]5$POLRC>=E.0;6588321%'4<>=ACCB@BAB@A>>><<<===;:::9888876667677776666787655667;;;<<=<:9::;<>@@?@@??>>==>>>>===>>==;;<;::;<<<<;;;;=:96<;::;<<9:97789998864452243777867::9:::9745733222234456666555555555444455668898:9895;>;<;82489989>=<<;;:979:87434E}���f�66=>@B@>>4/�2699864:?><:<<<�/25;><97772014589::88898::997877668:;;99:::;;;<<;><>LLJHIII;>AB9@KD>96@:310//4.+3$�������������������6J\rU\XYmrpkMO^jXKKPRRQJ?+")1;89778: ,1?<>@>;:/398%362.02/0,--,*)*('*)).243-)++('')())+-,+)+2312345832679:;506)*)*+-5765456;88:=AA?=9<@ACEAAACAAA@>><;<=>=;:;:9888877677677776655557:64146;>==?=====::<>@A>>>??>>==>==>>===<=>=:::;;:;=;99;<9=9759;99::;99888998976:634422//64440::::::9:787643222344556665555555555544437,58659;978:><:852247::9=<<<=<;;99986434$3��_u[28ABBCBAA8/��5889866A@@<<;9�2148<<;6641014787:;:54446776889:988:<<:;<;::<<<<<;<>@CEE@AEDAA@>>@@?@?BD=:>GKJJJI:;>?BE;<87<>4211154E994����������������>���|lf\`cVur�j\Y[`aYZGLODGDG706B=<7:;=)#45?9:>E=89;@>896420-*-,)+))*(''))'*/1-,+./.*&&)**+,,,-.0/.-24463568><9/1;')*+--232169:=@A@AA?<<;;ABCCBBBA>==@?=;;;<=<;<;;::998777777766653434696489367<=;:>@?>?>;;=<======>>>>>>>>==>>>>>?:;;::;896568:9D<986:877879766579888557753455555432389988999:>9953222344554455555555566666655566745651/-00/../122789;:79;<=<:986512256yɰ215;=@A888:+���55668BA><<62-11348::8643421235643433154675323:;;<<=>>======<>>>>>>>?????A?A?>@?>?B?>A>=;;>=<889755<:6956:)��������������������'SU[dhXjfehhbXTaSTVUZTJQNTWQHE>1323348,6:;@>9774:A><8JF<.+0+-./&$%')))...*+/0,)(&%')(++)+,,(+1.+..---/20002.,,,-/0.3335=A@ACEEA@;:?>??@@BAAA@?==>=;:@>><;;;<==>?????>>>>??>?>::;9:9785538766:879?@=;988656767;9:64441345665245468::98799:9::63322344555555555555566777666666676666431211///0478;<825;<<<<;9631223F��d1349<:96<:@/���23;<89>;;<43//4663398965<223332544211045775326:<===>=>>>?@?>?=??>??>?@?@?>@@A==<=ABA>==?>;:975555458:;8854776-CJV_aPT\d^^e\ZOMc`TNVLNye^OJIH91436>>65>>?<;:45::CFNLF66)(;KNC$%$)**)+01.-..))('*'))**+*))&)+((+-*-///..-,-.,,-.105568DFDB@?A?=;>@AC>@AAB@@A@>=>@B@;;=<;;;:9::99888777774477766=9:;998567<==>>>==>>>>=<:;=>>==?==<=>@@@@?@?@<;:9:8976544663789:86432/115766666679:<:98:99988743334444545555555556667777777766666443333222111588<=9324:=<><;960.03}�j6124::88;<>?8#��35<>:78:643!022.2189:87Y7222224533232668:86546::;<<;<<=>=<=>>@@???AAAA@>><<<<;:9>AAA@>>><99765454368964465547556793342������������������6&<<649<<:B><5;=<9>;DD?BL<66.;%(7/'-+),(.360./.*('(('%-+++++*&('&'*-,.-,,,+-/.-8;D?1067:=EA@?>?@>=BA?AA@DDCAAA@A@AA??AB@?<9::98999998777765465568<<>?<985467;=>>=>>??>=<:88;;7<>>>;;<==??@AA@=>=:::88754467:9879::@@@AA?;<<97567A?757530.0678:7666779=;:9:974998532234444455544555666677788877778895443886321015789:83223:<<<;;92002��N3358:869@>>;=4+��583;752052-%0/-06.2::97MJ44333334323367::89978978889:;=<<;<=??AABBBAB@>===>>>=;:;>???>=<;:7765564479855767663334522100-��������������������CCOZ`cYVUZY`hTYSPSQ>I]TEIeobKMFGG9369:=>?ACCA@@;?@D=?GDB:;-%$5761+(*(.583/231.,+*+$2*(*+,,('('(*,,()*-..-/04559:/19:>B@@A>?@><<>?>;::87445766;=;;9:??>?A@BA==>764454545676559:;>@???B??A>87355>GA;5431.1655;=<8:6668;:9998799864323344444444444555667778888888876477655431100224777444379;<>==8536Ǧ:348;:988?=>556+��3687252.11+.,*,50-;;98ZYQ7334334434367778::87776659::;:<<==?@CCDCAAA@?>>??>@?=<;:;;===987655665468:87998776333483<461)���������������������DFQ_`V\\UTLPVZ_QZYGAARBCHWhrSKJJD=8946=;D>@=A?><@KAYNGGD<.*&&H:1*'((/4:43557:7600)(1,**+,+)('(+,,)(+1/...-+,//,03:;>E>>?><<=;?CDA???BA@DECDCCB?==@@@A::96888886766566875775689;;;855425:;:<:=;987764223676545854::;=>??@>;:877753455654455:;==>==>?>>==:67459>BC9761.05544;89>:788999::9988875433344444433444455667778888888778889676543210223444376515;<=><:745ƒ82678:988?>=2342��/6/./62320"�/+-.251<;;=P]V844346764445545865445655599:::<=>?@ACDBBA@@A@??@?@>==<>><;=?<85697668557997;:869953344?9?JE85���������������������F89@=;8:>@@mlAF1;53*&%(')&((+179246:A>854+'+1.++++*(((*++,*.0-,--*+.-46<><=>B@?>:7;<>ADACA?>ABDEFECCBB@>>>>@@:866554455456789978656579874/13544423433356553123688876334356329=>>>97568:845334421368:9:;<<<<<<;:89565679:=;730/55434569<<=;888999898775443334445444455566677777888888888:9:9676622565443455>77225;<==:98F�{53699::89?;94323)��51+,5101,"�01-.2546:;:6E5LP=<4653345412242234555569988====?@ABBCBA@?@????>?@><=?>><;;:53378768555945976476433445688874$��������������������L?JMOXZMLKRQVRUlSGOQQIE?@E=;DPPOIHB<687FF99A@:5<<>Cgt^FICC@7,')(+()*.006585=B=>76,&$#$')++*+)')--/,+,,+-*//,--/:5=<<@?>>=??=@CB?@AAAADEDDFECDB@>=AB@@78:644455557787;875786465,-03355456667666732113555788877676542439998212678823302236665689:;;<=;;7439665664677640045444336;;=?;989988987765433444555555666667887787888888888987998659:<;;96557G=;4349;;=:98V�635579::=<=794334. �/3,+751,%��13..656/9<:6349_jK22202546421/013354455657:<>??@?@ABBABA@@@@??=<==<<>>?<;;;74467788756:789644443543333310//#��������������������EOYMLOVQJHGMKVm]_\GINMKRLGFDBIKUMNGB?F@:B?>>@AC@??>>??>B?ADCBDEDEECA@A@=78::67797778754333324577776555554545666534324688776778999888865779766443678522223336687447<>><;4610/01537;<<6464555655334499;<<<<:9889:9987533444555566666677787778888888889:;;:::;;87878;98756201142+��6/022/'�-02-0./748><74483356657623455433222234459;;;<=???ACEDDCEDBAB@@@?===<8:::<=;;:97656767775775555433333333310011��������������������=LMN[ZWQSJJLLKWTV\OPMMJCLDDG><=6=BC@FP^9<<<>D?dq[GHLA>?@??<@A??F:.-@A54847200362126*&)''-/*&*)(+220..140-+/011/-9?@AB@????@ADCBABCBADCEEAB@<867897777877777642356777776655454355656543445457:9:8788899::9886555:7664439;:86444355456557;>?>=<211/0/0168??@8345678775323499::==<::98::9997634444557666667777777778888888889:::889:98888:;:8::9867665;;9=<:7�^33897<;::CA42442340+��2./13(#"(+,1+1:?8759:9799866541/0232111223459:;;=?@@A@A@B@AMVU@@?@??>???>569::::::8536657664555555453454333321111+%�������������������,@GTRZ^UKMNMLLMQNUNIFVRBB@>?4<>:983HFIHH>@8:9>VOOIHVKD:<:>9??@@EA>8EE@9>670/1/14300-,()&&)*%.1--11244440/04111/-:>?A@=?>ACB@>??B=AEEDCCB866798875689977743256677776555556655679987766774688;;989999:::99768876654428876556555676666:<===;200////12344:=734778865633346899<===<::8999875444445577667678877777788887788889:9999988889::88898:;77569<9==>;��93798;<;;AC833420110"��0/06#&"��%!,,*.*2;=@=9:;9::9656843204-//1123469;<<>?@@?@@?>>@QONUO=7A>;648988;::8675644555655555545874332221113)�������������������!.?I?LUWYVSTOJINNQVS?EEE=;99,37??@?HFBGKIJB?=9IEI_NAGVB9<::=@>;=A@>EDA<>:851013761202*(''(&(.232012245323241/.--;>>@?>>?=ACAA@C>89?><:889ADBCBA@645447867888778643235577755554544577:9::;986788778:;<<999:999:::867765655436886433445666677;<:;84//0//1222112:;633777765633334469;;<<;;:999887644444557666779<;:887777788777778999988889:999:779899:;986699<=;:��64799::;;@B:6100/0/.*D�311'�������..,545?=CG988896766453544041/133457;<<==>>=>@?==>@M?@[>6@:78=@>=<;9579:;;;98764345776556544459963322221112!�������������������,738EYaXSXZQOOKKIJL6:A@><:287<;;?@CCBDCB:889987788<>@BA?:53313447:;87678533123466555555543459<;8<<;:98799;<;;<<:999999:::977896555434564333345544468;<:::-+00/123520.089443566565544334455;;;;:99998877644444558668:;<>==<:977777777777788888888999:9986889:;==:76669;;:��E468::99:=?>81.../4..���+������� +0-+2347>=E=957996445467:61200233578<<<<<:9599;=>=9:=I<@:\;28;:<<><<8998:::9977645686656654465<>;>:5222111/�������������������������LR]f_UPIFIMRGC>=>?A=7:<:=>A>DCACBFD@BFJQLREBBPK7>GG@E75449753212789:98>36983,/'&&&%&%%$')*+,../0.+*,-0.-/<>>=<:9>=ABBDD<88887657657;>=:763211224:8678677221112344565554433227::;>>>=<89:<<=<;;;98899999:::866776655322351136556344579:<:<,,11122211//0255865555655443344549<;999:87888774444456668:<<<====>=;7666776777788888888899:99888979:=>=96665:;;��F5789:999=@@9320/-//D3������������//84337<@>;7784410348;535301234669;::98:5567;><;:89999:FMH:::::=<:988899:999:6764885456644757;>OQ;3323000&�������������������������2:DfsaX<=>AEECFEB===:<<29899;>?@CCC98876555554887665422223337668989322222335665543433443368:>=<:::<=>>=:::::::89::::::96455665555557<:344415544678>==86666666777778877799999997887699<>?<65668;<��7689989::;=?9411/--4&0"�����������./532489>@=:84352357633200024456699::864668;=;==645;FH<{a@57:=;::::77799;:::9886577546533446:>@@DHF=9>?==>?9;=5;=BAEIZRMKOUaHJGD2200,+-'-<:/-+....2010257=7/21-,*,-*)%&&&'-..//,+)(,/.+*--.;;=>=<;66;>>=97666885522100.1211223455644343211221145566643224443344565655566>=;965655667777878878667424577:;86:::<6555789Z��sA>:9;;77788768999:899865466555454446>72111471010/(�#"���������������������CGLNTE=IFIRC=<><<;67::37;><=JLFBBMZKoGCFEEBEF2)+,))*+,-02H3-,/00137:88;3:9055-.,*)''*.*+.,+,.///-,-,1/:89:8764476788667797421121/..1122233566543423322212445466533344333334556656547<867889888999:;<===<;:766654432223443:J;3237667=>>4001//110//0132697975544444324454677777777787664445556789::99:;;<>==:9655556677777777776663205246988:9:;86557898Q�ǓH79>?>@@A@71012/-.21'�������������10-/113>FB@B<876;743111001224866866333489;:87654445843TeXKD989::999:9664565555567555321112310010/ �����������������������@HIGEFJC>?@BC:120/-.0/0 �������������6.--2138ELFB@?;7753221010112378885233489;946554133:;75@Q?;5?=7D86689:;:::::999:865356554555423221111651110/ /������������������������8=DDFFHC>B@:>BGC;@;64356/*"%+1),TdFUrIGFD?87854296*(0271275:GM/2147746445420-./.,.2/.-.0/..344//03/15544556977776666533322121.0/,1223333445563622120125646359888655432344555666765888867:989::;;;<==<<;;:9876554434<;B67463337755;>C=20/431222210//4:89786456752245756656667777766644455567789:::;;;;<<<765555566667778777799G034896477889:::6556789788��\9<<<>>AB?33/.222.0�������������#+0.---49CFE@B=:6142312321122478433356689974564224864439F99F=6>>5679<<<:;::;9::976456655484432222115321121024(-�������������������������79BFDEB874799:;886848(!��- �9EPPOHLQIAC>;4488@K;8955643150/25996:462365312110211/,-/0243540/201/756556656666555234353212200/-.1122344344656623441026764647=<;:86642225545688666886565:8;<<=;::;<<;;::::987654444>QK4454444865479@B>435754322/..27:;;:96567763235667766666667876643455567789:;;;<<<==<65555556666677776768692588875777:9;;:6655689889Wʟ;;;>??@AA83.,0/1//*���������������0-0./4566JC@C>98775633332222323345577:9744556434974126C;PPD4786458:;=<;::;;::98644556645543222243;5643210/10����������������������������<=:;>;<6668745440)$.,����,:IGKENPEDHE>99501332036764723;98549::97675533452210--/6631/./31012466555555544433134442121110.02221233334679<547883009;:6776?=>=:9844445654555654676666::=>><:::::9988899997655446^TA45654555555456?=?;4455010//0689:;;:7565752246777777666677777643455567899;<=<===>>;66655566666666565666<[5656777666:9;<;8655699889:��:;?A>?;2.-/--/1)���������������&//12547A?B@@:6;7986433333433444376685545698434411003=K:97735=555:<==<9;8:::::854555754444332594513512914:30.!�����������������������;AAH=<>;<952555201,,)+.23#����:G?AF>69?>E@?879445:<667645456414775789754435643431..0.//03231010315555235444422333544332002++.5221011166:6545789:820<<<:588:==>=;;55554334343434565569<><<;:;;:998777677898668:97@875665555544434379=>5442.,/13789:;;::856573225667777767767887764345556789:<=>==>>>=;66665666666666665657<<556666656699:;;:7557899897Q�\;9<@@A@>:2.////01&0���������������A/633668BA=;;;9:9895422235542592/034644356795453333013>547@7687439;<=;8858:;;::7655655545544336832251292110-5&"�����������������������B3�688/52,.-,53/$!$/--%�������><=@C>448@?:=B;:56688857683243356887687777797665431*)./244232.-./225555344322334421232111.01/01.15440/06786/1654779:44779:;89;:77554543452.01448768:><:;::9875574349>;9=>:55555K957777777765434554227<740-..1315678998556343366666656665688666754455669;=?@@@==866655566666666666667656=63124556675479999::77787756766��\:=BCBA878///-+,3.����������������3331/36FC@=?>=458:9766863567752./4223545544453/-/000147BB@C:4345=;<>;:87889====<;:998556676544334431221222.>3$�����������������������������D��481�������������0845=?4-.110329?@@C<<:99:;::;;;>>?=;=<:932467775443001210000/,*-0..554324443231111121100//0.-.413633-/12:53.3:;9:=92711;>97;;;;<77736655554401442257789<::;:9864322577898765444557?D767776656543233222018636./14324654577555442245556555554577566754455667889<;;:87755555566666666666666679>53214556774467989:87777665765P�e9>=<:9866767764444432222222203/#����������������������������������������������������%8018<8.,/1.0573668:5:566999:<>B:998:;8868<:656200/121/000/-*-0461/44433345334.11110111..02/,.542140/165=30-3:9<<<<897218::;<;9<=<<86788756435543466789<;9:96432459:99:9856F655566FA86566656564434432200211;30033433334555555222234555555345665677544556667889:99878655555566666665556677664565335557654468989766566546767x�i:???=:58340,)(20(����������������3/00/,?>A546>31-,0785,.0258=6372223453356510/-.--//.48HO|\E78976;>99796768988==>=<;:7788777456554422223210004������������������������������������������������+���$103=;96/52,-3314641552333368::;89;==<5518875554310./0//1/+)*.1-+-444555544332/00123311..11//088443237<6>//.23.35::=8;:98<:;><::>=<8777853541566563568:=:9974214;:::88865557556665DF6755;44433444554332../0872123212234445664223323455553356666777444555567985766778765555566666665556666563355622446776566888655467656775?��c<>B=56;66-,+)5.1�����������������/,../7988;>41.,.0643/32.17;77522113111654/-.-,,,.00468FiT?97769<<76677779768;==<<;9899764656633323333211.16��@/�����������������������������=�����2�������3*�����+,+4-34,,+++1/202/..05233556434578577025;;447<930,*,,-./,,-/.-*+19445542344210113443231.00103784016::;89+--+-/0046:<9<;=<<>?>;;<<;88777655657543421369<::86316:;;::9875555;>:5666;;4F;8@44444445554442////4523321233344545633222234555434566667764445567896643156888755565555655544455433642445523456677767995544556566778:f~Bo��rB;66/-*+-75���������������'*,031496:D9//,-//2122300086530./0/102411.--+,--/000.0;OBD@63555;D:5367797998:<==<<:988776554444333334011/ +L.: ���1&������������������������������������8/(����*)(/,0%111789=82///..232987<=<;<:;<;:5=7872223/.,**,,-,-,)(,*)*,26444423433111223332343/1223659533<:78=7)*+.20//./9=8:<=:9=<<;99:88966665689:854312357;:875369:;;:9876D=555635666644=>5444544445664440//.012/222223344444554333223355444556666776344567855521014565::8555666655554343355255332176444455566779444455666678887���C8qu:760.+.*+.)�����������������-./.0/0;=@A/.//2130012411422/-1001-/.//..-,+.0./000002>;<=66@878@;6545788889:CB<<;::98877455566554743011-"�B6!��4<-������������������������������������;4,�����++('-.1=?>3:743-40025322133498656744421//04(:?5/22122021-./,..,+0654333333211102334343432223378767977;85567798989;;;<<::998887665557895433320*#�03��9<:.&����������������������6[�����������95,�����-.)+4549751-+*124-/1534430--12120143320-+*-$.0//.--./220///010,+0/444343331114344555455442379235868=<=80*-//.0005//0;:<=4568;=9;hl:97202/./-/./)����������������(),-016751),24./,9:9;<52+"#)120,./12//00,,*+,,0112223239:9;@<;7;=967676:;;;:78?<<;::::9887656675333331222+����V/4785/��������������������������������;A78'�������" 1812@51../,.54..168531006B4-2<3110.++,/0.--///01110.-.10-*((*/443333211011324444454541666//479=<<;12),-./561222//05:9:9<;=>;5268:833686467648622225775468:97459<=<;;;:7653322344545544453575556687445301534545555554554332222222235222566543344556754/++/347988999:7567765554444444402354331018P574577753112345676678998>?>�u>8:9842--.-.-.-!����������������*++/41/00/+.6*+.9;97214....2/-+.1210//1//-./13133111221A:<;9<7:<<59987899;89:==<;:;;::9976455453431263240 ���,1475%������<������������������������1:9:9;=88<;:9=;<>=:;:::==<;::8875665444425;6234���(2463 ������������������������������9==7..2������������"+*).8../-89*50-123010-,+13132.../0/00124/,**(*,+*++153333320-0/1114/145215443561008898862,.---.0156.442.-0059<:=<:875433344555444479878747653666841253543244467543222222222222234456422243334764465355:87::99765677554434555544112255421.2HI6:045756233D46677777899886c:::972752-....-)(����������������$$+-3764./010,-,2997985420/40//,..,./.--.,,/4420.12211034BJE:9::9;=:=<;:;9<>8:;:;?==;;:988788545442>=<:975443445555455489789646442587744564443455664332222223332323345456422345544556666687:999985555577644344455554422255520.12541/03763223<5756777789978:�?978:2761--/-.*(*�����������������'*-865330/100/./367753101/31..,+-.0..---,.13111/143/0/137FGB9:<:7=877<9788;;79=8@D==<::989;;6553435B91343/�/004/2-����������������������������49A;:;440������������������������� �+#/$!�;,(*-)(&,/*,+++*,)(&&&+#'#%%()&''*25-((443332342/.2450.36366054443129873861-01.-..0340*'02101322213-51016767677899;::6122448:87657869:::;;=>>=<9875443556455444699779345421565534565464456544333233333333333455445223334566576666898989:852445678644444445454553345552..345100.0/0324B6866778889:88:jW774567541,,.,++.�����������������1/54345.*-./-,02354621/..22.,++----.,..-/051110123-./115FKL=:;=6=;65::6779<77:69:>>><;;99;:75555549423441!�-.1222/��������������������+�������<=?8;:1-.��������������������������������2/(()$%%*+)+3/##% #**�����!%( �������43334444443432.122356456622207632320/1/0/68.00-./21333221120.,-/58:<;875556:9<8431538:565588689:;;==;:996554445665444455777767454544343344665553664344332333433333334455443333444543/.468798888888874677786544444444445543334443133333331235336347756889::;885=]l:797441((+**-.)'�����������������<534772433.043313//021//-/0--.0///0--,...2//-//1.1///001:8A799<<7;<49=4676>:69878>??<<;:7896365654243342,//01--+���������������������������<568<6458,�������������������������������������. �6$���/�������������������������33344432444462///23245555101074221/.011/058.00.-/30443/02121-/3533468::865589;:21377795546776889:;;:9877544455556954445477876455344422333/%255467643332233333433333344443333334445432034775779778<::99987765444444554455443445543323234424344352245678899999765Df98987741+*&'++,&����������������'/0155962400672321./1332020.//,-...0.-./-0200/0330/.../..?@=57<9;6244397476997656:<>>><<;66872554433433433)-/0/..*�������������������������1//567:6553,�����������������������������������������������������������������������4444454645555560464148551001082210.-/00107611//12444321002322354111143588666899224678655566778879:99775544455544584454346875543354330011.(2444755433233344334443333444433333344456776559:8878779<:9888887655445554575544433454552333354343544723457789999988456N<89:8751-++)*),/���������������0)01228520-33131202013:2111/00.-0./0,,/0/.10./0342--/.//.9>;888679546248795676:658;;>>=<;64773454543444432/-1110.&������������������������<94266892245..�����������������������������������������������������������������������444455444455657646636856301316210/...10115842/.4344322212333223312113339757578;34358766567778878999855554455544555454443585542333221000.*)+34566543223344443344444444443333444445566669:::7799:99:7666898765445466446666444344553443555444354473356678899:889456:8887765123*./,*.,����������������./0/44564.16@:5433./37100/11110/1/.++..//01001332.-//0/01:?>;7757=::73::78118:636;:===<;856832545545444421*201/)������������������������13;74567::78:66!����������������������������������������������������������������������444455523507433666436779554433231/.0434101663//44443222123232344122234457345479433467487667889999887555444444445555544258654323210010.,+*,045654443333444444454444444443333444445799789::977:;:9:945556877765555554445674344344333336547354544502466668888889467:;987666331,,-.+++���������������2,--/3423338=92/09201/.012/1141////-,-.///3124531///0////03G?@6578=<@;78950/78454:;=>=<;956733545554444432-11(% �����������������������*.;464568;<8:9;93����������������������������������������������������������������������5555565534312411674335<>845352241//0444021461072124311024223444334222233672546853312758746777999777654444445445444435547854422100000.-,+,0245543445344454454555555443443344343445679889898688;;:9;89235667766545554345465446457645336656555433431455655654689267GeA56344510+++,,*,)����������������*.-,/15;A4655.,./753253230.1620//...-.00/56541000//0.-././;B314568A@=:87352595659:<>==<965532555554444431,.2/$&M����0��������������1-7845588@:623:201/0/1646/015620621342/0//221334444432112455343568621267764767889876544443444445555532117861110/-/0/..-,++023554335564444445355555555444433444443443/46788776778:99983544467767556554334456546445332245766555333420345434435796356MuoZ812554/--+-/..,�����������������--0-.215/1282/11053/110//.023/122/-..00/0351//00100...//.35413344?BA>;=;558::57579==<<964622595654444441.+//(/F����*�0������������!?56445589:<;86=:312���������������������������������������������������������������������66666676555443334663496>>5234422000/975./46431431110/,.13112333333330025434015645241686456998887443333334445544555244540-,,,+*.0.--,,./02454334444555423465544765777544444414445434446555566678:7710523767876556655433433433344432356655334323324535521134695223>687877851../2130,+)!���������������.'++--./24==9222653.,,--/0//0000...-./0///1001/00.,-0/./)+/1+*.35>;159987<@J>=@:@=>=<864922345554554442//00/0%#��R���2����+��4/�.../58698:<<@43/24152B��������������������������������������������������������������������677777765655552334766868823444222//0672/17554211201///023533322212322014422/024454314954678998764333333344444332221.00/--+++)-0/-,,,/002345344445466432456646466776554444445545554458>95544447797760622564777656775443533344466434444455334323434445401453578523;366798810/-../1//**$!�����������������&12/,./29686/09553+,,,/0/..------,-011/00.120//-.-...-+-/2..-.47=:986557JE78777E?==754A314555555444420020.-+%�T��$)+ $#!(/+*/222126985423345225003153//34211214701265541235552420033243222/--3661325777986765533332333233132200/0224.-,',+,..---./01234444445555322112/47656435565444444455554444344544435779:8554020586798756665576684656455645445444454363322456632344477035222453352.-/-+*(+--)&!)������������������%+,,.062143,)32342,-/-,,//--+,-.-02//./012/01/--+-+**.1223313797696577I823237;<=<75:939B45445554444/021.-,,*/--001112000.0/03442357878;?>90.497543209Y�����������������������������������������������������������������8888877778777655775679655223443310122640/35202145466763650135430212244233210/--.361222877875655433223333332333200/1333/--(*,,.--..//012443444554441/11220056621345564444444566444433345444367688620,+,05777:9766884786464646455646434434653344212467752444475135*2224334640./-,**)-,$����������������������&,,/0/-,,'(3367.,/--././--+,-.-//./.00/1./10./--+,,,/.24242555675456F634138;<=<75339=@45545554444002//,+--.-100233232233333445679899;@A<335>57=5434K`H���������������������������������������������������������������788888777787665664446976632233330.0/33353663342322655662.234321112232443321101-..36644466777656422123332233112211233210.-,*,-.//00011244444455522212223201343588665544440443654333443455447898854)02.,,3634578767;6765455545444545336555664442323666864565553)+-%54234303677553///./(����������������������(+,*/4-8:/64551(+0/0-+,+**+)-0.+-.,/0.2233/./../0.,./.132334545;456;5440259<=<9754112124453443331/221/--.0102322234556877566788888886331>>:5<;8685.,05879941�����������������������������������������������������888777777776665566569=8652222222221323076761032444554632324734320/1110133333431..13677654565665311233221212444234554311/.//0000010002235333333330/01230013353266744555.+2443566333665556886697644223456665888775677755445545434333446544655335#747776656682248-3(45444551/288771/0/0)(�����������������������%(,//,3746754.*,0//..0-,,-++..-),.+-.-1240//0///.,,..012/03357657743330259<=<9654B92011244233332/0110.,--./133333556679999988899758<=;51/.<<=6134::<9;;A86<743"����������������������������������������������������8888777777777655665569:76113323212134727745/.//2565754/04446642300241102333222122311377655555542232212212445422565543210//0111111111123333323330-0.-.30022261477754453,+244356633366556577:9786651335566439:866568765544455544454344546678656458555322444653336A$.43455630.8;7750../,"�����������������������*,1./00478751.-/-./0000-/,,.,,*,++.01/1420//.//-,+,./01331356545533330248<=;85435=4222132122232122110,,,-.1233345666798::999::9755:>>70//458114<:9A9:B74380111����������������������������������������������������777776777788775656778;8630223332221289638511-.12445764..03366673325311022232112322134576545454333322222245444335665434421//01/11111112223332222.32-/,10131366879644464.-355566533454455575:9875364346667557;73667766544444454455533454456:775591244565666644426 +221324311564674/,-,'������������������������#+.-,-168867820-./030///-,*+++**++-.0/131010-/1-,+,-002422248>76833320238==<854321/23324313333322321/,,,+/13344567677::9::::::8556786112/3:<87::?@>B6:34665225����������������������������������������������������6677666777877766649;:;875122232133138:44:64./234558755/.021666413652/00222232224435655685443354333233224556654598644345320010/10111022222221211.22-./0012257766744456654557665433444445556:97643554566766217547566565554444544444333444458852339::58;676756653,232/322311363423/-+-*������������������������**..0895888734,.--2100.,-(+,))*,-+-0,0/1011--.-,++./01300255456833310247<<;865203/22234544444333321/+,,,01234556678222334447:<:654433343324444 /232220//2..14011.*+**)*+����������������������4*(/,/1;987988.0111/.0*-+*))+++.-.0.-,//0/--/02/.001/1300643643432103569<;;8863200466535544444333210,,--2334566779====<<<:887545311100..//2357:54584733485973AA?.������������������������������������������������66778887777776788776554332222234221/432302767757673566/022235776123454454444445553345544566644322345543366788779765453211010//002410011102/011101300.//.067678776567421366666445554554566986555775466227877666766555577666446C?S?22344449;;;7645333434"(*/%./4534/...21123221,.,)(**(''(���������������������**+089:9:8777.../11..*),,,*+)*---///,-,-00.//0///00/1200643644321114459;;;9853311366535544444433211,-,-233456677:?>>>>=<=<;9779532220-..01357:6241353221241/>A@8������������������������������������������������7777777677766666677665432322243034439864467777623675352223321332123354555654444443344444555845313445444566887778655552111121011123111111./12223100/010--07888866466362/06766545665554445798655566626724866676566665556666644RG?P42234435;;::6445333324,.''035633364126423100-,*(**(((++%��������������������++*/717799:96///0/.**'&+-++*(),/.///.++,,//./100///2540342464422111544::::9853312455445544455443221---.223567778;>???>>>???=;9:65220/.-../2347502165134311448?@6693���������������������������������������������7777777766666666566655432334431/3432387996654552203424225211012333456555555444444313443434564553444345666777699655543110131000//22111111.03533211/-*.0./5799776726746412566654555455544579765456644753587667656555555566665AF54<62233335;9:64444333324* +(.//1366544/1121-0/.../+*%*&(**+,*'&��������������������.,01078;==95100//.-,('*-++*('*./0///,+**-00/0/1201452/2535573310125338;:88754322344445544555443321..-.123568878:>@@@>?@BBBA><:864210/../1/14620/722033446;63683983���������������������������������������������455566666655656666666644444331//32125521001223122502342211233324665466555555334554423433233574645555777666678984445546322244312223321000102120241//0//0257777777563/4654567554565554446669755562.267999986666667775555666544454?;8310/.--,-+--*)((()/0.1/./,+**).0//.035540262577843//23436:;967555333444355544555443321/.-.1347889989?>@@@BACBCCDA@;82110/./2302115990.../02689:7:83359���������������������������������������������444576666775566666656643545422212212121//00000020302433322454334776666675555333555333433333365654256667878799964444434543434534334421/12110311251111030155576776673035455675454544545666697666731667999976666667875555666544544843332248996443334324 32//14454345565235.11/20,24.-3.++,-,,.+,' ����������������)+,-0037;>>;931/.--,+.--,,*),,+/0///..+,,+,*,2./06652/155667345//22447<:766445433443335444554443321/.-.1347899::;@=?@ABBBCDCBBA;83210//0000000243/.--,/279996<<207%���������������������������������������������323456656676666665566665543223233211001/010././12212431233454446767776675454434565443443324355553424677788:998644444444443445443222332234201123301//..12443767766643466566545434455555667;76567770/4899666777667766666666545554433333338765443344234044345557534445538510/,,++12/-20/,--,//---+��������������),.,./66:<;9520/..-,+.,-.,,,**+///./..-,,,---10./555102465785670122357:9755335433333346444554433322/.-/13479::;;<@@@@ABBABCBA@?<93100////000011110..,//1699966=530����������������������������������������������3124456566665666554464555232222144322220010/../232334433345555677777666554454445555444333234345611477877:::9776343455454543457554443332355331421//1-/203544667666665556566544434555555569<8667874/0555457778777876566776654465444332234977564443345.3544565775344433./:31//-,02751430/+,.2/0..)���������������)/-22244787460/...-,+.,-/./0,++.-,..--0.,,-../010562/1555675443222346798642244434333255555665433322/-./23479::;<=@A@@BBBAAAB???=:52010///0/011111//-(/0/4799666360����������������������������������������������53441256656666676554426642332233443232211110///1222334444656667777776655445555555555543334322447534678899::7665343453354332245435543311267631321000-./1255667766765545546543344566555457:<976886302445577777788876566776655565333222246978665432344 2555566654555531,.2455564.28626521,+,.,0.-( ���������������).,02647761152-.-./3.1-.0/11-,+/.+-/..0.--/0/...2640/6457793432122356887532444444443355555564433322/../2347:;;<=>CB@ABCB@@@A>??=:52010000000011110/.+.0146797533-����������������������������������������������333322445556657777665465522222124433222223201102213344555666776789887665434567666655445533211236867788988986554453334423211254435443333321512121101/01664466665676543455544444554456556787767665355677877776676655556666665555323332448788976444444112332354434564423-0//8:115/56530./)+.2261**(����������������/-+/0-/17;40,,+,+2520124112.,,+.,*0211000///1//12/.246664332233343788752133444442334555555543333220//023479;:>?ACAAABAB>=>??==><960001000/000110000/./257987343������������������������������������������������33444455555655666776555662122113443423222232111321334456666676778876655444445666665555554322124686577898:87553345544332231026533434424..0/23421.120/0467266566667544446665534553334444456677763335425677666777555455556765555433322245877786655444522133224543246552,,-:378-+,,<8683/++*+/130,*' ����������������50-0-./1=;42/-.,+24302622220.-,-,*-220100//001121004555534313333447985211443344422455555555543322210/023479:9>@AC@AABB@>=<>>>==<:71112//100100001000/0176687352������������������������������������������������44434444445445666676555673222223332333222322222333043356766777767766654343445666665554554223224277677999976543445544353122026432123343/0.-/221/**+112541165575567542346667754433333333567778851154686776666866655455556655555323322235886677766435130/122235535566530.5862/62-1<;55/-,-,+/013,)'*����������������*/42+,/5C>46/.+,-22112633332/-,-,,.34/012//123331015758233233234459975212333334323555555555543332210013346878>@AB@AAA@@=<::====;:81/3300001110000000./17666633 ������������������������������������������������34334334445455666665545553322223322323323322222433234444654567777876544555446666655544554333314478778876865544445533362022/0552//0234442**,0-,.),.31361//6569766755456766654433444344455666883324686777679:7776445555665555543343333467766687653550.,-110/3545666433/04:764=775<945+**4/-1431-)''����������������#,-*,/26>=97--,,,132045333312/,..-.33.01200020240034668452123345569864213333333435545555555443322210013346766>@AA@ABA@?;:;;<<<<=<85123-/0012211000001/167765'�������������������������������������������������333444344345564556644544443222353333333233332223334455544534566766655557544556666555455443332124778787687544433345334511101238410/032011,++--.+)*,2156112878988775537776654443354344555555777775566768887:;9876455566665554443333444355467677545310*+-12114444631323/567<=<66729754**++-.2343.))''���������������*+**3.28L<7931/-+,/01764445020-+,-,-0//120//1/240044433542212345579762133332233355545445544443322110123346644=@AA@B@??>;999<=<<==<9732-.2311130021000/25566$���������������������������������������������������4455314554544444455445554423334433332223344433444445554445543656667766655444445555555554544322122467678754411223333554543531/25610./,.,0*+*--..*,,4796.0456657899877765655444444444455455677788789997778:9::9654456776545433334333334644799652212-*.,(.010343322.0//,.1.29=1/6<83///13,-2512/..-*&$��������������(,)+/4=JXWSC630,)/59;:4344..4.+**++-200./..00430246534332233456795421233224534654554333343333222111223346534>@AA@AA?=<:877:;;;=<==977533444221//111111466*����������������������������������������������������3465345435643345644434455522334433332234445543545445554446444554566543553333444555555555544322213455777754412204445564432342/01701///-+.*().//0*),4673/34476567885567566454344434445555556777689::989:;;999888844666665444432333433346449875512---***)-./-3433221......-3=>2/3:74/2/02--3;/,,11-*�����������������++-46@ACLNL>41.+089997581..1,,***+,/5//1//13320446523344333466873211221246766544543333333333222111223346645=@AA?@@?>=:8889::<<;==<;;55323552104333322463����������������������������������������������������45453454345433456553343444223334333343356654445554455553354343344545445433344445555555554543211244325655543232134366443442322/232200///0&(,)+/0,*1440-05657555677445677544433333334565566666688:;899<<<:998<;7666556665444333334433455448765511..-*(,+,--/2212011-..//..06984577540.,,,03:.*.32/,!����������������),-0255?P\A80.49667553..,--,*-,+-,-/010145795557433212333575331001014676655454332222233233221122223346567=?A??@><;:8888775888865425565766443322323010%�����������������������������������������������������665455545435548874;55555510232444445545766555555554444553334664444344443333344444555555545542222334554445332023344665524443210254565400.''&+-.-)0642/-45665446577556665544433333344444555566899999:;;;<<:9:;;:85555544444444344443347;7575535....,+,,0,-/0232210251/+,.---0429189;:.--.0084/0/0/,'�����������������/4,*,356=FSTK404:776340.---+,-0//../013246@B>==<;;;89868863575220002223444431233/..,"�'����������������������������������������������������7755467656577479:9<4667653101/145555567765555555544433453324532343445554444444444544455545654334444443333321233545444455543341554445454.))////-,/4542/5557544566565554445544444334443321245556667778999836799:7555454433333333344457876675430..*)*,,-/,-..1202122330-,.,.083//.8:;/...1234:7322/,)������������������(/+,-135<>AcYA74363/110,/-,++/-,,-..-0ARKFKK7>957569:;9:7;78664213124566656766665665544443332225443203365555555544433334555544555445444443211222333244453455535531134222344420/.1/22).0220004575446666665444455444433333232434555445546657764556987566554433444445444444565555440-62-*,-.//.-../112002131/.../04>70232-.,-/2967872831+'�������������������(-007::=yZ@2/0/-.-.,..++,///.11033@fjB8;5664:20135797321333456555455532111111112222222222333334579;=>@@?;:52235<;86532222200//00/333343123'�����&"����������������������������������������������������5675789A?;5887768899=8:5772232235656667656766443445543331464441587765666655444433355554445555444454412112223433545545544444001242101344430/1111)),0/1223654456666655555543314333343444445555445456556632456975555554333554334333334566665442/02-,-,/01.-..01121223310////026103/.-.-//3<9:86573/-&����������������������0+-489;hfM;01/--,+-,,**-////11344:LhF@9556432268888555765566555655531111111111222222223333444678:<=>@@>;86366786761020011//.../0434450033%����������������������������������������������������������6665797:<:7797446::???>:88487;5674.//0000.--,-.0644650144������������������������������������������������������������56557875:>=<;988677:955345443344545654455555544455665442025866:;7777556445533323443444455455577::21122221223324656543334454112441/3/0003662033/..*//03/154456665655555544333334446455556664656555444544445776554444332454444444444467655433130/.......-../01222223322221/0.///000143405<88688773/+#�������������������������,-.19EG>SQMA5203-.++*+-0/-0477565bgDQ6679;;;;:9878887655655553111111111111112222233444456679::==>??=;88:;:<;95542100010--.-/1577674433*������������������������������������������������������������77558579>?<::888566966555666577687765445544555456666544310375464333445644554333233234455445569;=51101111134434565544444454212554215000026640231/-)00140114556555555555543443332455555556665455444553433444589677663333554444344444466545433170/--......-./12223333733220.0.022167;:76/57636895331..��������������������������+423=<:=QNA0111-/,-,-0.-,1579888==6579:<<<<::876776655555643211111111111111212233444456778::;=>>??<;;:;;7796554211//1-,,--0177777542/�������������������������������������������������������������4533358;?=888888677<4656678999888888776677764666676655232317933542132454434443323223445456667:=:31100111235545655543445565324853756200/03553343/.-,./233344565455545544434434556565555566654554445544334455899987633345443444555444665564330GL0,-/./..///012313337:42220/-.001138=8531A7863384471/$��������������������������/1189;=??==>=>>>656566511000.-,,--1368777662.�������������������������������������������������������������4545567<=<;88776778;5766789::9888888887777766777776665332553733441222454334443333333454556789=8621101111334555555533345575445644753000/.1331232/0//,0-34565555555455544454244455585556664555544444445444456789966744444443455555555655673232@\4-..-//.../012122237991110/-/001114945002/552133232+���������������������������,05857=8EA2330./132../,/57777669978:;<<;;;877656666655663211112111111121112233334556789::<<<>???=?BBA>;533765311/0/-,++,-56888886781�������������������������������������������������������������2455879>>>:899887656556678888888888888777778777777665522245565;94133343432333333333456555789:87621101111334545555433345544555554520./..-0001112//2//1/145666555545554444432455654765665556443444434455555676688777444544345445433765557643307S:0././//../011111237781001//13134653400/3214226763..+���������������������������27756;;>H5230/0040.2/,14778866::87:;;<:::8776666665566522111222111111111322223456778::;;<==?@AC@?BDB?;43565111100.-++,-/86788886788+�������������������������������������������������������������556777779<:;=8;655777777788666655555665556777778786654332248788958<=;74753333323444555776765544432111122333445644334564433333434410/..-,,,-10110.-00233465545655554444553334454566665555555544444444556666567446656544444565532358873444411,/FD-/000/////0/0111/.0..83120//32138420336;7545885745,,&���������������������������377789?AEEFGGFDDA>:5663222110/---,-0277899:;::75.�������������������������������������������������������������566888:9::;9;7;7667666778887767656766666667677888876542333479>9;9;<:;9556543343444455677644443422111122234445543334554211222222230//0//--././///0/10246664545555554555544444444323545666655544443344555644453455667544445566632358:74444421+/=@../00/0///0/00(.00.//0700/./34348432225::767754331+)���������������������������12357D9CN843-,02-042008;877563567;;:;8998777666666653322223333322222233333446789:;<===>@BCEEEFFED@?;88881121010.-,,-0136779;:;<;865������������������������������������������������������������7889:79<;:9767:7558777888678887766667777666667788887653332485;9:9;=:96666763333344445676543332221111322244444433225421011111222210//00001210/0///021335654456677644554453320034233313566555445444344555444564556665544455566533448:85443421,/5:./12000///0/01,.110///60/001124355412565464598483-)(&���������������������������8756?;>Q<32.-./-/220/9::96563898;:::898877776566664432223444544444556654455789:<=?@?@@BDDDDDEBCB@<:88850011110.,,,/132567;;9;;9889$������������������������������������������������������������688997:=9:9767;977978889987776676666666676777776788876543248;<9:878;9997688544433333456543333222222132234444333224310001111121110///0111220100/1003334555446666654665555431204445564233555444544333334335665455777675445566654444788444432/,045/013////000 10522..09001002210134669;<789978:72/+*'��"��������������������������<;6379EVC34211.0//2/.67986654996:;:999977877666665333233444554567899987656699;?ACCCBABDCBBDDC@B?;::97430001322.-+-0244577<97841169������������������������������������������������������������8778;87<:987789777989899988887776666667667777776778888775238:<88865;;:96;;;444334433466543333433221132235443322233000/011111211100//0000011221001/3434654456666553455555775247535676457765555543322223336655456575665455666554445676644332..1262223101001. &(120//.-/9110/0110112367<=<==96446421/*,����������������������� ���631255:RQ95643110/40056686654895:;;999977777666665433344545566679:<<:988899<=>CCDDDDCBCBAACDBC@=99985331122454.++.233237764/%�����������������������������������������������������������������799:;:9:99:9995::7789::9877888779777778678777777886655888885::9:;7678778;<;74333253444444222233333323344332122200000011110111111////00//0123234./.256754566557544555667888756999878887765555543343332234656555544466566666679445566954442101005333333221220100121671-.4100110/32.0677:4777::841.150-4+/"������������������������+30283;JJ9333101213/,563013458:68899867767777565444455566677889;<<;99::;<;==??===<9::9787789;=;8767775545433543/.11356676"���������������������������������������������������������������������8::9999;<<:8997::8789:987776676797676787777767778766555689868:9899587689;;=:54233323444321222232344333332211210000111110011111110/..../01223234113467644466556644555678866689:86875624654575533443323344566765444466776656778665568:54442212115334233422GC2101331870..62/021./23036499157842330242/-/9:?������������������������,16:58<=0/341221340.342067659;879898875466777653455566667888:;=<<:9::::9::;;7666655566666677:975576786333445532124687561����������������������������������������������������������������������8:9;<99;<=:889:9:779:;8876677796877868:88787666777665557:8739:999738977;;:<:443332223421122222223444332211110000000111211211222211////0112232333445665465666566675557776677877899::85254666544433333444445554556656776666677767557:944442222223@44333537?2121430201/048611000/26566573458531133331.*,=Ha6-�����������������������59:869=7.153400213442238677;<:::998776446777534556677889999:<=;:9::9976777755554545565556678635598896324466557645798982����������������������������������������������������������������������9::<:98;:;:888;:9979;:777777677578787667789777787666557799966=:89989989:86::5432332344111211222233333222111000000001111112212212100../022223222233554457855676666346767689988::;977657766654444333334443434455566677667766777787689744443334334D5533444G52121.24500/345:32100/2448664476773001..-1-*,7Gc9�(����������������������789=>BJJ920452/125620327779<;;;9:9988766566644557778999;:::;;::99988756776544644455465546779575587875344378466847799:;0����������������������������������������������������������������������::9<;98989:88;:99769:87887656555766677766888898776766656697:9<:8;;:9898984555433333332111111123333322211000000000000011012222222211//011111333234433455665566667667889:::;:99988777876566544334432344444334455555675567765767997787744445434344A444456D8212100333200//36F2/221366765435664301.,/31-..@BHA75621310643467:878306779999986545678:;;:<==<;::88776897873454644555435455444545;775766888:99=7658999;9:;=9�����������������������������������������������������������������������;:<>;:98;;::;;:9:98999898:99897777677777788;;<;9879998888998988899:88775345543233322111110011222322111111100000001121112334434343222210000022332326689544455666656665666897766644432344443322332334455555445555765555665567787767864555577877665566?54=65544464321121335:5122316423221121.04445771/36.0.3/,/*!�����������������������<:59;:<::977666898875423754454444555354444865468;8:;;99>7679:99::;9=�����������������������������������������������������������������������8;>;::89:9:<<9999:9999999:88877866557887999;<=<;87999988888::;9::988966744444333432111110001122210/0111111000001112222122334343432110000//122234234555444444476555545565554433312223124431223422244455555554456675556466557787767755786467776655555344565345488742122244:=223226514454585324587891132-15:3/3/%�����������������������;;;689:@=;=;;8213023122668986467778877655679;=<=>>>==999:5466477665422564156443446556543466555788;::99=7:8<1'$'.���������������������������������������������������������������������������;>989999;9;:9998999998:988988676545788889988;=<;:999999898::8<<99878:766644333443311110010111211..//00111100000001122233222223332100000/00012333333565455543364443334555442433433323333231333322334555556555456655566467767886778855565467777666555456445444446676322323>E4334354435458665787878426310121@97//)������������������������667:7>><<=<=133/1220.0345667898998766568;<===>>>=<<8999777557855442774546645446555444575558778687:;<=::9:&�������������������������������������������������������������������������������?:79988:9:;;;;;;9;:99::::99878979<<979:8:;<;=?@==<<<::788777:9<;7549473402223554321010111000///////0120011111112223333433322210011111111...223333445475566666564222123222303311221422211/122234335555567655555666678866777888778:655555577799875685467653346776654343443I>:33212456556889866887421620213??D8.*+�������������������������+386>@UGHFA:53054;320256657888786546::;89;;::;;:888<:97644446976899679:867656769:989;<98578=@@?A@>������- P������������������������������������������������������������������������;8:;:877899;;;;;;;::::::9;;;;9:9<:88999:9::;=>?>@>:;:;99::8:;:9966677652122244543211100000//000/000011111111122222334554432221001122////001223333455786645556553221111324513221/0102110/023234344555566655455557777787777888878886665456778::8654;7477665246789765554554?A;21012367867;897789886400.34249<@6.**#�������������������������#;>==:22;<501233557998778458:;<;359::998758><::899:;;;;:899897887554312234243322110000/0000//00000000112122222233444332222100000/.//11112234345577655555554433222211323222101/000000022323445455566655445666877776678987877778766555688:;946;;<84666653687::98556644B641DK812465;8::97568866330204456<7I81,,&��������������������������/2<9?SRPQ=<:85531241146897666569;;<<:3588898546?:AA@> ��������������������������������������������������������������������������������>:9;;9998:;:::<<<;<===;:;<<;;::;:9:;;<<>??>==@@<;=689:<;:89<;9::9;;85322232554543421210000000//000000001111122223333323321001//...//23332223334555767755644343344333242355433134232001223334445545456656555678777787777898888787656655679:<:9649;<95567665999:;:9566565R0010?A=3666;;:<5665688312222946752781.,+�������������������������12>:=[VFM=;8456343304678765555:<;;;;94578975339=DCCBBBB@<97577:=<9999<9::9767===<;::::9:=<<;==<:88;===<9;:;;;;==<>==?><>@>?=968:=<;;;<;:;:;;;8421332253344332211000000/00000000001112233333223322000////..012334111..00//4666676456444433322343454554324434453114223445555556666666668987788:777689998898766665678:;;8978><935667665989689945954AI101./-48895:;::766665933343233561/1300+,"�������������������������(-C=>IMMD><7357431057876455559;;;:::7336843336<:DDBBBBBB=978:89;77:;;:<;;;887?>>?@@A?BBCBB?\UWTSIA91+���������������������������������������������������������������������������������?@===@>;9:;79;8:9;<;:9?@@???>;:988:;=;=<=>>?=?@>===<:5::;?>=;;899;85112444453454321111553////11111111112333444323333232223332110../0...,*+****+.047764565444544444435457544687388865563333445666667766776678887988777777787777666556688965573575;936668789879677787755KA011/./2289854885993124578543753231262/.*(��������������������#,'����;?>DR]O:AD8963442303232239;;::9764202112334<9?CBCCECBA;879:9997<=<<:<=;:768=>=ABCCGC=<>HB-&����*%����������������������������������������������������������������������������������=>==<>;;8768;:;<<=;:=>><9?@?>9:::<==<;<<>==>===?<;<:::59:?>>?:;:95421244545444543320123340000122111123443334433222222221122211110/../-+*++*+++-1577645667546666555445666577:77648:64452433457666677666766888877998888888777776656667788666564676681654887788968779::6<[944400/324876539973437578864277304469;1.*)������������������/"+,���496?Pem=;<984288411431334::98642222001223398?A?@ABADBE:3�������������������������������������������������������������������������������������������>>986<@;:89;=<>>=<=<<<>======<=>=>>>;;>?<2.*+&�������������������--+. �597@:;;<<<<====?ET]\B���������������������������������������������������������������������������������������������??<898;:;:;<===>:=;<>=:>?>=?>>>>>===@?>>>?>@?>=<<;<=;==;<9;;:44336666566555554312232223211232222344444544422222211211122110000000/../12233557654555565666766565789::9778997876566774443677666665665459::88779998888866766665435546655665:87896:<780588999989D:8;G@?<9>87012/644625445897587787535632:7<>M:3.++'��������������������#*1/���48B=DQ;99;;:736622320366964531111201214==@BBBCCCD=;:998678866889;<=>@>>><;<<=>EMMM:������������������������������������������������������������������������������������������������?=;9459==98:5216><4:>=:<=>;86=>=>???>>??@?@>?@??>><::;>=;=<<:754568856767666554332433332233331223333445554444332221111111222211111102422434566655556776677887788766688:;=;<86868665556556676666653676889:8998889987799866666655445334565422898987567759;88=>;99MM:89;<=:9851312354243559<:989;79===;9610/1133433330000000001=CCCDDDDDDADCC=;65447784478:=>=??>?=::7)�����������������������������������������������������������������������������������������������������9:99467;=::32:6564:5;=>633>>??@@@==??@AA?@?==??<;;:<<;97886667875677777665543344434331543332245544544333433322200001012222222222134444444456665578976688888776889:9:::<:669988632653456767665578889:9:98::889996799876566655554345654343799;:9787856988<;:8>J999:<<;;795543224423557=>?E:988976696569<<==;72.-(��-�%���������������� '��;7;;:@<<><>=?>><101023241120000000./13ADDDDCFDDDDEDCDCCC:445656679=?>??==;6'������������������������������������������������������������������������������������������������������977:7679::;4566;@@11238::?525?@@BA@??>>A@A@???>>>==;::9;::988888876687777765543344555533366432335554333333343333210011001112233333344445434455686647888778999877689<:;;9:8889988763035436766776568978999::9:9889:9667887656655555985776444689:;<;686736798;99;@::9:=;<:;9;7996555434558=>@RC>;:88778457;>?<:662-,+�������������������� #���89<;@:@@=>>>@B91.002223111//00000.14>DCCCDDDFFEECBADEFE@633354879=<:;5586/�������������������������������������������������������������������������������������������������������99:9953555443576:>24339::>435<>@B@@@?>?????>??>=><><=:999<;:9988876878877765432454565542265444444444333333333333321111000112233344555556445666677556889988898768:9:;<=;899888::7876545446757866788:879:;::::977::96777777776555455587876426789;;87997259:9988JC;9::<;;<;;<9:::775555759>?BIJ<<;9;87:55:CFD@>9653/+$*���)����������������/ ��79;=;3,0..233001100000/149BBBCDDDCEDEDDDBDDEDE?;5445567=?:92458!��������������������������������������������������������������������������������������������������������=:998679:;979;99<=51696:;A737;=;ABA>@>A@>>@???>=<<9:<<99::::8789::9888766654455556566652/868654445444554554433332322221110012333344445776678754666778999888878:<::::;<;:99;:;;989:55446564445879::9:998;;79999:::9767877865665666665678887767677:78:<5699;;9:L<;;:9:<>=<>=;9;:8889;;J;7=>ALC?;;<;<9=988:EGNH:?<32/-�%.�G1&+�������������������::==QHBA=<=<���������������������������������������������������������������������������������������������������������=@;8687:;<98;95:>?;77<98;A::<;<:@BB@>AB>?B@?@@@?>=;;=><;;<==;99:::9998986565555556567665276776665555456555544443332233334333333333444555778755767788899888988:;<::;<<:;98899989989:8322556668759::89::9=8569;;;::99868777567866675546688::978987::9::979:<99:V@;:;;:<=<=??=;::7;;:=EI9:?=AQO>;;:><<=7:99<>>?6/10./15100122/./0012267>@>>DCBCCBDEHIKKFDFGGGA?=;<=>=>?AF���������������������������������������������������������������������������������������������������������<=BA998967955456:?>>;8=@@A?@A=>@@@@A?@><:9:;=<<<>>>;;<;9:9;<;997777655657787666777776656654566555444333322234455554533223354557896776787899999789:::<<:;:::<;:899888<:9:;:548<>@;787547:99;;8757::;;<;:::8796666677666864557779877768:9::;<;789>:;><==>=@@>9:<9=<==DGA=@?>LL><:;?:==:88;DAAC@FIM8210���2:.�;������������)-=7;<>B?KFD@D9<<;=664+-./3-,,/00/.-022336689BFA-��������������������������������������������������������������������������������������������������������>??F98;:;78:65856;?>:69:;@;;>?@?;4425A>AABA@?<998;<=>=?>===<;;::::;9977888668788876666787866655554565554443344333344555655332236766678787667799;::9989:;<=>>>:::;<=;:<9:99=:;<==77;867::7677799899654779;<=;;<;:87876666766688656797768679:88:<:<;898;9;:PJ=?=;>=?=<@?=9;>:=>?@QRK;@AAMO=<;;=?>@NOJ�>73,���1:9&�� ������������ %55NM==NNE???:;888966457><66:88@?A?@BB>>AB?200637<<>?@?<:::<=>>>?><;<=;;<<<<:::>;89999:888666676776666665555566655555454444455466666663347676577786767899998999;;?=>=;:::<<=;:99;;;::;>?=>9<;889777788999::9647:9<99<===;866:765877667:8656788878979:8;;?;;:5459;;=<>?=<>??GVb9@BBBK?<;:=<>;=@B@>>?=EJG~ZC:29�5/.,8 �(�������������/;==?APPI@DME9;76570185/++,//00112?;8?=@@@AA@AACA>==?>:9/+���������������������������������������������������������������������������������������������������������>?>@BCA??>====>:98:?<:<=7@@>8ABB=@A<><:/3A=?@@A@?>?><>?=>?><<=>>;<<<=<::987767667877787877888676655555465565556777887:869766765466777889:;9:<<>;<<;=<=<<===>;;<;<:;>??>=>==;=87888<;;<;;<:8:66:=:749;<98986887788787::86577:<989;<<=;7779:53288;==?BAAGS@=BDBCJBB@@CBC@><<>>@@LJGM]lu=<0?B<�������������������"3GLShBDB?<:::=:64546/.-11/*))/116CBDCBA?CBCBA@B@@ADHDEDA@ABC?=<:8=?;0�������������������������������������������������������������������������������������������������������������<>>@?><=:995<><965<>>==85@B?=>??>>>?><=:>A;::;=>>><<;===<<;:9898788887788888877777656555566656655687878;:;:777755687799;:<:::<;=@>=?=>;<>><=?=?>=<=>>?@>>=??>><:8988<<=<:::::<8;<;743:;547766999988898;;96656=<999:=??85766965667;<;HOXOK@@?ACCBADBBB@@ABBA?GJ;@AB@ECC@?=GG?>ECFFC?OCAIJBB@9?4+(������������������ (4BOTN@BBQ@?89:97665/1/00/-+)*.245CBA@B=???CB@@A@?BFECCB@>=>===??>;8,��������������������������������������������������������������������������������������������������������������=@BDD@;===:8<::8669?>9><8AB?;;<<:;7657827>9CB@A@>>>>@>>??>>?@>>==<;:;<;<>=<;>=;<;;<<;:999::9887788898877877655677776666666678988;:;:8766457699:=>;=;<>@?@@@<>>===>>?>?>>A>>?@@@>=>@>>>>;9998:;<;<:::<=;98==6553344:435779:88::9=;96777>=99:=?@=868877879:8:<:@ACEIMGB?BAABGGOKGKPGJDGDBlI=A?21�����������������'!::>EBKLGFJE<6899877102/1//,+,-56:AB>?:>B=>@BAA@@?AA?>???==<>B=:1/����������������������������������������������������������������������������������������������������������������<@>CA><=>>;7<:5668:@?6>>:AAA;=<:=76657<8;=AD=???=@@A@@?@@=>@>>?>?<8;;=<>>=><=<:;<=<<<;:9:::988889::987878877676888987777888889:9:9:<96566769===<;;?>?>@@B>@;A=;>@==?AA@?>?=<;?==>><;;87998;:988<>=>;>?978;69::71148879:::;><9867;><97:=<7<976789958798<78?Q_eTIHAABDCBCCCCDCBAFDC@F<;AEEEDD?EAAEDEELOMNLJJGJIILKdIB>,*�����������������563<=?>;><;>AA@@A?><=<>@>=?=99/4�������������������������������������������������������������������������������������������������������������������:956:=?;966669987579<588>@>:;:;:><@=997>888>=<<:@DBA@AAAAABCC@@A><;:>=>ADAB@@>=>??>=;<;;:;;;::::::;::;;:::88998::::::::;::::;<;;:;;:678;;9:;=>=;<<===?==?<==<<=>=>E?=@?ACDC@AAA>99==;8;:;<:8;<9;<;79:=;:>;55898:;=;<@<<<<<<<=:<<=;877645<;779:;:0-;EO_ZVXCA@ADEBCBCABCCDEEEA54@FIHIBAGIKMIJLQSPFCDEINKLGE?B3477752����������������4=EIG><@CDE9:;656978<955251+,88@@?@@@@??@A?>>=;99;<:::+1.������������������������������������������������������������������������������������������������������������������������BCEECBA?@=9;;<=88<8=<564<:??DG@C?<>AAABA@??@@?@>===<;::;;;;<;;;;;;;:;;::;;;;;;;;;<<<;;;;<=<<<=<;87:<;;:=?@><>>>>=?=?A@?>=:<@?A@A@ACCEDC@@@?@<<<=<<::;:;==<<=;;899=<>===<>@A??=;>=83488<>>?A=;<=>==>><=<<<<:8875<<<8;;;8.=<=ES_[SBAA@EDCCCCDCEDHFEEC<6CHKHGDFFHLMKMNNUQEFGGEFDIGLUD3,15461 "&���������������@C=AYD=DIKG<8966999:9:57651,*9?@??@?>?@AA@<89603*���������������������������������������������������������������������������������������������������������������������������BB>ACCIA?<698<:65779<4;;;<878::=<><79=;9=;:<=<<@BCDBABDBACCDDDDDA=@??????@BA@@>>??@?=>==<;<<;<<<;;<<;;;;;;:<<=<<;;<;;<<;;<<<<=>?>>=;8:<=<<=@??@A@@>@@??AB>>=;>?@AABBCBAEIHDBBABA?;;=?>><;;=>><>=;:88;<;;:67:==@??<89;:99:9>>=A@><<=>?=?><<<==;:9778;;>;;;=:C=>?DOcg^HABCDCBDDDEHGFGHFEC?ADKNJJHHJFJKLTTSQPMKKJHHEDE@JB4,1052-0000��������������<;DEhXLHIHIA=;::<:::993442/-,2;>??>?@AACA?@BA@B989,!�����������������������������������������������������������������������������������������������������������������������������ADADDCDFE@<::@=;=>@;<>?CABBBBABAABCEDDEECBDC?@AA@BDBA?>??=>====<<<===;;;<<<;;:99<>@?>=<<=<==;<<;<=?A@?><;<<==>@@?@@@BBA?AA>?@@A@>>?AB?A@?>@FKIDBC@ABB?=>??@<>???>><=>=99;=<=>879<:>><==88:7<<;<>>>@A@>>??>?A><<:>?<<=>=<=<>;;;>C0>?=88?;B>;=BCBFGM==;@>:;;:;9?::;@?9???>>?>>==>?>??>===<99<@AA@@??>>>>=>??==<<<<<>>@BCA@B@ABCBDB>A@@??=>?@A?>@BAAEEBEIFDBCBBBACB@?@?AAABCE@<=:<9:9<;:<=@<><=<=;8:879;<:A?@ACBA>>BAA@?@?;@A@@@@AA===@@AB9ADOUUVY\acMBBQDCEFFGGGHGFEFIGEGHJKLPNMFJNKIIKSORRECCOKKIWTG@5.).,1.184,%�����������A5EEFEFFCEHDC=<:999;A?::;::78667<<;9/007:==:8/3"���������������������������������������������������������������������������������������������������������������������������������>?=>CEEFDEG<8<@;;89:8898CB@DCD@ACC@A@BBABEEBAA@@@@@ADD@@>????>??>??=>?@@?>>>>==<6<>FTKG88;?BAAA???>?>>=>><<><<<=@AABCBAAAABEDB?@C@CB?>?ACA@BDBABDEFEEEDGECCCDGBBA?BCBCEEG??@;=:8:;;<>===<<=>?>:989:;9;?@@ACCC?>B@@CAB?@AAABACCDA@AACCCH6EMQSVEOYhcHCTHBFHHHFFFFGFKRG@CGHIINOKLQLMLMOTRPMFMO\V[\^[_K71-2.-2002-(#����������00;>JAFDBNPLH><9;:9:=>;::9877567=>==3001.42868.����������������������������������������������������������������������������������������������������������������������������������=>?>@@A?>@EA;?B=A<:=:;:89F@<;B=>=<<>:66>=;9=IZZYH98:<<=;<==ABBABBBCBABEGC@BACBCB>@CBGFEFAAFFGDEEDFHEFDDDEEDBADDFFGEDACCA;;8;=>?;;<>=?>>=;<<;;<=9>>@BABDC@ABBDCBECAAABBBBBCCDDECCDJ9HMTWWCBHYaM@LNEEGGGHHGFFGEKEAEIJEHJMMTOMSTRTX]V[]_PVU^[XUUK=2/30-1.+4,)$$�����������A@;A:88810///09;.$����������������������������������������������������������������������������������������������������������������������������������EFGFD@A=>CBD:9B;=>=>;=9;;@>@?A<;BA<==999@@>B@BH<@?ADCECCDDDDCABECCCCCBAB@AEBCDCBABBAABA@@@AAA???>=<=>@C?9:;CBBCBA@@??@??>==>:=<=BCBAABBCDEDBDCBACEABB?EIDEHHGDIEDDCDEDKJLPHDBCDCABEBBFCFGADFC==>==??>=>?>?<98:==<:>@=??AA@BECBDBAECCCCB?ABCCCBCDEFDCADGE5PQRVABAX^F@BNPGJKJHGGFEEEHDABHIILOSOOQSTTPRT]Y]YNHHOHCKPVa?40*)0..,1-,()��������)��<24?=AEV]^PIB?>>>;888468FDJG8799;48620/0/-'/(�����������������������������������������������������������������������������������������������������������������������������������CACEDDCEDCED<:@==><>>=AAABB?FHG??C>>;75CIDECEEDEEEAEDFDCBDEEFDDEECCDBADCCBCDECBBEDBDECBAAA@?@@@@@BA?==@PVS_GOGABCCDCCBABBA?>?A?=>=>AABCBDDCCFHHFGGCDCCBABDDEFEEDCEBBDFEFCDLNNOIECAAABFDDFFEIHNGCA?@@A@B>A@@@>=?><;<9;A?@?@AB@EDDDFFHFIGEBAC?BCAACCBDGD@DGHFNNEHQWCDC`XQOBBFIGIGFGFGFCDJFDDGHFMMPSaj`OQX`WVYYVUQOGBHPXWnA.-0/(*-00./.+,+'��������88>>GemaVIFC@?=A@?<<;888732111..-4%��������������������������������������������������������������������������������������������������������������������������������������������DDEFDB=A?@BBD;:@??>@=>F??BCGJHI@?DEB??<@KFCFBFEGHE@DDDDCDDCEDFEEDCBBABCCBDDDDCDDDDDEDCBAB@???@@AAB@@==?C���\\TEBBCDDCCBEDAA@AB>>??ABCDEDFGEEGHKGIGFDBDGCBCCBCECCGABCFGFEEEMLLLLFC@ACIHCEGEFGCFOB@?@@BBA=BCA>AA?<9:;;@B?@?AAA@EDBEJIHHIFEBBB?CDBBBDDGHGBFHGFJW@NOWDDDNWTQCABJFFGFJIIKGEIECDEIHLOTS_ehSQO[]WZSRUWOKRPNNSX6.0/0$$)//./23+.,�����&���54ABMUOJJGC@==:96876530/-+)++(���������������������������������������������������������������������������������������������������������������������������������������������CCDCDA@?=B;=;:>E@@EFHJHJADDDGHB>CEEFCGEFXSJHHHGFECDDGEGCDCA?>BDDCCEEDFEFEEDECBCCBABA@??@ABABA>@@J����kOICCCCDDCDECBCACBBACDEEDEFFFFGFFGKIIGEDDHDGGDEEDBABFDDGEGEDDDKIHGJDCCDGHKEGEDEEEGBA@=@ABCC?AAB?@C?;9::=@@@ABBCDBFDFEJIGGGHGCCCAEDDBCFEHGGEGGFFHXLMPSEEJ_[[ID>AFYMGHGFFHGECCABDIIPUTWilt[TU\VSURPROL\ZQRLK\;30/=*%**---17%$$'��� ����6>DFFG@ICA?;<9?IB64656442/0,*(%!!���������������������������������������������������������������������������������������������������������������������������������������������FGCCDA??<;?;<>AB@=?A@AF@CFHJKIIFFGLIFA>BJDCAJ?'HHMLIHHMDBACFFHHDCEACDDFCBEDEEGHFGFEDDDCBCCCBAABBBBCCGJHJ|���hQMPQLLKECFEDCBBBDDCDDECEEFEFGIGHHGIFEECDDGEFIFCDBCCFFEHEFFCEFGJIHGHGEFEBBGEDAEGIMGE=@@ACA@?CBC<>??F@?CCCDDFFEEGGGHGHHEDCCECDACDGGGHHIHGGFG]JGGCDCGLNLLBAVOOHFEFFDDEBA@BGIJSSYZacof[Z_Z_hrFHEJTV[KMIM@7.*3,%+()0205)" &)��������8@@@NQFBC@:8>>4J=45599931/2+!�����������������������������������������������������������������������������������������������������������������������������������������������NJEBEBDHECFA?ACD@F>@ACFGIKIGIEEEDJQPNICCHHNB;;BGIHLKLGHKECEJHGHIHGHIHJGGGFEFFGIIGFEEDEFGFEDECEGFEEHNEEDBK��y���xvZZXSDCCEDCDDDDCDFFEFEEFGHIIHFIJIHHGHHGHGFEFGHEGIGFEFHHGJMKJMNHIGDDDCFNJIGKMHCIHA@>@@B@F?DA@>>DBBBB?@=@@@FEEIJHIDBDEDEDGHGFFEEEDDGFEFFHJJFFEHLJ[CCCDEIPLGDHGFQFEEDABB@BBCOSQUXZZioxncdjm{{kZE>>DNONKI@<50++/.,"0%,78=1,&#-������?;00&"���������������������������������������������������������������������������������������������������������������������������������������������������������KGHJHEFEGMJEAABEBB>BBEGIKKIJKDEFFKKOTHEAIOG@>AHHLKUSMRGFGQOIJKIHGHHHFGGIHEFHIJJKJJFFEFFFFDFFFFGHFPQFFFEDFh������gNxf_SDEDFEEEEEFFEFEFFFGHIIJKKHFGHFGGHEGHDHJLLFHHKKGEKLILMNKJMLLIFFEGIGHGMKNRMBDFB@B@ABBBBDB=@FCBEDCA>AA@ACCGJJJIEBACEEFGGGDFGIGEDDEFFFGGHGEFHU[ZJEEGLCCWGGIHKILHBBABCGE@DSUTTUX[^`fjhhx~vabU>4NINVMIH=>62//25%2():<;1+-*. �����916DGH@605+++O3������������������������������������������������������������������������������������������������������������������������������������������������������������GLKILKJKMMLIHDFECECFFFEGFLLKEBHELIHKKHFQOLFIJTSTVKKJKSHH@GHFIJKKGHLLIJJKJJIHJIIJIIJJGFGGHIIIHNOQULIFGHGDEFHe���Dd�ygKFGHHHHHHIKLMLHOdYIHIRMFHGHHOMPPLIICEIIJNIFKLIKIGFGJHJKMNRLKIIHIGHHIHOU]QEIB@ACEDECBBA@@CEEBDDBCBAEEEFHJJJG>??DDCEEFFEECFGFEHDDDEDFHEFGGGIGkNFGEDDRFDDDDCRaHFDGJ?CFFUVWg[[_f[]bcjp``[^;9BLGP`[\aFP>9212.443)'*:64:7965-&1���047;;=>B10.13+*.)�������������������������������������������������������������������������������������������������������������������������������������������������������������NHHNSOOTPMLONJKMLIFKKHIJLLLHHJFEGGDMGIKIFFEMKO]U^ZFGFKJHJILEFGFHHJLKMKKJIIIHIIJP^jJVSMILMOONPKKKLMKIIJIJJHGr���WcpRQOMKKKKJHJLJJIFHFGIVsGOMKKJNMKLNONNKJKGJKKJKLHHKHEHIIPKHJKOMLKJLGBEEHGGGKTNJKIBBECCDCBBBDCECAFFCDDDCBBBEEHFFEFBBF>@BDBBFIHEFIIHJHJFGFFGFGGFFFIFFGHKGDGPEEDBIGLQTHHECGKQUZ[\hdeiieki[VGGOHB=?MTWbez�A;@2A68;50+3/-78<>5777-,+���/14740***)'.$����������������������������������������������������������������������������������������������������������������������������������������������������������������KJKNTSSRQPRTOOLKLNHMPLJLPOQKJFGGDCKLGJKFBFITULQVHHFGFJNJKGEEFGGHLMMNMLLKKJIJJKLNkY]_WLFGHIJJKKMMNNMLJJKLJIIo���cmZNKLLKJJHHIKLJLJMGDLPSe*LLLIKNOMJMQONLMJHLPPKJKIMMLDAGGMGFEGRSJKOKKNOJJLJHFFKIIEDCDCCEECBEABBECEFEEEEEAABIFHBDGIBBEBBDDDFFGFHMHIIGGFCGFECCDEFGKGFFGLIEEFGQEEDEdUSRRNLHKNSVXX\ebjoosv^XVCJMHIJIHOXafytLCD8E9889636:984<<876744-0��,+.411)'�%/�����������������������������������������������������������������4����������������������������������������������������������������������������������������������LNRO\WSQPTXTSUMLOOQQTNLQTPNMLIGJIEHDMMGAEHDLXONGFIGIHKTKHHGFHIGHMNQONLKMLKIJKJK_qUaZPHGIKLLKIKKNPQOPJKLMKIKT���NMKPKHJKJJIHIIKJIJLKFLTOTYQNQNOOOPMVWYULJJLMSVNMJJHIIKKOKKIECIRPJNNQPXUNNRQOMDDEEDDCBDEFDECABCCCDDFEGHHDBBCFGGC>CGHFDFIGEFHHHFGHIJIHGGEFGFGDEFJHOHCEFLKGDFJQPEBF^a[SSTPOMNQW[`^hjiemm]^]LEIMLLQPJMUbj�ySJA@N77568779<93<:7555450*&��!�)1/&&#����������������������������������������������������������������������QE50[O@����������������������������������������������������������������������������������������LQUUXTROOUVXSSOPUZTQONOUUQPQNIMKJKFDIKGSMHHUYYMB8NGGGK\VMGGIIJLMLUQPNOMLLLKJKLKchV[OJIJIIIIHHKLKPPONNLNMKIKLS`gKJGGJGHKIHIIIIJLLILLGKRPNRTQNNMLPQXVW[VIHJNRRWNNKMLMMOOMJLMLPSROKLLOPPMSSSSQJEEEDBEBBEDDCCBBDEDCEDHGIIHEGGEEEELHICBCCEEDAFIHIKOLIJHHFGFEEGGEDDGHLMGGFJIFCGJNIIOYTWcYZ\ZXVSSX[^cklj[\^ibWMHAHJLORWUVZ^�fXM>7@:933573879;=:7643454..$����!������������������������������������������������������������������������@OIELxaK:2��������������������������������������������������������������������������������������QPPUPSWRYTSSQTXTUV[YXWQRWWUQPTMNOIIHGJPSSPXWLJLGHGEFGEV]N?>GHIJJJKPLOOOMNMLRLSQNQRJKLOLOJKJIKKNNROQPPPOMNKJGEDGFFDEFHEMMMKKILIKKLMNTJKOUQQJJMQPOQPPNTSHONSUSPSSPOLMNOPONNQPTUSMHFJNMLPW[_YOOHIHGEFEEFGEFFEABCEFEFEIHFFHGGFHHKLOOJFGEFEFFKJMN\VLLNKIJHIFFFEHGGEHUJEGGLMIEKGGRIJZTQ]QQS\TVZ]f[Y^^hgZ[XS^YMLGGIRZYdic^qn\^G?=;:78846996777857776101/-$������������������������������������������������������������������������������ISQI<.!����������������������������������������������������������������������������������������QONPRRUY\\ZVYXRTROTSVVUTZTQSSTOKNQOLIQZMQR[NHIKJHGFHHFL[JKKIJIJKILLJKOPLLOTVNJINQIJJLKNJKKKIJLNOPQQQPQPKMKLIFDEHHGEHHGLKMNKKLNRLOOOSLLQYONHDFJIJJLRPLQUWSSTTOQPQRKONPQOKPOONOOJFEHKJKV_dcZNLIJJHFEDCEFGGEGFGGFGIGEHGGEDEGFGGILLJIEFFEHLGIMMTRNUKOLNKHFEEFEFIGEISGKIIKIFEGGFQIKUXOZSSU[SVZcvfY_`ca5T&2^_\RKIMTVVabbddgSZBDB;>888666769;;:889;31/261)�������������������������������������������������������������������. ��������������������������������������������������������������������������������������������������������PMLPSUQY\ZZWVWTUTVUWUUVTWTRUSUQPMOSMTROIIMMKKKLLKIGGJIISOLKIIHIIIJKEJJLRQTRKLJILJKMKKNLKMJJIKOPPQSRMQRMHQJKHGFFFHGFIJEHIMNMKLNPQPMQQMMPOMKLKFFGNMMSQNOPVUUTPPPMPROQQPNMLOMNJPLIFFIHILTbc]SKJIIIIDEDCEEHGHILHHEEHHGHHEFJIFIJFHIGHHFDEOOOKOMNVTMORPOSOIIHIFEFEEILYJJIKJLHGGHGMMLNWQSURTVYZY`ogdib]T  PcfUWSNOSXa_jrao[rCB>=;98368767:>98:654210351/��������������������������������������������������������������������1/������S������������������������������������������������������������������������������������������������MONQPMUZQQTVYVUXYTTVZWWVTQXXWURRLKPQ\TOMNOOOMOMPNJIIJKDMMGFHJHJKLFFHLQOOONPLKJLKKKJJOOKNOMLLMNPQQQPOTTMHMKJFFJFFFGMYKHJILLMMMNPQRPPQPMOPLMKNIKLKLLVVPQVSTTTPQSMPRSQQPPNQQOQNMHIKIKHHKMU[UKIKJLKHHDHDGFHJIIIHGDGHGHGFHILHGJICBDDDHFHJRUUMTILPMKMXPNQQOKIHGGCACFG_KJIMKIJGDDGRJLLTSQSTVYTUYYZZ[eaVNQYXUXVSOTT]ecezmgBA9=>66352989;:8586592159530�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������TSQQQPTUTVUUPNOPVX]a_[UTQTRVUUUWVRTWa_\ZZSUQOPVZURMIHFIKKHFGIJKKMRPMPNPSTSSNPRSNLIOPJLPNMJLMMLNOPOOOOMOROMHIIHHLGFZibXLMQQMMONRRPPPPRQOQRSQROPOQSPPOPRVVURX[YVUTVXVSRNRTWURTNLQNMOMNMNTSOOJILIHGIJILLIIJKNJIHGHIHHIKKMONNLHA@DEFJHPTSXVRJIJLOOQOOMOSXPKKIIFHIJJ`VGIGFFFHHDFJDMORUWWY\ac`b]b[Zic[W \jairkff[aTVdhk�pucPC>@=7:6/.0/47157718=7<75%���������������������������������������������������������:?BHF/����������������������������������������������������������������������������������������������������������������RTSSPUTSWX\TSPQSW]dd`[XWQTSWSSWZWUUYYVVXUQTRQNYTZWMKJIHIFGGHIC@KRcTPNMOPOSRNQTSNMKRKKLKMLIKMLKNOQPPRNLMOOMKKHIJKMJ__`\OQQPMNMNPRSOOOROQPTQQPQPOQQPRRSTYWSR[^WUUUSSQSRQQSRNOQNUPONMPRQRSOMOJILJIHKHILIJIIKJNFGHJJKKKNMNPMMLFDDEGKLOQRV[YRRNLQSUTMOOPNJKTQKIEIMLKYUGIGGHIJHEC1CMPTW[^_dbil_a[Zc^f[id0RRXbhwwv`_eattny�ohVIEIC<979PF26966756/8PE7� ��������������������������������������������������������@ABCE8����������������������������������������������������������������������������������������������������������������QQQUTURWZYXTQORY[dYY]\VWUUWXURRQTRNSVXXUUORRPOQPQPLLOSIIIHHKKCBMPcUPMMNMNTWNOSNMMLKKNMHLIJGHGILQQOPSONNKPNNKMKLMNK\Z[\PSRPNONOQTRQONRPRRVTTQPQVSTTUWVQXTRUX[WRTVUSURTTYSRPNPNOPTOSRSOPUWMJNLJMKHIJKMGKKKLJKHIJJMMNOONNOJLIJDBLJJOQQSSSSPONNQPQTPPMQKROLKIIHLONM[ROOGFHJMHGF2ENUTYa`_felkif^[_id[k]?YWY\glha[U`jjmhau�d\JJDA?43-5Yp>Q78335)"42����������������������������������������������������������DPWTIJL7����������������������������������������������������������������������������������������������������������������UUVWUSRUYYRSQQU_giVSY^YYZTTUUUVSSPLQUUPUSKKQPNURTRNR\YKIJKKKIJNPKMLLLMNORSOOPRNIHHLNNNKMOHHFJMOQOPQROPMLNNOJNONRPLY\T[UTSQONONQRSSORUTPQSXWURTWXVVY]XVUUUZ\XUTVWSSRSURUUWRPTNRQXTTRPQRXUKKJKLOKKHKFKHIKKLKLHJKMOPQNLPKOMHEGCHJJLMMORQQPTNMOSVRSRTQNLJMNNNJLMMNW\IKNKLHGFKOL;#CNUUY\]_bqvq{ud_fm^XfadaZWT[Z`WNKQX``^\ite_KGA<:44&8Qnt�7441140/0100"���������������������������������������������������68ADFJNMJG�����������������������������������������������������������������������������������������������������������������UVUWVWUSY^URUW`baaVWXZWUTQW\Yc^XWSMJUTSWVPRPOOOUUPUjgbSOLJLMILMPF;FNMLMLOPNPNOLJMKJMOQMOOIIHLNMPSTXTRSTUUVTKNKM[WT[^TSbTTTQTUURQSUTXW[STXVTZUWVWYX[[[[YZWPLRTTTTRTUWXVYUTSSTTSTROPTORRTSPNOLLSMIIKKMMLJMMMMMOPTQNPNOMLLKIFHJIJIMMLJMOLLNONPPVWUQSRMLKOOQNMLRTL`RP[OKEHHL``RNMLDRX_a_be^s�tjdfihceprm]WTOYUKKFELRXdf`ehpaTSN;899`574_k887/5365215554 ��������������������������������������78(���9@DKOHABCBCFNND�����������������������������������������������������������������������������������������������������������������TVYUVWVSWVWUZ[ghc_YZYYTYXU[YYfcZTSQMVSSQPQPOORPTWSYpb\OOMKNMLSNQKKNZONNMMLLNOSPPNQQQMPRNJJIFKMMORPRRSTXUWSPHHJNVXVVZPUbZUVZWWWVSQSWVXZZV\[X\Z\^_\]\[[[YXTUSQRPPQST]`SUSRSXXWVRPSRRQSVRSOPNJONMMMMLLLLMLNOOOMOPRQNNSRRNPMKJMMJKKLOLMMNLMKNMRSSZXRSRNKMOMLORPOQQdWXXKIIHKPRONOOLMPSTY_Z`Zfekgiilipfsphb[SQSRMJGLTd`agfu�xhaYK>9:M>65/DP#,97464/336633�������������������������������������AA>@=68;?CCCJLDDDA@DGH=�����������������������������������������������������������������������������������������������������������������SUWX@TXSSQTWW`glaZYZZVUXa\YWZ[[XYYWSTWYUUQQRRUR[nmgl_VQQPSPNMRQQNOOaTUWZQVPOPSQRNKWSOQURPJHHJJLNOPNORQWVVSRMJKNSU\STOQ_WUXWWWUZXSRTVYXZYZ_Y]\^[`]YZY[ZZXUWXRTOSTTYb]QUSSTXWVSQRQRQOSVUTROMKNRPLMQJLQNPPOPPPMOQTUVYPUTTPONJOOKKLLMMLMKQTNONOTSWTSUTMNMSLOOSRNOU_UTZSMKLMRNPOSOQOVSU\]dija`kyjiktvuvsm[]X[WSTSNRVcheonjxne_fF>;;67977445965343377462����������������������������155%229747559:32434.344346PekAA?@?>?BBEE@?>DFFFJKLIFFEAA>�����������������������������������������������������������������������������������������������������������������WTTROOKPV_ezz�i_`cdYWX_dc]cgj^\[WW[aP[^dVUY^Z`VVguhdglmkea[^X\]SW[YXmlfgdg^WWX]SSXUSU]a_TPIJQLLNMMMPOOPRQUYRTSNNNPUYhdR^Y[`]XZYZXYZ[UZXY^\\]\^]__`VUXX[\UYYSPOOSVVTY\UUVRX_]XWWZVWXXXT^\[YWTTZXWW\XWOORVXRQRTUTYVVSQPTUTTW\YPOLOLRYRKMMNRRRRRSPLONMRUSUU[[RX]WQTYPLOSSSTPWSY[_a^jjkhl��oozkol{mnszylz���\[XPR_jjcoscZ`����VQ`AC?544676998965./53337=EH������������772'�485G��_H9769745;>??>;;??@EIGJHEDC?A<�����������������������������������������������������������������������������������������������������������������ZXWVLOQT[`k��oggqc]XYkqmdmrnjcjYYVXR]]\ZXX\Z^]XYT[jnfngc[\]^WUWVfZQcl`cbaZVW[_YXQZYYYj^S8IIONJNNPRQOPRNRWYSSTMOTTYX]\QZ]dhee[XXZ\Y`Z[][[ZXZYVX^_`WVWV\\W][TYPVTRW[XXTTVV]_ZZWUXTOVYWX^^\]UTU]^YYVX[VSW_`\URWYU[YWSRTWVUUWYTQXUPQQRRMRWRORPWUJLNOPNPVOU]]XWZVPQRSNPOQSPPRSR]]cl[cdcjq���{�wmkaaigk{o~z��[WWNTY\\fjf][V�y��P_TDA<9658866877762254348>DHD)��������116566666479;@=<:8<@>CGEEDB?@A<�����������������������������������������������������������������������������������������������������������������VTUYMQT[`kj��{}ort`_Y[s�wltrnnbaZY>Ydb[Z\Y[^_]\^]h|rmmmdPbfbb\X_bWO]m`hg_ZWZ\\ZWP\[XUZSN-OPNOLLMSTSRSRQ]YTQPNMNSQTZ[WQV_ba`aXUWV[[[Y[_dbWZZ\UR_[_VVWXX]YYUSSTXVR\YYUUUWY``TWUXTRSZZ^[[a_dTTU\\XSWXWWXad]`^UUZYZXXUUYXUVVWWRS[]TSQQQLQXYPQRXQMOOMNOS[UP\\WaURQRTTMUSQPOROPUZ^ioima\mu|�~s��}�v]^knju~to^`WST[[bd]X`e]�}z�SJLJ?<>7:;:98583134507678>EMMO.�������687:>>FHB=B?=AFB<:9576989:;77:<=;69;<@ACBBA?A@8����������������������������������������������������������������������������������������������������������������VSTRQVWZqu|}|pptihckklpeqrpqjptY_>_]TTTRVZ]Z[_Z`qsnixptppf\z{ir]WSjhf][ZUNVZVPaSONLPNMPOSRNORRSRSTURPZWXUQPNQSVVZX[\T\\_aa^\W^^[_ba]``caa^\UW]aa^]YZ[`VRPPX^\`]^_YWZZab\WXYY^]egde`^W\c_UN[]`a_^\adccc`^UY`bdd_`c^YXWZXYVV^US[ab^P[YOYZYNSSQWQPOVYUZMJT^e]WX\VRVPPPQRa[\fw�sihgjvnf����vmng_abgfdkkmce\VVXXXUZ\^ebc�rdu@Ci=@E>98;9B>:98867:?AC=8436=8<=9>>AA><;9965;;>>ABA7-9:���������������������������������������������������������������������������������������������������������������RONOORUZl}�~~tmqujlomheeqlooopqa`S[\T[WOV\YY\ZYb|rwwxry��ui��{vdXZ]iafYZZSRVWTTWPNOPKONNQOSOTXUTSQUSSP]WUWSONNPRSXWWYVT^_^^ZXXWZ\Z_ca]afdc\YXYb^db\YWZWQPRS^a__]a`XX_bb^a_[\eWOUc[[YWUX\ZW^]\e\[__\__aa_]]Z\]bb_aca_ZZZ\XWW`XUY^]]W_YZWSSQSWYZSSRSU]`QJS]aXRVVSVOOTQMPX_^dw�~idensh]g������okfeV^^\Y_\[YU[\\[Y]Z]cjh�ebiF[TEED;8;:?KC<><<9;>@BDFIEGE>CA@=:<679;;:=>;CMA<;>:59;==<;;<>;@BDC>������������������������������������������������������������������������������������������������������������������TOMPWU_]krh����yv�rtlhfcgoptukuulpY3UUTWYO[c]]YZ]epijo������|}��ka\be__bZY\[a]XY\XPNVUOOLJNQPMVQSSOOYSTY[ZUXSOQSSTQVXZVWT\\ZXZYYXZXZ]^cc_ac\\\ZXZ`c`[UP8XSSXY`_[X]]YZ]dh`ec_Z^`USVbc`YZZ]]Yabcda[\Z_YY_\YZ]`\WX[agl``a]YXZ[YYd]VZ^`_^\[\c[VWWXYQTXUSQXdd^a_baSWXXTTRROPRSVU`jjoneuhd`fiw�����lpr_[__\]W\Z\\^]XX[][_``|hppBZ�KG@=<>;9>BCA<>9:88:;><=A<768;<>===@ABA?@A>?@@BBBBB������������������������������������������������������������������������������������������������������������������SRORYY]^pnhy{�����tpvistnijokrtneT/\RMXUMVj\\[^erkecApv�������z`Ygeeab[XWYXZ\^[YPQZRONLKKKJKVRVUQSTVVbXWYVUTUQRRRRUWSZ[UU[[`[VV[[Za``bb`a^da_Ya_]YX^]?_`YY[`^]`[XXW_fb`ad``de_a^eed^]][]ac`]a^\[X^[__\XZZ_b^YWX[Z\``c]\^][Z]a]ZbgiZa[PYTZUZWUXYY^UW\W]jadg[WSTVY\\TRSXRV[]nph{wxlhhd^nz��������c\^][VW__b`a[Y^c`^]]Vwam�Lq�QGGC=A><>?:<;<<@BHHNA@KIDKNLDC@=@OLFYnO>;=I8>A?AA?98<>>:679;==<=>??BBBBBCCC@@AB@D������������������������������������������������������������������������������������������������������������������TSTVZ]cppli~st����w�}�}soomvvz~]\`Towx{�\Yah`[ajlaUdUi������q|ri]ahRYlg]\ZZ[\dj`YWSPMNOPWZWLSRSUSQ]b]\[]VXSSTSTXZVTVUYZ[SROX]WVWXSU]_\]`a^me]`d]Z`cbY[`^djda`^`[\\aY_`\^^g`hnlpmljllhhhhj`baca^\Z^_^^[][XZ\`__WW]ljpr][Z^]][^_ad`ibbLKPSNPOa[XXZWWU[XL[[W^\VHITX^ecguYQUX]nh{zgfx��nppjri�����~~�|�rdkogkgdip[^`]_^X^xoqwbOOCFID;===CB?CDJWLJOJFCGFCC=?@?>=>@>??B>�������������������������������������������������������������������������������������������������������������������RTUZ^]iuigf~u}���~�����~rrs|rsxbca^`mpieZUU[d^]gl\\h^f�����tx~|empkVVgd^TV\Y\ee^ZSZONPXY_QRU;QSRPPUfd][ZXWRRTUW\_WVVYZ[YPQNM[ST\URR\^\`\`bfdd^][]e]X^V]Xegfa]^^ZVee_jef[\dfiqonokmklopklscbja]]^[\bZY_^YVWXZY\ZZ_fgkb[WXWZ[[^^\`^aabaTRRSNN__Z[Z_KJUSWZY]WSOMKV]WchiYQRW[_wsz�wrr�|ynjhh����yw�{{�qsx}orfijehi\[Z`_i���gfH?AAFF=@A@HJGFDDNTTYOHFGEDCBAA@>A@@ACAB@BBDFFFDBC>=>758;==<6556::;<<>>;<>=<>@@?@C@�������������������������������������������������������������������������������������������������������������������RQ[Xacnxsln~~�����������jxnuukoqobVSV^fdZnU_iY[ah[tlgjpz���qi]hdka^RWZ`aWVYUYb`]bXTQQNUXX[]PKQVTTQPZXVYWWZU[\C[_d]VUX[]]SOMMMQUVMRX^_\^]^n^ag^b``W[[Y`ddfcca\^aX^c_jl^`bdopsvptmumihnroqnkldYY[a]^c``aa]ZX\[VWba]adaa][WXZZYaged^_\djacc[SOTWVb`]TSRRZZ\SXONMJQ[bol_TJMT]f{tv}tns��yykqstxtpct��{~�psozmsrlheaig]bof`��xYwLA?IGD>@DCPXSX_fw^VUOKIKHFGIECA@BA?@??CCEECBFFECD;;<8779=<;;>>;?BBCACGDCAB@:::788:<;<=;9;<<;>?==<=?==<=>@E�������������������������������������������������������������������������������������������������������������������_cfh���������������q��������lirmlulR[a^pwuswic^gfelpq~va���vxprqs�[OI`ab\WUTTb^\ZafcVTWYSRUZRRS\_YXVXUTUTXXXX^WTVZZZ[`W_[[\RMV`acb_^bfaabhmfhk_Zdgefd__]Z\bgc`dcfhdemuxotvyrptsnusz���}qhbbahpijlhheh^]^[Z\\aa`gcb_]\[U]ff[SR_peeSgltxxnx���ilfidgbSSVRRSLKMMV_UM\Zg_Z`o��������������`bacm����|�ty�ws|�}nb^geiiw{hV_�HO`o|z��~{sorpom_`TMMORNMLJIBDB@>ABBC?AACEFFFBAA??>:;89;;:;@@A@=;;<>>>>>===>>>>@D!�������������������������������������������������������������������������������������������������������������������gmmz������������������~~~���njiuuox0dakrju��ecdcbbbgkmvsy��h���wmz�^[Uiy�oWQSW`^^Z]b\b\SUTYYVTUPZVZTY]ZWXWV[_^YXV[a^[giU]X\^[SO\]]`^_``_`acbjkjhmqlmfdaa[bjf]Z]gkkhxz{}{uvtvww{zvux����uunqgegopjstjcnnhgjdacaba\bbaa\``^jup]Zk�udW]mmjjW`n���mryrmib\ccdUQQMQUYkjcsnh{c\d�����x���������`m{����u�w�}��~{z{ikuca^dYuou_yoI\|pqvw�}vs|ijkcf^dYKQPPPPKJFDA@@@A??@>EEEHEHE=>?=<::<;;>>:=BDB@@A>;>?=>@?==>?@?B;��������������������������������������������������������������������������������������������������������������������gls��������������������}�|y{uervquhoW^cihnz�~bjabeeaesllll~xl���v���rtutfk|dV_]c``bb]VURXYXUSUYZYXY^VbcdXXV\][[WUZfkjdhg`_VV[YSQY^_abb`^ecgkheddlikfed]_^eigeojhxzyx�|yxxz|yyvv��������~y~qpqruqonp^ekuqllhie]Z[[\^^^^\]]cb`[^s�faZ_fh^afle���u�ynliimmhaVRROXXZ}xl[VVSV[q���������������i������jr|ry{rpirhk}g`ZeengoblW^mxlqnt�zpowmggneye\QSPPNPMKA@>??ABBCA?FGHHCHDBCCC>;<;>><<=>BA?=>@B?>>=?@??=>@ABD1��������������������������������������������������������������������������������������������������������������������lh|����������������������|xtwlryz�ngc^sorntm\h`acmell\ba]b_p�~��������g`dg^Y^[^aakiZ]T[^TRMQQVV]cW\U[`eVSGZ[ZZXVbikhechge^^YUZZX^^dkld_iiiqllnllnqkffccbhnwu}k`o|��krvx|�~{������������~}|uvz~wxqdhjpnqtnd_ll`ZZZY]XZX[W[b_fk|��ikam`\Zojk�������ymlqlrgec`\Xhrp�}�[P^Zah���������������|q���xuvhszx{okzhgpkq\WUXf__jutWm�nx�}wxzquvto^toriccTOPNFHGBA@@ABEEFDDGIJJHEICCAA>==9<=9?BACCA?=ADF@@=?B@A>@ABC>3��������������������������������������������������������������������������������������������������������������������q����������������������~������������{rpeol_bYQS\Ycwjhq_ncwjjiu��~�{otqdaXT\WWVcdcekqgY[^cSSSVW__b^dagkh[_Z-"R\ZX[lngf`cbbaZZUVTV`___b__dffhomjhghumhcnrv{�|vy���{|rw{������������������{�������w{zsqvtrrieaejfjiga\_]bca^]]mj}����gZknda��������qkfQYfokUj}sv|��~�e_n`im���Ȳ�������������yw��jn���~kdY]\\if^ZaUU[b�aiekw|~wtuqrmxvscx|saa_\ZWPLGHHBCBAABEGGGGILMMNMIEIEA=;76769968;;;>@CCDECBAABA@BD,����������������������������������������������������������������������������������������������������������������������m������������������������������������vnqqo}jZ]ciain�~w}���vutrw���whb�{Z[[T\YW\a^[da`[^du}fiWYV[[Zcqedgk`V\YRZWZ[_gc`d`_]ff`_]WWWbab^b]]chjkmnljlhkoem�vx�qvsw}�|~~�����������������������������xsqvytmjlrmkriji^W`fgaa]_`bjs����fsnhc��������ocZahgboY`�|{�kgnysem`^ad��Ύ�y�����ɧ�����v��|�����odYZX[^_[UhZbjtemipq�wvrlnmberpth�|pda\a\`TKHIJGBBCFDFHIJJKKMOMKIFED@=:87658768:::<>@AECEABCCCAAA�����������������������������������������������������������������������������������������������������������������������t��������������������������������������|�����yrrddrjmv����gcpnq�uje^^ngXX[U\WU]\`hVYY]oi}��{fWVX\rkqb]^kbV`jh_ZYZ]dd^^b]]^]a^\SW]`^^`fdgmkhqsomsrsqq{rsqs}�����}��}������������������������������swrtvy}kuwomsmkfke``^]cafafdf�����kgvks��������hWUhff_e]^toqd][k�ldlbX`��ȁqs{�������������������po^Y[Zj^]XS`a�|cv�{rvhnndmpjl|roo�woeaYZY_XOMMGFBBEEEGJKKJMMKNLMMIHG@?>:98898779<:=@@@BBBBBCEFDE7�����������������������������������������������������������������������������������������������������������������������rr����|��������������������������`��������������sz�����np���iie\V^_^\]_Y\X\lYXTX`ikr������TXd_XV]onndZ^hnjsqajhfgaWodd\^flb\Zglrk\[bntqn{npv�������y�usnz�������������������������������������{tvwy}{ryyxxxsptuvuyycmcZbeammikc`d��~�jphv�������hYb[fZfbPR[pcc`md\cssyl`r}��x|������������������ta_bm��xq}r`lXHydt�����dhk~zukhx{xra{vtmi^VNPU[RROGGIIGEHJNKLMMNNQPMOMLKGA?>?;;;;::<=>>AABACDBAAABDD$�����������������������������������������������������������������������������������������������������������������������_\��½���������������������~����������������������vz�����xr{���ej^Z]__]\`a^Z`he[[U_luz�������^_xa]]]ju}}teypubY^Z_elarnj]\^tdoqrvwmbYcjvrnx{u�x|����uxxxzz~{��������������������������������������yz����xzzwz�|}{{ykhccbgffc^e^[ek}z��|dj�������pcW[\abYTPNdae_hk`kjfbh`faov�{���»���˺��������a^i{��|zn^^/Qq^i�����ocdurphfsqszqejptseQTUQPPTQMGMPSKHJMKLLMLMPPOOLNNMIDC@>??=<;;<=>@@ABADEC@?ACD<������������������������������������������������������������������������������������������������������������������������[w������}�����������������~�����������������������vw|����}�x{ykivofifd\aadmfYU[W\dqxv�������^cpjgd]hjhc|rcukhoeeckjefppnfihuhkoyurpegrutu����|x����r�����������������z����������������������������|z���zvtswyy|�zlhggbfhfafa_^^dov����q�������vkU\V[]VVX[RU_ec]cce[dcbcjlmukv��ȼ�������������yjm�����i[`W_iPVbo����nggikefefuv�}kife\[QSaSMKMUOJJLRQQJMKIHKILONNOLLNMJJGDA@AA>>=<>@@ADDEGGEEEDDE<������������������������������������������������������������������������������������������������������������������������Xz��Ĺ��{��������u����������|���j������������������zu�����w�||}�{vrmghd^ejh^_a__a`gpuk�������]hnrsrpokpr|rvzoppjlouunfboqkgizkghloqri{��uuw���~������������������������������������������������������������prv}|y�}}nlljdmrhkia`_]]_V_�{���������ojd[VYWT]`PONQdbaimmfvtpjjfph|�������������������sy������f]eeZ]Ped`yx��ntidkjrvno{�{lcY[ULOZWOHHKMIONHKNLMKMHIILMMNNNPMLEIIFEDBB?@=>>@BDDDFGGDDEDD6*������������������������������������������������������������������������������������������������������������������������m���ȸ�|z|�����|rs��������������������Դ�����ĭ�������ytw����yp�vjhrgddgiifkkvk|w|upidq�������wr{���xrss�xw��������~~wccnpchkem}qmfz}��|xy����������������������������������������������������������������zy~w������yrrtrkoljppdbc^VZubbs|��������xldMKUfXZ[RUWRfmw��g}~spqk�������������ӳ��������h�����{b[/Rp���h^v}��zu�onyxt��~~z[TNUMIGIMTLGFGQMLMLLMLJKJHKOOQRQQNMLKIIJHHGEDD@@BCEGIKIHHHGGI���������������������������������������������������������������������������������������������������������������������������1c���¦�xx~�������������������u�����������˻��������y{t��������xzstqqf`fecajnok�x�vsegy�������������txy�{m�������|kadgdolinlu��{���}}������������������������������Ĵ���������������������������������������������}xrtztlkoqsegfdZYdx^\k}������u�utdW[SZSUTQSQ[ajjrv}�pcds������������ӽ�Į���˔qs�l���or_XVXfkn�k^ao��ltwvvy{u���{gdVNJMMJJLPZRGDGINOOILONMMKKLMOOORROOPOMIIJHGDDEABDDEFIJKIHFHFD���������������������������������������������������������������������������������������������������������������������������58���Ĺ�������������������������������ɼ������������v����������|kfcjebjecffhikj���������������������}}���������rkvwjie^cckwz�~y������������������������������������˻�����������������������������������������������t~xxvenqsjafea]a{f^X[j������|~yxakZVTPSQSRMYfe`_jv}}swv���������������˹�����emvy��xtkeYYbSU]g�ec^]c{zyr��w|����q`_SKHLGLIOQUPFECDGMRMJOMNOQOONOQRRSSPNMNKJJIGFEDCDEEGGIJKIGGFEF���������������������������������������������������������������������������������������������������������������������������9E�����ŝ�����������������������������ǽ����ǻ�����������������}z}~}qjihjqjnpmg}�������������������������������pik{}vprebcrwq���x~����������������������������������������ɽ���İ�������������������������������������w��wnvssmgfksvmhbaXX[|���������_s`UQXWRRPS_aTa`ml�fvu�����������������Ǹ���|z����ois]4:e^hmpfgrqcz�vr��y�����sa[MHJNIJKZYSOHECLHLQPOOOMTWRSQQUTTSSPONNMIGGGIGCCEHJKIJJIGFCDGB���������������������������������������������������������������������������������������������������������������������������4,���˿���p����������������������������´��δ������������������}��wqhinghlmrstq�{�����������¡�����������|p�y������~z~�d\t�����������������������������������������������������������������������������������������������wx}ytmavusap`WUR\][q�������sQWROPSRSTTTm``i_^`xvjun~���������������ƿ�����o��lfccL_�~}}ggqvsw���u��yu]duk]dlZLKJMQPXSY]SXGCEEFRRX^ZSSRRVUVTSSSRNMMLOMKKGCBGGIIJKJIIGEDH8����������������������������������������������������������������������������������������������������������������������������G6���ʼ����������������������ug���������������Ů����������������{xvxmkolkksv�s�z�Ǽ��������������������rt���������|}}no���������������������������������������������������˼�þ�Ž��������������������������������������z{ztsqkoodcyjXeNZ\mg�������jOXRORQQSSUXe^nfke_kzpgr�|�������������ʴ�Ż�������tgabidr���g[]nh\Wdfq[�~�Z]e]QTZXNMNWYQWQTaXPUIIKLSVW`SVUSPRUSQPRPOLLLNMJJJDDDIHHILLLKDFFHA$����������������������������������������������������������������������������������������������������������������������������T>���ƻ���������������������������������������ȳ����������������vyzonppmpnx���x����ͭ���������y����������}���|������w|��������}��������������������������������������������ʸ��ɿ����ľ�����������������������������x����||}uukslm_ee[Xa[[z�{�������eQSSPTRUXYVZil|�zrj}wn~w�����¼������ؿ����������ocehss��~gVUZ�_UUXhq]|��zh]_VQRPOOQsxTJSVX\gYWYJOSXWV\XYTQPRRPMNPONMLLLLJIHDFGIIIJKJJGEIE@.�����������������������������������������������������������������������������������������������������������������������������$S����¡�����������������{�����������������������������������������{tpmcjru��������Ƭ�����ę����������������yu���������������������������������������������������������������˿�ž����������������������������������}�������zsrwrokkga_f[W���v������[RSQURSXWYZXnmy����uuv|xx������ͯ������ʼ���������e`y�����u`TKj�n^Y]v���zvgvaWPPRROP^xmNGR]]]qieXUMSZZ\WUYWSPPRMKPQPNLLJKJFFFBEIIIIIJJKEDCC3������������������������������������������������������������������������������������������������������������������������������%���÷���������������������������������������ɵ����x��������������yq}xjv�������ƻ��ƻnp�������������������������}��x�������������������������������������½�������������������������ƿ��������������������������w����������~���|popmfzw]���tx����rNNPOQVSVXUTQn`mkhj^pdhnjcq��������������������{|����k{���Ęx~���y������s�l_a[WRNNMMQNReUIDMXX`qhd`WQNTSVVUVSRQOPNNONLHIHFFEFDFEIIHJIILJHEDA>�����������������������������������������������������������������������������������������������������������������������������������Ʒ������������������������������±������������xy��������������|||��������Ʊ��˦�lju������������������������������������������������������������������ĺ��������������������������������������������������������������������zlqqncoj\��ykc�����KNMMNZSRVVVRkkewp_jscqS[is��������������˺׹�������xe����DZ������j�������xjccimzRMLSPOUYLINZ`VZfcaWNKOSTRIMRRRPQMLLMKGKCFDCFDCIHHHHGHLJIFF@9�������������������������������������������������������������������������������������������������������������������������������O}��¾��������������������ov�����������������������z{������������z����y�������������¯��w�������������������������������������������������������������������ÿ����¼���ľ�Ƽ���������ƿ��������������������������������������~����~{xrpi|gbh}cbd^ix��^MNNL\URYWSRkh^{~riqgoPZbi��麬��������ɷ�������}���v����ͽ��ž����������lojq}��MIGooMO\UGMTW[LXZRNMJKPPROPSQONRMLLKIIICB>?FB?IJIEFEAIHGGC<3��������������������������������������������������������������������������������������������������������������������������������dz���Ƽ�����������������EPyn��������������������ȱ��}�������xvmxvv}����������������������������������������������Ǹ�������������������������������������ſ��»�����������ɹ���������ż�������ï����������������������������������zqpqogefakikWd[ih���_QXZWQRSNTYTTYao{h\SSPY\\s����ò��������������yws9Wvw�����������޳��os�yqaaaa[RFCBHDCJOPRJMNFFGHGGGFRJMTSROLMONLMMOLJKGCAEHIGEEFCDDCFGDC@0������������������������������������������������������������������������������������������������������������������������������������������ƛ��{���/w�����R@Y~��h����������Į����ȽȞ������w�wxx|u|����������´��������������������������������������Ĺ�������������������������������������ĺ��ú������������������������ż�ž������������������������������������}zrkinrbkfee_`^`k��u[\^WRSZQSYPSVWhaaQUSZjh[hp����������������|pj~�`N�|����������Ϫ�q{zmikibYcacjKCBGD@DHHKHGIGFEGJKMKQKLRPMMKJJIKONJJJLJHCIJIFFDCDECBECCG; ��������������������������������������������������������������������������������������������������������������������������������������������������;���o1�C�����������������{�����“�������~��xtx�}|����ʬ�����²�����������������������������������ý�Žĥ������������������������¸��������������������������������ź����ű��ļ��ƹ��������������������������������|wnpstule_dOJOV^~�hq^WZ]VPPT^OVRWgb[Za\`l{�hk����������������{kp��o������������ǝ��sq�ti_aiZ`ji~RFECJJGFHGDFHHBIGJLONJIMMOLLLKJJNMMKJLNMKEJJJFEDEEFBBCCGK27����������������������������������������������������������������������������������������������������������������������������������������z���t�:#��������������������������ǘ��ɹ�������������kr��������������ȶ������������������������ĺ��������ų����ǰ�������������������������²����������ü�������������������̺�����¿��ü������������������������������������~vy|x}g`cWQONW^n�gihTVTPNOVRKOWV\[ba_Z����}����������������zw|����1��ȶ�������ƙ���wryp`\\d]hgizybfINSOPNDEEGHDCBNPOLJKLLKLLLMMLPNNMKLKMKHJHHHFGFGECDDFBG8����������������������������������������������������������������������������������������������������������������������������������������nh���������������������������������Ϯ��Ѻ���à��������x|{������������ʸ�������������ȹ���¿������ƾ�����������º���������˽��������������þ¿���Ʋ���´�����������������������������������ȶ��Ź��ƾ����������������������������vqxwpl`\`b`]__lfPS[UXPMKGQ_QRY[VPP]{�{u�������w��mo{��������+W����������������umrforZW]^Xcji���u_]MIKKMCGFHKKJLOGGJLMKIGMMNLKIIFCKJKGHDDECBBDECDC@B����������������������������������������������������������������������������������������������������������������������������������������u5��������������������������������������ӭ������������{z�����·������ڶó�����������Ȳ������������´���������ÿ�����������Ƽ���������������������������������������������������ǫ�����������������ƾ���������������������������w}rsssrm`_S_^]e�pXQVYVPOKLIIRPUW[RPX]u�rjhk|�}��{��`�~�������uq������÷����������z{hj{dUSW^ccc~|�vsydSKJIFEFEDFJLIJLGHJMNLOPOONJKIIMJLLNIFEBGBCBCC@ECA?�������������������������������������������������������������������������������������������������������������������������������������Z��]�����������������������������������������������������������������м���ù���d������Ļ˸�����������ſ�����������������������ɾ�������������˰�����������������������������������������ŵ������Ƕ������������������������������������}}uvyvcb^bY]]��^SWUKKMMMQLO\XZ\URX^s{w|ig{��|����}���������m������ƾ������������rlk\VRR^nwimuyqljrTLKJHGHDDDLGFFJHHKKNOPPNLLJIIKJKIKKHFDBBCDDDE>DBA5����������������������������������������������������������������������������������������������������������������������������������o]fNw�p���������������������������������m��ſ�����������������������������DZ�ƨ������������̺�����������ŸĿ���������������ȷ���ľ�������������д��Ⱦ��������������������������������������´ĺ��ƹ����������³Ľ�����������°��������������t��adcdXWbn�mUSNPPLNPOOKZccinZSWr|�yla��t����������w���H�������ͷ������}{xn�����f]WWdzwmcninbekLKJIJFECEIEEGGIEELNMMNPPLLHIFHJHLKMHCED??BDHKCFB;����������������������������������������������������������������������������������������������������������������������������������������������������������������������������uw������ϱ�������������������������̬����������������ͽ����������ȹ�����������½���̸�Ҹ������������������������������������������ɿ�������������������Į����������ž�Ź����¿��������������������|x��rmfaib^ar~WNGNQMKLRR^z{tit���o�{�~�����������|�����z������œ�������{��~x���sqe[WhsjZ^`qvvbTTQMLLILMMEDKSSEEFDHKMNNMKHGLGFLQQPIHGGCACKM@=:ZTJDCGJMLNMNJHLJGMSSPLJJHCCDJLHF;,������������������������������������������������������������������������������������������������������������������������������������������������������������������������������|�������ɴ�������������uz�������������������»��ļ��������������̨������������ʺ�ú����´�������¿ƿ�ļ¿��ư��������ľ�º����������������������������ľ������������������������������Ĭ�������������vqjtrccc�oQIV[JNLKfkt�wiiYal��sj��zr|�zqw{~w�ytwg��{��������������f\b��x|{y�yuwq`hadZYRSSQYTPPKKMJFGD=;:9AiNHAEFMKMLLJHMKKMOSLNPNJCJIIPMJ4�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������η����������;�¼�������������ù�����������������Ͻ�������������������ν����������¸�������������������������������ij����ƹ��������ʿ�������ɻ�Ż��������ª����������baeo|]QMQX^MKSbr[bb^ext�u|{e`[XX[_MbZ[q�nw���~����invx�������}w]qlelspr_[_g\S_]UXXJINOMKJIDCB@@?<:;:<@EPHDCGIJJLNMHFIJKMKKMQNOQNVIK<���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������u�����������������Ѿ����ɹ�����˭���������������º�����������������������Ŀ��ȹ��������ɽ������������ȳ���ι�������������������Ͽ���ɶ�����ĺ�þ�����ſ������ĺ����������ļ����������{tqtkv^UJOcbMORe�iddlpzrr��xug\ZaVTSU[U`\w��tw~��t�y��grt��������w�{vkkw�yl\cotYNQXRR]KIJLGEEDED@>?><;;=@@ATLDBFIIIMMNLIJHNMLOSSPOQTUS@5��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������zz�������w�����������������ʷ�����ƺ�Ҭ���ý�������ܹ�������������ξ����������ü���Ѷ�������ʼ����������ľ��������Ƚ��������������������������м������������������ʽ�������������ö���������qfcpkdcVUZmdYSVuxqlojz|z��zba\rl\fyLJLab�����y�������wxxsy���������pukh�lemgotjZTSURNLIFEEEDFEB?=>>@?@CBBBDHCBEIMONPQMLHGMMLOSUSESW[A1���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������Ǿ����Yz������������������Ǻ�����������Ķ������̽��������彻������������������ſͽ�Ⱥû����������Ƚ��������������������������������������;�������������̼��Ȼ�����������þ�����������Ѯ����������nb_seofa_^z�kQTpj�ntqvwp�sj\Xspbu�B@?Qj���{oZ����b���zoorw����x���vfl{w`f^[^YZVSRQMJCEADEDGC@=>=?IJHGDEDCDFFFFKLNRSOOLJONLUVVSPYFP:4�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������v_i\|�����{t`N|{�����������������˫������Ĵ���ľ϶����Ȭ���������������������Ź��̿������Ķ���������������������������������������������������������������������������ɵ�������������ó���������sjjk}�sob_i��umw`towusmnYXYJLWaQPWEAN�_]]t?4@���������ov��������y�z{��ilee_]TRJJIDBA?=CDEB=:HQX]XVTOKJGEEDEINPORRPNLFPWTPRRSZQA9 ������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������~bj^`dS�}�����I�}x���̷�ë��ȼ����Ϋ�������������վ���î����º��������������������ż�����ǻ����������������õ¿����������������������������������������ͺ��������������̽�����ǿ���ʹ�˽���������tuhp���}ogx��aqik|�����hTRJDGMTK@GJLt�\c\f:;h���������ud������|���jiuiirc^]^ZRKKHIB@ADBJHC?;=R[^]YYURPPKHDCDKLQTPSQNPNQSOPPTVYCB"�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������ͻ�q�vnx�y}v��v�������_�����ļ�����������͡�������������������˶�����Ƕ�������������������ŽȺ����κ���ƻ�������ҿ��������������������������������������������ҹ������������������������������²�ij����plfl����pno}zf]h�����r~j_VOIOOS@?KaZP-JX]nS~���������rrn������y����o_paaqqfa\\YQMHIJC@ADBIE?<������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������zL����������������������������������Ȼ��ŷ�������������������������ƿ����ż��������������������Ĺ�������������¹�����������������������������������������������ú����ƾ����������ÿ�����������������~tuz�������������������i[JPSWKJDDY[HBJ`p��}�z|����qu~������|dknpwi[iyncd[WTYMMLKGDDC@F@>>?EGJLNRROORSSOLHIFGIHTTSYYVYQNS[M<;(����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������[���������������м����Ƚ����������������������������������������������̭��������Թ�Ƿ����������������������������������������������ɽ���¼��������������Ğ�����������������������������~wmmqm���pxZHTHBB?EXM85@Gbo�{VRR[VWW��|s�������r{vekmjcmiZXSPGMHHJFGGGHDBGD@A=DNMMPQNSMNPTOPKNJLRVST]ZYVUWWWGBC.����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������{��|����������̢ž����ŷ����������������������������������������������߼������������ø���������������������������������������������Ĺ�������������������������������������������������Ǜpiepgas��z�cHOFEE@K^P-:A@]`joNJGUM]bw��}w{������udhcg]df[SRNPMKOPQQIFFIEEIGDGKNOOKNSSUVS[WTLQLHQWTTZ^\T^WcMDB>����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������ʡ�����|u�������Ǿ��������Ⱥ��������������������������������������������������ض������þ���Ľ����������������������������������������������Ͻ�����̱����������˸�����������֩����������x������meg���}��{~y_WIDkTABHQ. +#IGFEFGMCFM^b_nbo�}�vp~vrsefhdytZQQQSZUTRPMMHHEGNBORPTZ[[dfcaa[b^[\YPOKT[^_SPSUhgU@:>������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������ӿ����������ÿ�������������������������������������������������������Կ��������������������ɬ����������|v}����fmg���w�p{c\NOHBBDFN'7HE@@@=GAHMOV_\bk���wsrzyrrlhhic{�kXUXWYYYYWRSLKJHHKDS[ZW`e`ikjeccf[ZZ[MLKO`^`RTUYk_K72�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������Ϳ��������Ⱥ������������������������������������������������������������������������Ƽ����ž�������������w�����������{pg|yhMPLDBDBC@> AA<869:=@HGP^f[`lu����t��yrkddgkqegc\_]]\ZXWVSRVLHKJWdclshthiihfgbj^YVWRLIQeb\`^_\YP<;������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������ǽ�������������������������������������������������������������������������������Ż�������ͺ������������������������������������������������������������������������ſ���ͼ�������z�{�|����������{����r_ypjcV^E>B<=:; 7>94358:@HJ\q|dS^p��������}od]d`]]kef^a_ad[XXUSRLIKUX^kxwtzmjgeehgga]WVUSNX`_YdgdbRC;�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������׻�����������������������������������������������������������������������������������������������������������������������������������������������������ýŮ���������������������vpevjg]bli\��^B@;:56;!4:64215?ERfd[ZIJOxlt��unliYUXXeaaabk`Z^a]YYXSNLITc`\_dVN\s�udbgheb`\[`^\T_cfmlfY@/����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������Ӵ���������������������������������������������������������������������������������������������������������������������������������������������������ڻ����ǿ���y�������������������|rw_`Vcmlaz��I>>>>BCL9:7639:==DCEZUUYyprrzefc]ZVVXW^X]`\[^\_^_]_]XUNNVggc`jkOYt��dbghc_][W`k_X_cjdTG7�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������Ժ������������������������������������������������������������������������������������������������������������������������������������������������������������������ú��{������������vYUTQVkibemE=NZs���RUPTQR`YVUVZZUab_\]_`_|jd]\]\opbjkfmbbk`Z\`ZWWQRLJNz{l]U<�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������˿����ukg|�����������������smnTS_ht���h]??977AaRPYURKMQULRf����trXNWNSNNTUST^YWUXW[dba\\a`dYRQXbZka^`a_a_a`^d_\XRQNXjebX6/�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������ļ������������������������������������������������������������������������������������������������������������������������������������������������������ʦ���w_���x����������������xka]]nsy��paPE?86D{ej]��xVWRO[]eU]\UFGCBACCIQSRQRNQTUZY`b_goniec`gvyprqo{|vtwwmdhM����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������Ż�ֽ������ʰ����������������������������������������������������������ة�����������������������������������������������������������������������������������㺺����o�����������M 'AKJKLHKWORU^jjddhc^]\bpikpzyvstuuffZH2������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������ç����ν��ҹ�������������������������������������������������������������������������������������������������������������������������������������������������к�0 \�ҳ�������������k������������r�����`��YGZ���ra|ǛmiMRiufYHD?@FD>@@DHBHHLU`TUZhfrmhzveadiogilt{xppuocIC���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������±ea��ɺ���������������v����������ot�����l�Tae����p�̿qj_`l[TSGACkaD>EDSGIFXX[Yb`\aihhu|ufepqmhsu��wtsnjP:%��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������˷�����������������������������������������������������������������������������������������������������������������������������������������������������͹�} ����ŵ������������������������srz����}��Wad����ii��dd`fm^RKMG�ia\XEJGLPPW^TPffYVaiq~pkjgpyr��y��||{yqP8&�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������~�������̾Ż����������������������������������������������������������޺������������������������������������������������������������������������������������������҇����ʻ����������������������������ց��������^Tn|�x��PU�j]_^`V`V^TLOJA@ALMS[gj]TUQVUYTV_orn_ghom��{��~vfUL!��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������˾½��ļ�����������������������ۺ��������������������������������������������������������������������������������������������������������������������������᩻���˼Ƚ��������»���������������������v����m[|���p��SNT�bkb`Y\_RKIJHHI=PT^gqm[RPKNRV^`noqa]osyu������mVL)���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������ֵ���Ͻ������������������������ع��������������������������������������������������������������������������������������������������������������������������ٹ������Į������ý����������������������nu���zz���y_u�TNP\Q_a_eY\TJGIJJKHEUolk`RLIIJNUXaaY[Y`gquv��~��vhM'����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������֯����������������������������������������������ܿ�������������������������������������������������������������������������������������������������ķ�������ж�������������������������������{�����K��}tozjhmYi^}ly��h`VQGLNQDMJ[`X]ZVSGIPUWVSRYW\ky�����wfaK���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������[����������������������ȳ�������۳������������������������������������������������Դ������������������������������������������������������������������������������������������������`o����з���¶����Ȼ�������o��������������������s��zr�l`��^}nve�~�~j\TKMNNONEHU\`VZUORVTe_X[`dm~������m_R ��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������Չ�������������������qs��������������������������������������������������������������������������������������������������������������������������������������������������������������Ki��⿾��µ��͹�xu���v�����O������������¸������}���}yqhk���jpu�}�|je[YRMOX``PKL[VYRPMS\ec_jiw~�����eW%���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������٦��������������������eo����������������������������������������������������������������������������������������������������������������������ʺ������������������������������������^���ͷ�Ͳ���������������Ǫh�|�8�r��������ų�����������|rn|�������{{plifZVTW^hYUYSGZXTW[_jd_bpf�q�y����ya1���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������İ���������������������������������������Ǥ������������������������������������������������ڬ�������������������������������ɱ������������������������������������������������������ζ���s�����������������E(03)Y?#**;�����������Ѱ���������������ğ���}pdjaTSZ[WSOKOVWRZ\`kolldr��}�}qpqvkU�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������� \ No newline at end of file diff --git a/test_data/raster/png/png_from_stream.png b/test_data/raster/png/png_from_stream.png index 0d8ff3c1d..8c15e1b28 100644 Binary files a/test_data/raster/png/png_from_stream.png and b/test_data/raster/png/png_from_stream.png differ diff --git a/test_data/wms/gaussian_blur.png b/test_data/wms/gaussian_blur.png index 155a9b555..ee636edfd 100644 Binary files a/test_data/wms/gaussian_blur.png and b/test_data/wms/gaussian_blur.png differ diff --git a/test_data/wms/get_map_ndvi.png b/test_data/wms/get_map_ndvi.png index 8bd0cb2ed..586616cb5 100644 Binary files a/test_data/wms/get_map_ndvi.png and b/test_data/wms/get_map_ndvi.png differ diff --git a/test_data/wms/ne2_rgb_colorizer.png b/test_data/wms/ne2_rgb_colorizer.png index f980cfa68..4ccfe01c4 100644 Binary files a/test_data/wms/ne2_rgb_colorizer.png and b/test_data/wms/ne2_rgb_colorizer.png differ diff --git a/test_data/wms/ne2_rgb_colorizer_gray.png b/test_data/wms/ne2_rgb_colorizer_gray.png index a2ca1e8f4..db3901ad4 100644 Binary files a/test_data/wms/ne2_rgb_colorizer_gray.png and b/test_data/wms/ne2_rgb_colorizer_gray.png differ