Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Partial implementation of the MVT writer #181

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

[WIP] Partial implementation of the MVT writer #181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ criterion = "0.5.1"
csv = "1.2.2"
dbase = "0.4"
diesel = { version = "2.1.0", default-features = false, features = ["postgres"] }
dup-indexer = "0.3"
dup-indexer = "0.4"
env_logger = "0.10.0"
flatgeobuf = "3.27.0"
futures-util = "0.3.28"
Expand Down
17 changes: 9 additions & 8 deletions geozero/src/mvt/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ pub(crate) mod conversion {
use crate::error::Result;
use crate::mvt::vector_tile::tile;
use crate::mvt::MvtWriter;
use crate::GeozeroGeometry;
use crate::GeozeroDatasource;

/// Convert to MVT geometry.
pub trait ToMvt {
Expand All @@ -31,7 +31,7 @@ pub(crate) mod conversion {
/// * `extent` - Size of MVT tile in tile coordinate space (e.g. 4096).
/// * `left`, `bottom`, `right`, `top` - Bounds of tile in map coordinate space, with no buffer.
fn to_mvt(
&self,
&mut self,
extent: u32,
left: f64,
bottom: f64,
Expand All @@ -40,32 +40,33 @@ pub(crate) mod conversion {
) -> Result<tile::Feature>;

/// Convert to MVT geometry with geometries in unmodified tile coordinate space.
fn to_mvt_unscaled(&self) -> Result<tile::Feature>;
fn to_mvt_unscaled(&mut self) -> Result<tile::Feature>;
}

impl<T: GeozeroGeometry> ToMvt for T {
impl<T: GeozeroDatasource> ToMvt for T {
fn to_mvt(
&self,
&mut self,
extent: u32,
left: f64,
bottom: f64,
right: f64,
top: f64,
) -> Result<tile::Feature> {
let mut mvt = MvtWriter::new(extent, left, bottom, right, top);
self.process_geom(&mut mvt)?;
self.process(&mut mvt)?;
Ok(mvt.feature)
}

fn to_mvt_unscaled(&self) -> Result<tile::Feature> {
fn to_mvt_unscaled(&mut self) -> Result<tile::Feature> {
let mut mvt = MvtWriter::default();
self.process_geom(&mut mvt)?;
self.process(&mut mvt)?;
Ok(mvt.feature)
}
}
}

mod mvt_error;

pub use mvt_error::MvtError;

#[cfg(feature = "with-wkb")]
Expand Down
82 changes: 63 additions & 19 deletions geozero/src/mvt/mvt_writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@
use crate::error::Result;
use crate::mvt::mvt_commands::{Command, CommandInteger, ParameterInteger};
use crate::mvt::vector_tile::{tile, tile::GeomType};
use crate::GeomProcessor;
use crate::mvt::TagsBuilder;
use crate::{ColumnValue, FeatureProcessor, GeomProcessor, PropertyProcessor};

use super::mvt_error::MvtError;

/// Generator for MVT geometry type.
#[derive(Default, Debug)]
#[derive(Debug)]
pub struct MvtWriter {
pub(crate) feature: tile::Feature,
// Extent, 0 for unscaled
Expand All @@ -24,6 +25,25 @@ pub struct MvtWriter {
last_y: i32,
line_state: LineState,
is_multiline: bool,
tags_builder: TagsBuilder,
}

impl Default for MvtWriter {
fn default() -> Self {
Self {
feature: tile::Feature::default(),
extent: 4096,
left: 0.0,
bottom: 0.0,
x_multiplier: 1.0,
y_multiplier: 1.0,
last_x: 0,
last_y: 0,
line_state: LineState::None,
is_multiline: false,
tags_builder: TagsBuilder::new(),
}
}
}

#[derive(Default, Debug, PartialEq)]
Expand Down Expand Up @@ -175,6 +195,38 @@ impl GeomProcessor for MvtWriter {
}
}

impl PropertyProcessor for MvtWriter {
fn property(&mut self, _idx: usize, name: &str, value: &ColumnValue) -> Result<bool> {
self.tags_builder.insert_ref(
name,
value
.try_into()
.map_err(|_| MvtError::UnsupportedKeyValueType(name.to_string()))?,
);
// match colval {
// ColumnValue::Byte(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::UByte(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::Bool(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::Short(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::UShort(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::Int(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::UInt(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::Long(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::ULong(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::Float(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::Double(v) => write_num_prop(self.out, colname, &v)?,
// ColumnValue::String(v) | ColumnValue::DateTime(v) => {
// write_str_prop(self.out, colname, v)?;
// }
// ColumnValue::Json(_v) => (),
// ColumnValue::Binary(_v) => (),
// };
Ok(false)
}
}

impl FeatureProcessor for MvtWriter {}

#[cfg(test)]
mod test_mvt {
use super::*;
Expand Down Expand Up @@ -400,28 +452,29 @@ mod test {

#[test]
fn point_geom() {
let geojson = GeoJson(r#"{"type": "Point", "coordinates": [25, 17]}"#);
let mut geojson = GeoJson(r#"{"type": "Point", "coordinates": [25, 17]}"#);
let mvt = geojson.to_mvt_unscaled().unwrap();
assert_eq!(mvt.geometry, [9, 50, 34]);
}

#[test]
fn multipoint_geom() {
let geojson = GeoJson(r#"{"type": "MultiPoint", "coordinates": [[5, 7], [3, 2]]}"#);
let mut geojson = GeoJson(r#"{"type": "MultiPoint", "coordinates": [[5, 7], [3, 2]]}"#);
let mvt = geojson.to_mvt_unscaled().unwrap();
assert_eq!(mvt.geometry, [17, 10, 14, 3, 9]);
}

#[test]
fn line_geom() {
let geojson = GeoJson(r#"{"type": "LineString", "coordinates": [[2,2], [2,10], [10,10]]}"#);
let mut geojson =
GeoJson(r#"{"type": "LineString", "coordinates": [[2,2], [2,10], [10,10]]}"#);
let mvt = geojson.to_mvt_unscaled().unwrap();
assert_eq!(mvt.geometry, [9, 4, 4, 18, 0, 16, 16, 0]);
}

#[test]
fn multiline_geom() {
let geojson = GeoJson(
let mut geojson = GeoJson(
r#"{"type": "MultiLineString", "coordinates": [[[2,2], [2,10], [10,10]],[[1,1],[3,5]]]}"#,
);
let mvt = geojson.to_mvt_unscaled().unwrap();
Expand All @@ -433,7 +486,7 @@ mod test {

#[test]
fn polygon_geom() {
let geojson =
let mut geojson =
GeoJson(r#"{"type": "Polygon", "coordinates": [[[3, 6], [8, 12], [20, 34], [3, 6]]]}"#);
let mvt = geojson.to_mvt_unscaled().unwrap();
assert_eq!(mvt.geometry, [9, 6, 12, 18, 10, 12, 24, 44, 15]);
Expand All @@ -457,7 +510,7 @@ mod test {
]
]
}"#;
let geojson = GeoJson(geojson);
let mut geojson = GeoJson(geojson);
let mvt = geojson.to_mvt_unscaled().unwrap();
assert_eq!(
mvt.geometry,
Expand All @@ -473,7 +526,7 @@ mod test {
"type": "Polygon",
"coordinates": [[[34876,37618],[37047,39028],[37756,39484],[38779,40151],[39247,40451],[39601,40672],[40431,41182],[41010,41525],[41834,41995],[42190,42193],[42547,42387],[42540,42402],[42479,42516],[42420,42627],[42356,42749],[42344,42770],[42337,42784],[41729,42461],[40755,41926],[40118,41563],[39435,41161],[38968,40882],[38498,40595],[37200,39786],[36547,39382],[34547,38135],[34555,38122],[34595,38059],[34655,37964],[34726,37855],[34795,37745],[34863,37638],[34876,37618]]]
}"#;
let geojson = GeoJson(geojson);
let mut geojson = GeoJson(geojson);
let mvt = geojson.to_mvt_unscaled().unwrap();
assert_eq!(
mvt.geometry,
Expand All @@ -487,19 +540,10 @@ mod test {
);
}

#[test]
#[cfg(feature = "with-geo")]
fn geo_screen_coords_to_mvt() -> Result<()> {
let geo: geo_types::Geometry<f64> = geo_types::Point::new(25.0, 17.0).into();
let mvt = geo.to_mvt_unscaled()?;
assert_eq!(mvt.geometry, [9, 50, 34]);
Ok(())
}

#[test]
#[cfg(feature = "with-geo")]
fn geo_to_mvt() -> Result<()> {
let geo: geo_types::Geometry<f64> = geo_types::Point::new(960000.0, 6002729.0).into();
let mut geo = GeoJson(r#"{"type": "Point", "coordinates": [960000.0, 6002729.0]}"#);
let mvt = geo.to_mvt(256, 958826.08, 5987771.04, 978393.96, 6007338.92)?;
assert_eq!(mvt.geometry, [9, 30, 122]);
let geojson = mvt.to_json()?;
Expand Down
26 changes: 16 additions & 10 deletions geozero/src/mvt/tag_builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use crate::mvt::tile_value::TileValue;
use dup_indexer::{DupIndexer, PtrRead};
use std::hash::Hash;
use dup_indexer::{DupIndexer, DupIndexerRefs, PtrRead};

/// A builder for key-value pairs, where the key is a `String` or `&str`, and the value is a
/// [`TileValue`] enum which can hold any of the MVT value types.
Expand All @@ -17,37 +16,44 @@ use std::hash::Hash;
/// let (keys, values) = builder.into_tags();
/// # }
#[derive(Debug)]
pub struct TagsBuilder<K> {
keys: DupIndexer<K>,
pub struct TagsBuilder {
keys: DupIndexerRefs<String>,
values: DupIndexer<TileValue>,
}

/// This is safe because all values are either simple bit-readable values or strings,
/// both of which are safe for `PtrRead`.
unsafe impl PtrRead for TileValue {}

impl<K: Default + Eq + Hash + PtrRead> Default for TagsBuilder<K> {
impl Default for TagsBuilder {
fn default() -> Self {
Self::new()
}
}

impl<K: Eq + Hash + PtrRead> TagsBuilder<K> {
impl TagsBuilder {
pub fn new() -> Self {
Self {
keys: DupIndexer::new(),
keys: DupIndexerRefs::new(),
values: DupIndexer::new(),
}
}

pub fn insert(&mut self, key: K, value: TileValue) -> (u32, u32) {
pub fn insert(&mut self, key: String, value: TileValue) -> (u32, u32) {
(
self.keys.insert(key) as u32,
self.keys.insert_owned(key) as u32,
self.values.insert(value) as u32,
)
}

pub fn into_tags(self) -> (Vec<K>, Vec<TileValue>) {
pub fn insert_ref(&mut self, key: &str, value: TileValue) -> (u32, u32) {
(
self.keys.insert_ref(key) as u32,
self.values.insert(value) as u32,
)
}

pub fn into_tags(self) -> (Vec<String>, Vec<TileValue>) {
(self.keys.into_vec(), self.values.into_vec())
}
}
Expand Down
33 changes: 33 additions & 0 deletions geozero/src/mvt/tile_value.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::mvt::tile::Value;
use crate::ColumnValue;
use std::hash::Hash;

/// A wrapper for the MVT value types.
Expand Down Expand Up @@ -72,6 +73,38 @@ impl TryFrom<Value> for TileValue {
}
}

impl TryFrom<&ColumnValue<'_>> for TileValue {
type Error = ();

fn try_from(v: &ColumnValue) -> Result<Self, Self::Error> {
// string_value - ColumnValue::String
// float_value - ColumnValue::Float
// double_value - ColumnValue::Double
// int_value - ColumnValue::Long
// uint_value - ColumnValue::ULong
// sint_value - ColumnValue::Long
// bool_value - ColumnValue::Bool

Ok(match v {
ColumnValue::Byte(v) => TileValue::Sint(*v as i64),
ColumnValue::UByte(v) => TileValue::Uint(*v as u64),
ColumnValue::Bool(v) => TileValue::Bool(*v),
ColumnValue::Short(v) => TileValue::Sint(*v as i64),
ColumnValue::UShort(v) => TileValue::Uint(*v as u64),
ColumnValue::Int(v) => TileValue::Sint(*v as i64),
ColumnValue::UInt(v) => TileValue::Uint(*v as u64),
ColumnValue::Long(v) => TileValue::Sint(*v),
ColumnValue::ULong(v) => TileValue::Uint(*v),
ColumnValue::Float(v) => TileValue::Float(*v),
ColumnValue::Double(v) => TileValue::Double(*v),
ColumnValue::String(v) => TileValue::Str(v.to_string()),
ColumnValue::Json(v) => TileValue::Str(v.to_string()),
ColumnValue::DateTime(v) => TileValue::Str(v.to_string()),
ColumnValue::Binary(_) => Err(())?,
})
}
}

// Treat floats as bits so that we can use as keys.
// It is up to the users to ensure that the bits are not NaNs, or are consistent.

Expand Down
27 changes: 1 addition & 26 deletions geozero/tests/mvt.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,12 @@
use geozero::mvt::{Message, Tile};
use geozero::{
ColumnValue, CoordDimensions, FeatureProcessor, GeomProcessor, GeozeroDatasource,
PropertyProcessor, ToJson, ToMvt,
PropertyProcessor,
};
use serde_json::json;
use std::env;
use std::fmt::Write;
use std::sync::Mutex;

#[test]
fn geo_screen_coords_to_mvt() {
let geo: geo_types::Geometry<f64> = geo_types::Point::new(25.0, 17.0).into();
let mvt = geo.to_mvt_unscaled().unwrap();
assert_eq!(mvt.geometry, [9, 50, 34]);
}

#[test]
fn geo_to_mvt() {
let geo: geo_types::Geometry<f64> = geo_types::Point::new(960000.0, 6002729.0).into();
let mvt = geo
.to_mvt(256, 958826.08, 5987771.04, 978393.96, 6007338.92)
.unwrap();
assert_eq!(mvt.geometry, [9, 30, 122]);
let geojson = mvt.to_json().unwrap();
assert_eq!(
serde_json::from_str::<serde_json::Value>(&geojson).unwrap(),
json!({
"type": "Point",
"coordinates": [15,61]
})
);
}

type GzResult = geozero::error::Result<()>;

struct Proc {
Expand Down
Loading