Skip to content

Commit db4556d

Browse files
committed
feat(geoparquet): read and write GeoParquet 2.0 (rc.1) via GEOMETRY logical types
1 parent 0d8ea04 commit db4556d

12 files changed

Lines changed: 666 additions & 98 deletions

File tree

Cargo.lock

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

python/Cargo.lock

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

rust/geoparquet/CHANGELOG.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,13 @@
1111
- Dataset schemas are compared by field name and data type; nullability merges as "nullable in any file".
1212
- The dataset metadata merge is order-independent per column: `geometry_types` union, and columns declared by only some files carry over.
1313
- Covering shape checks moved to the row-filter path; row-group pruning and bounds work for flat top-level coverings.
14-
- New `GeoParquetVersion` enum: `GeoParquetWriterOptions` takes a target version (default 1.1). Native encodings and coverings require 1.1, M or ZM geometries 2.0. `GeoParquetMetadata::known_version` interprets the file's version string.
14+
- New `GeoParquetVersion` enum: `GeoParquetWriterOptions` takes a target version (default 1.1). Native encodings require 1.1, coverings 1.1 or later, M or ZM geometries 2.0. `GeoParquetMetadata::known_version` interprets the file's version string.
15+
- GeoParquet 2.0 support targets 2.0.0-rc.1; the 2.0 reader and writer behavior can change until the specification is final.
16+
- The reader accepts files with GEOMETRY and GEOGRAPHY logical types and no `geo` key, as GeoParquet 2.0 expects, synthesizing metadata from the logical types: WKB, unknown geometry types, CRS from the `crs` property, edges from the GEOGRAPHY algorithm.
17+
- Row-group pruning and bounds fall back to the column's native geospatial statistics when no covering is declared.
18+
- The dataset merge no longer compares version strings, so datasets written across a specification transition read.
19+
- The `edges` metadata maps all five 2.0 edge algorithms and survives a missing CRS.
20+
- The writer emits GeoParquet 2.0 on request: WKB with the GEOMETRY or GEOGRAPHY logical type, via `GeoParquetRecordBatchEncoder::target_parquet_schema` and `ArrowWriterOptions::with_parquet_schema`. The parquet `geospatial` feature is enabled.
1521

1622
## 0.7.0 - 2026-01-04
1723

rust/geoparquet/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ geo-types = { workspace = true }
2424
geoarrow-array = { workspace = true }
2525
geoarrow-schema = { workspace = true }
2626
indexmap = { workspace = true }
27-
parquet = { workspace = true, features = ["arrow"] }
27+
parquet = { workspace = true, features = ["arrow", "geospatial"] }
2828
serde = { workspace = true, features = ["derive"] }
2929
serde_with = { workspace = true }
3030
serde_json = { workspace = true }

rust/geoparquet/src/metadata.rs

Lines changed: 211 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,18 @@ use geoarrow_schema::{
1414
LineStringType, Metadata, MultiLineStringType, MultiPointType, MultiPolygonType, PointType,
1515
PolygonType,
1616
};
17-
use parquet::file::metadata::FileMetaData;
17+
use parquet::basic::{EdgeInterpolationAlgorithm, LogicalType};
18+
use parquet::file::metadata::{FileMetaData, KeyValue};
19+
use parquet::schema::types::SchemaDescriptor;
1820
use serde::{Deserialize, Serialize};
1921
use serde_json::Value;
2022
use serde_with::{DeserializeFromStr, SerializeDisplay};
2123

2224
use crate::writer::GeoParquetWriterEncoding;
2325

26+
// https://github.com/geoarrow/geoarrow-rs/pull/1159#issuecomment-2904610370
27+
pub(crate) const INFERRED_PRIMARY_COLUMN_NAMES: [&str; 2] = ["geometry", "geography"];
28+
2429
/// The actual encoding of the geometry in the Parquet file.
2530
///
2631
/// In contrast to the _user-specified API_, which is just "WKB" or "Native", here we need to know
@@ -531,6 +536,64 @@ impl GeoParquetMetadata {
531536
GeoParquetVersion::from_metadata_string(&self.version)
532537
}
533538

539+
/// Synthesize metadata from the Parquet GEOMETRY and GEOGRAPHY logical types.
540+
///
541+
/// GeoParquet 2.0 expects readers to read files that carry only these types and no `geo`
542+
/// key. Each top-level geometry-typed column becomes a WKB column with unknown geometry
543+
/// types. Returns `None` when no such column exists.
544+
///
545+
/// GeoParquet 2.0 is at release candidate 2.0.0-rc.1; this behavior can change until the
546+
/// specification is final.
547+
pub fn from_logical_types(
548+
parquet_schema: &SchemaDescriptor,
549+
key_value_metadata: Option<&Vec<KeyValue>>,
550+
) -> Option<GeoArrowResult<Self>> {
551+
let mut columns: HashMap<String, GeoParquetColumnMetadata> = HashMap::new();
552+
for field in parquet_schema.root_schema().get_fields() {
553+
let Some(logical_type) = field.get_basic_info().logical_type_ref() else {
554+
continue;
555+
};
556+
let (crs, edges) = match logical_type {
557+
LogicalType::Geometry(geometry) => (geometry.crs.clone(), None),
558+
LogicalType::Geography(geography) => {
559+
let edges = match geography.algorithm().map(edges_name_for_algorithm) {
560+
Some(Ok(name)) => Some(name),
561+
Some(Err(err)) => return Some(Err(err)),
562+
None => None,
563+
};
564+
(geography.crs.clone(), edges)
565+
}
566+
_ => continue,
567+
};
568+
let column = match synthesized_column(crs.as_deref(), edges, key_value_metadata) {
569+
Ok(column) => column,
570+
Err(err) => return Some(Err(err)),
571+
};
572+
columns.insert(field.name().to_string(), column);
573+
}
574+
575+
if columns.is_empty() {
576+
return None;
577+
}
578+
579+
let primary_column = INFERRED_PRIMARY_COLUMN_NAMES
580+
.iter()
581+
.find(|name| columns.contains_key(**name))
582+
.map(|name| name.to_string())
583+
.unwrap_or_else(|| {
584+
let mut names: Vec<&String> = columns.keys().collect();
585+
names.sort();
586+
names[0].clone()
587+
});
588+
589+
Some(Ok(Self {
590+
// Files carrying the geospatial logical types belong to the 2.0 ecosystem.
591+
version: GeoParquetVersion::V2_0.as_str().to_string(),
592+
primary_column,
593+
columns,
594+
}))
595+
}
596+
534597
/// Merge another file's metadata into this one
535598
///
536599
/// Expands each column's bbox, unions its geometry types, and carries over columns only
@@ -584,12 +647,9 @@ impl GeoParquetMetadata {
584647

585648
/// Assert that this metadata is compatible with another metadata instance, erroring if not
586649
pub fn try_compatible_with(&self, other: &GeoParquetMetadata) -> GeoArrowResult<()> {
587-
if self.version.as_str() != other.version.as_str() {
588-
return Err(GeoArrowError::GeoParquet(
589-
"Different GeoParquet versions".to_string(),
590-
));
591-
}
592-
650+
// The version string is deliberately not compared: a dataset written across a spec
651+
// transition (1.1 files next to 2.0 files) merges on the per-column metadata, and the
652+
// merged result keeps the first-seen version string.
593653
if self.primary_column.as_str() != other.primary_column.as_str() {
594654
return Err(GeoArrowError::GeoParquet(
595655
"Different GeoParquet primary columns".to_string(),
@@ -678,7 +738,8 @@ pub enum GeoParquetVersion {
678738
V1_1,
679739
/// GeoParquet 2.0.0
680740
///
681-
/// The 2.0 specification is a release candidate. Writing 2.0 output is not implemented yet.
741+
/// The 2.0 specification is at release candidate 2.0.0-rc.1; this crate's 2.0 behavior can
742+
/// change until it is final.
682743
V2_0,
683744
}
684745

@@ -889,23 +950,104 @@ impl GeoParquetColumnMetadata {
889950

890951
impl From<GeoParquetColumnMetadata> for Metadata {
891952
fn from(value: GeoParquetColumnMetadata) -> Self {
892-
let edges = if let Some(edges) = value.edges {
893-
if edges.as_str() == "spherical" {
894-
Some(Edges::Spherical)
895-
} else {
896-
None
897-
}
898-
} else {
899-
None
953+
let edges = value.edges.as_deref().and_then(edges_from_name);
954+
let crs = match value.crs {
955+
// A JSON string appears only in metadata synthesized from the Parquet logical
956+
// types, where it carries an authority code.
957+
Some(Value::String(authority_code)) => Crs::from_authority_code(authority_code),
958+
Some(projjson) => Crs::from_projjson(projjson),
959+
None => Crs::default(),
900960
};
901-
if let Some(crs) = value.crs {
902-
Metadata::new(Crs::from_projjson(crs), edges)
903-
} else {
904-
Metadata::default()
905-
}
961+
Metadata::new(crs, edges)
962+
}
963+
}
964+
965+
/// The `geo` metadata `edges` names, aligned with the Parquet edge interpolation algorithms.
966+
fn edges_from_name(name: &str) -> Option<Edges> {
967+
match name {
968+
"spherical" => Some(Edges::Spherical),
969+
"vincenty" => Some(Edges::Vincenty),
970+
"thomas" => Some(Edges::Thomas),
971+
"andoyer" => Some(Edges::Andoyer),
972+
"karney" => Some(Edges::Karney),
973+
_ => None,
906974
}
907975
}
908976

977+
fn edges_name_for_algorithm(algorithm: EdgeInterpolationAlgorithm) -> GeoArrowResult<&'static str> {
978+
match algorithm {
979+
EdgeInterpolationAlgorithm::SPHERICAL => Ok("spherical"),
980+
EdgeInterpolationAlgorithm::VINCENTY => Ok("vincenty"),
981+
EdgeInterpolationAlgorithm::THOMAS => Ok("thomas"),
982+
EdgeInterpolationAlgorithm::ANDOYER => Ok("andoyer"),
983+
EdgeInterpolationAlgorithm::KARNEY => Ok("karney"),
984+
// Reading an unknown algorithm as planar would silently change geometry semantics.
985+
other => Err(GeoArrowError::GeoParquet(format!(
986+
"Unknown edge interpolation algorithm: {other:?}"
987+
))),
988+
}
989+
}
990+
991+
fn synthesized_column(
992+
crs: Option<&str>,
993+
edges: Option<&str>,
994+
key_value_metadata: Option<&Vec<KeyValue>>,
995+
) -> GeoArrowResult<GeoParquetColumnMetadata> {
996+
let mut column = serde_json::json!({
997+
"encoding": "WKB",
998+
// The empty list explicitly signals that the geometry types are not known.
999+
"geometry_types": [],
1000+
});
1001+
if let Some(crs_value) = parquet_crs_to_geo_crs(crs, key_value_metadata)? {
1002+
column["crs"] = crs_value;
1003+
}
1004+
if let Some(edges) = edges {
1005+
column["edges"] = Value::String(edges.to_string());
1006+
}
1007+
serde_json::from_value(column).map_err(|err| GeoArrowError::GeoParquet(err.to_string()))
1008+
}
1009+
1010+
/// Interpret the Parquet logical-type `crs` property.
1011+
///
1012+
/// Four forms: inline PROJJSON, `projjson:<key>` naming a file metadata key that holds
1013+
/// PROJJSON, `srid:<identifier>`, and `<authority>:<code>`. An `srid:` identifier is not
1014+
/// resolvable without a CRS database and maps to no CRS; an authority code stays a JSON
1015+
/// string for the [`Metadata`] conversion.
1016+
fn parquet_crs_to_geo_crs(
1017+
crs: Option<&str>,
1018+
key_value_metadata: Option<&Vec<KeyValue>>,
1019+
) -> GeoArrowResult<Option<Value>> {
1020+
// An absent crs property means OGC:CRS84, which is also the `geo` metadata default.
1021+
let Some(crs) = crs else { return Ok(None) };
1022+
let crs = crs.trim();
1023+
if crs.starts_with('{') {
1024+
let value = serde_json::from_str(crs).map_err(|err| {
1025+
GeoArrowError::GeoParquet(format!("Invalid PROJJSON in Parquet crs property: {err}"))
1026+
})?;
1027+
return Ok(Some(value));
1028+
}
1029+
if let Some(key) = crs.strip_prefix("projjson:") {
1030+
let value = key_value_metadata
1031+
.and_then(|kvs| kvs.iter().find(|kv| kv.key == key))
1032+
.and_then(|kv| kv.value.as_deref())
1033+
.ok_or_else(|| {
1034+
GeoArrowError::GeoParquet(format!(
1035+
"Parquet crs property references missing file metadata key {key}"
1036+
))
1037+
})?;
1038+
let value = serde_json::from_str(value).map_err(|err| {
1039+
GeoArrowError::GeoParquet(format!(
1040+
"Invalid PROJJSON under file metadata key {key}: {err}"
1041+
))
1042+
})?;
1043+
return Ok(Some(value));
1044+
}
1045+
if crs.strip_prefix("srid:").is_some() {
1046+
return Ok(None);
1047+
}
1048+
Ok(Some(Value::String(crs.to_string())))
1049+
}
1050+
9091051
// TODO: deduplicate with `resolve_types` in `downcast.rs`
9101052
pub(crate) fn infer_geo_data_type(
9111053
geometry_types: &HashSet<GeoParquetGeometryTypeAndDimension>,
@@ -1043,6 +1185,54 @@ mod test {
10431185
assert_eq!(serde_json::to_value(&bbox).unwrap(), xyzm);
10441186
}
10451187

1188+
#[test]
1189+
fn logical_types_synthesize_geo_metadata() {
1190+
let descr = crate::test::geometry_schema_descr(LogicalType::geography(
1191+
Some("EPSG:32633".to_string()),
1192+
Some(EdgeInterpolationAlgorithm::KARNEY),
1193+
));
1194+
let meta = GeoParquetMetadata::from_logical_types(&descr, None)
1195+
.unwrap()
1196+
.unwrap();
1197+
assert_eq!(meta.version, "2.0.0");
1198+
assert_eq!(meta.primary_column, "geometry");
1199+
let column = &meta.columns["geometry"];
1200+
assert!(matches!(column.encoding, GeoParquetColumnEncoding::WKB));
1201+
assert!(column.geometry_types.is_empty());
1202+
assert_eq!(column.edges.as_deref(), Some("karney"));
1203+
1204+
let geoarrow_meta = Metadata::from(column.clone());
1205+
assert_eq!(geoarrow_meta.edges(), Some(Edges::Karney));
1206+
assert_eq!(
1207+
geoarrow_meta.crs(),
1208+
&Crs::from_authority_code("EPSG:32633".to_string())
1209+
);
1210+
1211+
let descr = crate::test::schema_descr("message schema { required binary name; }");
1212+
assert!(GeoParquetMetadata::from_logical_types(&descr, None).is_none());
1213+
}
1214+
1215+
#[test]
1216+
fn parquet_crs_property_forms() {
1217+
let inline = parquet_crs_to_geo_crs(Some(r#"{"type": "GeographicCRS"}"#), None).unwrap();
1218+
assert_eq!(inline, Some(serde_json::json!({"type": "GeographicCRS"})));
1219+
1220+
let kv = vec![KeyValue::new(
1221+
"my_crs".to_string(),
1222+
r#"{"a": 1}"#.to_string(),
1223+
)];
1224+
let referenced = parquet_crs_to_geo_crs(Some("projjson:my_crs"), Some(&kv)).unwrap();
1225+
assert_eq!(referenced, Some(serde_json::json!({"a": 1})));
1226+
assert!(parquet_crs_to_geo_crs(Some("projjson:absent"), Some(&kv)).is_err());
1227+
1228+
assert_eq!(parquet_crs_to_geo_crs(Some("srid:0"), None).unwrap(), None);
1229+
assert_eq!(parquet_crs_to_geo_crs(None, None).unwrap(), None);
1230+
assert_eq!(
1231+
parquet_crs_to_geo_crs(Some("EPSG:4326"), None).unwrap(),
1232+
Some(Value::String("EPSG:4326".to_string()))
1233+
);
1234+
}
1235+
10461236
#[test]
10471237
fn version_strings_map_to_known_versions() {
10481238
use GeoParquetVersion::*;

0 commit comments

Comments
 (0)