Skip to content

Commit 697abce

Browse files
wirybeaverdannycjonesCTTY
authored
feat: recognize geometry and geography types [1/3] (#2933)
feat: recognize geometry and geography types Co-authored-by: Danny C Jones <4976211+dannycjones@users.noreply.github.com> Co-authored-by: Shawn Chang <42792772+CTTY@users.noreply.github.com>
1 parent 8b59642 commit 697abce

18 files changed

Lines changed: 735 additions & 15 deletions

File tree

‎crates/catalog/glue/src/schema.rs‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,10 @@ impl SchemaVisitor for GlueSchemaBuilder {
148148
}
149149

150150
fn primitive(&mut self, p: &PrimitiveType) -> Result<Self::T> {
151+
// Glue column types are informational for Iceberg tables; readers use Iceberg metadata as
152+
// the source of truth. Keep the existing Glue-compatible spellings and use the Iceberg
153+
// display representation for geospatial types. A non-failing fallback for all other types
154+
// is tracked in https://github.com/apache/iceberg-rust/issues/3375.
151155
let glue_type = match p {
152156
PrimitiveType::Unknown => {
153157
return Err(Error::new(
@@ -173,6 +177,7 @@ impl SchemaVisitor for GlueSchemaBuilder {
173177
"string".to_string()
174178
}
175179
PrimitiveType::Binary | PrimitiveType::Fixed(_) => "binary".to_string(),
180+
PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => p.to_string(),
176181
PrimitiveType::Decimal { precision, scale } => {
177182
format!("decimal({precision},{scale})")
178183
}
@@ -189,7 +194,9 @@ impl SchemaVisitor for GlueSchemaBuilder {
189194
#[cfg(test)]
190195
mod tests {
191196
use iceberg::TableCreation;
192-
use iceberg::spec::{FormatVersion, Schema, TableMetadataBuilder};
197+
use iceberg::spec::{
198+
FormatVersion, GeometryType, NestedField, PrimitiveType, Schema, TableMetadataBuilder, Type,
199+
};
193200

194201
use super::*;
195202

@@ -560,4 +567,26 @@ mod tests {
560567
assert_eq!(result, expected);
561568
Ok(())
562569
}
570+
571+
#[test]
572+
fn test_schema_with_geospatial_type() {
573+
let schema = Schema::builder()
574+
.with_fields(vec![
575+
NestedField::required(
576+
1,
577+
"geom",
578+
Type::Primitive(PrimitiveType::Geometry(GeometryType::default())),
579+
)
580+
.into(),
581+
])
582+
.build()
583+
.unwrap();
584+
let metadata = create_metadata_with_format_version(schema, FormatVersion::V3).unwrap();
585+
586+
let result = GlueSchemaBuilder::from_iceberg(&metadata).unwrap().build();
587+
588+
assert_eq!(result, vec![
589+
create_column("geom", "geometry(OGC:CRS84)", "1", false,).unwrap()
590+
]);
591+
}
563592
}

‎crates/catalog/hms/src/schema.rs‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,12 @@ impl SchemaVisitor for HiveSchemaBuilder {
124124
"string".to_string()
125125
}
126126
PrimitiveType::Binary | PrimitiveType::Fixed(_) => "binary".to_string(),
127+
PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => {
128+
return Err(Error::new(
129+
ErrorKind::FeatureUnsupported,
130+
format!("Conversion from {p:?} is not supported"),
131+
));
132+
}
127133
PrimitiveType::Decimal { precision, scale } => {
128134
format!("decimal({precision},{scale})")
129135
}
@@ -142,7 +148,7 @@ impl SchemaVisitor for HiveSchemaBuilder {
142148
#[cfg(test)]
143149
mod tests {
144150
use iceberg::Result;
145-
use iceberg::spec::Schema;
151+
use iceberg::spec::{GeometryType, NestedField, PrimitiveType, Schema, Type};
146152

147153
use super::*;
148154

@@ -486,4 +492,22 @@ mod tests {
486492

487493
Ok(())
488494
}
495+
496+
#[test]
497+
fn test_schema_with_geospatial_type_is_unsupported() {
498+
let schema = Schema::builder()
499+
.with_fields(vec![
500+
NestedField::required(
501+
1,
502+
"geom",
503+
Type::Primitive(PrimitiveType::Geometry(GeometryType::default())),
504+
)
505+
.into(),
506+
])
507+
.build()
508+
.unwrap();
509+
510+
let err = HiveSchemaBuilder::from_iceberg(&schema).unwrap_err();
511+
assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}");
512+
}
489513
}

‎crates/iceberg/public-api.txt‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1490,6 +1490,71 @@ pub fn iceberg::sensitive::SensitiveString::fmt(&self, f: &mut core::fmt::Format
14901490
impl core::marker::StructuralPartialEq for iceberg::sensitive::SensitiveString
14911491
pub mod iceberg::spec
14921492
pub use iceberg::spec::ByteBuf
1493+
pub mod iceberg::spec::geospatial
1494+
pub enum iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1495+
pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Andoyer
1496+
pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Karney
1497+
pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Spherical
1498+
pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Thomas
1499+
pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Vincenty
1500+
impl core::clone::Clone for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1501+
pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::clone(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1502+
impl core::cmp::Eq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1503+
impl core::cmp::PartialEq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1504+
pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::eq(&self, other: &iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> bool
1505+
impl core::default::Default for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1506+
pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::default() -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1507+
impl core::fmt::Debug for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1508+
pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
1509+
impl core::hash::Hash for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1510+
pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::hash<__H: core::hash::Hasher>(&self, state: &mut __H)
1511+
impl core::marker::Copy for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1512+
impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1513+
impl serde_core::ser::Serialize for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1514+
pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
1515+
impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1516+
pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
1517+
pub struct iceberg::spec::geospatial::GeographyType
1518+
impl iceberg::spec::geospatial::GeographyType
1519+
pub fn iceberg::spec::geospatial::GeographyType::algorithm(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm
1520+
pub fn iceberg::spec::geospatial::GeographyType::crs(&self) -> &str
1521+
pub fn iceberg::spec::geospatial::GeographyType::new(crs: core::option::Option<alloc::string::String>, algorithm: iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> iceberg::Result<Self>
1522+
impl core::clone::Clone for iceberg::spec::geospatial::GeographyType
1523+
pub fn iceberg::spec::geospatial::GeographyType::clone(&self) -> iceberg::spec::geospatial::GeographyType
1524+
impl core::cmp::Eq for iceberg::spec::geospatial::GeographyType
1525+
impl core::cmp::PartialEq for iceberg::spec::geospatial::GeographyType
1526+
pub fn iceberg::spec::geospatial::GeographyType::eq(&self, other: &iceberg::spec::geospatial::GeographyType) -> bool
1527+
impl core::default::Default for iceberg::spec::geospatial::GeographyType
1528+
pub fn iceberg::spec::geospatial::GeographyType::default() -> Self
1529+
impl core::fmt::Debug for iceberg::spec::geospatial::GeographyType
1530+
pub fn iceberg::spec::geospatial::GeographyType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
1531+
impl core::hash::Hash for iceberg::spec::geospatial::GeographyType
1532+
pub fn iceberg::spec::geospatial::GeographyType::hash<__H: core::hash::Hasher>(&self, state: &mut __H)
1533+
impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeographyType
1534+
impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeographyType
1535+
pub fn iceberg::spec::geospatial::GeographyType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
1536+
impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeographyType
1537+
pub fn iceberg::spec::geospatial::GeographyType::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
1538+
pub struct iceberg::spec::geospatial::GeometryType
1539+
impl iceberg::spec::geospatial::GeometryType
1540+
pub fn iceberg::spec::geospatial::GeometryType::crs(&self) -> &str
1541+
pub fn iceberg::spec::geospatial::GeometryType::new(crs: core::option::Option<alloc::string::String>) -> iceberg::Result<Self>
1542+
impl core::clone::Clone for iceberg::spec::geospatial::GeometryType
1543+
pub fn iceberg::spec::geospatial::GeometryType::clone(&self) -> iceberg::spec::geospatial::GeometryType
1544+
impl core::cmp::Eq for iceberg::spec::geospatial::GeometryType
1545+
impl core::cmp::PartialEq for iceberg::spec::geospatial::GeometryType
1546+
pub fn iceberg::spec::geospatial::GeometryType::eq(&self, other: &iceberg::spec::geospatial::GeometryType) -> bool
1547+
impl core::default::Default for iceberg::spec::geospatial::GeometryType
1548+
pub fn iceberg::spec::geospatial::GeometryType::default() -> iceberg::spec::geospatial::GeometryType
1549+
impl core::fmt::Debug for iceberg::spec::geospatial::GeometryType
1550+
pub fn iceberg::spec::geospatial::GeometryType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
1551+
impl core::hash::Hash for iceberg::spec::geospatial::GeometryType
1552+
pub fn iceberg::spec::geospatial::GeometryType::hash<__H: core::hash::Hasher>(&self, state: &mut __H)
1553+
impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeometryType
1554+
impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeometryType
1555+
pub fn iceberg::spec::geospatial::GeometryType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
1556+
impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeometryType
1557+
pub fn iceberg::spec::geospatial::GeometryType::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
14931558
pub enum iceberg::spec::DataContentType
14941559
pub iceberg::spec::DataContentType::Data = 0
14951560
pub iceberg::spec::DataContentType::EqualityDeletes = 2
@@ -1735,6 +1800,8 @@ pub iceberg::spec::PrimitiveType::Decimal::scale: u32
17351800
pub iceberg::spec::PrimitiveType::Double
17361801
pub iceberg::spec::PrimitiveType::Fixed(u64)
17371802
pub iceberg::spec::PrimitiveType::Float
1803+
pub iceberg::spec::PrimitiveType::Geography(iceberg::spec::geospatial::GeographyType)
1804+
pub iceberg::spec::PrimitiveType::Geometry(iceberg::spec::geospatial::GeometryType)
17381805
pub iceberg::spec::PrimitiveType::Int
17391806
pub iceberg::spec::PrimitiveType::Long
17401807
pub iceberg::spec::PrimitiveType::String
@@ -2112,6 +2179,47 @@ impl serde_core::ser::Serialize for iceberg::spec::FieldSummary
21122179
pub fn iceberg::spec::FieldSummary::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
21132180
impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::FieldSummary
21142181
pub fn iceberg::spec::FieldSummary::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
2182+
pub struct iceberg::spec::GeographyType
2183+
impl iceberg::spec::geospatial::GeographyType
2184+
pub fn iceberg::spec::geospatial::GeographyType::algorithm(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm
2185+
pub fn iceberg::spec::geospatial::GeographyType::crs(&self) -> &str
2186+
pub fn iceberg::spec::geospatial::GeographyType::new(crs: core::option::Option<alloc::string::String>, algorithm: iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> iceberg::Result<Self>
2187+
impl core::clone::Clone for iceberg::spec::geospatial::GeographyType
2188+
pub fn iceberg::spec::geospatial::GeographyType::clone(&self) -> iceberg::spec::geospatial::GeographyType
2189+
impl core::cmp::Eq for iceberg::spec::geospatial::GeographyType
2190+
impl core::cmp::PartialEq for iceberg::spec::geospatial::GeographyType
2191+
pub fn iceberg::spec::geospatial::GeographyType::eq(&self, other: &iceberg::spec::geospatial::GeographyType) -> bool
2192+
impl core::default::Default for iceberg::spec::geospatial::GeographyType
2193+
pub fn iceberg::spec::geospatial::GeographyType::default() -> Self
2194+
impl core::fmt::Debug for iceberg::spec::geospatial::GeographyType
2195+
pub fn iceberg::spec::geospatial::GeographyType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
2196+
impl core::hash::Hash for iceberg::spec::geospatial::GeographyType
2197+
pub fn iceberg::spec::geospatial::GeographyType::hash<__H: core::hash::Hasher>(&self, state: &mut __H)
2198+
impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeographyType
2199+
impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeographyType
2200+
pub fn iceberg::spec::geospatial::GeographyType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
2201+
impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeographyType
2202+
pub fn iceberg::spec::geospatial::GeographyType::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
2203+
pub struct iceberg::spec::GeometryType
2204+
impl iceberg::spec::geospatial::GeometryType
2205+
pub fn iceberg::spec::geospatial::GeometryType::crs(&self) -> &str
2206+
pub fn iceberg::spec::geospatial::GeometryType::new(crs: core::option::Option<alloc::string::String>) -> iceberg::Result<Self>
2207+
impl core::clone::Clone for iceberg::spec::geospatial::GeometryType
2208+
pub fn iceberg::spec::geospatial::GeometryType::clone(&self) -> iceberg::spec::geospatial::GeometryType
2209+
impl core::cmp::Eq for iceberg::spec::geospatial::GeometryType
2210+
impl core::cmp::PartialEq for iceberg::spec::geospatial::GeometryType
2211+
pub fn iceberg::spec::geospatial::GeometryType::eq(&self, other: &iceberg::spec::geospatial::GeometryType) -> bool
2212+
impl core::default::Default for iceberg::spec::geospatial::GeometryType
2213+
pub fn iceberg::spec::geospatial::GeometryType::default() -> iceberg::spec::geospatial::GeometryType
2214+
impl core::fmt::Debug for iceberg::spec::geospatial::GeometryType
2215+
pub fn iceberg::spec::geospatial::GeometryType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
2216+
impl core::hash::Hash for iceberg::spec::geospatial::GeometryType
2217+
pub fn iceberg::spec::geospatial::GeometryType::hash<__H: core::hash::Hasher>(&self, state: &mut __H)
2218+
impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeometryType
2219+
impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeometryType
2220+
pub fn iceberg::spec::geospatial::GeometryType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
2221+
impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeometryType
2222+
pub fn iceberg::spec::geospatial::GeometryType::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
21152223
pub struct iceberg::spec::ListType
21162224
pub iceberg::spec::ListType::element_field: iceberg::spec::NestedFieldRef
21172225
impl iceberg::spec::ListType

‎crates/iceberg/src/arrow/schema.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -777,6 +777,10 @@ impl SchemaVisitor for ToArrowSchemaConverter {
777777
.unwrap_or(DataType::LargeBinary),
778778
)),
779779
PrimitiveType::Binary => Ok(ArrowSchemaOrFieldOrType::Type(DataType::LargeBinary)),
780+
PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => Err(Error::new(
781+
ErrorKind::FeatureUnsupported,
782+
format!("Converting {p} to an Arrow schema is not supported yet"),
783+
)),
780784
}
781785
}
782786

@@ -1246,6 +1250,9 @@ pub(crate) fn primitive_type_to_arrow_type_with_ree(primitive_type: &PrimitiveTy
12461250
PrimitiveType::Uuid => make_ree(DataType::Binary),
12471251
PrimitiveType::Fixed(_) => make_ree(DataType::Binary),
12481252
PrimitiveType::Binary => make_ree(DataType::Binary),
1253+
// Match the LargeBinary storage used by schema_to_arrow_schema in the read layer.
1254+
// Its 64-bit offsets cover the array's total WKB bytes, not only one geometry's size.
1255+
PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => make_ree(DataType::LargeBinary),
12491256
PrimitiveType::Decimal { precision, scale } => {
12501257
make_ree(DataType::Decimal128(*precision as u8, *scale as i8))
12511258
}
@@ -2371,6 +2378,24 @@ mod tests {
23712378
);
23722379
}
23732380

2381+
#[test]
2382+
fn test_geospatial_arrow_schema_conversion_is_unsupported() {
2383+
let schema = Schema::builder()
2384+
.with_fields(vec![
2385+
NestedField::optional(
2386+
1,
2387+
"geom",
2388+
Type::Primitive(PrimitiveType::Geometry(Default::default())),
2389+
)
2390+
.into(),
2391+
])
2392+
.build()
2393+
.unwrap();
2394+
2395+
let err = schema_to_arrow_schema(&schema).unwrap_err();
2396+
assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}");
2397+
}
2398+
23742399
#[test]
23752400
fn test_type_conversion() {
23762401
// test primitive type

‎crates/iceberg/src/arrow/value.rs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,10 @@ impl SchemaWithPartnerVisitor<ArrayRef> for ArrowArrayToIcebergStructConverter {
346346
Err(invalid_data!("The partner is not a binary array"))
347347
}
348348
}
349+
PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => Err(Error::new(
350+
ErrorKind::FeatureUnsupported,
351+
format!("Converting {p} Arrow array to an Iceberg literal is not supported yet"),
352+
)),
349353
}
350354
}
351355

@@ -1106,6 +1110,31 @@ mod test {
11061110
);
11071111
}
11081112

1113+
#[test]
1114+
fn test_arrow_geospatial_to_literal_is_unsupported() {
1115+
let values = Arc::new(LargeBinaryArray::from_vec(vec![b"wkb".as_slice()])) as ArrayRef;
1116+
let struct_array = Arc::new(StructArray::from(vec![(
1117+
Arc::new(
1118+
Field::new("geom", DataType::LargeBinary, false).with_metadata(HashMap::from([(
1119+
PARQUET_FIELD_ID_META_KEY.to_string(),
1120+
"1".to_string(),
1121+
)])),
1122+
),
1123+
values,
1124+
)])) as ArrayRef;
1125+
let ty = StructType::new(vec![
1126+
NestedField::required(
1127+
1,
1128+
"geom",
1129+
Type::Primitive(PrimitiveType::Geometry(Default::default())),
1130+
)
1131+
.into(),
1132+
]);
1133+
1134+
let err = arrow_struct_to_literal(&struct_array, &ty).unwrap_err();
1135+
assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}");
1136+
}
1137+
11091138
#[test]
11101139
fn test_find_field_by_id() {
11111140
// Create Arrow arrays for the nested structure

0 commit comments

Comments
 (0)