diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index c53593be74..566190b3dc 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -858,6 +858,7 @@ mod tests { use datafusion_proto::physical_plan::AsExecutionPlan; use datafusion_proto::protobuf; use delta_kernel::path::{LogPathFileType, ParsedLogPath}; + use delta_kernel::schema::ArrayType; use futures::{stream::BoxStream, StreamExt}; use object_store::ObjectMeta; use object_store::{ @@ -1850,6 +1851,49 @@ mod tests { assert_eq!(expected, actual); } + #[tokio::test] + async fn test_push_down_filter_panic_2602() -> DeltaResult<()> { + use crate::kernel::schema::{DataType, PrimitiveType}; + let ctx = SessionContext::new(); + let table = crate::DeltaOps::new_in_memory() + .create() + .with_column("id", DataType::Primitive(PrimitiveType::Long), true, None) + .with_column( + "name", + DataType::Primitive(PrimitiveType::String), + true, + None, + ) + .with_column("b", DataType::Primitive(PrimitiveType::Boolean), true, None) + .with_column( + "ts", + DataType::Primitive(PrimitiveType::Timestamp), + true, + None, + ) + .with_column("dt", DataType::Primitive(PrimitiveType::Date), true, None) + .with_column( + "zap", + DataType::Array(Box::new(ArrayType::new( + DataType::Primitive(PrimitiveType::Boolean), + true, + ))), + true, + None, + ) + .await?; + + ctx.register_table("snapshot", Arc::new(table)).unwrap(); + + let df = ctx + .sql("select * from snapshot where id > 10000 and id < 20000") + .await + .unwrap(); + + let _ = df.collect().await?; + Ok(()) + } + /// Records operations made by the inner object store on a channel obtained at construction struct RecordingObjectStore { inner: ObjectStoreRef, diff --git a/python/tests/test_writer.py b/python/tests/test_writer.py index cff8e01015..43126adaa6 100644 --- a/python/tests/test_writer.py +++ b/python/tests/test_writer.py @@ -2518,10 +2518,11 @@ def test_dots_in_column_names_2624(tmp_path: pathlib.Path): """ import pyarrow as pa + initial = pa.Table.from_pydict( { - "Product.Id": ['x-0', 'x-1', 'x-2', 'x-3'], - 'Cost' : [10, 11, 12, 13], + "Product.Id": ["x-0", "x-1", "x-2", "x-3"], + "Cost": [10, 11, 12, 13], } ) @@ -2533,8 +2534,8 @@ def test_dots_in_column_names_2624(tmp_path: pathlib.Path): update = pa.Table.from_pydict( { - "Product.Id": ['x-1'], - 'Cost' : [101], + "Product.Id": ["x-1"], + "Cost": [101], } ) @@ -2543,16 +2544,16 @@ def test_dots_in_column_names_2624(tmp_path: pathlib.Path): data=update, partition_by=["Product.Id"], mode="overwrite", - predicate="\"Product.Id\" = 'x-1'" + predicate="\"Product.Id\" = 'x-1'", ) dt = DeltaTable(tmp_path) expected = pa.Table.from_pydict( { - "Product.Id": ['x-0', 'x-1', 'x-2', 'x-3'], - 'Cost' : [10, 101, 12, 13], + "Product.Id": ["x-0", "x-1", "x-2", "x-3"], + "Cost": [10, 101, 12, 13], } ) # Sorting just to make sure the equivalency matches up - actual = dt.to_pyarrow_table().sort_by('Product.Id') + actual = dt.to_pyarrow_table().sort_by("Product.Id") assert expected == actual