Skip to content

Commit ad836bd

Browse files
ruihe774ruihe774
authored
feat: add "future" versioning (#17421)
1 parent e7915d8 commit ad836bd

File tree

79 files changed

+501
-265
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+501
-265
lines changed

‎crates/polars-core/src/chunked_array/array/mod.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,9 @@ impl ArrayChunked {
4444
) -> PolarsResult<ArrayChunked> {
4545
// Rechunk or the generated Series will have wrong length.
4646
let ca = self.rechunk();
47-
let field = self.inner_dtype().to_arrow_field("item", true);
47+
let field = self
48+
.inner_dtype()
49+
.to_arrow_field("item", CompatLevel::newest());
4850

4951
let chunks = ca.downcast_iter().map(|arr| {
5052
let elements = unsafe {
@@ -66,8 +68,10 @@ impl ArrayChunked {
6668
let out = out.rechunk();
6769
let values = out.chunks()[0].clone();
6870

69-
let inner_dtype =
70-
FixedSizeListArray::default_datatype(out.dtype().to_arrow(true), ca.width());
71+
let inner_dtype = FixedSizeListArray::default_datatype(
72+
out.dtype().to_arrow(CompatLevel::newest()),
73+
ca.width(),
74+
);
7175
let arr = FixedSizeListArray::new(inner_dtype, values, arr.validity().cloned());
7276
Ok(arr)
7377
});

‎crates/polars-core/src/chunked_array/builder/fixed_size_list.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ impl FixedSizeListBuilder for AnonymousOwnedFixedSizeListBuilder {
124124
.finish(
125125
self.inner_dtype
126126
.as_ref()
127-
.map(|dt| dt.to_arrow(true))
127+
.map(|dt| dt.to_arrow(CompatLevel::newest()))
128128
.as_ref(),
129129
)
130130
.unwrap();

‎crates/polars-core/src/chunked_array/builder/list/anonymous.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ impl<'a> AnonymousListBuilder<'a> {
8989

9090
let inner_dtype_physical = inner_dtype
9191
.as_ref()
92-
.map(|dt| dt.to_physical().to_arrow(true));
92+
.map(|dt| dt.to_physical().to_arrow(CompatLevel::newest()));
9393
let arr = slf.builder.finish(inner_dtype_physical.as_ref()).unwrap();
9494

9595
let list_dtype_logical = match inner_dtype {
@@ -157,7 +157,7 @@ impl ListBuilderTrait for AnonymousOwnedListBuilder {
157157
let slf = std::mem::take(self);
158158
let inner_dtype_physical = inner_dtype
159159
.as_ref()
160-
.map(|dt| dt.to_physical().to_arrow(true));
160+
.map(|dt| dt.to_physical().to_arrow(CompatLevel::newest()));
161161
let arr = slf.builder.finish(inner_dtype_physical.as_ref()).unwrap();
162162

163163
let list_dtype_logical = match inner_dtype {

‎crates/polars-core/src/chunked_array/builder/list/primitive.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ where
3939
) -> Self {
4040
let values = MutablePrimitiveArray::<T::Native>::with_capacity_from(
4141
values_capacity,
42-
values_type.to_arrow(true),
42+
values_type.to_arrow(CompatLevel::newest()),
4343
);
4444
let builder = LargePrimitiveBuilder::<T::Native>::new_with_capacity(values, capacity);
4545
let field = Field::new(name, DataType::List(Box::new(logical_type)));

‎crates/polars-core/src/chunked_array/builder/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ where
6666
T: PolarsNumericType,
6767
{
6868
fn from_slice(name: &str, v: &[T::Native]) -> Self {
69-
let arr = PrimitiveArray::from_slice(v).to(T::get_dtype().to_arrow(true));
69+
let arr = PrimitiveArray::from_slice(v).to(T::get_dtype().to_arrow(CompatLevel::newest()));
7070
ChunkedArray::with_chunk(name, arr)
7171
}
7272

‎crates/polars-core/src/chunked_array/builder/primitive.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ where
4141
{
4242
pub fn new(name: &str, capacity: usize) -> Self {
4343
let array_builder = MutablePrimitiveArray::<T::Native>::with_capacity(capacity)
44-
.to(T::get_dtype().to_arrow(true));
44+
.to(T::get_dtype().to_arrow(CompatLevel::newest()));
4545

4646
PrimitiveChunkedBuilder {
4747
array_builder,

‎crates/polars-core/src/chunked_array/cast.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ pub(crate) fn cast_chunks(
5151
let check_nulls = matches!(options, CastOptions::Strict);
5252
let options = options.into();
5353

54-
let arrow_dtype = dtype.try_to_arrow(true)?;
54+
let arrow_dtype = dtype.try_to_arrow(CompatLevel::newest())?;
5555
chunks
5656
.iter()
5757
.map(|arr| {

‎crates/polars-core/src/chunked_array/collect.rs‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use crate::chunked_array::ChunkedArray;
1818
use crate::datatypes::{
1919
ArrayCollectIterExt, ArrayFromIter, ArrayFromIterDtype, DataType, Field, PolarsDataType,
2020
};
21+
use crate::prelude::CompatLevel;
2122

2223
pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
2324
#[inline]
@@ -26,7 +27,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
2627
T::Array: ArrayFromIterDtype<Self::Item>,
2728
{
2829
let field = Arc::new(Field::new(name, dtype.clone()));
29-
let arr = self.collect_arr_with_dtype(field.dtype.to_arrow(true));
30+
let arr = self.collect_arr_with_dtype(field.dtype.to_arrow(CompatLevel::newest()));
3031
ChunkedArray::from_chunk_iter_and_field(field, [arr])
3132
}
3233

@@ -36,7 +37,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
3637
T::Array: ArrayFromIterDtype<Self::Item>,
3738
{
3839
let field = Arc::clone(&name_dtype_src.field);
39-
let arr = self.collect_arr_with_dtype(field.dtype.to_arrow(true));
40+
let arr = self.collect_arr_with_dtype(field.dtype.to_arrow(CompatLevel::newest()));
4041
ChunkedArray::from_chunk_iter_and_field(field, [arr])
4142
}
4243

@@ -47,7 +48,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
4748
Self: TrustedLen,
4849
{
4950
let field = Arc::new(Field::new(name, dtype.clone()));
50-
let arr = self.collect_arr_trusted_with_dtype(field.dtype.to_arrow(true));
51+
let arr = self.collect_arr_trusted_with_dtype(field.dtype.to_arrow(CompatLevel::newest()));
5152
ChunkedArray::from_chunk_iter_and_field(field, [arr])
5253
}
5354

@@ -58,7 +59,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
5859
Self: TrustedLen,
5960
{
6061
let field = Arc::clone(&name_dtype_src.field);
61-
let arr = self.collect_arr_trusted_with_dtype(field.dtype.to_arrow(true));
62+
let arr = self.collect_arr_trusted_with_dtype(field.dtype.to_arrow(CompatLevel::newest()));
6263
ChunkedArray::from_chunk_iter_and_field(field, [arr])
6364
}
6465

@@ -73,7 +74,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
7374
Self: Iterator<Item = Result<U, E>>,
7475
{
7576
let field = Arc::new(Field::new(name, dtype.clone()));
76-
let arr = self.try_collect_arr_with_dtype(field.dtype.to_arrow(true))?;
77+
let arr = self.try_collect_arr_with_dtype(field.dtype.to_arrow(CompatLevel::newest()))?;
7778
Ok(ChunkedArray::from_chunk_iter_and_field(field, [arr]))
7879
}
7980

@@ -87,7 +88,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
8788
Self: Iterator<Item = Result<U, E>>,
8889
{
8990
let field = Arc::clone(&name_dtype_src.field);
90-
let arr = self.try_collect_arr_with_dtype(field.dtype.to_arrow(true))?;
91+
let arr = self.try_collect_arr_with_dtype(field.dtype.to_arrow(CompatLevel::newest()))?;
9192
Ok(ChunkedArray::from_chunk_iter_and_field(field, [arr]))
9293
}
9394

@@ -102,7 +103,8 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
102103
Self: Iterator<Item = Result<U, E>> + TrustedLen,
103104
{
104105
let field = Arc::new(Field::new(name, dtype.clone()));
105-
let arr = self.try_collect_arr_trusted_with_dtype(field.dtype.to_arrow(true))?;
106+
let arr =
107+
self.try_collect_arr_trusted_with_dtype(field.dtype.to_arrow(CompatLevel::newest()))?;
106108
Ok(ChunkedArray::from_chunk_iter_and_field(field, [arr]))
107109
}
108110

@@ -116,7 +118,8 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
116118
Self: Iterator<Item = Result<U, E>> + TrustedLen,
117119
{
118120
let field = Arc::clone(&name_dtype_src.field);
119-
let arr = self.try_collect_arr_trusted_with_dtype(field.dtype.to_arrow(true))?;
121+
let arr =
122+
self.try_collect_arr_trusted_with_dtype(field.dtype.to_arrow(CompatLevel::newest()))?;
120123
Ok(ChunkedArray::from_chunk_iter_and_field(field, [arr]))
121124
}
122125
}

‎crates/polars-core/src/chunked_array/from.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,10 @@ where
217217
#[cfg(debug_assertions)]
218218
{
219219
if !chunks.is_empty() && !chunks[0].is_empty() && dtype.is_primitive() {
220-
assert_eq!(chunks[0].data_type(), &dtype.to_arrow(true))
220+
assert_eq!(
221+
chunks[0].data_type(),
222+
&dtype.to_arrow(CompatLevel::newest())
223+
)
221224
}
222225
}
223226
let field = Arc::new(Field::new(name, dtype));
@@ -234,7 +237,10 @@ where
234237
}
235238

236239
pub fn full_null_like(ca: &Self, length: usize) -> Self {
237-
let chunks = std::iter::once(T::Array::full_null(length, ca.dtype().to_arrow(true)));
240+
let chunks = std::iter::once(T::Array::full_null(
241+
length,
242+
ca.dtype().to_arrow(CompatLevel::newest()),
243+
));
238244
Self::from_chunk_iter_like(ca, chunks)
239245
}
240246
}

‎crates/polars-core/src/chunked_array/logical/categorical/from.rs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,25 @@ use arrow::datatypes::IntegerType;
33

44
use super::*;
55

6-
fn convert_values(arr: &Utf8ViewArray, pl_flavor: bool) -> ArrayRef {
7-
if pl_flavor {
6+
fn convert_values(arr: &Utf8ViewArray, compat_level: CompatLevel) -> ArrayRef {
7+
if compat_level.0 >= 1 {
88
arr.clone().boxed()
99
} else {
1010
utf8view_to_utf8::<i64>(arr).boxed()
1111
}
1212
}
1313

1414
impl CategoricalChunked {
15-
pub fn to_arrow(&self, pl_flavor: bool, as_i64: bool) -> ArrayRef {
15+
pub fn to_arrow(&self, compat_level: CompatLevel, as_i64: bool) -> ArrayRef {
1616
if as_i64 {
17-
self.to_i64(pl_flavor).boxed()
17+
self.to_i64(compat_level).boxed()
1818
} else {
19-
self.to_u32(pl_flavor).boxed()
19+
self.to_u32(compat_level).boxed()
2020
}
2121
}
2222

23-
fn to_u32(&self, pl_flavor: bool) -> DictionaryArray<u32> {
24-
let values_dtype = if pl_flavor {
23+
fn to_u32(&self, compat_level: CompatLevel) -> DictionaryArray<u32> {
24+
let values_dtype = if compat_level.0 >= 1 {
2525
ArrowDataType::Utf8View
2626
} else {
2727
ArrowDataType::LargeUtf8
@@ -32,7 +32,7 @@ impl CategoricalChunked {
3232
let dtype = ArrowDataType::Dictionary(IntegerType::UInt32, Box::new(values_dtype), false);
3333
match map {
3434
RevMapping::Local(arr, _) => {
35-
let values = convert_values(arr, pl_flavor);
35+
let values = convert_values(arr, compat_level);
3636

3737
// SAFETY:
3838
// the keys are in bounds
@@ -44,7 +44,7 @@ impl CategoricalChunked {
4444
.map(|opt_k| opt_k.map(|k| *reverse_map.get(k).unwrap()));
4545
let keys = PrimitiveArray::from_trusted_len_iter(iter);
4646

47-
let values = convert_values(values, pl_flavor);
47+
let values = convert_values(values, compat_level);
4848

4949
// SAFETY:
5050
// the keys are in bounds
@@ -53,8 +53,8 @@ impl CategoricalChunked {
5353
}
5454
}
5555

56-
fn to_i64(&self, pl_flavor: bool) -> DictionaryArray<i64> {
57-
let values_dtype = if pl_flavor {
56+
fn to_i64(&self, compat_level: CompatLevel) -> DictionaryArray<i64> {
57+
let values_dtype = if compat_level.0 >= 1 {
5858
ArrowDataType::Utf8View
5959
} else {
6060
ArrowDataType::LargeUtf8
@@ -65,7 +65,7 @@ impl CategoricalChunked {
6565
let dtype = ArrowDataType::Dictionary(IntegerType::Int64, Box::new(values_dtype), false);
6666
match map {
6767
RevMapping::Local(arr, _) => {
68-
let values = convert_values(arr, pl_flavor);
68+
let values = convert_values(arr, compat_level);
6969

7070
// SAFETY:
7171
// the keys are in bounds
@@ -89,7 +89,7 @@ impl CategoricalChunked {
8989
.map(|opt_k| opt_k.map(|k| *reverse_map.get(k).unwrap() as i64));
9090
let keys = PrimitiveArray::from_trusted_len_iter(iter);
9191

92-
let values = convert_values(values, pl_flavor);
92+
let values = convert_values(values, compat_level);
9393

9494
// SAFETY:
9595
// the keys are in bounds

0 commit comments

Comments
 (0)