Skip to content

Commit cf2315c

Browse files
Subham SinghalSubham Singhal
authored andcommitted
Fix build failure
1 parent 5db09e8 commit cf2315c

6 files changed

Lines changed: 33 additions & 6 deletions

File tree

datafusion/common/src/config.rs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1147,6 +1147,12 @@ config_namespace! {
11471147
///
11481148
/// Default: true
11491149
pub enable_sort_pushdown: bool, default = true
1150+
1151+
/// When set to true, the optimizer will extract leaf expressions
1152+
/// (such as `get_field`) from filter/sort/join nodes into projections
1153+
/// closer to the leaf table scans, and push those projections down
1154+
/// towards the leaf nodes.
1155+
pub enable_leaf_expression_pushdown: bool, default = true
11501156
}
11511157
}
11521158

@@ -3070,6 +3076,22 @@ config_namespace! {
30703076
/// If not specified, the default level for the compression algorithm is used.
30713077
pub compression_level: Option<u32>, default = None
30723078
pub schema_infer_max_rec: Option<usize>, default = None
3079+
/// The JSON format to use when reading files.
3080+
///
3081+
/// When `true` (default), expects newline-delimited JSON (NDJSON):
3082+
/// ```text
3083+
/// {"key1": 1, "key2": "val"}
3084+
/// {"key1": 2, "key2": "vals"}
3085+
/// ```
3086+
///
3087+
/// When `false`, expects JSON array format:
3088+
/// ```text
3089+
/// [
3090+
/// {"key1": 1, "key2": "val"},
3091+
/// {"key1": 2, "key2": "vals"}
3092+
/// ]
3093+
/// ```
3094+
pub newline_delimited: bool, default = true
30733095
}
30743096
}
30753097

datafusion/common/src/file_options/parquet_writer.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,7 @@ mod tests {
461461
skip_arrow_metadata: defaults.skip_arrow_metadata,
462462
coerce_int96: None,
463463
max_predicate_cache_size: defaults.max_predicate_cache_size,
464+
field_id_read_enabled: defaults.field_id_read_enabled,
464465
}
465466
}
466467

@@ -575,6 +576,7 @@ mod tests {
575576
binary_as_string: global_options_defaults.binary_as_string,
576577
skip_arrow_metadata: global_options_defaults.skip_arrow_metadata,
577578
coerce_int96: None,
579+
field_id_read_enabled: global_options_defaults.field_id_read_enabled,
578580
},
579581
column_specific_options,
580582
key_value_metadata,

datafusion/datasource-parquet/src/metadata.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ pub struct DFParquetMetadata<'a> {
8383
/// field IDs at all nesting levels. See PARQUET_FIELD_ID_IMPLEMENTATION.md
8484
/// for details on nested schema support.
8585
fn add_field_ids_to_arrow_schema(
86-
arrow_schema: Schema,
86+
arrow_schema: &Schema,
8787
parquet_schema: &SchemaDescriptor,
8888
) -> Result<Schema> {
8989
use arrow::datatypes::Field;
@@ -244,7 +244,8 @@ impl<'a> DFParquetMetadata<'a> {
244244

245245
// Add field IDs if requested
246246
if self.enable_field_ids {
247-
schema = add_field_ids_to_arrow_schema(schema, file_metadata.schema_descr())?;
247+
schema =
248+
add_field_ids_to_arrow_schema(&schema, file_metadata.schema_descr())?;
248249
}
249250

250251
// Apply INT96 coercion if configured

datafusion/physical-expr/src/utils/mod.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -295,10 +295,10 @@ fn find_field_index(
295295
// TODO: For nested schemas, this needs to recursively match field IDs
296296
// through the struct hierarchy
297297
for (idx, target_field) in target_schema.fields().iter().enumerate() {
298-
if let Some(target_field_id) = get_field_id(target_field) {
299-
if source_field_id == target_field_id {
300-
return Ok(idx);
301-
}
298+
if let Some(target_field_id) = get_field_id(target_field)
299+
&& source_field_id == target_field_id
300+
{
301+
return Ok(idx);
302302
}
303303
}
304304
}

datafusion/proto-common/src/from_proto/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1090,6 +1090,7 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
10901090
max_predicate_cache_size: value.max_predicate_cache_size_opt.map(|opt| match opt {
10911091
protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v) => Some(v as usize),
10921092
}).unwrap_or(None),
1093+
field_id_read_enabled: false, // Default value
10931094
})
10941095
}
10951096
}

datafusion/proto/src/logical_plan/file_formats.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,7 @@ mod parquet {
525525
max_predicate_cache_size: proto.max_predicate_cache_size_opt.as_ref().map(|opt| match opt {
526526
parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(size) => *size as usize,
527527
}),
528+
field_id_read_enabled: false,
528529
}
529530
}
530531
}

0 commit comments

Comments
 (0)