Skip to content

Commit e950592

Browse files
authored
perf: Change default memory prefetch to MADV_WILLNEED (#21056)
1 parent 635a215 commit e950592

File tree

2 files changed

+8
-5
lines changed

2 files changed

+8
-5
lines changed

Diff for: crates/polars-stream/src/nodes/io_sources/parquet/mem_prefetch_funcs.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@ pub(super) fn no_prefetch(_: &[u8]) {}
66
pub(super) fn get_memory_prefetch_func(verbose: bool) -> fn(&[u8]) -> () {
77
let memory_prefetch_func = match std::env::var("POLARS_MEMORY_PREFETCH").ok().as_deref() {
88
None => {
9-
// Sequential advice was observed to provide speedups on Linux.
10-
// ref https://github.com/pola-rs/polars/pull/18152#discussion_r1721701965
11-
#[cfg(target_os = "linux")]
9+
// madvise_willneed performed the best on both MacOS on Apple Silicon and Ubuntu on x86-64,
10+
// using PDS-H query 3 SF=10 after clearing file cache as a benchmark.
11+
#[cfg(target_family = "unix")]
1212
{
13-
madvise_sequential
13+
madvise_willneed
1414
}
15-
#[cfg(not(target_os = "linux"))]
15+
#[cfg(not(target_family = "unix"))]
1616
{
1717
no_prefetch
1818
}

Diff for: crates/polars-utils/src/mem.rs

+3
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ pub fn force_populate_read(slice: &[u8]) {
8282

8383
#[cfg(target_family = "unix")]
8484
fn madvise(slice: &[u8], advice: libc::c_int) {
85+
if slice.is_empty() {
86+
return;
87+
}
8588
let ptr = slice.as_ptr();
8689

8790
let align = ptr as usize % *PAGE_SIZE;

0 commit comments

Comments
 (0)