Skip to content

Commit 91a7e95

Browse files
committed
Covered all build keys in the null-aware pushdown guard.
Indexing on[0] leaned on the single-key invariant without saying so. Iterating all keys survives a future multi-key relaxation. In addition, added allow/reject unit tests to pin both directions of the guard.
1 parent 64a60a8 commit 91a7e95

1 file changed

Lines changed: 82 additions & 4 deletions

File tree

  • datafusion/physical-plan/src/joins/hash_join

datafusion/physical-plan/src/joins/hash_join/exec.rs

Lines changed: 82 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -871,10 +871,15 @@ impl HashJoinExec {
871871
return false;
872872
}
873873

874-
// A null-aware anti join emits a build-side NULL only when the probe is truly empty. The
875-
// pushed filter can empty the probe by pruning every row, which would surface that NULL
876-
// wrongly. A NOT NULL build key cannot produce such a NULL, so the filter stays there.
877-
if self.null_aware && self.on[0].0.nullable(&self.left.schema()).unwrap_or(true) {
874+
// A null-aware anti join emits a build-side NULL only when the probe
875+
// is truly empty. The pushed filter can empty the probe by pruning
876+
// every row, which would surface that NULL wrongly. A NOT NULL build
877+
// key cannot produce such a NULL, so the filter stays there.
878+
if self.null_aware
879+
&& self.on.iter().any(|(build_key, _)| {
880+
build_key.nullable(&self.left.schema()).unwrap_or(true)
881+
})
882+
{
878883
return false;
879884
}
880885

@@ -6773,6 +6778,79 @@ mod tests {
67736778
Ok(())
67746779
}
67756780

6781+
#[test]
6782+
fn test_dynamic_filter_pushdown_rejects_null_aware_nullable_build_key() -> Result<()>
6783+
{
6784+
let left = build_table_two_cols(
6785+
("a1", &vec![Some(1), None]),
6786+
("b1", &vec![Some(1), Some(2)]),
6787+
);
6788+
let right = build_table_two_cols(
6789+
("a2", &vec![Some(2), Some(3)]),
6790+
("b2", &vec![Some(1), Some(2)]),
6791+
);
6792+
let on = vec![(
6793+
Arc::new(Column::new_with_schema("a1", &left.schema())?) as _,
6794+
Arc::new(Column::new_with_schema("a2", &right.schema())?) as _,
6795+
)];
6796+
6797+
let mut session_config = SessionConfig::default();
6798+
session_config
6799+
.options_mut()
6800+
.optimizer
6801+
.enable_join_dynamic_filter_pushdown = true;
6802+
6803+
let join = HashJoinExec::try_new(
6804+
left,
6805+
right,
6806+
on,
6807+
None,
6808+
&JoinType::LeftAnti,
6809+
None,
6810+
PartitionMode::CollectLeft,
6811+
NullEquality::NullEqualsNothing,
6812+
true,
6813+
)?;
6814+
6815+
assert!(!join.allow_join_dynamic_filter_pushdown(session_config.options()));
6816+
6817+
Ok(())
6818+
}
6819+
6820+
#[test]
6821+
fn test_dynamic_filter_pushdown_allows_null_aware_non_null_build_key() -> Result<()> {
6822+
// A NOT NULL build key cannot surface a build-side NULL, so the
6823+
// pushdown must stay enabled.
6824+
let left = build_table(("a1", &vec![1]), ("b1", &vec![1]), ("c1", &vec![1]));
6825+
let right = build_table(("a2", &vec![2]), ("b2", &vec![2]), ("c2", &vec![2]));
6826+
let on = vec![(
6827+
Arc::new(Column::new_with_schema("a1", &left.schema())?) as _,
6828+
Arc::new(Column::new_with_schema("a2", &right.schema())?) as _,
6829+
)];
6830+
6831+
let mut session_config = SessionConfig::default();
6832+
session_config
6833+
.options_mut()
6834+
.optimizer
6835+
.enable_join_dynamic_filter_pushdown = true;
6836+
6837+
let join = HashJoinExec::try_new(
6838+
left,
6839+
right,
6840+
on,
6841+
None,
6842+
&JoinType::LeftAnti,
6843+
None,
6844+
PartitionMode::CollectLeft,
6845+
NullEquality::NullEqualsNothing,
6846+
true,
6847+
)?;
6848+
6849+
assert!(join.allow_join_dynamic_filter_pushdown(session_config.options()));
6850+
6851+
Ok(())
6852+
}
6853+
67766854
#[test]
67776855
fn test_partitioned_dynamic_filter_pushdown_rejects_range_partitioning() -> Result<()>
67786856
{

0 commit comments

Comments
 (0)