fix: Handle null-aware joins correctly in FilterNullJoinKeys when its enabled (apache#23848)

## Which issue does this PR close?

<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes apache#123` indicates that this PR will close issue apache#123.
-->

- Closes apache#23847 .

## Rationale for this change

Fixes a correctness bug, not sure when that config used/desirable.

This is another thing I ran into during
apache#21585


## What changes are included in this PR?

1. New SLT test
2. Fix for bug in `datafusion-optimizer`

## Are these changes tested?

Existing tests, and a new SLT test verifying the config change doesn't
affect the query result and basic unit test.

## Are there any user-facing changes?

None

---------

Signed-off-by: Adam Gutglick <adamgsal@gmail.com>

Author: AdamGS

datafusion/optimizer/src/filter_null_join_keys.rs

@@ -52,6 +52,7 @@ impl OptimizerRule for FilterNullJoinKeys {
         match plan {
             LogicalPlan::Join(mut join)
                 if !join.on.is_empty()
+                    && !join.null_aware
                     && join.null_equality == NullEquality::NullEqualsNothing =>
             {
                 let (left_preserved, right_preserved) =
@@ -359,4 +360,50 @@ mod tests {
         let t2 = table_scan(Some("t2"), &schema, None)?.build()?;
         Ok((t1, t2))
     }
+
+    #[test]
+    fn null_aware_left_mark_join_keys_not_filtered() -> Result<()> {
+        let (t1, t2) = test_tables()?;
+        let plan = build_null_aware_plan(t1, t2, JoinType::LeftMark)?;
+
+        assert_optimized_plan_equal!(plan, @r"
+        LeftMark Join: t1.id = t2.optional_id null_aware
+          TableScan: t1
+          TableScan: t2
+        ")
+    }
+
+    #[test]
+    fn null_aware_left_anti_join_keys_not_filtered() -> Result<()> {
+        let (t1, t2) = test_tables()?;
+        let plan = build_null_aware_plan(t1, t2, JoinType::LeftAnti)?;
+
+        assert_optimized_plan_equal!(plan, @r"
+        LeftAnti Join: t1.id = t2.optional_id null_aware
+          TableScan: t1
+          TableScan: t2
+        ")
+    }
+
+    /// A join whose nullable right key would get an `IS NOT NULL` filter if it
+    /// were not null-aware.
+    fn build_null_aware_plan(
+        left_table: LogicalPlan,
+        right_table: LogicalPlan,
+        join_type: JoinType,
+    ) -> Result<LogicalPlan> {
+        LogicalPlanBuilder::from(left_table)
+            .join_detailed_with_options(
+                right_table,
+                join_type,
+                (
+                    vec![Column::from_qualified_name("t1.id")],
+                    vec![Column::from_qualified_name("t2.optional_id")],
+                ),
+                None,
+                NullEquality::NullEqualsNothing,
+                true,
+            )?
+            .build()
+    }
 }

datafusion/sqllogictest/test_files/null_aware_anti_join.slt

@@ -70,6 +70,20 @@ query IT rowsort
 SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
 ----
 
+# Regression test
+
+statement ok
+set datafusion.optimizer.filter_null_join_keys = true;
+
+# The subquery NULL must reach the join: every row's NOT IN is UNKNOWN or
+# FALSE, so the result stays empty.
+query IT rowsort
+SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
+----
+
+statement ok
+reset datafusion.optimizer.filter_null_join_keys;
+
 # Verify the result is empty even though there are rows in outer_table
 # that don't match the non-NULL value (2) in the subquery.
 # This is correct null-aware behavior: if subquery contains NULL, result is unknown.