test: Add more push down support

Author: Xuanwo

src/lance_scan.cpp

@@ -90,7 +90,7 @@ struct LanceScanBindData : public TableFunctionData {
   ArrowTableSchema arrow_table;
   vector<string> names;
   vector<LogicalType> types;
-  string lance_complex_filter_sql;
+  string lance_pushed_filter_sql;
 
   ~LanceScanBindData() override {
     if (dataset) {
@@ -413,7 +413,7 @@ static bool TrySerializeLanceExpr(const LogicalGet &get,
       }
       out_sql = "(" + child_sql +
                 (op.type == ExpressionType::OPERATOR_IS_NULL ? " IS NULL)"
-                                                             : " IS NOT NULL)");
+                                                              : " IS NOT NULL)");
       return true;
     }
     if (op.type == ExpressionType::COMPARE_IN ||
@@ -467,7 +467,7 @@ static bool TrySerializeLanceExpr(const LogicalGet &get,
 }
 
 static void
-LancePushdownComplexFilter(ClientContext &, LogicalGet &get,
+LancePushdownComplexFilter(ClientContext &context, LogicalGet &get,
                            FunctionData *bind_data,
                            vector<unique_ptr<Expression>> &filters) {
   if (!bind_data || filters.empty()) {
@@ -493,11 +493,11 @@ LancePushdownComplexFilter(ClientContext &, LogicalGet &get,
     return;
   }
   auto pushed_sql = StringUtil::Join(predicates, " AND ");
-  if (scan_bind.lance_complex_filter_sql.empty()) {
-    scan_bind.lance_complex_filter_sql = std::move(pushed_sql);
+  if (scan_bind.lance_pushed_filter_sql.empty()) {
+    scan_bind.lance_pushed_filter_sql = std::move(pushed_sql);
   } else {
-    scan_bind.lance_complex_filter_sql =
-        "(" + scan_bind.lance_complex_filter_sql + ") AND (" + pushed_sql + ")";
+    scan_bind.lance_pushed_filter_sql =
+        "(" + scan_bind.lance_pushed_filter_sql + ") AND (" + pushed_sql + ")";
   }
 }
 
@@ -583,13 +583,15 @@ LanceScanInitGlobal(ClientContext &context, TableFunctionInitInput &input) {
   }
 
   auto table_filter_sql = BuildLanceFilterSQL(bind_data, input);
-  if (bind_data.lance_complex_filter_sql.empty()) {
+  auto pushed_filter_sql = bind_data.lance_pushed_filter_sql;
+
+  if (pushed_filter_sql.empty()) {
     scan_state.lance_filter_sql = std::move(table_filter_sql);
   } else if (table_filter_sql.empty()) {
-    scan_state.lance_filter_sql = bind_data.lance_complex_filter_sql;
+    scan_state.lance_filter_sql = std::move(pushed_filter_sql);
   } else {
-    scan_state.lance_filter_sql = "(" + bind_data.lance_complex_filter_sql +
-                                  ") AND (" + table_filter_sql + ")";
+    scan_state.lance_filter_sql =
+        "(" + pushed_filter_sql + ") AND (" + table_filter_sql + ")";
   }
   return state;
 }

test/sql/lance.test

@@ -14,7 +14,11 @@ SELECT 1
 statement ok
 SELECT * FROM lance_scan('test/test_data.lance') LIMIT 1
 
-# Test parallel scan + filter pushdown
+# Baseline: materialize once and evaluate filters in DuckDB
+statement ok
+CREATE TEMP TABLE baseline AS SELECT * FROM 'test/test_data.lance'
+
+# Test parallel scan + filter pushdown correctness
 statement ok
 PRAGMA threads=4
 
@@ -33,7 +37,7 @@ SELECT count(*) FROM 'test/test_data.lance' WHERE age > 30 AND score < 90
 ----
 2
 
-# Test complex filter forms (OR across columns, NOT, BETWEEN, NOT IN)
+# Test complex filter forms
 query I
 SELECT count(*) FROM 'test/test_data.lance' WHERE age > 30 OR score < 90
 ----
@@ -62,6 +66,42 @@ Charlie
 David
 Eve
 
+# Equivalence: baseline table (DuckDB evaluation) matches expected results
+query I
+SELECT count(*) FROM baseline WHERE age > 30
+----
+3
+
+query I
+SELECT count(*) FROM baseline WHERE name IN ('Alice', 'Eve')
+----
+2
+
+query I
+SELECT count(*) FROM baseline WHERE age > 30 AND score < 90
+----
+2
+
+query I
+SELECT count(*) FROM baseline WHERE age > 30 OR score < 90
+----
+4
+
+query I
+SELECT count(*) FROM baseline WHERE NOT (age > 30)
+----
+2
+
+query I
+SELECT count(*) FROM baseline WHERE age BETWEEN 26 AND 40
+----
+3
+
+query I
+SELECT count(*) FROM baseline WHERE name NOT IN ('Alice', 'Eve')
+----
+3
+
 # Test replacement scan for SELECT * FROM '.../dataset.lance'
 statement ok
 SELECT * FROM 'test/test_data.lance' LIMIT 1