Merge pull request ClickHouse#110695 from ClickHouse/prewhere-cost-with-selectivity

Combine I/O cost with selectivity in PREWHERE condition ordering

Author: Avogar

src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp

@@ -110,6 +110,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
         if (it != column_sizes.end())
             total_size_of_queried_columns += it->second;
     }
+
+    if (estimator)
+        total_rows = estimator->getTotalRows();
 }
 
 void MergeTreeWhereOptimizer::optimize(SelectQueryInfo & select_query_info, const ContextPtr & context) const
@@ -509,6 +512,22 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree
                 pk_positions.emplace(cond.min_position_in_primary_key);
             }
 
+            /// Combine I/O cost with selectivity using the classic conjunctive filter ordering rule:
+            /// sort by cost / (1 - selectivity), i.e. cost per rejected row.
+            const double rejected_rows = static_cast<double>(total_rows) - static_cast<double>(cond.estimated_row_count);
+            if (total_rows == 0)
+                /// No statistics: fall back to pure I/O cost.
+                cond.cost_with_selectivity = static_cast<double>(cond.columns_size);
+            else if (rejected_rows <= 0)
+                /// Rejects no rows, so it is useless in PREWHERE regardless of its cost: schedule it last.
+                cond.cost_with_selectivity = std::numeric_limits<double>::infinity();
+            else if (cond.columns_size == 0)
+                /// Compact parts don't track per-column compressed sizes: fall back to pure selectivity,
+                /// otherwise every condition collapses to cost 0 and keeps its original position.
+                cond.cost_with_selectivity = static_cast<double>(cond.estimated_row_count);
+            else
+                cond.cost_with_selectivity = static_cast<double>(cond.columns_size) / rejected_rows;
+
             res.emplace_back(std::move(cond));
         }
     }
@@ -534,6 +553,7 @@ MergeTreeWhereOptimizer::Conditions MergeTreeWhereOptimizer::analyze(const RPNBu
             Condition cond({conjunct});
             cond.table_columns = columns;
             cond.columns_size = getColumnsSize(columns);
+            cond.cost_with_selectivity = static_cast<double>(cond.columns_size);
             cond.viable =
                 !has_invalid_column
                 && !columns.empty()

src/Storages/MergeTree/MergeTreeWhereOptimizer.h

@@ -82,6 +82,12 @@ class MergeTreeWhereOptimizer : private boost::noncopyable
         /// the lower the better
         UInt64 estimated_row_count = 0;
 
+        /// Combined I/O cost and selectivity score (lower is better): cost per rejected row,
+        /// columns_size / (total_rows - estimated_row_count). A condition that rejects no rows
+        /// gets +inf (scheduled last), and when per-column sizes are unavailable (columns_size=0,
+        /// e.g. compact parts) it falls back to estimated_row_count so selectivity ordering is kept.
+        double cost_with_selectivity = 0;
+
         /// Does the condition contain primary key column?
         /// If so, it is better to move it further to the end of PREWHERE chain depending on minimal position in PK of any
         /// column in this condition because this condition have bigger chances to be already satisfied by PK analysis.
@@ -99,19 +105,20 @@ class MergeTreeWhereOptimizer : private boost::noncopyable
             }
             return fmt::format(
                 "Condition(exp:{} viable: {}, good: {}, min_position_in_primary_key: {}, estimated_row_count: {}, "
-                "columns_size: {}, table_columns.size: {})",
+                "columns_size: {}, cost_with_selectivity: {}, table_columns.size: {})",
                 names,
                 viable,
                 good,
                 min_position_in_primary_key,
                 estimated_row_count,
                 columns_size,
+                cost_with_selectivity,
                 table_columns.size());
         }
 
         auto tuple() const
         {
-            return std::make_tuple(!viable, !good, -min_position_in_primary_key, estimated_row_count, columns_size, table_columns.size());
+            return std::make_tuple(!viable, !good, -min_position_in_primary_key, cost_with_selectivity, table_columns.size());
         }
 
         /// Is condition a better candidate for moving to PREWHERE?
@@ -186,6 +193,7 @@ class MergeTreeWhereOptimizer : private boost::noncopyable
     LoggerPtr log;
     std::unordered_map<std::string, UInt64> column_sizes;
     UInt64 total_size_of_queried_columns = 0;
+    UInt64 total_rows = 0;
 };
 
 

src/Storages/Statistics/ConditionSelectivityEstimator.h

@@ -101,6 +101,9 @@ class ConditionSelectivityEstimator : public WithContext
     };
     using AtomMap = std::unordered_map<std::string, void(*)(RPNElement & out, const String & column, const Field & value)>;
     static const AtomMap atom_map;
+
+    UInt64 getTotalRows() const { return total_rows; }
+
 private:
     friend class ColumnStatistics;
 

tests/queries/0_stateless/03580_improve_prewhere.sql

@@ -20,7 +20,9 @@ CREATE TABLE test_improve_prewhere (
     value UInt32 STATISTICS(TDigest),
     date Date STATISTICS(CountMin),
 ) ENGINE = MergeTree()
-ORDER BY primary_key;
+ORDER BY primary_key
+-- Pin compact parts (per-column sizes = 0) so PREWHERE ordering is by selectivity alone, stable under CI-randomized part-type/serialization settings.
+SETTINGS min_bytes_for_wide_part = 1000000000000, min_rows_for_wide_part = 1000000000000;
 
 INSERT INTO test_improve_prewhere
 SELECT

tests/queries/0_stateless/04053_prewhere_statistics_decimal_overflow.sql

@@ -16,7 +16,8 @@ CREATE TABLE test_prewhere_decimal_overflow
 )
 ENGINE = MergeTree()
 ORDER BY (ts, id)
-SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, auto_statistics_types = '';
+-- Pin compact parts (per-column sizes = 0) so PREWHERE ordering is by selectivity alone, stable under CI-randomized serialization settings.
+SETTINGS min_bytes_for_wide_part = 1000000000000, min_rows_for_wide_part = 1000000000000, auto_statistics_types = '';
 
 -- Single INSERT so all rows land in one part (needed for selectivity estimates to differ).
 INSERT INTO test_prewhere_decimal_overflow VALUES

tests/queries/0_stateless/04266_statistics_basic_prewhere.sql

@@ -31,7 +31,8 @@ CREATE TABLE test_basic_prewhere
     range_probe Int64 STATISTICS(tdigest)
 ) ENGINE = MergeTree()
 ORDER BY id
-SETTINGS auto_statistics_types = '';
+-- Pin compact parts (per-column sizes = 0) so PREWHERE ordering is by selectivity alone, stable under CI-randomized part-type/serialization settings.
+SETTINGS auto_statistics_types = '', min_bytes_for_wide_part = 1000000000000, min_rows_for_wide_part = 1000000000000;
 
 INSERT INTO test_basic_prewhere SELECT
     number,

tests/queries/0_stateless/04304_prewhere_statistics_column_grouping.sql

@@ -8,7 +8,9 @@ DROP TABLE IF EXISTS prewhere_stats_group;
 CREATE TABLE prewhere_stats_group (
     a UInt64 STATISTICS(tdigest, countmin),
     b UInt64 STATISTICS(countmin)
-) ENGINE = MergeTree ORDER BY tuple();
+) ENGINE = MergeTree ORDER BY tuple()
+-- Pin compact parts (per-column sizes = 0) so PREWHERE ordering is by selectivity alone, stable under CI-randomized part-type/serialization settings.
+SETTINGS min_bytes_for_wide_part = 1000000000000, min_rows_for_wide_part = 1000000000000;
 
 INSERT INTO prewhere_stats_group SELECT number, number % 200 FROM numbers(5000) SETTINGS materialize_statistics_on_insert = 1;
 

tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.reference

@@ -0,0 +1,5 @@
+-- with statistics: cheap filter is placed before the expensive Map predicate
+1
+-- correctness: result is the same regardless of ordering
+0
+0

tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.sql

@@ -0,0 +1,40 @@
+-- Regression test: when statistics are enabled, the PREWHERE optimizer must
+-- combine selectivity (estimated_row_count) with I/O cost (columns_size)
+-- using the classic cost / (1 - selectivity) rule, not sort by selectivity alone.
+--
+-- Without this fix, the auto-collected Uniq statistic on `modality` (2 distinct
+-- values → 50% selectivity) makes the optimizer place the expensive Map-key
+-- predicate first because its default 1% selectivity looks "more selective",
+-- ignoring that the Map column is ~500x more expensive to read.
+
+SET enable_analyzer = 1;
+SET optimize_functions_to_subcolumns = 0;
+SET optimize_move_to_prewhere = 1;
+SET query_plan_optimize_prewhere = 1;
+SET allow_reorder_prewhere_conditions = 1;
+SET use_statistics = 1;
+SET explain_query_plan_default = 'legacy';
+
+DROP TABLE IF EXISTS t_prewhere_stats_cost;
+CREATE TABLE t_prewhere_stats_cost (id UInt64, modality LowCardinality(String), h Map(String, String))
+ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0;
+
+-- Split `modality` 50/50: the reject-count ratio is then ~2x while the Map column dwarfs the
+-- scalar, so the cost gap dominates and cheap-filter-first stays stable under randomized serialization.
+INSERT INTO t_prewhere_stats_cost
+SELECT number, if(number % 2 = 0, 'active', ''), map('k', repeat('v', 300), 'k2', repeat('w', 300))
+FROM numbers(200000);
+OPTIMIZE TABLE t_prewhere_stats_cost FINAL;
+
+SELECT '-- with statistics: cheap filter is placed before the expensive Map predicate';
+SELECT position(explain, 'modality') > 0 AND position(explain, 'modality') < position(explain, 'arrayElement') AS cheap_first
+FROM (
+    EXPLAIN actions = 1 SELECT count() FROM t_prewhere_stats_cost WHERE modality = '' AND h['k'] = 'nope'
+) WHERE explain LIKE '%Prewhere filter column%';
+
+SELECT '-- correctness: result is the same regardless of ordering';
+SELECT count() FROM t_prewhere_stats_cost WHERE modality = '' AND h['k'] = 'nope';
+SELECT count() FROM t_prewhere_stats_cost WHERE modality = '' AND h['k'] = 'nope'
+SETTINGS allow_reorder_prewhere_conditions = 0;
+
+DROP TABLE t_prewhere_stats_cost;

tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.reference

@@ -0,0 +1,5 @@
+-- compact part: selective condition ordered first despite zero column sizes
+1
+-- correctness: result is the same regardless of ordering
+10
+10