Merge 'EXPLAIN QUERY PLAN: fix incorrect join order, add missing annotations, convert some snapshot tests to EQP-only' from Jussi Saurio

## EXPLAIN QUERY PLAN didn't show correct join order
- EXPLAIN QUERY PLAN was showing original join order, not the actual
join order after optimizer pass. Highly annoying and clankers get mega
confused by it too.
- Fixing the above issue revealed that left join ordering fuzz test
(based on EQP output) was not accounting for LEFT->INNER optimizations,
so fix that by only asserting on LEFT JOINs that survived the optimizer
pass
## EXPLAIN QUERY PLAN was missing stuff that SQLite has
- EXPLAIN QUERY PLAN was missing annotations for GROUP BY, DISTINCT,
subqueries + it had an incorrect `USE TEMP B-TREE FOR ORDER BY`
annotation because we never use temp b-tree for sorting, so instead
print `USE SORTER FOR ORDER BY`. Also add turso-custom prints like `USE
HASH TABLE FOR count(DISTINCT)` (sqlite doesn't have hash table)
## Add new snapshot format that only prints EQP instead of full
bytecode, in cases where we don't much care about the full bytecode
- Add new `snapshot-eqp` snapshot test format that just prints the EQP,
not the bytecode
- Convert a bunch of snapshots to use `snapshot-eqp` (for example, most
TPC-H tests; in many of them we just care about the join order and maybe
order by elimination)
- Note: the massive diff is due to the above - main change commits are
smaller

Reviewed-by: Preston Thorpe <preston@turso.tech>

Closes #5511

Author: jussisaurio

core/translate/group_by.rs

@@ -17,11 +17,12 @@ use crate::translate::{
     optimizer::Optimizable,
 };
 use crate::{
+    emit_explain,
     schema::PseudoCursorType,
     translate::collate::{get_collseq_from_expr, CollationSeq},
     util::exprs_are_equivalent,
     vdbe::{
-        builder::{CursorType, ProgramBuilder},
+        builder::{CursorType, ProgramBuilder, QueryMode},
         insn::Insn,
         BranchOffset,
     },
@@ -156,6 +157,7 @@ pub fn init_group_by<'a>(
             columns: column_count,
             order_and_collations,
         });
+        emit_explain!(program, false, "USE SORTER FOR GROUP BY".to_owned());
         let pseudo_cursor = group_by_create_pseudo_table(program, column_count);
         GroupByRowSource::Sorter {
             pseudo_cursor,

core/translate/main_loop.rs

@@ -21,6 +21,7 @@ use super::{
     },
 };
 use crate::{
+    emit_explain,
     schema::{Index, Table},
     translate::{
         collate::{get_collseq_from_expr, resolve_comparison_collseq, CollationSeq},
@@ -37,7 +38,7 @@ use crate::{
         affinity::{self, Affinity},
         builder::{
             CursorKey, CursorType, HashBuildSignature, MaterializedBuildInputModeTag,
-            ProgramBuilder,
+            ProgramBuilder, QueryMode,
         },
         insn::{to_u16, CmpInsFlags, HashBuildData, IdxInsertFlags, Insn},
         BranchOffset, CursorID,
@@ -147,6 +148,11 @@ pub fn init_loop(
                 label_on_conflict: program.allocate_label(),
             }),
         };
+        emit_explain!(
+            program,
+            false,
+            format!("USE HASH TABLE FOR {}(DISTINCT)", agg.func)
+        );
     }
     // Include hash-join build tables so their cursors are opened for hash build.
     let mut required_tables: HashSet<usize> = join_order

core/translate/order_by.rs

@@ -188,9 +188,11 @@ pub fn emit_order_by(
         + if has_sequence { 1 } else { 0 }
         + remappings.iter().filter(|r| !r.deduplicated).count();
 
-    // TODO: we need to know how many indices used for sorting
-    // to emit correct explain output.
-    emit_explain!(program, false, "USE TEMP B-TREE FOR ORDER BY".to_owned());
+    if use_heap_sort {
+        emit_explain!(program, false, "USE TEMP B-TREE FOR ORDER BY".to_owned());
+    } else {
+        emit_explain!(program, false, "USE SORTER FOR ORDER BY".to_owned());
+    }
 
     let cursor_id = if !use_heap_sort {
         let pseudo_cursor = program.alloc_cursor_id(CursorType::Pseudo(PseudoCursorType {

core/translate/subquery.rs

@@ -852,21 +852,24 @@ pub fn emit_from_clause_subqueries(
         }
     }
 
-    // Include hash-join build tables so EXPLAIN reflects all tables that feed the loop.
-    let mut required_tables: HashSet<usize> = join_order
+    // Build the iteration order: join_order first (execution order), then any
+    // hash-join build tables that aren't already in the join order.
+    let mut visit_order: Vec<usize> = join_order
         .iter()
         .map(|member| member.original_idx)
         .collect();
+    let visit_set: HashSet<usize> = visit_order.iter().copied().collect();
     for table in tables.joined_tables().iter() {
         if let Operation::HashJoin(hash_join_op) = &table.op {
-            required_tables.insert(hash_join_op.build_table_idx);
+            let build_idx = hash_join_op.build_table_idx;
+            if !visit_set.contains(&build_idx) {
+                visit_order.push(build_idx);
+            }
         }
     }
 
-    for (table_index, table_reference) in tables.joined_tables_mut().iter_mut().enumerate() {
-        if !required_tables.contains(&table_index) {
-            continue;
-        }
+    for table_index in visit_order {
+        let table_reference = &mut tables.joined_tables_mut()[table_index];
         emit_explain!(
             program,
             true,
@@ -1530,6 +1533,29 @@ pub fn emit_non_from_clause_subquery(
     is_correlated: bool,
 ) -> Result<()> {
     program.nested(|program| {
+        let subquery_id = program.next_subquery_eqp_id();
+        let correlated_prefix = if is_correlated { "CORRELATED " } else { "" };
+        match query_type {
+            SubqueryType::Exists { .. } => {
+                // EXISTS subqueries don't get a separate EQP annotation in SQLite;
+                // instead the SEARCH/SCAN line gets an "EXISTS" suffix handled elsewhere.
+            }
+            SubqueryType::In { .. } => {
+                emit_explain!(
+                    program,
+                    true,
+                    format!("{correlated_prefix}LIST SUBQUERY {subquery_id}")
+                );
+            }
+            SubqueryType::RowValue { .. } => {
+                emit_explain!(
+                    program,
+                    true,
+                    format!("{correlated_prefix}SCALAR SUBQUERY {subquery_id}")
+                );
+            }
+        }
+
         let label_skip_after_first_run = if !is_correlated {
             let label = program.allocate_label();
             program.emit_insn(Insn::Once {
@@ -1586,6 +1612,10 @@ pub fn emit_non_from_clause_subquery(
                 });
             }
         }
+        // Pop the parent explain for LIST/SCALAR SUBQUERY annotations.
+        if !matches!(query_type, SubqueryType::Exists { .. }) {
+            program.pop_current_parent_explain();
+        }
         if let Some(label) = label_skip_after_first_run {
             program.preassign_label_to_next_insn(label);
         }

core/vdbe/builder.rs

@@ -185,6 +185,8 @@ pub struct ProgramBuilder {
     /// references in non-recursive CTEs and to prevent fallthrough to schema
     /// resolution for same-named tables/views.
     ctes_being_defined: Vec<String>,
+    /// Counter for subquery numbering in EXPLAIN QUERY PLAN output.
+    next_subquery_eqp_id: usize,
 }
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
@@ -413,9 +415,16 @@ impl ProgramBuilder {
             materialized_ctes: HashMap::default(),
             cte_reference_counts: HashMap::default(),
             ctes_being_defined: Vec::new(),
+            next_subquery_eqp_id: 1,
         }
     }
 
+    pub fn next_subquery_eqp_id(&mut self) -> usize {
+        let id = self.next_subquery_eqp_id;
+        self.next_subquery_eqp_id += 1;
+        id
+    }
+
     pub fn alloc_hash_table_id(&mut self) -> usize {
         let id = self.next_hash_table_id;
         self.next_hash_table_id = self

testing/runner/src/parser/ast.rs

@@ -152,6 +152,9 @@ pub struct SnapshotCase {
     pub name_span: Range<usize>,
     /// SQL to execute (EXPLAIN will be prepended)
     pub sql: String,
+    /// If true, only run EXPLAIN QUERY PLAN (no bytecode).
+    /// Set by the `snapshot-eqp` directive.
+    pub eqp_only: bool,
     /// Common modifiers (setups, skip, backend, requires)
     pub modifiers: CaseModifiers,
 }

testing/runner/src/parser/lexer.rs

@@ -142,6 +142,10 @@ pub enum Token {
     #[token("snapshot")]
     Snapshot,
 
+    /// `snapshot-eqp` keyword (EXPLAIN QUERY PLAN only, no bytecode)
+    #[token("snapshot-eqp")]
+    SnapshotEqp,
+
     /// `expect` keyword
     #[token("expect")]
     Expect,
@@ -233,6 +237,7 @@ impl fmt::Display for Token {
             Token::Setup => write!(f, "setup"),
             Token::Test => write!(f, "test"),
             Token::Snapshot => write!(f, "snapshot"),
+            Token::SnapshotEqp => write!(f, "snapshot-eqp"),
             Token::Expect => write!(f, "expect"),
             Token::Error => write!(f, "error"),
             Token::Pattern => write!(f, "pattern"),

testing/runner/src/parser/mod.rs

@@ -77,7 +77,8 @@ impl Parser {
                     | Token::AtRequires
                     | Token::AtBackend
                     | Token::Test
-                    | Token::Snapshot,
+                    | Token::Snapshot
+                    | Token::SnapshotEqp,
                 ) => {
                     // Could be test or snapshot with decorators, peek ahead
                     let item = self.parse_test_or_snapshot()?;
@@ -258,8 +259,9 @@ impl Parser {
 
         // Now check if it's a test or snapshot
         match self.peek() {
-            Some(Token::Snapshot) => {
-                self.expect_token(Token::Snapshot)?;
+            Some(Token::Snapshot | Token::SnapshotEqp) => {
+                let eqp_only = matches!(self.peek(), Some(Token::SnapshotEqp));
+                self.advance();
                 let (name, name_span) = self.expect_identifier_with_span()?;
                 let sql = self.expect_block_content()?.trim().to_string();
 
@@ -269,6 +271,7 @@ impl Parser {
                     name,
                     name_span,
                     sql,
+                    eqp_only,
                     modifiers: CaseModifiers {
                         setups: test_setups,
                         skip,

testing/runner/src/runner/mod.rs

@@ -154,37 +154,40 @@ impl Runnable for SnapshotCase {
     }
 
     fn queries_to_execute(&self) -> Vec<String> {
-        // Run both EXPLAIN QUERY PLAN and EXPLAIN
-        vec![
-            format!("EXPLAIN QUERY PLAN {}", self.sql),
-            format!("EXPLAIN {}", self.sql),
-        ]
+        if self.eqp_only {
+            vec![format!("EXPLAIN QUERY PLAN {}", self.sql)]
+        } else {
+            vec![
+                format!("EXPLAIN QUERY PLAN {}", self.sql),
+                format!("EXPLAIN {}", self.sql),
+            ]
+        }
     }
 
     async fn evaluate_results(
         &self,
         results: Vec<QueryResult>,
         options: &RunOptions,
     ) -> TestOutcome {
-        // results[0] = EXPLAIN QUERY PLAN
-        // results[1] = EXPLAIN
-        let eqp_result = results.first().expect("should have two query results");
-        let explain_result = results.get(1).expect("should have two query results");
+        let eqp_result = results.first().expect("should have EQP result");
 
-        // Check for errors in query execution
         if let Some(err) = &eqp_result.error {
             return TestOutcome::Error {
                 message: format!("EXPLAIN QUERY PLAN failed: {err}"),
             };
         }
-        if let Some(err) = &explain_result.error {
-            return TestOutcome::Error {
-                message: format!("EXPLAIN failed: {err}"),
-            };
-        }
 
-        // Format both outputs
-        let actual_output = format_snapshot_content(&eqp_result.rows, &explain_result.rows);
+        let actual_output = if self.eqp_only {
+            format_eqp_snapshot_content(&eqp_result.rows)
+        } else {
+            let explain_result = results.get(1).expect("should have EXPLAIN result");
+            if let Some(err) = &explain_result.error {
+                return TestOutcome::Error {
+                    message: format!("EXPLAIN failed: {err}"),
+                };
+            }
+            format_snapshot_content(&eqp_result.rows, &explain_result.rows)
+        };
 
         // Build snapshot info with metadata
         let db_location_str = options.db_config.location.to_string();
@@ -223,6 +226,16 @@ impl Runnable for SnapshotCase {
     }
 }
 
+/// Format EQP-only snapshot content (no bytecode).
+fn format_eqp_snapshot_content(eqp_rows: &[Vec<String>]) -> String {
+    use crate::snapshot::format_explain_query_plan_output;
+
+    let mut output = String::new();
+    output.push_str("QUERY PLAN\n");
+    output.push_str(&format_explain_query_plan_output(eqp_rows));
+    output
+}
+
 /// Format the combined snapshot content with both EXPLAIN QUERY PLAN and EXPLAIN output.
 fn format_snapshot_content(eqp_rows: &[Vec<String>], explain_rows: &[Vec<String>]) -> String {
     use crate::snapshot::{format_explain_output, format_explain_query_plan_output};

testing/runner/tests/orderby/orderby_plan.sqltest

@@ -1,26 +1,26 @@
 @database :memory:
 @skip-file-if mvcc "mvcc has slightly different cursor ids, so skipping it for now"
 
-setup orderby_temp_btree {
+setup orderby_plan {
     CREATE TABLE t(a);
 }
 
-@setup orderby_temp_btree
-snapshot orderby_a_uses_temp_btree {
+@setup orderby_plan
+snapshot orderby_a_uses_sorter {
     SELECT * FROM t ORDER BY a;
 }
 
-@setup orderby_temp_btree
-snapshot orderby_a_desc_uses_temp_btree {
+@setup orderby_plan
+snapshot orderby_a_desc_uses_sorter {
     SELECT * FROM t ORDER BY a DESC;
 }
 
-@setup orderby_temp_btree
-snapshot orderby_rowid_no_temp_btree {
+@setup orderby_plan
+snapshot orderby_rowid_no_sorter {
     SELECT * FROM t ORDER BY rowid;
 }
 
-@setup orderby_temp_btree
-snapshot orderby_rowid_desc_no_temp_btree {
+@setup orderby_plan
+snapshot orderby_rowid_desc_no_sorter {
     SELECT * FROM t ORDER BY rowid DESC;
 }