refactor: move examples to the planner crate (#1014)

# Rationale for this change
We need to use `proof-of-sql-planner` to process queries now. Hence the
examples need to be moved. Unsupported features such as ORDER BY are
removed.


# What changes are included in this PR?

<!--
There is no need to duplicate the description in the ticket here but it
is sometimes worth providing a summary of the individual changes in this
PR.

Example:
- Add `NestedLoopJoinExec`.
- Speed up `HashJoinExec`.
- Route joins to `NestedLoopJoinExec` if the outer input is sufficiently
small.
-->

# Are these changes tested?
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code

If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?

Example:
Yes.
-->

Author: iajoiner

.github/workflows/lint-and-test.yml

@@ -132,21 +132,21 @@ jobs:
         - name: Install stable toolchain
           run: curl https://sh.rustup.rs -sSf | bash -s -- -y --profile minimal && source ~/.cargo/env && rustup toolchain install
         - name: Run hello_world example (With Blitzar)
-          run: cargo run --example hello_world --features="test"
+          run: cargo run --example hello_world --features="proof-of-sql/test"
         - name: Run hello_world example (Without Blitzar and With Rayon)
-          run: cargo run --example hello_world --no-default-features --features="rayon test"
+          run: cargo run --example hello_world --no-default-features --features="proof-of-sql/rayon proof-of-sql/test"
         - name: Run hello_world example (Without Blitzar and Without Rayon)
-          run: cargo run --example hello_world --no-default-features --features="test"
+          run: cargo run --example hello_world --no-default-features --features="proof-of-sql/test"
         - name: Run space example
           run: cargo run --example space
         - name: Run dog breeds example
           run: cargo run --example dog_breeds
         - name: Run wood types example
           run: cargo run --example wood_types
         - name: Run posql_db example (With Blitzar)
-          run: bash crates/proof-of-sql/examples/posql_db/run_example.sh
+          run: bash crates/proof-of-sql-planner/examples/posql_db/run_example.sh
         - name: Run posql_db example (Without Blitzar)
-          run: bash crates/proof-of-sql/examples/posql_db/run_example.sh --no-default-features --features="rayon"
+          run: bash crates/proof-of-sql-planner/examples/posql_db/run_example.sh --no-default-features --features="proof-of-sql/rayon"
 
   clippy:
     name: Clippy

Cargo.toml

@@ -21,7 +21,6 @@ ark-serialize = { version = "0.5.0" }
 ark-std = { version = "0.5.0", default-features = false }
 arrayvec = { version = "0.7", default-features = false }
 arrow = { version = "51.0.0" }
-arrow-csv = { version = "51.0.0" }
 bincode = { version = "2.0.0-rc.3", default-features = false }
 bit-iter = { version = "1.1.1" }
 bigdecimal = { version = "0.4.5", default-features = false, features = ["serde"] }
@@ -62,7 +61,7 @@ proptest-derive = { version = "0.5.1" }
 rand = { version = "0.8", default-features = false }
 rand_chacha = { version = "0.3.1" }
 rand_core = { version = "0.6", default-features = false }
-rayon = { version = "1.5" }
+rayon = { version = "1.11.0" }
 serde = { version = "1", default-features = false }
 serde_json = { version = "1", default-features = false, features = ["alloc"] }
 sha2 = { version = "0.10.8", default-features = false }

crates/proof-of-sql-planner/Cargo.toml

@@ -14,11 +14,14 @@ crate-type = ["cdylib", "rlib"]
 [dependencies]
 ahash = { workspace = true }
 arrow = { workspace = true }
+clap = { workspace = true, features = ["derive", "env"]}
 datafusion = { workspace = true }
 # getrandom and uuid must be compiled with js feature
 getrandom = { workspace = true, features = ["js"] }
 indexmap = { workspace = true }
+postcard = { workspace = true }
 proof-of-sql = { workspace = true, features = ["arrow"] }
+rand = { workspace = true, default-features = false }
 serde = { workspace = true }
 snafu = { workspace = true }
 sqlparser = { workspace = true }
@@ -34,5 +37,70 @@ hex = { workspace = true }
 itertools = { workspace = true }
 proof-of-sql = { workspace = true, features = ["arrow", "hyperkzg_proof"] }
 
+[package.metadata.cargo-udeps.ignore]
+normal = ["clap", "postcard"]
+
 [lints]
 workspace = true
+
+[features]
+cpu-perf = ["proof-of-sql/cpu-perf"]
+
+[[example]]
+name = "hello_world"
+required-features = ["proof-of-sql/test"]
+
+[[example]]
+name = "posql_db"
+required-features = ["proof-of-sql/utils"]
+
+[[example]]
+name = "space"
+
+[[example]]
+name = "dog_breeds"
+
+[[example]]
+name = "wood_types"
+
+[[example]]
+name = "dinosaurs"
+
+[[example]]
+name = "books"
+
+[[example]]
+name = "programming_books"
+
+[[example]]
+name = "brands"
+
+[[example]]
+name = "census"
+
+[[example]]
+name = "plastics"
+
+[[example]]
+name = "avocado-prices"
+
+[[example]]
+name = "sushi"
+
+[[example]]
+name = "stocks"
+
+[[example]]
+name = "tech_gadget_prices"
+
+[[example]]
+name = "albums"
+
+[[example]]
+name = "vehicles"
+
+[[example]]
+name = "countries"
+
+[[example]]
+name = "rockets"

…/proof-of-sql/examples/albums/albums.csv …f-sql-planner/examples/albums/albums.csvcrates/proof-of-sql/examples/albums/albums.csv renamed to crates/proof-of-sql-planner/examples/albums/albums.csv crates/proof-of-sql/examples/albums/albums.csv renamed to crates/proof-of-sql-planner/examples/albums/albums.csv

@@ -2,10 +2,17 @@
 //! To run this, use `cargo run --release --example albums`.
 //!
 //! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed,
-//! you can run `cargo run --release --example albums --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation.
+//! you can run `cargo run --release --example albums --no-default-features --features="cpu-perf"` instead. It will be slower for proof generation.
 
-use arrow::datatypes::SchemaRef;
-use arrow_csv::{infer_schema_from_files, ReaderBuilder};
+use datafusion::{
+    arrow::{
+        csv::{infer_schema_from_files, ReaderBuilder},
+        datatypes::SchemaRef,
+        record_batch::RecordBatch,
+        util::pretty::pretty_format_batches,
+    },
+    config::ConfigOptions,
+};
 use proof_of_sql::{
     base::database::{
         arrow_schema_utility::get_posql_compatible_schema, OwnedTable, OwnedTableTestAccessor,
@@ -14,11 +21,11 @@ use proof_of_sql::{
     proof_primitive::dory::{
         DynamicDoryEvaluationProof, ProverSetup, PublicParameters, VerifierSetup,
     },
-    sql::{
-        parse::QueryExpr, postprocessing::apply_postprocessing_steps, proof::VerifiableQueryResult,
-    },
+    sql::proof::VerifiableQueryResult,
 };
+use proof_of_sql_planner::sql_to_proof_plans;
 use rand::{rngs::StdRng, SeedableRng};
+use sqlparser::{dialect::GenericDialect, parser::Parser};
 use std::{fs::File, time::Instant};
 
 // We generate the public parameters and the setups used by the prover and verifier for the Dory PCS.
@@ -38,14 +45,16 @@ fn prove_and_verify_query(
     // Parse the query:
     println!("Parsing the query: {sql}...");
     let now = Instant::now();
-    let query_plan = QueryExpr::try_new(sql.parse().unwrap(), "albums".into(), accessor).unwrap();
+    let config = ConfigOptions::default();
+    let statements = Parser::parse_sql(&GenericDialect {}, sql).unwrap();
+    let query_plan = &sql_to_proof_plans(&statements, accessor, &config).unwrap()[0];
     println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.);
 
     // Generate the proof and result:
     print!("Generating proof...");
     let now = Instant::now();
     let verifiable_result = VerifiableQueryResult::<DynamicDoryEvaluationProof>::new(
-        query_plan.proof_expr(),
+        query_plan,
         accessor,
         &prover_setup,
         &[],
@@ -56,15 +65,17 @@ fn prove_and_verify_query(
     // Verify the result with the proof:
     print!("Verifying proof...");
     let now = Instant::now();
-    let result = verifiable_result
-        .verify(query_plan.proof_expr(), accessor, &verifier_setup, &[])
+    let result: RecordBatch = verifiable_result
+        .verify(query_plan, accessor, &verifier_setup, &[])
+        .unwrap()
+        .table
+        .try_into()
         .unwrap();
-    let result = apply_postprocessing_steps(result.table, query_plan.postprocessing());
     println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.);
 
     // Display the result
     println!("Query Result:");
-    println!("{result:?}");
+    println!("{}", pretty_format_batches(&[result]).unwrap());
 }
 
 fn main() {
@@ -73,7 +84,7 @@ fn main() {
     let prover_setup = ProverSetup::from(&public_parameters);
     let verifier_setup = VerifierSetup::from(&public_parameters);
 
-    let filename = "crates/proof-of-sql/examples/albums/albums.csv";
+    let filename = "crates/proof-of-sql-planner/examples/albums/albums.csv";
     let schema = get_posql_compatible_schema(&SchemaRef::new(
         infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(),
     ));
@@ -96,15 +107,15 @@ fn main() {
 
     // Query 1: Count number of albums by genre
     prove_and_verify_query(
-        "SELECT genre, COUNT(*) AS album_count FROM albums.collection GROUP BY genre ORDER BY genre",
+        "SELECT genre, COUNT(*) AS album_count FROM albums.collection GROUP BY genre",
         &accessor,
         &prover_setup,
         &verifier_setup,
     );
 
     // Query 2: Find all albums from the 1970s
     prove_and_verify_query(
-        "SELECT artist, album, year FROM albums.collection WHERE year >= 1970 AND year < 1980 ORDER BY year",
+        "SELECT artist, album, year FROM albums.collection WHERE year >= 1970 AND year < 1980",
         &accessor,
         &prover_setup,
         &verifier_setup,
@@ -120,7 +131,7 @@ fn main() {
 
     // Query 4: List all rock albums after 1975 (using exact matches for Rock genres)
     prove_and_verify_query(
-        "SELECT artist, album, year FROM albums.collection WHERE (genre = 'Rock' OR genre = 'Hard Rock' OR genre = 'Progressive Rock') AND year > 1975 ORDER BY year DESC",
+        "SELECT artist, album, year FROM albums.collection WHERE (genre = 'Rock' OR genre = 'Hard Rock' OR genre = 'Progressive Rock') AND year > 1975",
         &accessor,
         &prover_setup,
         &verifier_setup,

…tes/proof-of-sql/examples/albums/main.rs …f-of-sql-planner/examples/albums/main.rscrates/proof-of-sql/examples/albums/main.rs renamed to crates/proof-of-sql-planner/examples/albums/main.rs crates/proof-of-sql/examples/albums/main.rs renamed to crates/proof-of-sql-planner/examples/albums/main.rs

@@ -2,19 +2,29 @@
 //! To run, use `cargo run --example avocado-prices`.
 //!
 //! NOTE: If this doesn't work because you do not have the appropriate GPU drivers installed,
-//! you can run `cargo run --release --example avocado-prices --no-default-features --features="arrow cpu-perf"` instead. It will be slower for proof generation.
-use arrow::datatypes::SchemaRef;
-use arrow_csv::{infer_schema_from_files, ReaderBuilder};
+//! you can run `cargo run --release --example avocado-prices --no-default-features --features="cpu-perf"` instead. It will be slower for proof generation.
+use datafusion::{
+    arrow::{
+        csv::{infer_schema_from_files, ReaderBuilder},
+        datatypes::SchemaRef,
+        record_batch::RecordBatch,
+        util::pretty::pretty_format_batches,
+    },
+    config::ConfigOptions,
+};
 use proof_of_sql::{
-    base::database::{OwnedTable, OwnedTableTestAccessor, TableRef},
+    base::database::{
+        arrow_schema_utility::get_posql_compatible_schema, OwnedTable, OwnedTableTestAccessor,
+        TableRef,
+    },
     proof_primitive::dory::{
         DynamicDoryEvaluationProof, ProverSetup, PublicParameters, VerifierSetup,
     },
-    sql::{
-        parse::QueryExpr, postprocessing::apply_postprocessing_steps, proof::VerifiableQueryResult,
-    },
+    sql::proof::VerifiableQueryResult,
 };
+use proof_of_sql_planner::sql_to_proof_plans;
 use rand::{rngs::StdRng, SeedableRng};
+use sqlparser::{dialect::GenericDialect, parser::Parser};
 use std::{fs::File, time::Instant};
 
 // We generate the public parameters and the setups used by the prover and verifier for the Dory PCS.
@@ -42,14 +52,16 @@ fn prove_and_verify_query(
     // Parse the query:
     println!("Parsing the query: {sql}...");
     let now = Instant::now();
-    let query_plan = QueryExpr::try_new(sql.parse().unwrap(), "avocado".into(), accessor).unwrap();
+    let config = ConfigOptions::default();
+    let statements = Parser::parse_sql(&GenericDialect {}, sql).unwrap();
+    let query_plan = &sql_to_proof_plans(&statements, accessor, &config).unwrap()[0];
     println!("Done in {} ms.", now.elapsed().as_secs_f64() * 1000.);
 
     // Generate the proof and result:
     print!("Generating proof...");
     let now = Instant::now();
     let verifiable_result = VerifiableQueryResult::<DynamicDoryEvaluationProof>::new(
-        query_plan.proof_expr(),
+        query_plan,
         accessor,
         &prover_setup,
         &[],
@@ -60,15 +72,17 @@ fn prove_and_verify_query(
     // Verify the result with the proof:
     print!("Verifying proof...");
     let now = Instant::now();
-    let result = verifiable_result
-        .verify(query_plan.proof_expr(), accessor, &verifier_setup, &[])
+    let result: RecordBatch = verifiable_result
+        .verify(query_plan, accessor, &verifier_setup, &[])
+        .unwrap()
+        .table
+        .try_into()
         .unwrap();
-    let result = apply_postprocessing_steps(result.table, query_plan.postprocessing());
     println!("Verified in {} ms.", now.elapsed().as_secs_f64() * 1000.);
 
     // Display the result
     println!("Query Result:");
-    println!("{result:?}");
+    println!("{}", pretty_format_batches(&[result]).unwrap());
 }
 
 fn main() {
@@ -77,16 +91,18 @@ fn main() {
     let prover_setup = ProverSetup::from(&public_parameters);
     let verifier_setup = VerifierSetup::from(&public_parameters);
 
-    let filename = "./crates/proof-of-sql/examples/avocado-prices/avocado-prices.csv";
-    let data_batch = ReaderBuilder::new(SchemaRef::new(
+    let filename = "crates/proof-of-sql-planner/examples/avocado-prices/avocado-prices.csv";
+    let schema = get_posql_compatible_schema(&SchemaRef::new(
         infer_schema_from_files(&[filename.to_string()], b',', None, true).unwrap(),
-    ))
-    .with_header(true)
-    .build(File::open(filename).unwrap())
-    .unwrap()
-    .next()
-    .unwrap()
-    .unwrap();
+    ));
+
+    let data_batch = ReaderBuilder::new(schema.clone())
+        .with_header(true)
+        .build(File::open(filename).unwrap())
+        .unwrap()
+        .next()
+        .unwrap()
+        .unwrap();
 
     // Load the table into an "Accessor" so that the prover and verifier can access the data/commitments.
     let accessor = OwnedTableTestAccessor::<DynamicDoryEvaluationProof>::new_from_table(
@@ -97,19 +113,19 @@ fn main() {
     );
 
     prove_and_verify_query(
-        "SELECT COUNT(*) AS total FROM prices",
+        "SELECT COUNT(*) AS total FROM avocado.prices",
         &accessor,
         &prover_setup,
         &verifier_setup,
     );
     prove_and_verify_query(
-        "SELECT Price, COUNT(*) AS total FROM prices GROUP BY Price ORDER BY total",
+        "SELECT Price, COUNT(*) AS total FROM avocado.prices GROUP BY Price",
         &accessor,
         &prover_setup,
         &verifier_setup,
     );
     prove_and_verify_query(
-        "SELECT Year, COUNT(*) AS total FROM prices WHERE Price > 100 GROUP BY Year ORDER BY total DESC LIMIT 5",
+        "SELECT Year, COUNT(*) AS total FROM avocado.prices WHERE Price > 100 GROUP BY Year",
         &accessor,
         &prover_setup,
         &verifier_setup,