Add cspuz_bench crate

Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "cspuz_rs",
     "cspuz_rs_puzzles",
     "cspuz_solver_backend",
+    "cspuz_bench",
 ]
 
 default-members = [

cspuz_bench/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "cspuz_bench"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "cspuz_bench"
+crate-type = ["lib"]
+
+[features]
+default = []
+experimental-backend-glucose-rs = ["cspuz_rs/experimental-backend-glucose-rs", "cspuz_rs_puzzles/experimental-backend-glucose-rs"]
+
+[dependencies]
+cspuz_core = { path = "../cspuz_core", default-features = false }
+cspuz_rs = { path = "../cspuz_rs", default-features = false, features = ["generator"]}
+cspuz_rs_puzzles = { path = "../cspuz_rs_puzzles", default-features = false }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+zstd = "0.13"
+rand = "0.8.5"

cspuz_bench/src/benchmarks/dbchoco.rs

@@ -0,0 +1,38 @@
+use cspuz_rs_puzzles::puzzles::dbchoco::{deserialize_problem, solve_doublechoco};
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct DoubleChocoTask {
+    pub color: Vec<Vec<i32>>,
+    pub num: Vec<Vec<Option<i32>>>,
+    pub expectation: Option<super::BoolInnerGridEdgesIrrefutableFacts>,
+}
+
+pub(super) fn run_benchmark(
+    task: &DoubleChocoTask,
+) -> Result<super::BenchResult, super::BenchmarkError> {
+    let color = task.color.clone();
+    let num = task.num.clone();
+    let expectation = task.expectation.as_ref().map(|e| e.to_cspuz_rs());
+
+    let (answer, bench_result) = super::run_with_bench_result(|| solve_doublechoco(&color, &num));
+    if answer != expectation {
+        Err(super::BenchmarkError::AnswerMismatch)
+    } else {
+        Ok(bench_result)
+    }
+}
+
+pub(super) fn materialize_solve_task(url: &str) -> super::Task {
+    let (color, num) = deserialize_problem(url).expect("Failed to deserialize problem");
+    let expectation = solve_doublechoco(&color, &num)
+        .as_ref()
+        .map(super::BoolInnerGridEdgesIrrefutableFacts::from_cspuz_rs);
+
+    super::Task::DoubleChoco(DoubleChocoTask {
+        color,
+        num,
+        expectation,
+    })
+}

cspuz_bench/src/benchmarks/mod.rs

@@ -0,0 +1,288 @@
+use cspuz_core::integration::{reset_thread_local_perf_stats, thread_local_perf_stats};
+use cspuz_rs::graph::BoolGridEdgesIrrefutableFacts as CspuzRsBoolGridEdgesIrrefutableFacts;
+use cspuz_rs::graph::BoolInnerGridEdgesIrrefutableFacts as CspuzRsBoolInnerGridEdgesIrrefutableFacts;
+use serde::{Deserialize, Serialize};
+
+mod dbchoco;
+mod nurikabe;
+mod slitherlink;
+mod yajilin;
+
+#[derive(Serialize, Deserialize)]
+pub struct BoolGridEdgesIrrefutableFacts {
+    pub horizontal: Vec<Vec<Option<bool>>>,
+    pub vertical: Vec<Vec<Option<bool>>>,
+}
+
+impl BoolGridEdgesIrrefutableFacts {
+    pub fn to_cspuz_rs(&self) -> CspuzRsBoolGridEdgesIrrefutableFacts {
+        CspuzRsBoolGridEdgesIrrefutableFacts {
+            horizontal: self.horizontal.clone(),
+            vertical: self.vertical.clone(),
+        }
+    }
+
+    pub fn from_cspuz_rs(facts: &CspuzRsBoolGridEdgesIrrefutableFacts) -> Self {
+        Self {
+            horizontal: facts.horizontal.clone(),
+            vertical: facts.vertical.clone(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BoolInnerGridEdgesIrrefutableFacts {
+    pub horizontal: Vec<Vec<Option<bool>>>,
+    pub vertical: Vec<Vec<Option<bool>>>,
+}
+
+impl BoolInnerGridEdgesIrrefutableFacts {
+    pub fn to_cspuz_rs(&self) -> CspuzRsBoolInnerGridEdgesIrrefutableFacts {
+        CspuzRsBoolInnerGridEdgesIrrefutableFacts {
+            horizontal: self.horizontal.clone(),
+            vertical: self.vertical.clone(),
+        }
+    }
+
+    pub fn from_cspuz_rs(facts: &CspuzRsBoolInnerGridEdgesIrrefutableFacts) -> Self {
+        Self {
+            horizontal: facts.horizontal.clone(),
+            vertical: facts.vertical.clone(),
+        }
+    }
+}
+
+pub struct BenchResult {
+    pub elapsed_time_seconds: f64,
+    pub sat_num_propagations: f64,
+}
+
+fn accumulate(results: &[BenchResult]) -> BenchResult {
+    let elapsed_time_seconds = results.iter().map(|r| r.elapsed_time_seconds).sum();
+    let sat_num_propagations = results.iter().map(|r| r.sat_num_propagations).sum();
+    BenchResult {
+        elapsed_time_seconds,
+        sat_num_propagations,
+    }
+}
+
+#[derive(Debug)]
+pub enum BenchmarkError {
+    AnswerMismatch,
+}
+
+fn run_with_bench_result<F, T>(f: F) -> (T, BenchResult)
+where
+    F: FnOnce() -> T,
+{
+    reset_thread_local_perf_stats();
+
+    let orig_default_config = cspuz_core::config::Config::default();
+    let updated_default_config = cspuz_core::config::Config {
+        record_perf_stats_thread_local: true,
+        ..orig_default_config
+    };
+    cspuz_core::config::Config::set_default(updated_default_config);
+
+    let start = std::time::Instant::now();
+    let answer = f();
+    let elapsed_time_seconds = start.elapsed().as_secs_f64();
+
+    let perf_stats = thread_local_perf_stats();
+    let sat_num_propagations = perf_stats.propagations();
+
+    cspuz_core::config::Config::set_default(orig_default_config);
+
+    (
+        answer,
+        BenchResult {
+            elapsed_time_seconds,
+            sat_num_propagations: sat_num_propagations as f64,
+        },
+    )
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum Task {
+    DoubleChoco(dbchoco::DoubleChocoTask),
+    Nurikabe(nurikabe::NurikabeTask),
+    Slitherlink(slitherlink::SlitherlinkTask),
+    Yajilin(yajilin::YajilinTask),
+}
+
+pub fn run_benchmark(task: &Task) -> Result<BenchResult, BenchmarkError> {
+    match task {
+        Task::DoubleChoco(task) => dbchoco::run_benchmark(task),
+        Task::Nurikabe(task) => nurikabe::run_benchmark(task),
+        Task::Slitherlink(task) => slitherlink::run_benchmark(task),
+        Task::Yajilin(task) => yajilin::run_benchmark(task),
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct BaseBenchmarkSet {
+    solve_tasks: Vec<BaseSolveTask>,
+    generate_tasks: Vec<BaseGenerateTaskSet>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct BaseSolveTask {
+    name: String,
+    puzzle_type: String,
+    url: String,
+    comment: Option<String>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct BaseGenerateTaskSet {
+    name: String,
+    puzzle_type: String,
+    height: usize,
+    width: usize,
+    seed: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+struct BenchmarkSet {
+    solve_tasks: Vec<SolveTask>,
+    generate_tasks: Vec<GenerateTaskSet>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct SolveTask {
+    name: String,
+    puzzle_type: String,
+    task: Task,
+}
+
+#[derive(Serialize, Deserialize)]
+struct GenerateTaskSet {
+    name: String,
+    puzzle_type: String,
+    tasks: Vec<Task>,
+}
+
+fn materialize_benchmark_set(base: BaseBenchmarkSet) -> BenchmarkSet {
+    let solve_tasks = base
+        .solve_tasks
+        .into_iter()
+        .map(|base_task| {
+            let task = match base_task.puzzle_type.as_str() {
+                "dbchoco" => dbchoco::materialize_solve_task(&base_task.url),
+                "nurikabe" => nurikabe::materialize_solve_task(&base_task.url),
+                "slitherlink" => slitherlink::materialize_solve_task(&base_task.url),
+                "yajilin" => yajilin::materialize_solve_task(&base_task.url),
+                _ => panic!("Unknown puzzle type: {}", base_task.puzzle_type),
+            };
+            SolveTask {
+                name: base_task.name,
+                puzzle_type: base_task.puzzle_type,
+                task,
+            }
+        })
+        .collect();
+
+    let generate_tasks = base
+        .generate_tasks
+        .into_iter()
+        .map(|base_task| match base_task.puzzle_type.as_str() {
+            "slitherlink" => slitherlink::materialize_generate_task(base_task),
+            "yajilin" => yajilin::materialize_generate_task(base_task),
+            _ => panic!("Unknown puzzle type: {}", base_task.puzzle_type),
+        })
+        .collect();
+
+    BenchmarkSet {
+        solve_tasks,
+        generate_tasks,
+    }
+}
+
+pub fn run_materialize(src_path: &str, dest_path: &str) -> Result<(), Box<dyn std::error::Error>> {
+    let base_benchmark_set: BaseBenchmarkSet =
+        serde_json::from_reader(std::fs::File::open(src_path)?)?;
+    let benchmark_set = materialize_benchmark_set(base_benchmark_set);
+    let dest_file = std::fs::File::create(dest_path)?;
+    let mut encoder = zstd::stream::write::Encoder::new(dest_file, 0)?;
+    serde_json::to_writer(&mut encoder, &benchmark_set)?;
+    encoder.finish()?;
+    Ok(())
+}
+
+pub fn run_benchmarks(src_path: &str) {
+    let src_file = std::fs::File::open(src_path).unwrap();
+    let decoder = zstd::stream::read::Decoder::new(src_file).unwrap();
+    let benchmark_set: BenchmarkSet = serde_json::from_reader(decoder).unwrap();
+    let mut results = vec![];
+
+    for solve_task in &benchmark_set.solve_tasks {
+        println!("Running solve benchmark: {}", solve_task.name);
+        let result = run_benchmark(&solve_task.task).unwrap();
+        results.push(("solve", solve_task.name.as_str(), result));
+    }
+
+    for generate_task_set in &benchmark_set.generate_tasks {
+        println!("Running generate benchmark set: {}", generate_task_set.name);
+
+        let mut task_results = vec![];
+        for task in &generate_task_set.tasks {
+            let result = run_benchmark(task).unwrap();
+            task_results.push(result);
+        }
+        let accumulated_result = accumulate(&task_results);
+        results.push((
+            "generate",
+            generate_task_set.name.as_str(),
+            accumulated_result,
+        ));
+    }
+
+    print_benchmark_results(&results);
+}
+
+fn print_benchmark_results(results: &[(&str, &str, BenchResult)]) {
+    let kind_width = results
+        .iter()
+        .map(|(kind, _, _)| kind.len())
+        .max()
+        .unwrap_or(0)
+        .max("Kind".len());
+    let name_width = results
+        .iter()
+        .map(|(_, name, _)| name.len())
+        .max()
+        .unwrap_or(0)
+        .max("Benchmark".len());
+    let elapsed_header = "Elapsed (s)";
+    let propagations_header = "SAT propagations";
+    let separator = format!(
+        "+-{}-+-{}-+-{}-+-{}-+",
+        "-".repeat(kind_width),
+        "-".repeat(name_width),
+        "-".repeat(elapsed_header.len()),
+        "-".repeat(propagations_header.len()),
+    );
+
+    println!("{separator}");
+    println!(
+        "| {:<kind_width$} | {:<name_width$} | {:>elapsed_width$} | {:>propagations_width$} |",
+        "Kind",
+        "Benchmark",
+        elapsed_header,
+        propagations_header,
+        elapsed_width = elapsed_header.len(),
+        propagations_width = propagations_header.len(),
+    );
+    println!("{separator}");
+    for (kind, name, result) in results {
+        println!(
+            "| {kind:<kind_width$} | {name:<name_width$} | {:>elapsed_width$.3} | {:>propagations_width$.0} |",
+            result.elapsed_time_seconds,
+            result.sat_num_propagations,
+            elapsed_width = elapsed_header.len(),
+            propagations_width = propagations_header.len(),
+        );
+    }
+    println!("{separator}");
+}

cspuz_bench/src/benchmarks/nurikabe.rs

@@ -0,0 +1,30 @@
+use cspuz_rs_puzzles::puzzles::nurikabe::{deserialize_problem, solve_nurikabe};
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct NurikabeTask {
+    pub clues: Vec<Vec<Option<i32>>>,
+    pub expectation: Option<Vec<Vec<Option<bool>>>>,
+}
+
+pub(super) fn run_benchmark(
+    task: &NurikabeTask,
+) -> Result<super::BenchResult, super::BenchmarkError> {
+    let clues = task.clues.clone();
+    let expectation = task.expectation.clone();
+
+    let (answer, bench_result) = super::run_with_bench_result(|| solve_nurikabe(&clues));
+    if answer != expectation {
+        Err(super::BenchmarkError::AnswerMismatch)
+    } else {
+        Ok(bench_result)
+    }
+}
+
+pub(super) fn materialize_solve_task(url: &str) -> super::Task {
+    let clues = deserialize_problem(url).expect("Failed to deserialize problem");
+    let expectation = solve_nurikabe(&clues);
+
+    super::Task::Nurikabe(NurikabeTask { clues, expectation })
+}