add benchmark codes

Author: semiexp

cspuz_bench/Cargo.toml

@@ -12,6 +12,9 @@ default = []
 experimental-backend-glucose-rs = ["cspuz_rs/experimental-backend-glucose-rs", "cspuz_rs_puzzles/experimental-backend-glucose-rs"]
 
 [dependencies]
-cspuz_rs = { path = "../cspuz_rs", default-features = false }
+cspuz_core = { path = "../cspuz_core", default-features = false }
+cspuz_rs = { path = "../cspuz_rs", default-features = false, features = ["generator"]}
 cspuz_rs_puzzles = { path = "../cspuz_rs_puzzles", default-features = false }
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+rand = "0.8.5"

cspuz_bench/src/benchmarks/mod.rs

@@ -0,0 +1,200 @@
+use cspuz_core::integration::{reset_thread_local_perf_stats, thread_local_perf_stats};
+use cspuz_rs::graph::BoolGridEdgesIrrefutableFacts as CspuzRsBoolGridEdgesIrrefutableFacts;
+use serde::{Deserialize, Serialize};
+
+mod slitherlink;
+
+#[derive(Serialize, Deserialize)]
+pub struct BoolGridEdgesIrrefutableFacts {
+    pub horizontal: Vec<Vec<Option<bool>>>,
+    pub vertical: Vec<Vec<Option<bool>>>,
+}
+
+impl BoolGridEdgesIrrefutableFacts {
+    pub fn to_cspuz_rs(&self) -> CspuzRsBoolGridEdgesIrrefutableFacts {
+        CspuzRsBoolGridEdgesIrrefutableFacts {
+            horizontal: self.horizontal.clone(),
+            vertical: self.vertical.clone(),
+        }
+    }
+
+    pub fn from_cspuz_rs(facts: &CspuzRsBoolGridEdgesIrrefutableFacts) -> Self {
+        Self {
+            horizontal: facts.horizontal.clone(),
+            vertical: facts.vertical.clone(),
+        }
+    }
+}
+
+pub struct BenchResult {
+    pub elapsed_time_seconds: f64,
+    pub sat_num_propagations: f64,
+}
+
+fn accumulate(results: &[BenchResult]) -> BenchResult {
+    let elapsed_time_seconds = results.iter().map(|r| r.elapsed_time_seconds).sum();
+    let sat_num_propagations = results.iter().map(|r| r.sat_num_propagations).sum();
+    BenchResult {
+        elapsed_time_seconds,
+        sat_num_propagations,
+    }
+}
+
+#[derive(Debug)]
+pub enum BenchmarkError {
+    AnswerMismatch,
+}
+
+fn run_with_bench_result<F, T>(f: F) -> (T, BenchResult)
+where
+    F: FnOnce() -> T,
+{
+    reset_thread_local_perf_stats();
+
+    let orig_default_config = cspuz_core::config::Config::default();
+    let updated_default_config = cspuz_core::config::Config {
+        record_perf_stats_thread_local: true,
+        ..orig_default_config
+    };
+    cspuz_core::config::Config::set_default(updated_default_config);
+
+    let start = std::time::Instant::now();
+    let answer = f();
+    let elapsed_time_seconds = start.elapsed().as_secs_f64();
+
+    let perf_stats = thread_local_perf_stats();
+    let sat_num_propagations = perf_stats.propagations();
+
+    cspuz_core::config::Config::set_default(orig_default_config);
+
+    (
+        answer,
+        BenchResult {
+            elapsed_time_seconds,
+            sat_num_propagations: sat_num_propagations as f64,
+        },
+    )
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum Task {
+    Slitherlink(slitherlink::SlitherlinkTask),
+}
+
+pub fn run_benchmark(task: &Task) -> Result<BenchResult, BenchmarkError> {
+    match task {
+        Task::Slitherlink(task) => slitherlink::run_benchmark(task),
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct BaseBenchmarkSet {
+    solve_tasks: Vec<BaseSolveTask>,
+    generate_tasks: Vec<BaseGenerateTaskSet>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct BaseSolveTask {
+    name: String,
+    puzzle_type: String,
+    url: String,
+}
+
+#[derive(Serialize, Deserialize)]
+struct BaseGenerateTaskSet {
+    name: String,
+    puzzle_type: String,
+    height: usize,
+    width: usize,
+    seed: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+struct BenchmarkSet {
+    solve_tasks: Vec<SolveTask>,
+    generate_tasks: Vec<GenerateTaskSet>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct SolveTask {
+    name: String,
+    puzzle_type: String,
+    task: Task,
+}
+
+#[derive(Serialize, Deserialize)]
+struct GenerateTaskSet {
+    name: String,
+    puzzle_type: String,
+    tasks: Vec<Task>,
+}
+
+fn materialize_benchmark_set(base: BaseBenchmarkSet) -> BenchmarkSet {
+    let solve_tasks = base
+        .solve_tasks
+        .into_iter()
+        .map(|base_task| {
+            let task = match base_task.puzzle_type.as_str() {
+                "slitherlink" => slitherlink::materialize_solve_task(&base_task.url),
+                _ => panic!("Unknown puzzle type: {}", base_task.puzzle_type),
+            };
+            SolveTask {
+                name: base_task.name,
+                puzzle_type: base_task.puzzle_type,
+                task,
+            }
+        })
+        .collect();
+
+    let generate_tasks = base
+        .generate_tasks
+        .into_iter()
+        .map(|base_task| match base_task.puzzle_type.as_str() {
+            "slitherlink" => slitherlink::materialize_generate_task(base_task),
+            _ => panic!("Unknown puzzle type: {}", base_task.puzzle_type),
+        })
+        .collect();
+
+    BenchmarkSet {
+        solve_tasks,
+        generate_tasks,
+    }
+}
+
+pub fn run_materialize(src_path: &str, dest_path: &str) -> Result<(), Box<dyn std::error::Error>> {
+    let base_benchmark_set: BaseBenchmarkSet =
+        serde_json::from_reader(std::fs::File::open(src_path)?)?;
+    let benchmark_set = materialize_benchmark_set(base_benchmark_set);
+    serde_json::to_writer_pretty(std::fs::File::create(dest_path)?, &benchmark_set)?;
+    Ok(())
+}
+
+pub fn run_benchmarks(src_path: &str) {
+    let benchmark_set: BenchmarkSet =
+        serde_json::from_reader(std::fs::File::open(src_path).unwrap()).unwrap();
+
+    for solve_task in &benchmark_set.solve_tasks {
+        println!("Running solve benchmark: {}", solve_task.name);
+        let result = run_benchmark(&solve_task.task).unwrap();
+        println!(
+            "Elapsed time: {:.3} seconds, SAT propagations: {}",
+            result.elapsed_time_seconds, result.sat_num_propagations
+        );
+    }
+
+    for generate_task_set in &benchmark_set.generate_tasks {
+        println!("Running generate benchmark set: {}", generate_task_set.name);
+
+        let mut results = vec![];
+        for task in &generate_task_set.tasks {
+            let result = run_benchmark(task).unwrap();
+            results.push(result);
+        }
+        let accumulated_result = accumulate(&results);
+        println!(
+            "Accumulated result for generate benchmark set {}: Elapsed time: {:.3} seconds, SAT propagations: {}",
+            generate_task_set.name, accumulated_result.elapsed_time_seconds, accumulated_result.sat_num_propagations
+        );
+    }
+}

cspuz_bench/src/benchmarks/slitherlink.rs

@@ -0,0 +1,94 @@
+use cspuz_rs::generator;
+use cspuz_rs_puzzles::puzzles::slitherlink::{deserialize_problem, solve_slitherlink};
+
+use rand::SeedableRng;
+use serde::{Deserialize, Serialize};
+use std::cell::RefCell;
+
+#[derive(Serialize, Deserialize)]
+pub struct SlitherlinkTask {
+    pub problem: Vec<Vec<Option<i32>>>,
+    pub expectation: Option<super::BoolGridEdgesIrrefutableFacts>,
+}
+
+pub(super) fn run_benchmark(
+    task: &SlitherlinkTask,
+) -> Result<super::BenchResult, super::BenchmarkError> {
+    let problem = task.problem.clone();
+    let expectation = task.expectation.as_ref().map(|e| e.to_cspuz_rs());
+
+    let (answer, bench_result) =
+        super::run_with_bench_result(|| solve_slitherlink(false, &problem));
+    if answer != expectation {
+        Err(super::BenchmarkError::AnswerMismatch)
+    } else {
+        Ok(bench_result)
+    }
+}
+
+pub(super) fn materialize_solve_task(url: &str) -> super::Task {
+    let (full, problem) = deserialize_problem(url).expect("Failed to deserialize problem");
+    assert_eq!(full, false);
+
+    let expectation = solve_slitherlink(false, &problem)
+        .map(|ans| super::BoolGridEdgesIrrefutableFacts::from_cspuz_rs(&ans));
+    super::Task::Slitherlink(SlitherlinkTask {
+        problem,
+        expectation,
+    })
+}
+
+pub(super) fn materialize_generate_task(
+    base: super::BaseGenerateTaskSet,
+) -> super::GenerateTaskSet {
+    assert_eq!(base.puzzle_type, "slitherlink");
+
+    let height = base.height;
+    let width = base.width;
+    let pattern =
+        vec![
+            vec![
+                generator::Choice::new(vec![None, Some(0), Some(1), Some(2), Some(3)], None);
+                width
+            ];
+            height
+        ];
+
+    let trajectory = RefCell::new(vec![]);
+    let solve_with_record = |problem: &Vec<Vec<Option<i32>>>| {
+        let answer = solve_slitherlink(false, problem);
+
+        trajectory
+            .borrow_mut()
+            .push((problem.clone(), answer.clone()));
+
+        answer
+    };
+
+    let mut rng = rand::rngs::StdRng::seed_from_u64(42);
+    let _ = generator::Generator::new(
+        solve_with_record,
+        pattern,
+        generator::default_uniqueness_checker(),
+        generator::default_scorer(None, 5.0),
+    )
+    .generate(&mut rng);
+
+    let tasks = trajectory
+        .into_inner()
+        .into_iter()
+        .map(|(problem, answer)| {
+            let expectation =
+                answer.map(|ans| super::BoolGridEdgesIrrefutableFacts::from_cspuz_rs(&ans));
+            super::Task::Slitherlink(SlitherlinkTask {
+                problem,
+                expectation,
+            })
+        })
+        .collect::<Vec<_>>();
+    super::GenerateTaskSet {
+        name: base.name,
+        puzzle_type: base.puzzle_type,
+        tasks,
+    }
+}

cspuz_bench/src/bin/bench.rs

@@ -0,0 +1,9 @@
+fn main() {
+    // cspuz_bench/tasks/base.json -> cspuz_bench/tasks/bench.json
+    // NOTE: relative to this file, not the current working directory
+
+    cspuz_bench::benchmarks::run_benchmarks(concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/tasks/bench.json"
+    ));
+}

cspuz_bench/src/bin/materialize_tasks.rs

@@ -0,0 +1,7 @@
+fn main() {
+    cspuz_bench::benchmarks::run_materialize(
+        concat!(env!("CARGO_MANIFEST_DIR"), "/tasks/base.json"),
+        concat!(env!("CARGO_MANIFEST_DIR"), "/tasks/bench.json"),
+    )
+    .unwrap();
+}

cspuz_bench/src/lib.rs

@@ -0,0 +1 @@
+pub mod benchmarks;

cspuz_bench/tasks/base.json

@@ -0,0 +1,12 @@
+{
+    "solve_tasks": [],
+    "generate_tasks": [
+        {
+            "name": "slitherlink_10x10",
+            "puzzle_type": "slitherlink",
+            "height": 10,
+            "width": 10,
+            "seed": 42
+        }
+    ]
+}