cli: support generating cu reports (#99)

Author: buffalojoec

Cargo.lock

@@ -56,9 +56,10 @@
 //! | bench3 | 2,811 | +2,361 |
 //! ```
 
-mod result;
+pub mod result;
 
 use {
+    chrono::Utc,
     mollusk_svm::{result::ProgramResult, Mollusk},
     result::{write_results, MolluskComputeUnitBenchResult},
     solana_account::Account,
@@ -113,6 +114,7 @@ impl<'a> MolluskComputeUnitBencher<'a> {
 
     /// Execute the benches.
     pub fn execute(&mut self) {
+        let table_header = Utc::now().to_string();
         let solana_version = get_solana_version();
         let bench_results = std::mem::take(&mut self.benches)
             .into_iter()
@@ -132,11 +134,11 @@ impl<'a> MolluskComputeUnitBencher<'a> {
                 MolluskComputeUnitBenchResult::new(name, result)
             })
             .collect::<Vec<_>>();
-        write_results(&self.out_dir, &solana_version, bench_results);
+        write_results(&self.out_dir, &table_header, &solana_version, bench_results);
     }
 }
 
-fn get_solana_version() -> String {
+pub fn get_solana_version() -> String {
     match Command::new("solana").arg("--version").output() {
         Ok(output) if output.status.success() => {
             String::from_utf8_lossy(&output.stdout).trim().to_string()

bencher/src/lib.rs

@@ -1,13 +1,12 @@
 //! Compute unit benchmarking results and checks.
 
 use {
-    chrono::{DateTime, Utc},
     mollusk_svm::result::InstructionResult,
     num_format::{Locale, ToFormattedString},
     std::path::Path,
 };
 
-pub(crate) struct MolluskComputeUnitBenchResult<'a> {
+pub struct MolluskComputeUnitBenchResult<'a> {
     name: &'a str,
     cus_consumed: u64,
 }
@@ -19,8 +18,9 @@ impl<'a> MolluskComputeUnitBenchResult<'a> {
     }
 }
 
-pub(crate) fn write_results(
+pub fn write_results(
     out_dir: &Path,
+    table_header: &str,
     solana_version: &str,
     results: Vec<MolluskComputeUnitBenchResult>,
 ) {
@@ -38,7 +38,7 @@ pub(crate) fn write_results(
         .map(|content| parse_last_md_table(content));
 
     // Prepare to write a new table.
-    let mut md_table = md_header(solana_version);
+    let mut md_table = md_header(table_header, solana_version);
 
     // Evaluate the results against the previous table, if any.
     // If there are changes, write a new table.
@@ -80,24 +80,23 @@ pub(crate) fn write_results(
     }
 }
 
-fn md_header(solana_version: &str) -> String {
-    let now: DateTime<Utc> = Utc::now();
+fn md_header(table_header: &str, solana_version: &str) -> String {
     format!(
-        r#"#### Compute Units: {}
+        r#"#### {}
 
 Solana CLI Version: {}
 
 | Name | CUs | Delta |
 |------|------|-------|
 "#,
-        now, solana_version,
+        table_header, solana_version,
     )
 }
 
 fn parse_last_md_table(content: &str) -> Vec<MolluskComputeUnitBenchResult> {
     let mut results = vec![];
 
-    for line in content.lines().skip(4) {
+    for line in content.lines().skip(6) {
         if line.starts_with("####") || line.is_empty() {
             break;
         }

bencher/src/result.rs

@@ -9,10 +9,12 @@ edition = { workspace = true }
 version = { workspace = true }
 
 [dependencies]
+chrono = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
+mollusk-svm-bencher = { workspace = true }
 mollusk-svm-fuzz-fixture = { workspace = true }
 mollusk-svm-fuzz-fixture-firedancer = { workspace = true }
 mollusk-svm = { workspace = true, features = ["fuzz", "fuzz-fd", "serde"] }

cli/Cargo.toml

@@ -8,6 +8,7 @@ use {
     clap::{Parser, Subcommand},
     config::ConfigFile,
     mollusk_svm::{result::Compare, Mollusk},
+    runner::CusReport,
     solana_pubkey::Pubkey,
     std::{fs, path::Path, str::FromStr},
 };
@@ -30,6 +31,14 @@ enum SubCommand {
         /// Path to the config file for validation checks.
         #[arg(short, long)]
         config: Option<String>,
+        /// Directory to write a compute unit consumption report.
+        #[arg(long)]
+        cus_report: Option<String>,
+        /// Table header for the compute unit consumption report.
+        ///
+        /// Note this flag is ignored if `cus_report` is not set.
+        #[arg(long)]
+        cus_report_table_header: Option<String>,
         /// Skip comparing compute unit consumption, but compare everything
         /// else.
         ///
@@ -72,6 +81,14 @@ enum SubCommand {
         /// Path to the config file for validation checks.
         #[arg(short, long)]
         config: Option<String>,
+        /// Directory to write a compute unit consumption report.
+        #[arg(long)]
+        cus_report: Option<String>,
+        /// Table header for the compute unit consumption report.
+        ///
+        /// Note this flag is ignored if `cus_report` is not set.
+        #[arg(long)]
+        cus_report_table_header: Option<String>,
         /// Skip comparing compute unit consumption, but compare everything
         /// else.
         ///
@@ -136,6 +153,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             fixture,
             program_id,
             config,
+            cus_report,
+            cus_report_table_header,
             ignore_compute_units,
             inputs_only,
             program_logs,
@@ -156,18 +175,24 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
             let fixtures = search_paths(&fixture, "fix")?;
 
-            Runner::new(checks, inputs_only, program_logs, proto, verbose).run_all(
-                None,
-                &mut mollusk,
-                &fixtures,
-            )?
+            Runner::new(
+                checks,
+                cus_report.map(|path| CusReport::new(path, cus_report_table_header)),
+                inputs_only,
+                program_logs,
+                proto,
+                verbose,
+            )
+            .run_all(None, &mut mollusk, &fixtures)?
         }
         SubCommand::RunTest {
             elf_path_source,
             elf_path_target,
             fixture,
             program_id,
             config,
+            cus_report,
+            cus_report_table_header,
             ignore_compute_units,
             program_logs,
             proto,
@@ -194,6 +219,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
             Runner::new(
                 checks,
+                cus_report.map(|path| CusReport::new(path, cus_report_table_header)),
                 /* inputs_only */ true,
                 program_logs,
                 proto,

cli/src/main.rs

@@ -1,11 +1,14 @@
 //! CLI runner. Many jobs share the same pattern but do different core actions.
 
 use {
+    chrono::Utc,
     clap::ValueEnum,
     mollusk_svm::{
         result::{Compare, Config, InstructionResult},
         Mollusk,
     },
+    mollusk_svm_bencher::{get_solana_version, result::MolluskComputeUnitBenchResult},
+    std::path::PathBuf,
 };
 
 #[derive(Clone, Debug, Default, ValueEnum)]
@@ -17,8 +20,26 @@ pub enum ProtoLayout {
     Firedancer,
 }
 
+pub struct CusReport {
+    pub path: String,
+    pub table_header: String,
+}
+
+impl CusReport {
+    pub fn new(path: String, table_header: Option<String>) -> Self {
+        let table_header = table_header.unwrap_or_else(|| Utc::now().to_string());
+        Self { path, table_header }
+    }
+}
+
+pub struct RunResult<'a> {
+    pub pass: bool,
+    pub bench_result: Option<MolluskComputeUnitBenchResult<'a>>,
+}
+
 pub struct Runner {
     checks: Vec<Compare>,
+    cus_report: Option<CusReport>,
     inputs_only: bool,
     program_logs: bool,
     proto: ProtoLayout,
@@ -28,13 +49,15 @@ pub struct Runner {
 impl Runner {
     pub fn new(
         checks: Vec<Compare>,
+        cus_report: Option<CusReport>,
         inputs_only: bool,
         program_logs: bool,
         proto: ProtoLayout,
         verbose: bool,
     ) -> Self {
         Self {
             checks,
+            cus_report,
             inputs_only,
             program_logs,
             proto,
@@ -66,12 +89,12 @@ impl Runner {
         }
     }
 
-    pub fn run(
+    fn run<'a>(
         &self,
         ground: Option<&mut Mollusk>,
         target: &mut Mollusk,
-        fixture_path: &str,
-    ) -> Result<bool, Box<dyn std::error::Error>> {
+        fixture_path: &'a str,
+    ) -> Result<RunResult<'a>, Box<dyn std::error::Error>> {
         // Disable stdout logging of program logs if not specified.
         if !self.program_logs {
             solana_logger::setup_with("");
@@ -141,6 +164,16 @@ impl Runner {
 
         let (target_result, effects) = self.run_fixture(target, fixture_path);
 
+        // Record a bench result for the CU report, if specified.
+        let bench_result = if self.cus_report.is_some() {
+            Some(MolluskComputeUnitBenchResult::new(
+                parse_fixture_name(fixture_path),
+                target_result.clone(),
+            ))
+        } else {
+            None
+        };
+
         if self.program_logs {
             println!();
         }
@@ -201,7 +234,7 @@ impl Runner {
             println!();
         }
 
-        Ok(pass)
+        Ok(RunResult { pass, bench_result })
     }
 
     pub fn run_all(
@@ -211,11 +244,16 @@ impl Runner {
         fixtures: &[String],
     ) -> Result<(), Box<dyn std::error::Error>> {
         let mut failures = 0;
+        let mut bench_results = Vec::new();
 
         for fixture_path in fixtures {
-            let result = self.run(ground.as_deref_mut(), target, fixture_path)?;
+            let mut result = self.run(ground.as_deref_mut(), target, fixture_path)?;
+
+            if let Some(bench_result) = result.bench_result.take() {
+                bench_results.push(bench_result);
+            }
 
-            if !result {
+            if !result.pass {
                 failures += 1;
             }
         }
@@ -227,6 +265,24 @@ impl Runner {
             std::process::exit(1);
         }
 
+        if let Some(cus_report) = &self.cus_report {
+            let solana_version = get_solana_version();
+            mollusk_svm_bencher::result::write_results(
+                &PathBuf::from(&cus_report.path),
+                &cus_report.table_header,
+                &solana_version,
+                bench_results,
+            );
+        }
+
         Ok(())
     }
 }
+
+fn parse_fixture_name(fixture_path: &str) -> &str {
+    fixture_path
+        .rsplit_once('/')
+        .map_or(fixture_path, |(_, name)| name)
+        .split_once('.')
+        .map_or_else(|| fixture_path, |(name, _)| name)
+}

cli/src/runner.rs

@@ -75,7 +75,7 @@ impl From<Result<(), InstructionError>> for ProgramResult {
 }
 
 /// The overall result of the instruction.
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct InstructionResult {
     /// The number of compute units consumed by the instruction.
     pub compute_units_consumed: u64,