feat: add blocktest subcommand to evm-spec-tester

.github/workflows/test.yml

@@ -183,4 +183,4 @@ jobs:
         run: cargo build --release --bin evm-spec-tester
 
       - name: Run EVM spec tests
-        run: cargo run --release --bin evm-spec-tester -- testdata/evm-spec-test
+        run: cargo run --release --bin evm-spec-tester -- statetest testdata/evm-spec-test

Cargo.toml

@@ -332,16 +332,19 @@ blst = "0.3"
 #rustls = "0.21"
 hashbrown = "0.7.1"
 
+# cli arg parsing
+clap = "2" # outdated, need to migrate to clap 4
+clap4 = { version = "4", package = "clap" }
+docopt = "1.0"
+clap-verbosity-flag = "3"
+
 # rand & rng
 rand = "0.7"
 rand_xorshift = "0.2"
 rand_08 = { package = "rand", version = "0.8" }
 rand_chacha = "0.2.1"
 
 # misc
-clap = "2" # outdated, need to migrate to clap 4
-clap4 = { version = "4", package = "clap" }
-structopt = { version = "0.3", default-features = false } # unmaintained, need to migrate to clap 4
 log = "0.4"
 log4rs = "1.3.0"
 env_logger = "0.11"
@@ -371,7 +374,6 @@ ipnetwork = "0.12.6"
 derivative = "2.0.2"
 edit-distance = "2"
 zeroize = "1"
-docopt = "1.0"
 vergen = "8.3.2"
 target_info = "0.1"
 bit-set = "0.4"

bins/evm-spec-tester/Cargo.toml

@@ -34,6 +34,8 @@ hex-literal = { workspace = true }
 
 log = { workspace = true }
 
-structopt = { workspace = true }
 env_logger = { workspace = true }
-itertools = { workspace = true }
+itertools = { workspace = true }
+
+clap = { version = "4.5", features = ["derive"] }
+clap-verbosity-flag = { workspace = true }

bins/evm-spec-tester/src/blocktest/mod.rs

@@ -0,0 +1,56 @@
+use crate::util::{find_all_json_tests, make_configuration};
+use cfx_config::Configuration;
+use clap::Args;
+use std::{path::PathBuf, sync::Arc};
+
+/// ethereum statetest doc: https://eest.ethereum.org/main/consuming_tests/state_test/
+#[derive(Args, Debug)]
+pub struct BlockchainTestCmd {
+    /// Paths to blockchain test files or directories
+    #[arg(required = true)]
+    pub(super) paths: Vec<PathBuf>,
+
+    /// Conflux client configuration
+    #[arg(short, long, value_parser = make_configuration, default_value = "", help = "Path to the configuration file")]
+    pub(super) config: Arc<Configuration>,
+
+    /// Only run tests matching this string
+    #[arg(short, long, value_name = "Matches")]
+    pub(super) matches: Option<String>,
+}
+
+impl BlockchainTestCmd {
+    pub fn run(&self) -> bool {
+        for path in &self.paths {
+            if !path.exists() {
+                panic!("Path not exists: {:?}", path);
+            }
+
+            let test_files = find_all_json_tests(path);
+
+            if test_files.is_empty() {
+                error!("No fixtures found in directory: {:?}", path);
+                continue;
+            }
+
+            if let Err(_) = self.run_file_tests(test_files, path) {
+                warn!("Failed to run tests in directory: {:?}", path);
+                continue;
+            }
+        }
+
+        true
+    }
+
+    fn run_file_tests(
+        &self, test_files: Vec<PathBuf>, path: &PathBuf,
+    ) -> Result<(), String> {
+        info!(
+            "Running {} TestSuites in {}",
+            test_files.len(),
+            path.display()
+        );
+
+        Ok(())
+    }
+}

bins/evm-spec-tester/src/cmd.rs

@@ -0,0 +1,36 @@
+use crate::{blocktest::BlockchainTestCmd, statetest::command::StateTestCmd};
+use clap::{Parser, Subcommand};
+use clap_verbosity_flag::{InfoLevel, Verbosity};
+
+/// A command line tool for running Ethereum spec tests
+#[derive(Parser, Debug)]
+#[command(infer_subcommands = true)]
+#[command(version, about, long_about = None)]
+#[command(propagate_version = true)]
+pub struct MainCmd {
+    /// Verbosity level (can be used multiple times)
+    /// Check detail at https://docs.rs/clap-verbosity-flag/3.0.2/clap_verbosity_flag/
+    #[command(flatten)]
+    pub verbose: Verbosity<InfoLevel>,
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand, Debug)]
+#[command(infer_subcommands = true)]
+#[allow(clippy::large_enum_variant)]
+pub enum Commands {
+    /// Execute state tests of ethereum execution spec tests
+    Statetest(StateTestCmd),
+    /// Execute blockchain tests of ethereum execution spec tests
+    Blocktest(BlockchainTestCmd),
+}
+
+impl MainCmd {
+    pub fn run(self) -> bool {
+        match self.command {
+            Commands::Statetest(cmd) => cmd.run(),
+            Commands::Blocktest(cmd) => cmd.run(),
+        }
+    }
+}

bins/evm-spec-tester/src/main.rs

@@ -1,37 +1,29 @@
 #[macro_use]
 extern crate log;
 
+mod blocktest;
+mod cmd;
 mod statetest;
+mod util;
 
-use statetest::command::StateTestCmd;
-use structopt::StructOpt;
-
-fn init_logger(verbosity: u8) {
-    use log::LevelFilter;
-
-    const BASE_LEVEL: u8 = 2;
-
-    let level = match BASE_LEVEL + verbosity {
-        0 => LevelFilter::Error,
-        1 => LevelFilter::Warn,
-        2 => LevelFilter::Info,
-        3 => LevelFilter::Debug,
-        _ => LevelFilter::Trace,
-    };
+use clap::Parser;
+use cmd::MainCmd;
+use log::LevelFilter;
 
+fn init_logger(level_filter: LevelFilter) {
     env_logger::Builder::new()
         .target(env_logger::Target::Stdout)
         .filter(None, LevelFilter::Off)
-        .filter_module("evm_spec_tester", level)
+        .filter_module("evm_spec_tester", level_filter)
         .format_timestamp(None) // Optional: add timestamp
         // .format_level(true)     // show log level
         // .format_module_path(true)  // show module path
         .init();
 }
 
 fn main() {
-    let cmd = StateTestCmd::from_args();
-    init_logger(cmd.verbose);
+    let cmd = MainCmd::parse();
+    init_logger(cmd.verbose.log_level_filter());
     let success = cmd.run();
     if !success {
         std::process::exit(1);