Merge pull request #3206 from Pana/feat/blockTest

feat: add blocktest subcommand to evm-spec-tester

Author: Pana

.github/workflows/test.yml

@@ -183,4 +183,4 @@ jobs:
         run: cargo build --release --bin evm-spec-tester
 
       - name: Run EVM spec tests
-        run: cargo run --release --bin evm-spec-tester -- testdata/evm-spec-test
+        run: cargo run --release --bin evm-spec-tester -- statetest testdata/evm-spec-test

Cargo.lock

@@ -332,16 +332,19 @@ blst = "0.3"
 #rustls = "0.21"
 hashbrown = "0.7.1"
 
+# cli arg parsing
+clap = "2" # outdated, need to migrate to clap 4
+clap4 = { version = "4", package = "clap" }
+docopt = "1.0"
+clap-verbosity-flag = "3"
+
 # rand & rng
 rand = "0.7"
 rand_xorshift = "0.2"
 rand_08 = { package = "rand", version = "0.8" }
 rand_chacha = "0.2.1"
 
 # misc
-clap = "2" # outdated, need to migrate to clap 4
-clap4 = { version = "4", package = "clap" }
-structopt = { version = "0.3", default-features = false } # unmaintained, need to migrate to clap 4
 log = "0.4"
 log4rs = "1.3.0"
 env_logger = "0.11"
@@ -371,7 +374,6 @@ ipnetwork = "0.12.6"
 derivative = "2.0.2"
 edit-distance = "2"
 zeroize = "1"
-docopt = "1.0"
 vergen = "8.3.2"
 target_info = "0.1"
 bit-set = "0.4"

Cargo.toml

@@ -34,6 +34,8 @@ hex-literal = { workspace = true }
 
 log = { workspace = true }
 
-structopt = { workspace = true }
 env_logger = { workspace = true }
-itertools = { workspace = true }
+itertools = { workspace = true }
+
+clap = { version = "4.5", features = ["derive"] }
+clap-verbosity-flag = { workspace = true }

bins/evm-spec-tester/Cargo.toml

@@ -0,0 +1,56 @@
+use crate::util::{find_all_json_tests, make_configuration};
+use cfx_config::Configuration;
+use clap::Args;
+use std::{path::PathBuf, sync::Arc};
+
+/// ethereum statetest doc: https://eest.ethereum.org/main/consuming_tests/state_test/
+#[derive(Args, Debug)]
+pub struct BlockchainTestCmd {
+    /// Paths to blockchain test files or directories
+    #[arg(required = true)]
+    pub(super) paths: Vec<PathBuf>,
+
+    /// Conflux client configuration
+    #[arg(short, long, value_parser = make_configuration, default_value = "", help = "Path to the configuration file")]
+    pub(super) config: Arc<Configuration>,
+
+    /// Only run tests matching this string
+    #[arg(short, long, value_name = "Matches")]
+    pub(super) matches: Option<String>,
+}
+
+impl BlockchainTestCmd {
+    pub fn run(&self) -> bool {
+        for path in &self.paths {
+            if !path.exists() {
+                panic!("Path not exists: {:?}", path);
+            }
+
+            let test_files = find_all_json_tests(path);
+
+            if test_files.is_empty() {
+                error!("No fixtures found in directory: {:?}", path);
+                continue;
+            }
+
+            if let Err(_) = self.run_file_tests(test_files, path) {
+                warn!("Failed to run tests in directory: {:?}", path);
+                continue;
+            }
+        }
+
+        true
+    }
+
+    fn run_file_tests(
+        &self, test_files: Vec<PathBuf>, path: &PathBuf,
+    ) -> Result<(), String> {
+        info!(
+            "Running {} TestSuites in {}",
+            test_files.len(),
+            path.display()
+        );
+
+        Ok(())
+    }
+}

bins/evm-spec-tester/src/blocktest/mod.rs

@@ -0,0 +1,36 @@
+use crate::{blocktest::BlockchainTestCmd, statetest::command::StateTestCmd};
+use clap::{Parser, Subcommand};
+use clap_verbosity_flag::{InfoLevel, Verbosity};
+
+/// A command line tool for running Ethereum spec tests
+#[derive(Parser, Debug)]
+#[command(infer_subcommands = true)]
+#[command(version, about, long_about = None)]
+#[command(propagate_version = true)]
+pub struct MainCmd {
+    /// Verbosity level (can be used multiple times)
+    /// Check detail at https://docs.rs/clap-verbosity-flag/3.0.2/clap_verbosity_flag/
+    #[command(flatten)]
+    pub verbose: Verbosity<InfoLevel>,
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand, Debug)]
+#[command(infer_subcommands = true)]
+#[allow(clippy::large_enum_variant)]
+pub enum Commands {
+    /// Execute state tests of ethereum execution spec tests
+    Statetest(StateTestCmd),
+    /// Execute blockchain tests of ethereum execution spec tests
+    Blocktest(BlockchainTestCmd),
+}
+
+impl MainCmd {
+    pub fn run(self) -> bool {
+        match self.command {
+            Commands::Statetest(cmd) => cmd.run(),
+            Commands::Blocktest(cmd) => cmd.run(),
+        }
+    }
+}

bins/evm-spec-tester/src/cmd.rs

@@ -1,37 +1,29 @@
 #[macro_use]
 extern crate log;
 
+mod blocktest;
+mod cmd;
 mod statetest;
+mod util;
 
-use statetest::command::StateTestCmd;
-use structopt::StructOpt;
-
-fn init_logger(verbosity: u8) {
-    use log::LevelFilter;
-
-    const BASE_LEVEL: u8 = 2;
-
-    let level = match BASE_LEVEL + verbosity {
-        0 => LevelFilter::Error,
-        1 => LevelFilter::Warn,
-        2 => LevelFilter::Info,
-        3 => LevelFilter::Debug,
-        _ => LevelFilter::Trace,
-    };
+use clap::Parser;
+use cmd::MainCmd;
+use log::LevelFilter;
 
+fn init_logger(level_filter: LevelFilter) {
     env_logger::Builder::new()
         .target(env_logger::Target::Stdout)
         .filter(None, LevelFilter::Off)
-        .filter_module("evm_spec_tester", level)
+        .filter_module("evm_spec_tester", level_filter)
         .format_timestamp(None) // Optional: add timestamp
         // .format_level(true)     // show log level
         // .format_module_path(true)  // show module path
         .init();
 }
 
 fn main() {
-    let cmd = StateTestCmd::from_args();
-    init_logger(cmd.verbose);
+    let cmd = MainCmd::parse();
+    init_logger(cmd.verbose.log_level_filter());
     let success = cmd.run();
     if !success {
         std::process::exit(1);

bins/evm-spec-tester/src/main.rs

@@ -1,61 +1,20 @@
-use cfx_config::{Configuration, RawConfiguration};
-use primitives::block_header::CIP112_TRANSITION_HEIGHT;
-use std::path::PathBuf;
-use structopt::StructOpt;
+use crate::util::make_configuration;
+use cfx_config::Configuration;
+use clap::Args;
+use std::{path::PathBuf, sync::Arc};
 
 /// ethereum statetest doc: https://eest.ethereum.org/main/consuming_tests/state_test/
-#[derive(StructOpt)]
-#[structopt(name = "statetest", about = "State test command")]
+#[derive(Args, Debug)]
 pub struct StateTestCmd {
-    /// Paths to test files or directories
-    #[structopt(parse(from_os_str), required = true)]
+    /// Paths to state test files or directories
+    #[arg(required = true)]
     pub(super) paths: Vec<PathBuf>,
 
-    /// Configuration
-    #[structopt(short, long, parse(try_from_str = make_configuration), required = true, default_value = "", help = "Path to the configuration file")]
-    pub(super) config: Configuration,
+    /// Conflux client configuration
+    #[arg(short, long, value_parser = make_configuration, default_value = "", help = "Path to the configuration file")]
+    pub(super) config: Arc<Configuration>,
 
     /// Only run tests matching this string
-    #[structopt(short, long)]
+    #[arg(short, long, value_name = "Matches")]
     pub(super) matches: Option<String>,
-
-    /// Verbosity level (can be used multiple times)
-    #[structopt(short, long, parse(from_occurrences))]
-    pub verbose: u8,
-}
-
-fn make_configuration(config_file: &str) -> Result<Configuration, String> {
-    let mut config = Configuration::default();
-    config.raw_conf = if config_file.is_empty() {
-        default_raw_configuration()
-    } else {
-        RawConfiguration::from_file(config_file)?
-    };
-
-    config.raw_conf.node_type = Some(cfxcore::NodeType::Full);
-
-    let cip112_height =
-        config.raw_conf.cip112_transition_height.unwrap_or(u64::MAX);
-    match CIP112_TRANSITION_HEIGHT.set(cip112_height) {
-        Err(e) if e != cip112_height => {
-            return Err(
-                "Duplicate setting for CIP-112 config with inconsistent value"
-                    .to_string(),
-            );
-        }
-        _ => {}
-    }
-
-    Ok(config)
-}
-
-fn default_raw_configuration() -> RawConfiguration {
-    let mut config = RawConfiguration::default();
-    config.mode = Some("dev".to_string());
-    config.default_transition_time = Some(1);
-    config.pos_reference_enable_height = 1;
-    config.align_evm_transition_height = 1;
-    config.chain_id = Some(2);
-    config.evm_chain_id = Some(1);
-    config
 }