Implement basic reporting facility (#18)

* wip

Signed-off-by: Cyrill Leutwiler <[email protected]>

* save to file after all tasks done

Signed-off-by: Cyrill Leutwiler <[email protected]>

* error out early if the workdir does not exist

Signed-off-by: Cyrill Leutwiler <[email protected]>

* the compiler statistics

Signed-off-by: Cyrill Leutwiler <[email protected]>

* allow compiler statistics per implementation

Signed-off-by: Cyrill Leutwiler <[email protected]>

* save compiler problems

Signed-off-by: Cyrill Leutwiler <[email protected]>

* add flag whether to extract compiler errors

Signed-off-by: Cyrill Leutwiler <[email protected]>

* whitespace

Signed-off-by: Cyrill Leutwiler <[email protected]>

---------

Signed-off-by: Cyrill Leutwiler <[email protected]>

Author: xermicus

Cargo.lock

@@ -20,6 +20,7 @@ revive-dt-format = { version = "0.1.0", path = "crates/format" }
 revive-dt-node = { version = "0.1.0", path = "crates/node" }
 revive-dt-node-interaction = { version = "0.1.0", path = "crates/node-interaction" }
 revive-dt-node-pool = { version = "0.1.0", path = "crates/node-pool" }
+revive-dt-report = { version = "0.1.0", path = "crates/report" }
 revive-dt-solc-binaries = { version = "0.1.0", path = "crates/solc-binaries" }
 
 anyhow = "1.0"
@@ -38,9 +39,9 @@ temp-dir = { version = "0.1.14" }
 tokio = { version = "1", default-features = false, features = ["rt-multi-thread"] }
 
 # revive compiler
-revive-solc-json-interface = { git = "https://github.com/paritytech/revive", rev = "497dae2494dabe12d1af32d6d687122903cb2ada" }
-revive-common = { git = "https://github.com/paritytech/revive", rev = "497dae2494dabe12d1af32d6d687122903cb2ada" }
-revive-differential = { git = "https://github.com/paritytech/revive", rev = "497dae2494dabe12d1af32d6d687122903cb2ada" }
+revive-solc-json-interface = { git = "https://github.com/paritytech/revive", rev = "3389865af7c3ff6f29a586d82157e8bc573c1a8e" }
+revive-common = { git = "https://github.com/paritytech/revive", rev = "3389865af7c3ff6f29a586d82157e8bc573c1a8e" }
+revive-differential = { git = "https://github.com/paritytech/revive", rev = "3389865af7c3ff6f29a586d82157e8bc573c1a8e" }
 
 [workspace.dependencies.alloy]
 version = "0.13.0"

Cargo.toml

@@ -48,8 +48,12 @@ pub struct CompilerInput<T: PartialEq + Eq + Hash> {
 
 /// The generic compilation output configuration.
 pub struct CompilerOutput<T: PartialEq + Eq + Hash> {
+    /// The solc standard JSON input.
     pub input: CompilerInput<T>,
+    /// The produced solc standard JSON output.
     pub output: SolcStandardJsonOutput,
+    /// The error message in case the compiler returns abnormally.
+    pub error: Option<String>,
 }
 
 impl<T> PartialEq for CompilerInput<T>
@@ -110,6 +114,7 @@ where
                         false,
                     ),
                     None,
+                    None,
                 ),
             },
             extra_options: Default::default(),
@@ -156,4 +161,9 @@ where
             input: self.input,
         })
     }
+
+    /// Returns the compiler JSON input.
+    pub fn input(&self) -> SolcStandardJsonInput {
+        self.input.clone()
+    }
 }

crates/compiler/src/lib.rs

@@ -41,12 +41,18 @@ impl SolidityCompiler for Resolc {
         let stderr = output.stderr;
 
         if !output.status.success() {
+            let message = String::from_utf8_lossy(&stderr);
             log::error!(
                 "resolc failed exit={} stderr={} JSON-in={} ",
                 output.status,
-                String::from_utf8_lossy(&stderr),
+                &message,
                 json_in,
             );
+            return Ok(CompilerOutput {
+                input,
+                output: Default::default(),
+                error: Some(message.into()),
+            });
         }
 
         let parsed: SolcStandardJsonOutput = serde_json::from_slice(&stdout).map_err(|e| {
@@ -59,6 +65,7 @@ impl SolidityCompiler for Resolc {
         Ok(CompilerOutput {
             input,
             output: parsed,
+            error: None,
         })
     }
 

crates/compiler/src/revive_resolc.rs

@@ -30,11 +30,22 @@ impl SolidityCompiler for Solc {
 
         let stdin = child.stdin.as_mut().expect("should be piped");
         serde_json::to_writer(stdin, &input.input)?;
+        let output = child.wait_with_output()?;
+
+        if !output.status.success() {
+            let message = String::from_utf8_lossy(&output.stderr);
+            log::error!("solc failed exit={} stderr={}", output.status, &message);
+            return Ok(CompilerOutput {
+                input,
+                output: Default::default(),
+                error: Some(message.into()),
+            });
+        }
 
-        let output = child.wait_with_output()?.stdout;
         Ok(CompilerOutput {
             input,
-            output: serde_json::from_slice(&output)?,
+            output: serde_json::from_slice(&output.stdout)?,
+            error: None,
         })
     }
 

crates/compiler/src/solc.rs

@@ -13,3 +13,5 @@ alloy = { workspace = true }
 clap = { workspace = true }
 semver = { workspace = true }
 temp-dir = { workspace = true }
+serde = { workspace = true }
+

crates/config/Cargo.toml

@@ -1,13 +1,17 @@
 //! The global configuration used accross all revive differential testing crates.
 
-use std::path::{Path, PathBuf};
+use std::{
+    fmt::Display,
+    path::{Path, PathBuf},
+};
 
 use alloy::{network::EthereumWallet, signers::local::PrivateKeySigner};
 use clap::{Parser, ValueEnum};
 use semver::Version;
+use serde::{Deserialize, Serialize};
 use temp_dir::TempDir;
 
-#[derive(Debug, Parser, Clone)]
+#[derive(Debug, Parser, Clone, Serialize, Deserialize)]
 #[command(name = "retester")]
 pub struct Arguments {
     /// The `solc` version to use if the test didn't specify it explicitly.
@@ -40,6 +44,7 @@ pub struct Arguments {
     ///
     /// We attach it here because [TempDir] prunes itself on drop.
     #[clap(skip)]
+    #[serde(skip)]
     pub temp_dir: Option<&'static TempDir>,
 
     /// The path to the `geth` executable.
@@ -83,6 +88,10 @@ pub struct Arguments {
     /// Determines the amount of tests that are executed in parallel.
     #[arg(long = "workers", default_value = "12")]
     pub workers: usize,
+
+    /// Extract problems back to the test corpus.
+    #[arg(short, long = "extract-problems")]
+    pub extract_problems: bool,
 }
 
 impl Arguments {
@@ -124,11 +133,20 @@ impl Default for Arguments {
 /// The Solidity compatible node implementation.
 ///
 /// This describes the solutions to be tested against on a high level.
-#[derive(Clone, Debug, Eq, Hash, PartialEq, ValueEnum)]
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum, Serialize, Deserialize)]
 #[clap(rename_all = "lower")]
 pub enum TestingPlatform {
     /// The go-ethereum reference full node EVM implementation.
     Geth,
     /// The kitchensink runtime provides the PolkaVM (PVM) based node implentation.
     Kitchensink,
 }
+
+impl Display for TestingPlatform {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Geth => f.write_str("geth"),
+            Self::Kitchensink => f.write_str("revive"),
+        }
+    }
+}

crates/config/src/lib.rs

@@ -18,6 +18,7 @@ revive-dt-config = { workspace = true }
 revive-dt-format = { workspace = true }
 revive-dt-node = { workspace = true }
 revive-dt-node-interaction = { workspace = true }
+revive-dt-report = { workspace = true }
 
 alloy = { workspace = true }
 anyhow = { workspace = true }

crates/core/Cargo.toml

@@ -8,6 +8,7 @@ use revive_dt_compiler::{Compiler, CompilerInput, SolidityCompiler};
 use revive_dt_config::Arguments;
 use revive_dt_format::{input::Input, metadata::Metadata, mode::SolcMode};
 use revive_dt_node_interaction::EthereumNode;
+use revive_dt_report::reporter::{CompilationTask, Report, Span};
 use revive_solc_json_interface::SolcStandardJsonOutput;
 
 use crate::Platform;
@@ -19,6 +20,7 @@ type Contracts<T> = HashMap<
 
 pub struct State<'a, T: Platform> {
     config: &'a Arguments,
+    span: Span,
     contracts: Contracts<T>,
     deployed_contracts: HashMap<String, Address>,
 }
@@ -27,37 +29,65 @@ impl<'a, T> State<'a, T>
 where
     T: Platform,
 {
-    pub fn new(config: &'a Arguments) -> Self {
+    pub fn new(config: &'a Arguments, span: Span) -> Self {
         Self {
             config,
+            span,
             contracts: Default::default(),
             deployed_contracts: Default::default(),
         }
     }
 
+    /// Returns a copy of the current span.
+    fn span(&self) -> Span {
+        self.span
+    }
+
     pub fn build_contracts(&mut self, mode: &SolcMode, metadata: &Metadata) -> anyhow::Result<()> {
+        let mut span = self.span();
+        span.next_metadata(
+            metadata
+                .file_path
+                .as_ref()
+                .expect("metadata should have been read from a file")
+                .clone(),
+        );
+
         let Some(version) = mode.last_patch_version(&self.config.solc) else {
-            anyhow::bail!("unsupported solc version: {:?}", mode.solc_version);
+            anyhow::bail!("unsupported solc version: {:?}", &mode.solc_version);
         };
 
-        let sources = metadata.contract_sources()?;
-        let base_path = metadata.directory()?.display().to_string();
+        let mut compiler = Compiler::<T::Compiler>::new()
+            .base_path(metadata.directory()?.display().to_string())
+            .solc_optimizer(mode.solc_optimize());
 
-        let mut compiler = Compiler::<T::Compiler>::new().base_path(base_path.clone());
-        for (file, _contract) in sources.values() {
+        for (file, _contract) in metadata.contract_sources()?.values() {
             log::debug!("contract source {}", file.display());
             compiler = compiler.with_source(file)?;
         }
 
-        let compiler_path = T::Compiler::get_compiler_executable(self.config, version)?;
-
-        let output = compiler
-            .solc_optimizer(mode.solc_optimize())
-            .try_build(compiler_path)?;
-
-        self.contracts.insert(output.input, output.output);
+        let mut task = CompilationTask {
+            json_input: compiler.input(),
+            json_output: None,
+            mode: mode.clone(),
+            compiler_version: format!("{}", &version),
+            error: None,
+        };
 
-        Ok(())
+        let compiler_path = T::Compiler::get_compiler_executable(self.config, version)?;
+        match compiler.try_build(compiler_path) {
+            Ok(output) => {
+                task.json_output = Some(output.output.clone());
+                task.error = output.error;
+                self.contracts.insert(output.input, output.output);
+                Report::compilation(span, T::config_id(), task);
+                Ok(())
+            }
+            Err(error) => {
+                task.error = Some(error.to_string());
+                Err(error)
+            }
+        }
     }
 
     pub fn execute_input(
@@ -102,12 +132,12 @@ where
         }
     }
 
-    pub fn execute(&mut self) -> anyhow::Result<()> {
+    pub fn execute(&mut self, span: Span) -> anyhow::Result<()> {
         for mode in self.metadata.solc_modes() {
-            let mut leader_state = State::<L>::new(self.config);
+            let mut leader_state = State::<L>::new(self.config, span);
             leader_state.build_contracts(&mode, self.metadata)?;
 
-            let mut follower_state = State::<F>::new(self.config);
+            let mut follower_state = State::<F>::new(self.config, span);
             follower_state.build_contracts(&mode, self.metadata)?;
 
             for case in &self.metadata.cases {

crates/core/src/driver/mod.rs

@@ -4,6 +4,7 @@
 //! provides a helper utilty to execute tests.
 
 use revive_dt_compiler::{SolidityCompiler, revive_resolc, solc};
+use revive_dt_config::TestingPlatform;
 use revive_dt_node::geth;
 use revive_dt_node_interaction::EthereumNode;
 
@@ -15,6 +16,9 @@ pub mod driver;
 pub trait Platform {
     type Blockchain: EthereumNode;
     type Compiler: SolidityCompiler;
+
+    /// Returns the matching [TestingPlatform] of the [revive_dt_config::Arguments].
+    fn config_id() -> TestingPlatform;
 }
 
 #[derive(Default)]
@@ -23,6 +27,10 @@ pub struct Geth;
 impl Platform for Geth {
     type Blockchain = geth::Instance;
     type Compiler = solc::Solc;
+
+    fn config_id() -> TestingPlatform {
+        TestingPlatform::Geth
+    }
 }
 
 #[derive(Default)]
@@ -31,4 +39,8 @@ pub struct Kitchensink;
 impl Platform for Kitchensink {
     type Blockchain = geth::Instance;
     type Compiler = revive_resolc::Resolc;
+
+    fn config_id() -> TestingPlatform {
+        TestingPlatform::Kitchensink
+    }
 }