Update retester CI to check expectations (#225)

* Add a report processing tool

* Add expectations tests to the CI action

* Fix an issue with CI

* Fix CI

* Fix the path of the workdir in CI

* Fix CI issue with the paths

* Update the format of the expectations file

Author: 0xOmarA

.github/actions/run-differential-tests/action.yml

@@ -41,6 +41,10 @@ inputs:
     description: "The id of the parachain to spawn with the polkadot-omni-node. This is only required if the polkadot-omni-node is one of the selected platforms."
     type: number
     required: false
+  expectations-file-path:
+    description: "Path to the expectations file to use to compare against."
+    type: string
+    required: false
 
 runs:
   using: "composite"
@@ -79,6 +83,12 @@ runs:
       run: |
         ${{ inputs['cargo-command'] }} build --locked --profile release -p pallet-revive-eth-rpc -p revive-dev-node --manifest-path ${{ inputs['polkadot-sdk-path'] }}/Cargo.toml
         ${{ inputs['cargo-command'] }} build --locked --profile release --bin polkadot-omni-node --manifest-path ${{ inputs['polkadot-sdk-path'] }}/Cargo.toml
+    - name: Installing retester
+      shell: bash
+      run: ${{ inputs['cargo-command'] }} install --path ./revive-differential-tests/crates/core
+    - name: Installing report-processor
+      shell: bash
+      run: ${{ inputs['cargo-command'] }} install --path ./revive-differential-tests/crates/report-processor
     - name: Running the Differential Tests
       shell: bash
       run: |
@@ -96,11 +106,12 @@ runs:
           )
         fi
 
-        ${{ inputs['cargo-command'] }} run --locked --manifest-path revive-differential-tests/Cargo.toml -- test \
+        retester test \
           --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/simple \
           --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/complex \
           --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/translated_semantic_tests \
           --platform ${{ inputs['platform'] }} \
+          --report.file-name report.json \
           --concurrency.number-of-nodes 10 \
           --concurrency.number-of-threads 10 \
           --concurrency.number-of-concurrent-tasks 100 \
@@ -110,22 +121,21 @@ runs:
           --eth-rpc.path ${{ inputs['polkadot-sdk-path'] }}/target/release/eth-rpc \
           --polkadot-omni-node.path ${{ inputs['polkadot-sdk-path'] }}/target/release/polkadot-omni-node \
           --resolc.path ./resolc \
-          "${OMNI_ARGS[@]}"
-    - name: Creating a markdown report of the test execution
+          "${OMNI_ARGS[@]}" || true
+    - name: Generate the expectation file
       shell: bash
-      if: ${{ always() }}
-      run: |
-        mv ./workdir/*.json report.json
-        python3 revive-differential-tests/scripts/process-differential-tests-report.py report.json ${{ inputs['platform'] }}
+      run: report-processor generate-expectations-file --report-path ./workdir/report.json --output-path ./workdir/expectations.json --remove-prefix ./revive-differential-tests/resolc-compiler-tests
     - name: Upload the Report to the CI
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
-      if: ${{ always() }}
       with:
-        name: report-${{ inputs['platform'] }}.md
-        path: report.md
-    - name: Posting the report as a comment on the PR
-      uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405
-      if: ${{ always() }}
+        name: ${{ inputs['platform'] }}-report.json
+        path: ./workdir/report.json
+    - name: Upload the Report to the CI
+      uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
       with:
-        header: diff-tests-report-${{ inputs['platform'] }}
-        path: report.md
+        name: ${{ inputs['platform'] }}.json
+        path: ./workdir/expectations.json
+    - name: Check Expectations
+      shell: bash
+      if: ${{ inputs['expectations-file-path'] != '' }}
+      run: report-processor compare-expectation-files --base-expectation-path ${{ inputs['expectations-file-path'] }} --other-expectation-path ./workdir/expectations.json

Cargo.lock

@@ -21,6 +21,7 @@ revive-dt-node-interaction = { version = "0.1.0", path = "crates/node-interactio
 revive-dt-node-pool = { version = "0.1.0", path = "crates/node-pool" }
 revive-dt-report = { version = "0.1.0", path = "crates/report" }
 revive-dt-solc-binaries = { version = "0.1.0", path = "crates/solc-binaries" }
+revive-dt-report-processor = { version = "0.1.0", path = "crates/report-processor" }
 
 alloy = { version = "1.4.1", features = ["full", "genesis", "json-rpc"] }
 ansi_term = "0.12.1"
@@ -81,7 +82,12 @@ zombienet-sdk = { git = "https://github.com/paritytech/zombienet-sdk.git", rev =
 
 [profile.bench]
 inherits = "release"
+codegen-units = 1
 lto = true
+
+[profile.production]
+inherits = "release"
 codegen-units = 1
+lto = true
 
 [workspace.lints.clippy]

Cargo.toml

@@ -23,6 +23,18 @@ pub struct Mode {
     pub version: Option<semver::VersionReq>,
 }
 
+impl Ord for Mode {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.to_string().cmp(&other.to_string())
+    }
+}
+
+impl PartialOrd for Mode {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 impl Display for Mode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         self.pipeline.fmt(f)?;

crates/common/src/types/mode.rs

@@ -1,10 +1,15 @@
-use std::{fmt::Display, path::PathBuf, str::FromStr};
+use std::{
+    fmt::Display,
+    path::{Path, PathBuf},
+    str::FromStr,
+};
 
 use anyhow::{Context as _, bail};
+use serde::{Deserialize, Serialize};
 
 use crate::types::Mode;
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum ParsedTestSpecifier {
     /// All of the test cases in the file should be ran across all of the specified modes
     FileOrDirectory {
@@ -34,6 +39,22 @@ pub enum ParsedTestSpecifier {
     },
 }
 
+impl ParsedTestSpecifier {
+    pub fn metadata_path(&self) -> &Path {
+        match self {
+            ParsedTestSpecifier::FileOrDirectory {
+                metadata_or_directory_file_path: metadata_file_path,
+            }
+            | ParsedTestSpecifier::Case {
+                metadata_file_path, ..
+            }
+            | ParsedTestSpecifier::CaseWithMode {
+                metadata_file_path, ..
+            } => metadata_file_path,
+        }
+    }
+}
+
 impl Display for ParsedTestSpecifier {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -131,3 +152,22 @@ impl TryFrom<&str> for ParsedTestSpecifier {
         value.parse()
     }
 }
+
+impl Serialize for ParsedTestSpecifier {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        self.to_string().serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for ParsedTestSpecifier {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let string = String::deserialize(deserializer)?;
+        string.parse().map_err(serde::de::Error::custom)
+    }
+}

crates/common/src/types/parsed_test_specifier.rs

@@ -1113,6 +1113,10 @@ pub struct ReportConfiguration {
     /// Controls if the compiler output is included in the final report.
     #[clap(long = "report.include-compiler-output")]
     pub include_compiler_output: bool,
+
+    /// The filename to use for the report.
+    #[clap(long = "report.file-name")]
+    pub file_name: Option<String>,
 }
 
 #[derive(Clone, Debug, Parser, Serialize, Deserialize)]

crates/config/src/lib.rs

@@ -0,0 +1,25 @@
+[package]
+name = "revive-dt-report-processor"
+description = "revive differential testing report processor utility"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[[bin]]
+name = "report-processor"
+path = "src/main.rs"
+
+[dependencies]
+revive-dt-report = { workspace = true }
+revive-dt-common = { workspace = true }
+
+anyhow = { workspace = true }
+clap = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+
+[lints]
+workspace = true