Allow Targets in Test Cases (#229)

* Add an optional `targets` field to cases.

This PR adds an optional `targets` field to cases which takes presence
over that same field in the `Metadata`. The hope from this is to allow
us to limit specific tests so that they only run on specific platforms
only.

* Update the resolc tests submodule

* Update the default resolc version to use

* Update the default heap-size and stack-size in the cli

* Update the report processor

Author: 0xOmarA

.github/actions/run-differential-tests/action.yml

@@ -29,6 +29,10 @@ inputs:
     default: true
     type: boolean
   # Test Execution Arguments
+  # TODO: We need a better way for people to pass arguments to retester. This way is not very good
+  # because we need to add support for each argument separately and support defaults and all of that
+  # perhaps having people pass in a JSON String of the arguments is the better long term solution
+  # for this.
   platform:
     description: "The identifier of the platform to run the tests on (e.g., geth-evm-solc, revive-dev-node-revm-solc)"
     required: true
@@ -121,10 +125,12 @@ runs:
           --eth-rpc.path ${{ inputs['polkadot-sdk-path'] }}/target/release/eth-rpc \
           --polkadot-omni-node.path ${{ inputs['polkadot-sdk-path'] }}/target/release/polkadot-omni-node \
           --resolc.path ./resolc \
+          --resolc.heap-size 128000 \
+          --resolc.stack-size 128000 \
           "${OMNI_ARGS[@]}" || true
     - name: Generate the expectation file
       shell: bash
-      run: report-processor generate-expectations-file --report-path ./workdir/report.json --output-path ./workdir/expectations.json --remove-prefix ./revive-differential-tests/resolc-compiler-tests
+      run: report-processor generate-expectations-file --report-path ./workdir/report.json --output-path ./workdir/expectations.json --remove-prefix ./revive-differential-tests/resolc-compiler-tests --include-status failed
     - name: Upload the Report to the CI
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
       with:

Cargo.lock

@@ -806,6 +806,7 @@ pub struct ResolcConfiguration {
     /// If unspecified, the revive compiler default is used
     #[clap(id = "resolc.heap-size", long = "resolc.heap-size")]
     pub heap_size: Option<u32>,
+
     /// Specifies the PVM stack size in bytes.
     ///
     /// If unspecified, the revive compiler default is used

crates/config/src/lib.rs

@@ -223,17 +223,24 @@ impl<'a> TestDefinition<'a> {
 
     /// Checks if the platforms all support the desired targets in the metadata file.
     fn check_target_compatibility(&self) -> TestCheckFunctionResult {
+        // The case targets takes presence over the metadata targets.
+        let Some(targets) = self
+            .case
+            .targets
+            .as_ref()
+            .or(self.metadata.targets.as_ref())
+        else {
+            return Ok(());
+        };
+
         let mut error_map = indexmap! {
-            "test_desired_targets" => json!(self.metadata.targets.as_ref()),
+            "test_desired_targets" => json!(targets),
         };
+
         let mut is_allowed = true;
         for (_, platform_information) in self.platforms.iter() {
-            let is_allowed_for_platform = match self.metadata.targets.as_ref() {
-                None => true,
-                Some(required_vm_identifiers) => {
-                    required_vm_identifiers.contains(&platform_information.platform.vm_identifier())
-                }
-            };
+            let is_allowed_for_platform =
+                targets.contains(&platform_information.platform.vm_identifier());
             is_allowed &= is_allowed_for_platform;
             error_map.insert(
                 platform_information.platform.platform_identifier().into(),

crates/core/src/helpers/test.rs

@@ -1,16 +1,22 @@
-use alloy::primitives::Address;
+use alloy::primitives::{Address, map::HashSet};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
 use revive_dt_common::{
     macros::define_wrapper_type,
-    types::{Mode, ParsedMode},
+    types::{Mode, ParsedMode, VmIdentifier},
 };
 
 use crate::steps::*;
 
 #[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq, JsonSchema)]
 pub struct Case {
+    /// An optional vector of targets that this Metadata file's cases can be executed on. As an
+    /// example, if we wish for the metadata file's cases to only be run on PolkaVM then we'd
+    /// specify a target of "PolkaVM" in here.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub targets: Option<HashSet<VmIdentifier>>,
+
     /// An optional name of the test case.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub name: Option<String>,

crates/format/src/case.rs

@@ -8,6 +8,7 @@ use std::{
     str::FromStr,
 };
 
+use alloy::primitives::map::HashSet;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
@@ -83,7 +84,7 @@ pub struct Metadata {
     /// example, if we wish for the metadata file's cases to only be run on PolkaVM then we'd
     /// specify a target of "PolkaVM" in here.
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub targets: Option<Vec<VmIdentifier>>,
+    pub targets: Option<HashSet<VmIdentifier>>,
 
     /// A vector of the test cases and workloads contained within the metadata file. This is their
     /// primary description.

crates/format/src/metadata.rs

@@ -18,6 +18,7 @@ revive-dt-common = { workspace = true }
 
 anyhow = { workspace = true }
 clap = { workspace = true }
+strum = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 

crates/report-processor/Cargo.toml

@@ -1,6 +1,6 @@
 use std::{
     borrow::Cow,
-    collections::{BTreeMap, BTreeSet},
+    collections::{BTreeMap, BTreeSet, HashSet},
     fmt::Display,
     fs::{File, OpenOptions},
     ops::{Deref, DerefMut},
@@ -9,11 +9,12 @@ use std::{
 };
 
 use anyhow::{Context as _, Error, Result, bail};
-use clap::Parser;
+use clap::{Parser, ValueEnum};
 use serde::{Deserialize, Serialize, de::DeserializeOwned};
 
 use revive_dt_common::types::{Mode, ParsedTestSpecifier};
 use revive_dt_report::{Report, TestCaseStatus};
+use strum::EnumString;
 
 fn main() -> Result<()> {
     let cli = Cli::try_parse().context("Failed to parse the CLI arguments")?;
@@ -23,11 +24,14 @@ fn main() -> Result<()> {
             report_path,
             output_path: output_file,
             remove_prefix,
+            include_status,
         } => {
             let remove_prefix = remove_prefix
                 .into_iter()
                 .map(|path| path.canonicalize().context("Failed to canonicalize path"))
                 .collect::<Result<Vec<_>>>()?;
+            let include_status =
+                include_status.map(|value| value.into_iter().collect::<HashSet<_>>());
 
             let expectations = report_path
                 .execution_information
@@ -73,7 +77,12 @@ fn main() -> Result<()> {
                         Status::from(status),
                     )
                 })
-                .filter(|(_, status)| *status == Status::Failed)
+                .filter(|(_, status)| {
+                    include_status
+                        .as_ref()
+                        .map(|allowed_status| allowed_status.contains(status))
+                        .unwrap_or(true)
+                })
                 .collect::<Expectations>();
 
             let output_file = OpenOptions::new()
@@ -143,6 +152,11 @@ pub enum Cli {
         /// Prefix paths to remove from the paths in the final expectations file.
         #[clap(long)]
         remove_prefix: Vec<PathBuf>,
+
+        /// Controls which test case statuses are included in the generated expectations file. If
+        /// nothing is specified then it will include all of the test case status.
+        #[clap(long)]
+        include_status: Option<Vec<Status>>,
     },
 
     /// Compares two expectation files to ensure that they match each other.
@@ -157,7 +171,21 @@ pub enum Cli {
     },
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    ValueEnum,
+    EnumString,
+)]
+#[strum(serialize_all = "kebab-case")]
 pub enum Status {
     Succeeded,
     Failed,