diff --git a/.github/actions/run-differential-tests/action.yml b/.github/actions/run-differential-tests/action.yml index 477fd81..587f40b 100644 --- a/.github/actions/run-differential-tests/action.yml +++ b/.github/actions/run-differential-tests/action.yml @@ -29,6 +29,10 @@ inputs: default: true type: boolean # Test Execution Arguments + # TODO: We need a better way for people to pass arguments to retester. This way is not very good + # because we need to add support for each argument separately and support defaults and all of that + # perhaps having people pass in a JSON String of the arguments is the better long term solution + # for this. platform: description: "The identifier of the platform to run the tests on (e.g., geth-evm-solc, revive-dev-node-revm-solc)" required: true @@ -121,10 +125,12 @@ runs: --eth-rpc.path ${{ inputs['polkadot-sdk-path'] }}/target/release/eth-rpc \ --polkadot-omni-node.path ${{ inputs['polkadot-sdk-path'] }}/target/release/polkadot-omni-node \ --resolc.path ./resolc \ + --resolc.heap-size 128000 \ + --resolc.stack-size 128000 \ "${OMNI_ARGS[@]}" || true - name: Generate the expectation file shell: bash - run: report-processor generate-expectations-file --report-path ./workdir/report.json --output-path ./workdir/expectations.json --remove-prefix ./revive-differential-tests/resolc-compiler-tests + run: report-processor generate-expectations-file --report-path ./workdir/report.json --output-path ./workdir/expectations.json --remove-prefix ./revive-differential-tests/resolc-compiler-tests --include-status failed - name: Upload the Report to the CI uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: diff --git a/Cargo.lock b/Cargo.lock index 4cb6a7b..9bfa7ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5886,6 +5886,7 @@ dependencies = [ "revive-dt-report", "serde", "serde_json", + "strum", ] [[package]] diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 993103b..33f5a68 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -806,6 +806,7 @@ pub struct ResolcConfiguration { /// If unspecified, the revive compiler default is used #[clap(id = "resolc.heap-size", long = "resolc.heap-size")] pub heap_size: Option, + /// Specifies the PVM stack size in bytes. /// /// If unspecified, the revive compiler default is used diff --git a/crates/core/src/helpers/test.rs b/crates/core/src/helpers/test.rs index 261f29f..c13d893 100644 --- a/crates/core/src/helpers/test.rs +++ b/crates/core/src/helpers/test.rs @@ -223,17 +223,24 @@ impl<'a> TestDefinition<'a> { /// Checks if the platforms all support the desired targets in the metadata file. fn check_target_compatibility(&self) -> TestCheckFunctionResult { + // The case targets takes presence over the metadata targets. + let Some(targets) = self + .case + .targets + .as_ref() + .or(self.metadata.targets.as_ref()) + else { + return Ok(()); + }; + let mut error_map = indexmap! { - "test_desired_targets" => json!(self.metadata.targets.as_ref()), + "test_desired_targets" => json!(targets), }; + let mut is_allowed = true; for (_, platform_information) in self.platforms.iter() { - let is_allowed_for_platform = match self.metadata.targets.as_ref() { - None => true, - Some(required_vm_identifiers) => { - required_vm_identifiers.contains(&platform_information.platform.vm_identifier()) - } - }; + let is_allowed_for_platform = + targets.contains(&platform_information.platform.vm_identifier()); is_allowed &= is_allowed_for_platform; error_map.insert( platform_information.platform.platform_identifier().into(), diff --git a/crates/format/src/case.rs b/crates/format/src/case.rs index 19114a9..f8d4afa 100644 --- a/crates/format/src/case.rs +++ b/crates/format/src/case.rs @@ -1,16 +1,22 @@ -use alloy::primitives::Address; +use alloy::primitives::{Address, map::HashSet}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use revive_dt_common::{ macros::define_wrapper_type, - types::{Mode, ParsedMode}, + types::{Mode, ParsedMode, VmIdentifier}, }; use crate::steps::*; #[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq, JsonSchema)] pub struct Case { + /// An optional vector of targets that this Metadata file's cases can be executed on. As an + /// example, if we wish for the metadata file's cases to only be run on PolkaVM then we'd + /// specify a target of "PolkaVM" in here. + #[serde(skip_serializing_if = "Option::is_none")] + pub targets: Option>, + /// An optional name of the test case. #[serde(skip_serializing_if = "Option::is_none")] pub name: Option, diff --git a/crates/format/src/metadata.rs b/crates/format/src/metadata.rs index 173e288..c73a6e9 100644 --- a/crates/format/src/metadata.rs +++ b/crates/format/src/metadata.rs @@ -8,6 +8,7 @@ use std::{ str::FromStr, }; +use alloy::primitives::map::HashSet; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -83,7 +84,7 @@ pub struct Metadata { /// example, if we wish for the metadata file's cases to only be run on PolkaVM then we'd /// specify a target of "PolkaVM" in here. #[serde(skip_serializing_if = "Option::is_none")] - pub targets: Option>, + pub targets: Option>, /// A vector of the test cases and workloads contained within the metadata file. This is their /// primary description. diff --git a/crates/report-processor/Cargo.toml b/crates/report-processor/Cargo.toml index ce71d10..f0f5ccd 100644 --- a/crates/report-processor/Cargo.toml +++ b/crates/report-processor/Cargo.toml @@ -18,6 +18,7 @@ revive-dt-common = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } +strum = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/report-processor/src/main.rs b/crates/report-processor/src/main.rs index 2d2b013..82f78ef 100644 --- a/crates/report-processor/src/main.rs +++ b/crates/report-processor/src/main.rs @@ -1,6 +1,6 @@ use std::{ borrow::Cow, - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, HashSet}, fmt::Display, fs::{File, OpenOptions}, ops::{Deref, DerefMut}, @@ -9,11 +9,12 @@ use std::{ }; use anyhow::{Context as _, Error, Result, bail}; -use clap::Parser; +use clap::{Parser, ValueEnum}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use revive_dt_common::types::{Mode, ParsedTestSpecifier}; use revive_dt_report::{Report, TestCaseStatus}; +use strum::EnumString; fn main() -> Result<()> { let cli = Cli::try_parse().context("Failed to parse the CLI arguments")?; @@ -23,11 +24,14 @@ fn main() -> Result<()> { report_path, output_path: output_file, remove_prefix, + include_status, } => { let remove_prefix = remove_prefix .into_iter() .map(|path| path.canonicalize().context("Failed to canonicalize path")) .collect::>>()?; + let include_status = + include_status.map(|value| value.into_iter().collect::>()); let expectations = report_path .execution_information @@ -73,7 +77,12 @@ fn main() -> Result<()> { Status::from(status), ) }) - .filter(|(_, status)| *status == Status::Failed) + .filter(|(_, status)| { + include_status + .as_ref() + .map(|allowed_status| allowed_status.contains(status)) + .unwrap_or(true) + }) .collect::(); let output_file = OpenOptions::new() @@ -143,6 +152,11 @@ pub enum Cli { /// Prefix paths to remove from the paths in the final expectations file. #[clap(long)] remove_prefix: Vec, + + /// Controls which test case statuses are included in the generated expectations file. If + /// nothing is specified then it will include all of the test case status. + #[clap(long)] + include_status: Option>, }, /// Compares two expectation files to ensure that they match each other. @@ -157,7 +171,21 @@ pub enum Cli { }, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + ValueEnum, + EnumString, +)] +#[strum(serialize_all = "kebab-case")] pub enum Status { Succeeded, Failed,