Benchmarking TS (#1325)

(This is #1287 but in a local branch to test bencher, and with some
commits's author fixed)

This PR introduces the following infra commands:
- `infra perf cargo`: this is the `infra perf` command from before
(flamegraphs).
- ~`infra perf fetch`: fetches a series of projects from solcify and
stores them in a specified local folder.~
- `infra perf npm`: runs comparisons of our TS library with solc and
(currently) solc-typed-ast, and ships them to bencher. The comparisons
are run twice, one cold and one hot.

Details:

This is a revamp of
https://github.com/NomicFoundation/slang/tree/perf-ts with several
improvements:

 - [x] ~Uses Benny instead of Jest.~ Removed Benny
 - [x] Count only the hot path.
 - [x] Moved to `solidity/testing/perf`
- [x] Improved solc interface to be capable of executing the test in any
file.
 - [x] Integration with `infra` script.
- [x] ~Add more examples by running on every file in certain projects
(e.g., openzeppelin's)~
- [x] Collect specific measures for slang (time for setup, time to
resolve each definition, etc).
 - [x] ~Add other examples specific for slang.~ Won't fix.
 - [x] ~Add memory test.~ Won't do now.
 - [x] Load to Bencher.
 
 
 The current architecture for the tests is as follow:
 
 ```mermaid
 sequenceDiagram
    box sienna Rust
      participant infra
      participant fetch
      participant perf/npm-benches
    end

   box olive External
      participant sourcify
      participant bencher
    end
        
   box midnightblue TS
      participant npm/main
    end
    
    infra->>perf/npm-benches: run
    
    perf/npm-benches->>fetch: for every project
    fetch->>sourcify: get project
    sourcify->>fetch: persist in FS (JSON)

    perf/npm-benches->>npm/main: for every project
    npm/main-->>perf/npm-benches: ouptut comparison in console (JSON)
    perf/npm-benches-->>infra: BMF
    infra->>bencher: send results
 ```

---------

Co-authored-by: github-actions <github-actions@users.noreply.github.com>
Co-authored-by: Gustavo Giráldez <ggiraldez@manas.tech>

Author: 3 people

.github/workflows/benchmark.yml .github/workflows/benchmark_cargo.yml.github/workflows/benchmark.yml renamed to .github/workflows/benchmark_cargo.yml

@@ -1,4 +1,4 @@
-name: "benchmark"
+name: "benchmark_cargo"
 
 on:
   # Run using manual triggers from GitHub UI:
@@ -11,6 +11,11 @@ on:
         required: true
         default: true
 
+      bencherProject:
+        description: "Bencher project:"
+        type: "string"
+        default: "slang-dashboard-cargo"
+
   # Run on pushes to 'main' branch:
   push:
     branches:
@@ -35,11 +40,15 @@ jobs:
       - name: "Restore Cache"
         uses: "./.github/actions/cache/restore"
 
+      - name: "Set bencher project"
+        run: |
+          echo "SLANG_BENCHER_PROJECT=${{ inputs.bencherProject }}" >> $GITHUB_ENV
+
       - name: "infra setup"
         run: "./scripts/bin/infra setup"
 
-      - name: "infra perf benchmark"
-        run: "./scripts/bin/infra perf benchmark ${{ inputs.dryRun == true && '--dry-run' || '' }}"
+      - name: "infra perf cargo"
+        run: "./scripts/bin/infra perf cargo ${{ inputs.dryRun == true && '--dry-run' || '' }}"
         env:
           BENCHER_API_TOKEN: "${{ secrets.BENCHER_API_TOKEN }}"
 

.github/workflows/benchmark_npm.yml

@@ -0,0 +1,56 @@
+name: "benchmark_npm"
+
+on:
+  # Run using manual triggers from GitHub UI:
+  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
+  workflow_dispatch:
+    inputs:
+      dryRun:
+        description: "Attempt a local run and report results here, without updating the bencher dashboard."
+        type: "boolean"
+        required: true
+        default: true
+
+      bencherProject:
+        description: "Bencher project:"
+        type: "string"
+        default: "slang-dashboard-npm"
+
+  # Run on pushes to 'main' branch:
+  push:
+    branches:
+      - "main"
+
+# Queue up benchmark workflows for the same branch, so that results are reported in order:
+concurrency:
+  group: "${{ github.workflow }}-${{ github.ref_name }}"
+  cancel-in-progress: false
+
+jobs:
+  benchmark:
+    runs-on: "ubuntu-22.04" # _SLANG_DEV_CONTAINER_BASE_IMAGE_ (keep in sync)
+
+    # Only run on the main repo (not forks), unless it is a dry run:
+    if: "${{ github.repository == 'NomicFoundation/slang' || inputs.dryRun == true }}"
+
+    steps:
+      - name: "Checkout Repository"
+        uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+
+      - name: "Restore Cache"
+        uses: "./.github/actions/cache/restore"
+
+      - name: "Set bencher project"
+        run: |
+          echo "SLANG_BENCHER_PROJECT=${{ inputs.bencherProject }}" >> $GITHUB_ENV
+
+      - name: "infra setup"
+        run: "./scripts/bin/infra setup"
+
+      - name: "infra check npm"
+        run: "./scripts/bin/infra check npm"
+
+      - name: "infra perf npm"
+        run: "./scripts/bin/infra perf npm ${{ inputs.dryRun == true && '--dry-run' || '' }}"
+        env:
+          BENCHER_API_TOKEN: "${{ secrets.BENCHER_API_TOKEN }}"

Cargo.toml

@@ -64,17 +64,23 @@ metaslang_bindings = { path = "crates/metaslang/bindings", version = "1.2.1" }
 metaslang_graph_builder = { path = "crates/metaslang/graph_builder", version = "1.2.1" }
 metaslang_cst = { path = "crates/metaslang/cst", version = "1.2.1" }
 
-slang_solidity = { path = "crates/solidity/outputs/cargo/crate", version = "1.2.1" }
-slang_solidity_cli = { path = "crates/solidity/outputs/cargo/cli", version = "1.2.1" }
-solidity_cargo_tests = { path = "crates/solidity/outputs/cargo/tests", version = "1.2.1" }
-solidity_cargo_wasm = { path = "crates/solidity/outputs/cargo/wasm", version = "1.2.1" }
-solidity_language = { path = "crates/solidity/inputs/language", version = "1.2.1" }
-solidity_npm_package = { path = "crates/solidity/outputs/npm/package", version = "1.2.1" }
-solidity_spec = { path = "crates/solidity/outputs/spec", version = "1.2.1" }
-solidity_testing_perf = { path = "crates/solidity/testing/perf", version = "1.2.1" }
-solidity_testing_snapshots = { path = "crates/solidity/testing/snapshots", version = "1.2.1" }
-solidity_testing_solc = { path = "crates/solidity/testing/solc", version = "1.2.1" }
-solidity_testing_sourcify = { path = "crates/solidity/testing/sourcify", version = "1.2.1" }
+slang_solidity = { path = "crates/solidity/outputs/cargo/crate", version = "1.2.0" }
+slang_solidity_cli = { path = "crates/solidity/outputs/cargo/cli", version = "1.2.0" }
+solidity_cargo_tests = { path = "crates/solidity/outputs/cargo/tests", version = "1.2.0" }
+solidity_cargo_wasm = { path = "crates/solidity/outputs/cargo/wasm", version = "1.2.0" }
+solidity_language = { path = "crates/solidity/inputs/language", version = "1.2.0" }
+solidity_npm_package = { path = "crates/solidity/outputs/npm/package", version = "1.2.0" }
+solidity_spec = { path = "crates/solidity/outputs/spec", version = "1.2.0" }
+solidity_testing_perf = { path = "crates/solidity/testing/perf", version = "1.2.0" }
+solidity_testing_snapshots = { path = "crates/solidity/testing/snapshots", version = "1.2.0" }
+solidity_testing_solc = { path = "crates/solidity/testing/solc", version = "1.2.0" }
+solidity_testing_sourcify = { path = "crates/solidity/testing/sourcify", version = "1.2.0" }
+
+slang_testlang = { path = "crates/testlang/outputs/cargo/crate", version = "1.2.0" }
+testlang_cargo_tests = { path = "crates/testlang/outputs/cargo/tests", version = "1.2.0" }
+testlang_cargo_wasm = { path = "crates/testlang/outputs/cargo/wasm", version = "1.2.0" }
+testlang_language = { path = "crates/testlang/inputs/language", version = "1.2.0" }
+testlang_npm_package = { path = "crates/testlang/outputs/npm/package", version = "1.2.0" }
 
 #
 # External

…a/cli/src/commands/perf/benchmark/mod.rs …infra/cli/src/commands/perf/cargo/mod.rscrates/infra/cli/src/commands/perf/benchmark/mod.rs renamed to crates/infra/cli/src/commands/perf/cargo/mod.rs crates/infra/cli/src/commands/perf/benchmark/mod.rs renamed to crates/infra/cli/src/commands/perf/cargo/mod.rs

@@ -7,18 +7,16 @@ use infra_utils::commands::Command;
 use infra_utils::github::GitHub;
 use infra_utils::paths::PathExtensions;
 
+use crate::toolchains::bencher::run_bench;
 use crate::utils::DryRun;
 
-// Source: https://github.com/bencherdev/bencher/blob/aa31a002842cfb0da9d6c60569396fc5261f5111/tasks/test_api/src/task/test/smoke_test.rs#L20
-const BENCHER_TEST_TOKEN: &str = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjo1OTkzNjQyMTU2LCJpYXQiOjE2OTg2NzQ4NjEsImlzcyI6Imh0dHBzOi8vZGV2ZWwtLWJlbmNoZXIubmV0bGlmeS5hcHAvIiwic3ViIjoibXVyaWVsLmJhZ2dlQG5vd2hlcmUuY29tIiwib3JnIjpudWxsfQ.9z7jmM53TcVzc1inDxTeX9_OR0PQPpZAsKsCE7lWHfo";
-
 #[derive(Clone, Debug, Parser)]
-pub struct BenchmarkController {
+pub struct CargoController {
     #[command(flatten)]
     dry_run: DryRun,
 }
 
-impl BenchmarkController {
+impl CargoController {
     pub fn execute(&self) -> Result<()> {
         Self::install_valgrind();
 
@@ -70,49 +68,16 @@ impl BenchmarkController {
     }
 
     fn run_iai_bench(&self, package_name: &str, bench_name: &str) {
-        let dry_run = self.dry_run.get();
-
-        let token = if dry_run {
-            // Use a dummy test token for dry runs:
-            // https://github.com/bencherdev/bencher/issues/468
-            BENCHER_TEST_TOKEN.to_string()
-        } else {
-            std::env::var("BENCHER_API_TOKEN").expect(
-              "BENCHER_API_TOKEN is not set. Either perform a '--dry-run', or set it to your Bencher API token: https://bencher.dev/console"
-            )
-        };
-
         let test_runner = format!("cargo bench --package {package_name} --bench {bench_name}");
 
-        let testbed = if GitHub::is_running_in_ci() {
-            "ci"
-        } else {
-            "dev"
-        };
-
-        let mut command = Command::new("bencher")
-            .arg("run")
-            .property("--project", "slang")
-            .property("--adapter", "rust_iai_callgrind")
-            .property("--testbed", testbed)
-            .secret("BENCHER_API_TOKEN", token);
-
-        if dry_run {
-            command = command.flag("--dry-run");
-        }
-
-        // Has to be the last argument:
-        command.arg(test_runner).run();
+        run_bench(self.dry_run.get(), "rust_iai_callgrind", &test_runner);
 
         let reports_dir = Path::repo_path("target/iai")
             .join(package_name)
             .join(bench_name);
 
         println!("
 
-Bencher Run is complete...
-Test Results: [https://bencher.dev/console/projects/slang/reports]
-
 Reports/Logs: {reports_dir:?}
 - Callgrind flamegraphs (callgrind.*.svg) can be viewed directly in the browser.
 - DHAT traces (dhat.*.out) can be viewed using the [dhat/dh_view.html] tool from the Valgrind release [https://valgrind.org/downloads/].

crates/infra/cli/src/commands/perf/mod.rs

@@ -1,9 +1,11 @@
-mod benchmark;
+mod cargo;
+mod npm;
 
 use anyhow::Result;
 use clap::{Parser, Subcommand};
 
-use crate::commands::perf::benchmark::BenchmarkController;
+use crate::commands::perf::cargo::CargoController;
+use crate::commands::perf::npm::NpmController;
 
 #[derive(Clone, Debug, Parser)]
 pub struct PerfController {
@@ -13,14 +15,17 @@ pub struct PerfController {
 
 #[derive(Clone, Debug, Subcommand)]
 enum PerfCommand {
-    /// Run benchmark tests, and report the results to <https://bencher.dev/console>
-    Benchmark(BenchmarkController),
+    /// Run benchmark tests for the rust api, and report the results to <https://bencher.dev/console>
+    Cargo(CargoController),
+    /// Run benchmark tests for the typescript api, and report the results to <https://bencher.dev/console>
+    Npm(NpmController),
 }
 
 impl PerfController {
     pub fn execute(&self) -> Result<()> {
         match &self.command {
-            PerfCommand::Benchmark(controller) => controller.execute(),
+            PerfCommand::Cargo(controller) => controller.execute(),
+            PerfCommand::Npm(controller) => controller.execute(),
         }
     }
 }

crates/infra/cli/src/commands/perf/npm/mod.rs

@@ -0,0 +1,51 @@
+use anyhow::Result;
+use clap::Parser;
+use infra_utils::cargo::CargoWorkspace;
+
+use crate::toolchains::bencher::run_bench;
+use crate::utils::DryRun;
+
+#[derive(Clone, Debug, Parser)]
+pub struct NpmController {
+    #[arg(short, long, default_value_t = String::from(".*"))]
+    /// A regex pattern to select which project(s) to run
+    pattern: String,
+
+    #[command(flatten)]
+    dry_run: DryRun,
+
+    #[arg(long, default_value_t = 2)]
+    /// The number of cold runs
+    cold: usize,
+
+    #[arg(long, default_value_t = 5)]
+    /// The number of hot runs
+    hot: usize,
+}
+
+impl NpmController {
+    fn install_deps() -> Result<()> {
+        CargoWorkspace::install_binary("bencher_cli")?;
+        Ok(())
+    }
+
+    fn execute_npm_benchmarks(&self) {
+        let test_runner = format!(
+            "cargo run --package {package} -- --pattern=\"{pattern}\" --cold={cold} --hot={hot}",
+            package = "solidity_testing_perf_npm",
+            pattern = &self.pattern,
+            cold = &self.cold,
+            hot = &self.hot,
+        );
+
+        run_bench(self.dry_run.get(), "json", &test_runner);
+    }
+
+    #[allow(clippy::unnecessary_wraps)]
+    pub fn execute(&self) -> Result<()> {
+        Self::install_deps()?;
+
+        self.execute_npm_benchmarks();
+        Ok(())
+    }
+}

crates/infra/cli/src/toolchains/bencher/mod.rs

@@ -0,0 +1,54 @@
+use infra_utils::commands::Command;
+use infra_utils::github::GitHub;
+
+// Use a dummy test token for dry runs:
+// https://github.com/bencherdev/bencher/issues/468
+// Source: https://github.com/bencherdev/bencher/blob/aa31a002842cfb0da9d6c60569396fc5261f5111/tasks/test_api/src/task/test/smoke_test.rs#L20
+const BENCHER_TEST_TOKEN: &str = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhcGlfa2V5IiwiZXhwIjo1OTkzNjQyMTU2LCJpYXQiOjE2OTg2NzQ4NjEsImlzcyI6Imh0dHBzOi8vZGV2ZWwtLWJlbmNoZXIubmV0bGlmeS5hcHAvIiwic3ViIjoibXVyaWVsLmJhZ2dlQG5vd2hlcmUuY29tIiwib3JnIjpudWxsfQ.9z7jmM53TcVzc1inDxTeX9_OR0PQPpZAsKsCE7lWHfo";
+
+// Default bencher project. Can be overriden with the env var SLANG_BENCHER_PROJECT
+const BENCHER_PROJECT: &str = "slang-extended-dashboard";
+
+fn get_bencher_project() -> String {
+    std::env::var("SLANG_BENCHER_PROJECT").unwrap_or(String::from(BENCHER_PROJECT))
+}
+
+pub(crate) fn run_bench(dry_run: bool, adapter: &str, test_runner: &str) {
+    let token = if dry_run {
+        BENCHER_TEST_TOKEN.to_string()
+    } else {
+        std::env::var("BENCHER_API_TOKEN").expect(
+            "BENCHER_API_TOKEN is not set. Either perform a '--dry-run', or set it to your Bencher API token: https://bencher.dev/console"
+        )
+    };
+
+    let testbed = if GitHub::is_running_in_ci() {
+        "ci"
+    } else {
+        "dev"
+    };
+
+    let project = get_bencher_project();
+
+    let mut command = Command::new("bencher")
+        .arg("run")
+        .property("--project", &project)
+        .property("--adapter", adapter)
+        .property("--testbed", testbed)
+        .secret("BENCHER_API_TOKEN", token);
+
+    if dry_run {
+        command = command.flag("--dry-run");
+    }
+
+    // Has to be the last argument:
+    command.arg(test_runner).run();
+
+    println!(
+        "
+
+Bencher Run is complete...
+Test Results: [https://bencher.dev/console/projects/{project}/reports]
+"
+    );
+}

crates/infra/cli/src/toolchains/mod.rs

@@ -1,3 +1,4 @@
+pub mod bencher;
 pub mod mkdocs;
 pub mod npm;
 pub mod pipenv;

crates/solidity/testing/perf/npm/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "solidity_testing_perf_npm"
+version.workspace = true
+rust-version.workspace = true
+edition.workspace = true
+publish = false
+
+[dependencies]
+serde = { workspace = true }
+serde_json = { workspace = true }
+infra_utils = { workspace = true }
+anyhow = { workspace = true }
+regex = { workspace = true }
+clap = { workspace = true }
+reqwest = { workspace = true, features = ["json"] }
+strum = { workspace = true }
+strum_macros = { workspace = true }
+
+[lints]
+workspace = true

crates/solidity/testing/perf/npm/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@slang-private/solidity-testing-perf-npm",
+  "private": true,
+  "type": "module",
+  "dependencies": {
+    "@nomicfoundation/slang": "file:../../../outputs/npm/package/",
+    "@openzeppelin/contracts": "^5.2.0",
+    "@solidity-parser/parser": "^0.20.1",
+    "command-line-args": "^6.0.1",
+    "command-line-usage": "^7.0.3",
+    "solc": "^0.8.29"
+  },
+  "devDependencies": {
+    "@types/command-line-args": "^5.2.3",
+    "@types/command-line-usage": "^5.0.4",
+    "@types/node": "^22.15.17"
+  }
+}