Nightly tests #1206
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 23 * * *" # run at 11pm UTC | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUSTFLAGS: "-C target-cpu=native" | |
| RUST_BACKTRACE: 1 | |
| JEMALLOC_SYS_WITH_MALLOC_CONF: "retain:true,background_thread:true,metadata_thp:always,dirty_decay_ms:10000,muzzy_decay_ms:10000,abort_conf:true" | |
| POWDR_OPENVM_SEGMENT_DELTA: 50000 | |
| jobs: | |
| bench: | |
| runs-on: warp-ubuntu-2404-x64-4x | |
| permissions: | |
| contents: write | |
| deployments: write | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: ⚡ Restore rust cache | |
| id: cache | |
| uses: WarpBuilds/cache/restore@v1 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| Cargo.lock | |
| key: ${{ runner.os }}-cargo-pr-tests | |
| - name: Install Rust toolchain nightly-2025-10-01 (with clippy and rustfmt) | |
| run: rustup toolchain install nightly-2025-10-01 --component clippy,rustfmt,rust-src | |
| - name: Install Rust toolchain 1.90 | |
| run: rustup toolchain install 1.90 | |
| - name: Install riscv target | |
| run: rustup target add riscv32imac-unknown-none-elf --toolchain nightly-2025-10-01 | |
| - name: Install test dependencies | |
| run: sudo apt-get update && sudo apt-get install -y binutils-riscv64-unknown-elf lld | |
| - name: Run benchmarks | |
| # we add `|| exit 1` to make sure the step fails if `cargo bench` fails | |
| run: cargo bench --workspace --features "metrics" -- --output-format bencher | tee output.txt || exit 1 | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: Benchmarks | |
| tool: "cargo" | |
| output-file-path: output.txt | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true | |
| alert-threshold: "120%" | |
| comment-on-alert: true | |
| summary-always: true | |
| test_apc: | |
| runs-on: server-dev | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: ⚡ Cache rust | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-release-apc-${{ hashFiles('**/Cargo.toml') }} | |
| - name: Build | |
| run: cargo build --release -p powdr-openvm | |
| - name: Install cargo openvm | |
| # Rust 1.90 is needed by fresher versions of dependencies of cargo-openvm. | |
| run: | | |
| rustup toolchain install 1.90 | |
| cargo +1.90 install --git 'http://github.com/powdr-labs/openvm.git' --rev "v1.4.2-powdr-rc.2" --locked cargo-openvm | |
| - name: Setup python venv | |
| run: | | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install -r openvm/scripts/requirements.txt | |
| pip install -r autoprecompiles/scripts/requirements.txt | |
| - name: Remove old results if present | |
| run: | | |
| rm -rf results | |
| mkdir -p results | |
| - name: Run guest benchmarks | |
| run: | | |
| source .venv/bin/activate | |
| bash ./openvm/scripts/run_guest_benches.sh | |
| - name: Patch benchmark | |
| uses: ./.github/actions/patch-openvm-reth-benchmark | |
| - name: Run reth benchmark | |
| run: | | |
| source .venv/bin/activate | |
| cd openvm-reth-benchmark | |
| RES_DIR=reth | |
| mkdir -p $RES_DIR | |
| echo "export RPC_1=${{ secrets.RPC_1 }}" >> .env | |
| # prove with no APCs | |
| ./run.sh --apc 0 --mode prove-stark || exit 1 | |
| # remove apc cache to not interfere with the next runs | |
| rm -rf apc-cache | |
| echo "Finished proving with no APCs" | |
| mv metrics.json $RES_DIR/apc000.json | |
| python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc000.png $RES_DIR/apc000.json > $RES_DIR/trace_cells_apc000.txt | |
| # prove with 3 APCs | |
| ./run.sh --apc 3 --mode prove-stark || exit 1 | |
| # remove apc cache to not interfere with the next runs | |
| rm -rf apc-cache | |
| echo "Finished proving with 3 APCs" | |
| mv metrics.json $RES_DIR/apc003.json | |
| python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc003.png $RES_DIR/apc003.json > $RES_DIR/trace_cells_apc003.txt | |
| # prove with 10 APCs | |
| ./run.sh --apc 10 --mode prove-stark || exit 1 | |
| # remove apc cache to not interfere with the next runs | |
| rm -rf apc-cache | |
| echo "Finished proving with 10 APCs" | |
| mv metrics.json $RES_DIR/apc010.json | |
| python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc010.png $RES_DIR/apc010.json > $RES_DIR/trace_cells_apc010.txt | |
| # prove with 30 APCs | |
| ./run.sh --apc 30 --mode prove-stark || exit 1 | |
| # remove apc cache to not interfere with the next runs | |
| rm -rf apc-cache | |
| echo "Finished proving with 30 APCs" | |
| mv metrics.json $RES_DIR/apc030.json | |
| python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc030.png $RES_DIR/apc030.json > $RES_DIR/trace_cells_apc030.txt | |
| # prove with 100 APCs, recording mem usage | |
| psrecord --include-children --interval 1 --log $RES_DIR/psrecord.csv --log-format csv --plot $RES_DIR/psrecord.png "./run.sh --apc 100 --mode prove-stark" || exit 1 | |
| # remove apc cache to not interfere with the next runs | |
| rm -rf apc-cache | |
| echo "Finished proving with 100 APCs" | |
| mv metrics.json $RES_DIR/apc100.json | |
| python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc100.png $RES_DIR/apc100.json > $RES_DIR/trace_cells_apc100.txt | |
| # The APC candidates would be the same for all runs, so just keep the last one | |
| mv apcs/apc_candidates.json $RES_DIR/apc_candidates.json | |
| python ../openvm/scripts/basic_metrics.py summary-table --csv $RES_DIR/apc000.json $RES_DIR/apc003.json $RES_DIR/apc010.json $RES_DIR/apc030.json $RES_DIR/apc100.json > $RES_DIR/basic_metrics.csv | |
| python ../openvm/scripts/basic_metrics.py plot $RES_DIR/apc000.json $RES_DIR/apc003.json $RES_DIR/apc010.json $RES_DIR/apc030.json $RES_DIR/apc100.json -o $RES_DIR/proof_time_breakdown.png | |
| python ../autoprecompiles/scripts/plot_effectiveness.py $RES_DIR/apc_candidates.json --output $RES_DIR/effectiveness.png | |
| mv $RES_DIR ../results/ | |
| - name: Save revisions and run info | |
| run: | | |
| echo "openvm-reth-benchmark: $(git -C openvm-reth-benchmark rev-parse HEAD)" > results/run.txt | |
| echo "powdr: $(git rev-parse HEAD)" >> results/run.txt | |
| echo "run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" >> results/run.txt | |
| - name: upload result artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-results | |
| path: | | |
| results/* | |
| - name: get the date/time | |
| id: date | |
| run: echo "value=$(date +'%Y-%m-%d-%H%M')" >> $GITHUB_OUTPUT | |
| - name: commit to bench results | |
| uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| personal_token: ${{ secrets.BENCH_RESULTS_TOKEN }} | |
| external_repository: powdr-labs/bench-results | |
| publish_dir: ./results | |
| destination_dir: results/${{ steps.date.outputs.value }}/ | |
| keep_files: true | |
| enable_jekyll: true | |
| test_apc_gpu: | |
| runs-on: [self-hosted, GPU] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: ⚡ Cache rust | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-release-apc-gpu-${{ hashFiles('**/Cargo.toml') }} | |
| - name: Install cargo openvm | |
| # Rust 1.90 is needed by fresher versions of dependencies of cargo-openvm. | |
| run: | | |
| rustup toolchain install 1.90 | |
| cargo +1.90 install --git 'http://github.com/powdr-labs/openvm.git' --rev "v1.4.2-powdr-rc.2" --locked cargo-openvm | |
| - name: Setup python venv | |
| run: | | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install -r openvm/scripts/requirements.txt | |
| pip install -r autoprecompiles/scripts/requirements.txt | |
| - name: Remove old results if present | |
| run: | | |
| rm -rf results | |
| mkdir -p results | |
| - name: Patch benchmark | |
| uses: ./.github/actions/patch-openvm-reth-benchmark | |
| - name: Run reth benchmark (GPU) | |
| run: | | |
| source .venv/bin/activate | |
| cd openvm-reth-benchmark | |
| RES_DIR=reth_gpu | |
| mkdir -p $RES_DIR | |
| echo "export RPC_1=${{ secrets.RPC_1 }}" >> .env | |
| # prove with no APCs | |
| ./run.sh --cuda --apc 0 --mode prove-stark || exit 1 | |
| # remove apc cache to not interfere with the next runs | |
| rm -rf apc-cache | |
| echo "Finished proving with no APCs" | |
| mv metrics.json $RES_DIR/apc000.json | |
| python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc000.png $RES_DIR/apc000.json > $RES_DIR/trace_cells_apc000.txt | |
| # prove with 10 APCs | |
| ./run.sh --cuda --apc 10 --mode prove-stark || exit 1 | |
| # remove apc cache to not interfere with the next runs | |
| rm -rf apc-cache | |
| echo "Finished proving with 10 APCs" | |
| mv metrics.json $RES_DIR/apc010.json | |
| python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc010.png $RES_DIR/apc010.json > $RES_DIR/trace_cells_apc010.txt | |
| # prove with 30 APCs | |
| ./run.sh --cuda --apc 30 --mode prove-stark || exit 1 | |
| # remove apc cache to not interfere with the next runs | |
| rm -rf apc-cache | |
| echo "Finished proving with 30 APCs" | |
| mv metrics.json $RES_DIR/apc030.json | |
| python ../openvm/scripts/plot_trace_cells.py -o $RES_DIR/trace_cells_apc030.png $RES_DIR/apc030.json > $RES_DIR/trace_cells_apc030.txt | |
| # The APC candidates would be the same for all runs, so just keep the last one | |
| mv apcs/apc_candidates.json $RES_DIR/apc_candidates.json | |
| python ../openvm/scripts/basic_metrics.py summary-table --csv $RES_DIR/apc000.json $RES_DIR/apc010.json $RES_DIR/apc030.json > $RES_DIR/basic_metrics.csv | |
| python ../openvm/scripts/basic_metrics.py plot $RES_DIR/apc000.json $RES_DIR/apc010.json $RES_DIR/apc030.json -o $RES_DIR/proof_time_breakdown.png | |
| python ../autoprecompiles/scripts/plot_effectiveness.py $RES_DIR/apc_candidates.json --output $RES_DIR/effectiveness.png | |
| mv $RES_DIR ../results/ | |
| - name: Save revisions and run info | |
| run: | | |
| echo "openvm-reth-benchmark: $(git -C openvm-reth-benchmark rev-parse HEAD)" > results/run.txt | |
| echo "powdr: $(git rev-parse HEAD)" >> results/run.txt | |
| echo "run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" >> results/run.txt | |
| - name: upload result artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-results-gpu | |
| path: | | |
| results/* | |
| - name: get the date/time | |
| id: date | |
| run: echo "value=$(date +'%Y-%m-%d-%H%M')" >> $GITHUB_OUTPUT | |
| - name: commit to bench results | |
| uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| personal_token: ${{ secrets.BENCH_RESULTS_TOKEN }} | |
| external_repository: powdr-labs/bench-results | |
| publish_dir: ./results | |
| destination_dir: results/${{ steps.date.outputs.value }}-gpu/ | |
| keep_files: true | |
| enable_jekyll: true | |