ci: Use codspeed for benchmarking
#28
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CodSpeed | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| branches: ["main"] | |
| merge_group: | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| bench-matrix: | |
| name: Determine bench matrix | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| benches: ${{ steps.benches.outputs.benches }} | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| persist-credentials: false | |
| - id: benches | |
| run: | | |
| # Create a GitHub matrix where each entry is a bench with its | |
| # associated crate and benchmarking mode taken from the Cargo | |
| # metadata. | |
| { | |
| echo -n "benches=[" | |
| first=true | |
| while IFS=: read -r crate manifest_path; do | |
| dir=$(dirname "$manifest_path") | |
| if [ -e "$dir/benches" ]; then | |
| for bench in "$dir/benches"/*.rs; do | |
| bench=$(basename -s .rs "$bench") | |
| # Set measurement mode from Cargo.toml metadata, default to "instrumentation". | |
| # The following jq pipeline: | |
| # - Selects the package with name == $crate | |
| # - Looks up .metadata.bench[$bench].codspeed.mode for the benchmark | |
| # - If not set, defaults to "instrumentation" | |
| mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') | |
| # FIXME: Always use "instrumentation" mode to eliminate CI VM effects. | |
| mode="instrumentation" | |
| if [ "$first" = true ]; then | |
| first=false | |
| else | |
| echo -n "," | |
| fi | |
| echo -n "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}" | |
| done | |
| fi | |
| done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') | |
| echo -n "]" | |
| } >> "${GITHUB_OUTPUT}" | |
| benchmarks: | |
| name: Run bench | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: | |
| bench: ${{ fromJson(needs.bench-matrix.outputs.benches) }} | |
| needs: bench-matrix | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: ./.github/actions/rust | |
| with: | |
| version: stable | |
| tools: cargo-codspeed | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - id: nss-version | |
| run: echo "minimum=$(cat neqo-crypto/min_version.txt)" >> "$GITHUB_OUTPUT" | |
| - uses: ./.github/actions/nss | |
| with: | |
| minimum-version: ${{ steps.nss-version.outputs.minimum }} | |
| - name: Build bench | |
| env: | |
| BENCH: ${{ matrix.bench.bench }} | |
| CRATE: ${{ matrix.bench.crate }} | |
| MODE: ${{ matrix.bench.mode }} | |
| run: cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "$MODE" | |
| - name: Run bench & upload results | |
| uses: CodSpeedHQ/action@4348f634fa7309fe23aac9502e88b999ec90a164 # v4.3.1 | |
| with: | |
| mode: ${{ matrix.bench.mode }} | |
| run: cargo codspeed run | |
| token: ${{ secrets.CODSPEED_TOKEN }} |