monorepo/.github/workflows/benchmark.yml at 74ce9409169abe956cf7ce687b7928a1650390a4 · commonwarexyz/monorepo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
name: Benchmark

on:
  workflow_dispatch:
    inputs:
      dry_run:
        description: "Generate synthetic benchmark output instead of running cargo bench"
        required: false
        default: true
        type: boolean
      publish:
        description: "Publish results to the benchmarks repository"
        required: false
        default: false
        type: boolean
  schedule:
    - cron: '0 10 * * *' # Run at 10:00 UTC every day

permissions:
  contents: read

jobs:
  has-new-commit:
    name: Has New Commit? # Check if there are new commits in the last 24 hours
    runs-on: ubuntu-latest
    steps:
      - name: Checkout default branch
        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
        with:
          fetch-depth: 1

      - name: Exit early if no new commits
        if: github.event_name == 'schedule'
        id: check-commits
        run: |
          count=$(git log --oneline --since '24 hours ago' | wc -l)
          if [ "$count" -eq 0 ]; then
            echo "No new commits found in the last 24 hours"
            exit 1
          fi

  benchmark:
    name: Run
    needs: has-new-commit # Only run if there are new commits
    runs-on: ubuntu-latest-8
    timeout-minutes: 720
    strategy:
      fail-fast: false
      matrix:
        include:
          - package: commonware-cryptography
            cargo_flags: ""
            benchmark_name: "commonware-cryptography"
          - package: commonware-storage
            cargo_flags: "--features test-traits"
            benchmark_name: "commonware-storage"
          - package: commonware-storage
            cargo_flags: "--features commonware-runtime/iouring-storage,test-traits"
            benchmark_name: "commonware-storage --features"
          - package: commonware-coding
            cargo_flags: ""
            benchmark_name: "commonware-coding"
          - package: commonware-formatting
            cargo_flags: ""
            benchmark_name: "commonware-formatting"
          - package: commonware-utils
            cargo_flags: ""
            benchmark_name: "commonware-utils"
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
      - name: Install nightly Rust toolchain
        run: rustup toolchain install nightly
      - name: Run setup
        uses: ./.github/actions/setup
      - name: Compile benchmarks
        if: github.event_name == 'schedule' || !inputs.dry_run || inputs.publish
        env:
          RUSTFLAGS: "--cfg full_bench"
        run: cargo bench ${{ matrix.cargo_flags }} --benches -p ${{ matrix.package }} --no-run
      - name: Run benchmarks
        if: github.event_name == 'schedule' || !inputs.dry_run || inputs.publish
        env:
          RUSTFLAGS: "--cfg full_bench"
        run: |
          cargo bench ${{ matrix.cargo_flags }} \
            --benches -p ${{ matrix.package }} \
            -- --output-format bencher \
            | tee "${{ matrix.benchmark_name }}.txt"
      - name: Generate dry-run benchmark result
        if: github.event_name == 'workflow_dispatch' && inputs.dry_run && !inputs.publish
        run: |
          output="${{ matrix.benchmark_name }}.txt"
          printf 'test dry_run::mechanics ... bench:           1 ns/iter (+/- 0)\n' | tee "$output"
      - name: Upload benchmark result
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: benchmark-${{ matrix.benchmark_name }}
          path: "${{ matrix.benchmark_name }}.txt"
          if-no-files-found: error

  publish:
    name: Publish
    needs: benchmark
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      max-parallel: 1 # Push benchmark results one package at a time.
      matrix:
        include:
          - package: commonware-cryptography
            benchmark_name: "commonware-cryptography"
          - package: commonware-storage
            benchmark_name: "commonware-storage"
          - package: commonware-storage
            benchmark_name: "commonware-storage --features"
          - package: commonware-coding
            benchmark_name: "commonware-coding"
          - package: commonware-formatting
            benchmark_name: "commonware-formatting"
          - package: commonware-utils
            benchmark_name: "commonware-utils"
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
      - name: Download benchmark result
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: benchmark-${{ matrix.benchmark_name }}
      - name: Validate benchmark result
        run: |
          output="${{ matrix.benchmark_name }}.txt"
          test -s "$output"
          grep -Eq '^test .+ \.\.\. bench:' "$output"
      - name: Publish benchmark result
        if: github.event_name == 'schedule' || inputs.publish
        uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7
        with:
          name: ${{ matrix.benchmark_name }}
          tool: 'cargo'
          output-file-path: "${{ matrix.benchmark_name }}.txt"
          github-token: ${{ secrets.BENCHMARKS_SECRET }}
          gh-repository: github.com/commonwarexyz/benchmarks
          gh-pages-branch: main
          benchmark-data-dir-path: 'docs'
          auto-push: true
          max-items-in-chart: 60