Skip to content

Commit 72eb4ed

Browse files
timkpainetimkpaine
committed
initialize benchmarks
1 parent 01ce9cf commit 72eb4ed

48 files changed

Lines changed: 2691 additions & 9 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/benchmarks.yaml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
name: Benchmarks
2+
3+
on:
4+
# Run on schedule - weekly benchmarks
5+
schedule:
6+
- cron: '0 2 * * 0' # Every Sunday at 2 AM UTC
7+
8+
# Manual trigger with options
9+
workflow_dispatch:
10+
inputs:
11+
use_hetzner:
12+
description: 'Use Hetzner Cloud for benchmarks'
13+
required: false
14+
default: 'true'
15+
type: choice
16+
options:
17+
- 'true'
18+
- 'false'
19+
server_type:
20+
description: 'Hetzner server type'
21+
required: false
22+
default: 'cx42'
23+
type: choice
24+
options:
25+
- 'cx22' # 2 vCPU, 4GB RAM
26+
- 'cx42' # 8 vCPU, 16GB RAM
27+
commit_range:
28+
description: 'Commit range to benchmark (e.g., HEAD~5..HEAD)'
29+
required: false
30+
default: 'HEAD^!'
31+
32+
# Run on pushes to main (for tracking performance over time)
33+
push:
34+
branches:
35+
- main
36+
paths:
37+
- 'csp_benchmarks/benchmarks/**'
38+
- 'csp_benchmarks/asv.conf.json'
39+
40+
concurrency:
41+
group: benchmarks-${{ github.ref }}
42+
cancel-in-progress: true
43+
44+
permissions:
45+
contents: write
46+
pull-requests: write
47+
48+
jobs:
49+
# Quick local benchmarks for PRs
50+
benchmark-local:
51+
if: github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && github.event.inputs.use_hetzner == 'false')
52+
runs-on: ubuntu-latest
53+
54+
steps:
55+
- uses: actions/checkout@v4
56+
with:
57+
fetch-depth: 0
58+
59+
- name: Set up Python
60+
uses: actions/setup-python@v5
61+
with:
62+
python-version: '3.11'
63+
64+
- name: Install system dependencies
65+
run: |
66+
sudo apt-get update
67+
sudo apt-get install -y build-essential cmake ninja-build libboost-all-dev
68+
69+
- name: Install dependencies
70+
run: |
71+
python -m pip install --upgrade pip
72+
python -m pip install -e ".[develop]"
73+
74+
- name: Configure ASV machine
75+
run: cp csp_benchmarks/asv-machine.json ~/.asv-machine.json
76+
77+
- name: Run quick benchmarks
78+
run: |
79+
python -m asv run --config csp_benchmarks/asv.conf.json --machine github-actions --python=same --quick --verbose
80+
81+
- name: Compare with main
82+
if: github.event_name == 'pull_request'
83+
run: |
84+
python -m asv continuous --config csp_benchmarks/asv.conf.json --verbose main HEAD || true
85+
86+
- name: Generate HTML report
87+
run: python -m asv publish --config csp_benchmarks/asv.conf.json
88+
89+
- name: Upload benchmark results
90+
uses: actions/upload-artifact@v4
91+
with:
92+
name: benchmark-results
93+
path: |
94+
csp_benchmarks/results/
95+
.asv/html/
96+
97+
# Full benchmarks on Hetzner Cloud
98+
benchmark-hetzner:
99+
if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.use_hetzner == 'true') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
100+
runs-on: ubuntu-latest
101+
102+
steps:
103+
- uses: actions/checkout@v4
104+
with:
105+
fetch-depth: 0
106+
107+
- name: Set up Python
108+
uses: actions/setup-python@v5
109+
with:
110+
python-version: '3.11'
111+
112+
- name: Install dependencies
113+
run: |
114+
python -m pip install --upgrade pip
115+
python -m pip install -e ".[develop,hetzner]"
116+
117+
- name: Set up SSH key
118+
run: |
119+
mkdir -p ~/.ssh
120+
echo "${{ secrets.HETZNER_SSH_PRIVATE_KEY }}" > ~/.ssh/hetzner_key
121+
chmod 600 ~/.ssh/hetzner_key
122+
123+
- name: Run benchmarks on Hetzner
124+
env:
125+
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
126+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
127+
run: |
128+
python -m csp_benchmarks.hetzner.cli run \
129+
--server-type ${{ github.event.inputs.server_type || 'cx32' }} \
130+
--ssh-key ~/.ssh/hetzner_key \
131+
--commits "${{ github.event.inputs.commit_range || 'HEAD^!' }}" \
132+
--push
133+
134+
- name: Cleanup on failure
135+
if: failure()
136+
env:
137+
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
138+
run: |
139+
python -m csp_benchmarks.hetzner.cli cleanup || true
140+
141+
# Publish results to GitHub Pages
142+
publish-results:
143+
needs: [benchmark-hetzner]
144+
if: always() && (needs.benchmark-hetzner.result == 'success' || needs.benchmark-local.result == 'success')
145+
runs-on: ubuntu-latest
146+
147+
steps:
148+
- uses: actions/checkout@v4
149+
with:
150+
fetch-depth: 0
151+
152+
- name: Set up Python
153+
uses: actions/setup-python@v5
154+
with:
155+
python-version: '3.11'
156+
157+
- name: Install dependencies
158+
run: |
159+
python -m pip install --upgrade pip
160+
python -m pip install asv
161+
162+
- name: Pull latest results
163+
run: |
164+
git pull origin main || true
165+
166+
- name: Generate HTML report
167+
run: python -m asv publish --config csp_benchmarks/asv.conf.json
168+
169+
- name: Deploy to GitHub Pages
170+
uses: peaceiris/actions-gh-pages@v4
171+
with:
172+
github_token: ${{ secrets.GITHUB_TOKEN }}
173+
publish_dir: .asv/html
174+
destination_dir: benchmarks

‎.github/workflows/build.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ jobs:
5454
- name: Test
5555
run: make coverage
5656

57+
- name: Run benchmarks
58+
run: |
59+
pip install csp asv
60+
cp csp_benchmarks/asv-machine.json ~/.asv-machine.json
61+
python -m asv run --config ${{ github.workspace }}/csp_benchmarks/asv.conf.json --machine github-actions --python=same --quick
62+
5763
- name: Upload test results (Python)
5864
uses: actions/upload-artifact@v6
5965
with:

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,3 +157,7 @@ csp_benchmarks/labextension
157157

158158
# Rust
159159
target
160+
# ASV (airspeed velocity) benchmarks
161+
.asv/
162+
csp_benchmarks/csp-benchmarks
163+
csp_benchmarks/csp

‎Makefile‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,35 @@ dist: clean dist-build dist-check ## build all dists
102102

103103
publish: dist ## publish python assets
104104

105+
##############
106+
# BENCHMARKS #
107+
##############
108+
.PHONY: benchmark benchmarks benchmark-quick benchmark-local benchmark-view benchmark-compare
109+
110+
ASV_CONFIG := $(CURDIR)/csp_benchmarks/asv.conf.json
111+
112+
benchmark: ## run benchmarks for current commit
113+
python -m asv run --config $(ASV_CONFIG) --verbose HEAD^!
114+
115+
benchmark-quick: ## run quick benchmark
116+
python -m asv run --config $(ASV_CONFIG) --quick --verbose HEAD^!
117+
118+
benchmark-local: ## run benchmark using local environment
119+
python -m asv run --config $(ASV_CONFIG) --python=same --verbose
120+
121+
benchmark-compare: ## compare benchmarks between commits
122+
python -m asv compare --config $(ASV_CONFIG) HEAD~1 HEAD
123+
124+
benchmark-view: ## generate and view benchmark results
125+
python -m asv publish --config $(ASV_CONFIG)
126+
python -m asv preview --config $(ASV_CONFIG)
127+
128+
benchmark-continuous: ## run benchmarks comparing HEAD to main
129+
python -m asv continuous --config $(ASV_CONFIG) main HEAD --verbose
130+
131+
# Alias
132+
benchmarks: benchmark
133+
105134
#########
106135
# CLEAN #
107136
#########

‎README.md‎

Lines changed: 137 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,142 @@
1-
# csp benchmarks
1+
# csp-benchmarks
22

3-
Benchmarks for csp
3+
Performance benchmarks for [csp](https://github.com/Point72/csp) using [airspeed velocity (ASV)](https://asv.readthedocs.io/).
44

5-
[![Build Status](https://github.com/csp-benchmarks/csp-benchmarks/actions/workflows/build.yaml/badge.svg?branch=main&event=push)](https://github.com/csp-benchmarks/csp-benchmarks/actions/workflows/build.yaml)
6-
[![codecov](https://codecov.io/gh/csp-benchmarks/csp-benchmarks/branch/main/graph/badge.svg)](https://codecov.io/gh/csp-benchmarks/csp-benchmarks)
7-
[![License](https://img.shields.io/github/license/csp-benchmarks/csp-benchmarks)](https://github.com/csp-benchmarks/csp-benchmarks)
8-
[![PyPI](https://img.shields.io/pypi/v/csp-benchmarks.svg)](https://pypi.python.org/pypi/csp-benchmarks)
5+
[![Build Status](https://github.com/csp-community/csp-benchmarks/actions/workflows/build.yaml/badge.svg?branch=main&event=push)](https://github.com/csp-community/csp-benchmarks/actions/workflows/build.yaml)
6+
[![Benchmarks](https://github.com/csp-community/csp-benchmarks/actions/workflows/benchmarks.yaml/badge.svg)](https://github.com/csp-community/csp-benchmarks/actions/workflows/benchmarks.yaml)
7+
[![License](https://img.shields.io/github/license/csp-community/csp-benchmarks)](https://github.com/csp-community/csp-benchmarks)
98

109
## Overview
1110

12-
> [!NOTE]
13-
> This library was generated using [copier](https://copier.readthedocs.io/en/stable/) from the [Base Python Project Template repository](https://github.com/python-project-templates/base).
11+
This repository contains performance benchmarks for the csp library, designed to:
12+
13+
- Track performance over time across commits
14+
- Detect performance regressions
15+
- Compare different implementations and configurations
16+
- Run on dedicated Hetzner Cloud machines for consistent results
17+
18+
## Benchmark Suites
19+
20+
### Core Benchmarks (bench_core.py)
21+
22+
- **GraphExecutionSuite**: Tests graph execution with varying node counts and tick rates
23+
- **NodeOverheadSuite**: Measures node invocation overhead
24+
25+
### Stats Benchmarks (bench_stats.py)
26+
27+
- **StatsBenchmarkSuite**: Tests statistical functions (median, quantile, rank)
28+
- **StatsScalingSuite**: Tests how stats scale with data size
29+
30+
### Baselib Benchmarks (bench_baselib.py)
31+
32+
- **BaselibSuite**: Tests built-in operations (filter, sample, delay, merge, flatten)
33+
- **CurveSuite**: Tests historical data loading
34+
35+
### Math Benchmarks (bench_math.py)
36+
37+
- **MathSuite**: Tests arithmetic and comparison operations
38+
- **AccumulatorSuite**: Tests accumulating operations (accum, count, diff)
39+
40+
## Quick Start
41+
42+
### Installation
43+
44+
```bash
45+
# Install with development dependencies
46+
pip install -e ".[develop]"
47+
48+
# For Hetzner Cloud integration
49+
pip install -e ".[develop,hetzner]"
50+
```
51+
52+
### Running Benchmarks Locally
53+
54+
```bash
55+
# Run quick benchmarks for the current commit
56+
make benchmark-quick
57+
58+
# Run full benchmarks
59+
make benchmark
60+
61+
# Run using local Python environment (no virtualenv)
62+
make benchmark-local
63+
64+
# View results
65+
make benchmark-view
66+
```
67+
68+
### Using ASV Directly
69+
70+
```bash
71+
# Initialize machine configuration
72+
python -m asv machine --yes
73+
74+
# Run benchmarks for current commit
75+
python -m asv run HEAD^!
76+
77+
# Compare with previous commit
78+
python -m asv compare HEAD~1 HEAD
79+
80+
# Generate and serve HTML report
81+
python -m asv publish
82+
python -m asv preview
83+
```
84+
85+
## Hetzner Cloud Integration
86+
87+
For consistent benchmark results, this repository supports running benchmarks on dedicated Hetzner Cloud servers.
88+
89+
### Setup
90+
91+
1. Create a Hetzner Cloud API token at <https://console.hetzner.cloud/>
92+
1. Set the token as a repository secret: `HCLOUD_TOKEN`
93+
1. (Optional) Add an SSH key to Hetzner and set `HETZNER_SSH_PRIVATE_KEY` secret
94+
95+
### Running on Hetzner
96+
97+
```bash
98+
# Set your Hetzner token
99+
export HCLOUD_TOKEN="your-token-here"
100+
101+
# Run benchmarks on Hetzner
102+
python -m csp_benchmarks.hetzner.cli run --push
103+
104+
# Clean up any leftover servers
105+
python -m csp_benchmarks.hetzner.cli cleanup
106+
```
107+
108+
### GitHub Actions
109+
110+
Benchmarks run automatically:
111+
112+
- **Weekly**: Full benchmarks on Hetzner Cloud (Sunday 2 AM UTC)
113+
- **On push to main**: Benchmarks for the new commit
114+
- **Manual trigger**: Via workflow_dispatch with custom options
115+
116+
## Configuration
117+
118+
The ASV configuration is in `asv.conf.json`. Key settings:
119+
120+
```json
121+
{
122+
"project": "csp",
123+
"repo": "https://github.com/Point72/csp.git",
124+
"branches": ["main"],
125+
"pythons": ["3.11"],
126+
"benchmark_dir": "benchmarks",
127+
"results_dir": "results"
128+
}
129+
```
130+
131+
## Results
132+
133+
Benchmark results are stored in the `results/` directory and published to GitHub Pages.
134+
135+
View the latest results at: <https://csp-community.github.io/csp-benchmarks/benchmarks/>
136+
137+
## Contributing
138+
139+
1. Add new benchmarks to the `benchmarks/` directory
140+
1. Follow ASV naming conventions (`bench_*.py`, class names ending in `Suite`)
141+
1. Use parameterized benchmarks for testing across different configurations
142+
1. Run `make benchmark-local` to test your benchmarks before submitting

0 commit comments

Comments
 (0)