diff --git a/.gitignore b/.gitignore index 0903760..f591311 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,10 @@ ENV/ # Generated docs figs docs/savefig/ + +# ASV benchmarks +benchmarks/results/ +benchmarks/html/ +benchmarks/env/ +benchmarks/virtualenv/ +benchmarks/.asv/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..fa4ceb4 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,178 @@ +# MembraneCurvature benchmarks + +Performance benchmarks for MembraneCurvature using [Airspeed Velocity] (ASV). + +ASV creates isolated environments with the [uv plugin] (See ``[environment_type: uv]`` in [asv.conf.json]). + +Each benchmark run uses Python **3.11** and **3.14**, the minimum and latest versions declared in [pyproject.toml]. + +## Prerequisites + +- [uv] on your `PATH` (`uv --version`) + +Install the required Python versions in your `uv` environment: + +```bash +uv python install 3.11 3.14 +``` + +From the repository root, sync the benchmark dependencies: + +```bash +uv sync --extra benchmarks +``` + +If this is the first time you are running the benchmarks, register the machine with: + +```bash +cd benchmarks +uv run asv machine --yes +``` + +## Usage + +> [!NOTE] +> Run all `asv` commands from the `benchmarks/` directory. + +### Benchmark the current commit + +Benchmarks `HEAD` on the current checkout against the `main` branch. Results are saved, but the HTML only generates data if that commit is on a branch listed in `asv.conf.json` in the `main` branch. + +```bash +cd benchmarks +uv run asv run HEAD --steps 1 +uv run asv publish +uv run asv preview --browser +``` + +Timings are saved under `results/`, but summary graphs in the published site only include branches listed in [asv.conf.json] (currently `main`). + +> [!WARNING] +> Do not open `html/index.html` via `file://`. Use `asv preview` to view the results in a local HTTP server. + +### History on `main` (recommended for the benchmark grid) + +With ASV benchmarks, users can pick a few representative commits from `main` to build a performance timeline. For example, use `--steps 3` to sample three commits evenly spaced along the branch. ASV will use the branch(es) listed in `asv.conf.json` (currently `main`) to generate the summary graphs. + +```bash +cd benchmarks +git checkout main +uv run asv run main --steps 3 +uv run asv publish +uv run asv preview --browser +``` + +> [!WARNING] +> A full run executes every benchmark on Python 3.11 and 3.14 and can take some time! Use the `--quick` flag to run a subset of benchmarks. + +### Compare two refs + +`asv continuous` benchmarks two git refs side by side and prints a summary of benchmarks that are faster, slower, or unchanged. `main` is the baseline; the second argument is the ref to test. + +With your PR branch checked out: + +```bash +uv run asv continuous main HEAD +``` + +Or compare `main` against a named branch without checking it out: + +```bash +uv run asv continuous main +``` + +In the example above, ASV benchmarks `` and compares it against main as the +baseline. + +In both cases, `HEAD` when your PR branch is checked out or name the branch explicitly, the +`main` branch is the baseline and the second reference is what gets benchmarked against it. + +For a quicker but less trustworthy check: + +```bash +uv run asv continuous main HEAD --quick +``` + +Useful flags: + +```bash +uv run asv run --quick -e # fast check; results not saved +uv run asv run --quick --show-stderr # fast smoke check; results not saved +uv run asv run --show-stderr -b Small # debug failures (regex on benchmark name) +``` + +## Clean previous outputs + +Delete generated ASV artifacts when environments go stale, results look inconsistent, or you want a fresh local run: + +```bash +cd benchmarks +rm -rf env results html virtualenv .asv +``` + +This removes benchmark environments (`env/`), saved timings (`results/`), and the published site (`html/`). ASV recreates them on the next run. The `virtualenv/` directory is a leftover from older configs. + +If you removed `.asv/`, you can re-register your machine before benchmarking again: + +```bash +uv run asv machine --yes +``` + +Note that all of these paths are gitignored. See [Output layout](#output-layout) for what each directory contains. + +## Output layout + +- `env/`: ASV uv virtualenvs. +- `results/`: Raw JSON timing data. +- `html/`: Published site (`asv publish`). +- `virtualenv/`: Leftover from older ASV configs. +- `.asv/`: Local machine metadata. + +## Benchmark modules + +- `MembraneCurvatureSmallBenchmark`: `MembraneCurvature.run()` on a nine-atom PO4 system (Fourier and binning). +- `MembraneCurvatureBenchmark`: `MembraneCurvature.run()` on ~900 PO4 atoms at grid sizes 25, 50, and 100. +- `MembraneCurvatureFourierModesBenchmark`: Fourier mode truncation `(2, 2)`, `(3, 3)`, and `(5, 5)`. +- `MembraneCurvatureTrajectoryBenchmark`: Multi-frame Fourier and binning runs, including peak memory. + +The file `membranecurvature.py` contains the benchmarks for the `MembraneCurvature.run()` method for different input sizes and for the two surface derivation methods. Peak memory benchmarks are also included to track RAM usage during multi-frame trajectory runs, where per-frame surface and curvature arrays grow with trajectory length and grid resolution. + +## Pull requests + +### Before opening or updating a PR + +1. **Check for correctness (fast, required for benchmark changes)** + Run the benchmarks with the `--quick` flag to check for correctness: + + ```bash + uv run asv run --quick --show-stderr + ``` + + With `--quick`, results are not saved. The run only checks that every benchmark completes. + +2. **Optional: full suite smoke on your branch** + + If you changed benchmark code or expect a performance impact, benchmark the current commit: + + ```bash + uv run asv run HEAD --steps 1 --show-stderr + ``` + + This runs the full suite on Python 3.11 and 3.14 and saves one commit's timings. **It is not required on every PR, but it is recommended to run it if you changed benchmark code or expect a performance impact.** + +3. **Optional: performance comparison vs `main`** + + When the PR may change speed, compare your branch against `main` (see [Compare two refs](#compare-two-refs)): + + ```bash + uv run asv continuous main HEAD + ``` + + ASV compares `HEAD` (your branch) against `main` and prints a summary table of benchmarks that are faster, slower, or unchanged. This runs locally and does **not** modify the public `main` history. This is a quick comparison to be included in the PR discussion (optional). + + +[uv]:https://docs.astral.sh/uv/ +[uv plugin]:https://asv.readthedocs.io/en/latest/autoapi/asv/plugins/uv/index.html +[Airspeed Velocity]:https://asv.readthedocs.io/ +[asv.conf.json]: asv.conf.json +[pyproject.toml]:../pyproject.toml diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json new file mode 100644 index 0000000..f905b55 --- /dev/null +++ b/benchmarks/asv.conf.json @@ -0,0 +1,46 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "membranecurvature", + + // The URL of the source code repository for the project being + // benchmarked + "repo": "..", + "dvcs": "git", + "repo_subdir": ".", + "branches": ["main"], + + // The base URL to show information about a particular commit. + "show_commit_url": "https://github.com/MDAnalysis/membrane-curvature/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.11", "3.14"], + "build_command": [ + "python -m pip wheel --no-deps -w {build_cache_dir} {build_dir}" + ], + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list indicates to just test against the default (latest) + // version. + "matrix": { + "numpy": [], + "scipy": [], + "mdanalysis": [] + }, + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": ".", + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": "env", + "environment_type": "uv", + "results_dir": "results", + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": "html", + "hash_length": 8 +} diff --git a/benchmarks/membranecurvature.py b/benchmarks/membranecurvature.py new file mode 100644 index 0000000..bdc47ae --- /dev/null +++ b/benchmarks/membranecurvature.py @@ -0,0 +1,150 @@ +import MDAnalysis as mda +import numpy as np + +from membrane_curvature.base import MembraneCurvature +from membrane_curvature.tests.datafiles import GRO_PO4, GRO_PO4_SMALL, XTC_PO4 + +PO4_SELECT = 'name PO4' + + +def po4_universe(gro=GRO_PO4): + """Universe with first frame loaded""" + universe = mda.Universe(gro) + universe.trajectory[0] + return universe + + +def po4_frame0_context(gro=GRO_PO4): + """Positions and box ranges for one PO4 frame""" + universe = po4_universe(gro) + atomgroup = universe.select_atoms(PO4_SELECT) + return { + 'positions': atomgroup.positions.copy(), + 'x_range': (0.0, float(universe.dimensions[0])), + 'y_range': (0.0, float(universe.dimensions[1])), + } + + +def dummy_height_grid(n_bins=100, seed=0): + """Smooth height field for gradient / Monge curvature micro-benchmarks.""" + rng = np.random.default_rng(seed) + x = np.linspace(0, 2 * np.pi, n_bins) + y = np.linspace(0, 2 * np.pi, n_bins) + xx, yy = np.meshgrid(x, y) + z = np.sin(xx) * np.cos(yy) + 0.1 * rng.standard_normal((n_bins, n_bins)) + return z, 0.1, 0.1 + + +class MembraneCurvatureSmallBenchmark: + def setup(self): + self.universe = po4_universe(GRO_PO4_SMALL) + + def time_run_fourier_low_modes(self): + # Nine PO4 atoms support at most fourier_m = fourier_n = 1. + MembraneCurvature(self.universe, select=PO4_SELECT, fourier_m=1, fourier_n=1).run() + + def time_run_binning_default_wrap(self): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=3, + n_y_bins=3, + ).run() + + def time_run_binning_no_wrap(self): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=3, + n_y_bins=3, + wrap=False, + ).run() + + +class MembraneCurvatureBenchmark: + """Benchmark for PO4-only membrane (~900 atoms), regular grid sizes.""" + + params = ([25, 50, 100],) + param_names = ['n_bins'] + + def setup(self, n_bins): + self.universe = po4_universe(GRO_PO4) + + def time_run_fourier(self, n_bins): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + n_x_bins=n_bins, + n_y_bins=n_bins, + ).run() + + def time_run_binning_wrap(self, n_bins): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=n_bins, + n_y_bins=n_bins, + ).run() + + def time_run_binning_no_wrap(self, n_bins): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=n_bins, + n_y_bins=n_bins, + wrap=False, + ).run() + + +class MembraneCurvatureFourierModesBenchmark: + """Benchmarks for Fourier modes with PO4 selection.""" + + params = ([(2, 2), (3, 3), (5, 5)],) + param_names = ['modes'] + + def setup(self, modes): + self.universe = po4_universe(GRO_PO4) + self.fourier_m, self.fourier_n = modes + + def time_run_fourier(self, modes): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + fourier_m=self.fourier_m, + fourier_n=self.fourier_n, + ).run() + + +class MembraneCurvatureTrajectoryBenchmark: + """Multi-frame trajectory""" + + def setup(self): + self.universe = mda.Universe(GRO_PO4, XTC_PO4) + + def time_run_fourier_trajectory(self): + MembraneCurvature(self.universe, select=PO4_SELECT).run() + + def time_run_binning_trajectory(self): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=3, + n_y_bins=3, + ).run() + + def peakmem_run_fourier_trajectory(self): + MembraneCurvature(self.universe, select=PO4_SELECT).run() + + def peakmem_run_binning_trajectory(self): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=3, + n_y_bins=3, + ).run() diff --git a/pyproject.toml b/pyproject.toml index 69438c4..b308d19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,11 @@ dev = [ "ty" ] +benchmarks = [ + # Use ASV from github to include the uv plugin + "asv @ git+https://github.com/airspeed-velocity/asv.git", +] + [tool.setuptools.packages.find] include = ["membrane_curvature*"]