From fa5895cfc29115918062604eba5949d6052f8ecf Mon Sep 17 00:00:00 2001 From: ojeda-e Date: Mon, 4 Jul 2022 17:54:54 -0400 Subject: [PATCH 1/5] Added initial AVS benchmark --- benchmarks/__init__.py | 0 benchmarks/asv.conf.json | 47 +++++++++++++++++++++++++++++++++ benchmarks/membranecurvature.py | 16 +++++++++++ 3 files changed, 63 insertions(+) create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/asv.conf.json create mode 100644 benchmarks/membranecurvature.py diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json new file mode 100644 index 00000000..b22713c9 --- /dev/null +++ b/benchmarks/asv.conf.json @@ -0,0 +1,47 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "membranecurvature", + + // The URL of the source code repository for the project being + // benchmarked + "repo": "..", + "dvcs": "git", + "repo_subdir":".", + "branches": ["main"], + + // The base URL to show information about a particular commit. + "show_commit_url": "https://github.com/MDAnalysis/membrane-curvature/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.8"], + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list indicates to just test against the default (latest) + // version. + "conda_channels": ["conda-forge"], + "matrix": { + "numpy": [], + "scipy": [], + "pytest": [], + "MDAnalysisTests": [], + }, + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": ".", + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": "virtualenv", + "environment_type": "conda", + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": "results", + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": "html" + +} \ No newline at end of file diff --git a/benchmarks/membranecurvature.py b/benchmarks/membranecurvature.py new file mode 100644 index 00000000..acfa98c5 --- /dev/null +++ b/benchmarks/membranecurvature.py @@ -0,0 +1,16 @@ +import MDAnalysis as mda +from membrane_curvature.tests.datafiles import GRO_PO4_SMALL +from membrane_curvature.base import MembraneCurvature + + +class MembraneCurvatureBenchmark(): + """ + Benchmark for MembraneCurvature class + """ + + def setup(self): + self.u = mda.Universe(GRO_PO4_SMALL) + self.sel = mda.select_atoms('name PO4') + + def time_surface(self): + MembraneCurvature(self.u, select=self.sel).run() From d3a87a760f0e7aaf9908c83b379327d048969adb Mon Sep 17 00:00:00 2001 From: ojeda-e Date: Wed, 27 May 2026 13:15:49 -0600 Subject: [PATCH 2/5] Feat: Add ASV with uv as environment_type --- .gitignore | 7 ++ benchmarks/README.md | 187 ++++++++++++++++++++++++++++++++ benchmarks/asv.conf.json | 23 ++-- benchmarks/membranecurvature.py | 152 ++++++++++++++++++++++++-- 4 files changed, 348 insertions(+), 21 deletions(-) create mode 100644 benchmarks/README.md diff --git a/.gitignore b/.gitignore index 09037606..f5913111 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,10 @@ ENV/ # Generated docs figs docs/savefig/ + +# ASV benchmarks +benchmarks/results/ +benchmarks/html/ +benchmarks/env/ +benchmarks/virtualenv/ +benchmarks/.asv/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..267aed4f --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,187 @@ +MembraneCurvature benchmarks +============================= + +Performance benchmarks for MembraneCurvature using [Airspeed Velocity] (ASV). + +ASV creates isolated environments with the [uv plugin] (See [environment_type: uv] in [asv.conf.json]). + +Each benchmark run uses Python **3.11** and **3.14**, the minimum and latest versions declared in [pyproject.toml]. + +Prerequisites +------------- + +- [uv] on your `PATH` (`uv --version`) + +Install the required Python versions in your `uv` environment: + +```bash +uv python install 3.11 3.14 +``` + +From the repository root, sync the benchmark dependencies: + +```bash +uv sync --extra benchmarks +``` + +If this is the first time you are running the benchmarks, register the machine with: + +```bash +cd benchmarks +uv run asv machine --yes +``` + +Usage +----- + +> [!NOTE] +> Run all `asv` commands from the `benchmarks/` directory. + +### Benchmark the current commit + +Benchmarks `HEAD` on the current checkout against the `main` branch. Results are saved, but the HTML only generates data if that commit is on a branch listed in `asv.conf.json` in the `main` branch. + +```bash +cd benchmarks +uv run asv run HEAD --steps 1 +uv run asv publish +uv run asv preview --browser +``` + +Timings are saved under `results/`, but summary graphs in the published site only include branches listed in [asv.conf.json] (currently `main`). + +> [!WARNING] +> Do not open `html/index.html` via `file://`. Use `asv preview` to view the results in a local HTTP server. + +### History on `main` (recommended for the benchmark grid) + +With ASV benchmarks, users can pick a few representative commits from `main` to build a performance timeline. For example, use `--steps 3` to sample three commits evenly spaced along the branch. ASV will use the branch(es) listed in `asv.conf.json` (currently `main`) to generate the summary graphs. + +```bash +cd benchmarks +git checkout main +uv run asv run main --steps 3 +uv run asv publish +uv run asv preview --browser +``` + +> [!WARNING] +> A full run executes every benchmark on Python 3.11 and 3.14 and can take some time! Use the `--quick` flag to run a subset of benchmarks. + +### Compare two refs + +`asv continuous` benchmarks two git refs side by side and prints a summary of benchmarks that are faster, slower, or unchanged. `main` is the baseline; the second argument is the ref to test. + +With your PR branch checked out: + +```bash +uv run asv continuous main HEAD +``` + +Or compare `main` against a named branch without checking it out: + +```bash +uv run asv continuous main +``` + +In the example above, ASV benchmarks `` and compares it against main as the +baseline. + +In both cases, `HEAD` when your PR branch is checked out or name the branch explicitly, the +`main` branch is the baseline and the second reference is what gets benchmarked against it. + +For a quicker but less trustworthy check: + +```bash +uv run asv continuous main HEAD --quick +``` + +Useful flags: + +```bash +uv run asv run --quick -e # fast check; results not saved +uv run asv run --quick --show-stderr # fast smoke check; results not saved +uv run asv run --show-stderr -b Small # debug failures (regex on benchmark name) +``` + +Clean previous outputs +---------------------- + +Delete generated ASV artifacts when environments go stale, results look inconsistent, or you want a fresh local run: + +```bash +cd benchmarks +rm -rf env results html virtualenv .asv +``` + +This removes benchmark environments (`env/`), saved timings (`results/`), and the published site (`html/`). ASV recreates them on the next run. The `virtualenv/` directory is a leftover from older configs. + +If you removed `.asv/`, you can re-register your machine before benchmarking again: + +```bash +uv run asv machine --yes +``` + +Note that all of these paths are gitignored. See [Output layout](#output-layout) for what each directory contains. + + +Output layout +------------- + +- `env/`: ASV uv virtualenvs. +- `results/`: Raw JSON timing data. +- `html/`: Published site (`asv publish`). +- `virtualenv/`: Leftover from older ASV configs. +- `.asv/`: Local machine metadata. + + +## Benchmark modules + + +- `MembraneCurvatureSmallBenchmark`: `MembraneCurvature.run()` on a nine-atom PO4 system (Fourier and binning). +- `MembraneCurvatureBenchmark`: `MembraneCurvature.run()` on ~900 PO4 atoms at grid sizes 25, 50, and 100. +- `MembraneCurvatureFourierModesBenchmark`: Fourier mode truncation `(2, 2)`, `(3, 3)`, and `(5, 5)`. +- `MembraneCurvatureTrajectoryBenchmark`: Multi-frame Fourier and binning runs, including peak memory. + +The file `membranecurvature.py` contains the benchmarks for the `MembraneCurvature.run()` method for different input sizes and for the two surface derivation methods. Peak memory benchmarks are also included to track RAM usage during multi-frame trajectory runs, where per-frame surface and curvature arrays grow with trajectory length and grid resolution. + +Pull requests +------------- + +### Before opening or updating a PR + +1. **Check for correctness (fast, required for benchmark changes)** + Run the benchmarks with the `--quick` flag to check for correctness: + + ```bash + uv run asv run --quick --show-stderr + ``` + + With `--quick`, results are not saved. The run only checks that every benchmark completes. + +2. **Optional: full suite smoke on your branch** + + If you changed benchmark code or expect a performance impact, benchmark the current commit: + + ```bash + uv run asv run HEAD --steps 1 --show-stderr + ``` + + This runs the full suite on Python 3.11 and 3.14 and saves one commit's timings. **It is not required on every PR, but it is recommended to run it if you changed benchmark code or expect a performance impact.** + +3. **Optional: performance comparison vs `main`** + + When the PR may change speed, compare your branch against `main` (see [Compare two refs](#compare-two-refs)): + + ```bash + uv run asv continuous main HEAD + ``` + + ASV compares `HEAD` (your branch) against `main` and prints a summary table of benchmarks that are faster, slower, or unchanged. This runs locally and does **not** modify the public `main` history. This is a quick comparison to be included in the PR discussion (optional). + + +[uv]:https://docs.astral.sh/uv/ +[uv plugin]:https://asv.readthedocs.io/en/latest/autoapi/asv/plugins/uv/index.html +[Airspeed Velocity]:https://asv.readthedocs.io/ +[asv.conf.json]: asv.conf.json +[pyproject.toml]:../pyproject.toml diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index b22713c9..f905b55e 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -10,7 +10,7 @@ // benchmarked "repo": "..", "dvcs": "git", - "repo_subdir":".", + "repo_subdir": ".", "branches": ["main"], // The base URL to show information about a particular commit. @@ -18,30 +18,29 @@ // The Pythons you'd like to test against. If not provided, defaults // to the current version of Python used to run `asv`. - "pythons": ["3.8"], + "pythons": ["3.11", "3.14"], + "build_command": [ + "python -m pip wheel --no-deps -w {build_cache_dir} {build_dir}" + ], // The matrix of dependencies to test. Each key is the name of a // package (in PyPI) and the values are version numbers. An empty // list indicates to just test against the default (latest) // version. - "conda_channels": ["conda-forge"], "matrix": { "numpy": [], "scipy": [], - "pytest": [], - "MDAnalysisTests": [], + "mdanalysis": [] }, // The directory (relative to the current directory) that benchmarks are // stored in. If not provided, defaults to "benchmarks" "benchmark_dir": ".", // The directory (relative to the current directory) to cache the Python // environments in. If not provided, defaults to "env" - "env_dir": "virtualenv", - "environment_type": "conda", - // The directory (relative to the current directory) that raw benchmark - // results are stored in. If not provided, defaults to "results". + "env_dir": "env", + "environment_type": "uv", "results_dir": "results", // The directory (relative to the current directory) that the html tree // should be written to. If not provided, defaults to "html". - "html_dir": "html" - -} \ No newline at end of file + "html_dir": "html", + "hash_length": 8 +} diff --git a/benchmarks/membranecurvature.py b/benchmarks/membranecurvature.py index acfa98c5..bdc47ae8 100644 --- a/benchmarks/membranecurvature.py +++ b/benchmarks/membranecurvature.py @@ -1,16 +1,150 @@ import MDAnalysis as mda -from membrane_curvature.tests.datafiles import GRO_PO4_SMALL +import numpy as np + from membrane_curvature.base import MembraneCurvature +from membrane_curvature.tests.datafiles import GRO_PO4, GRO_PO4_SMALL, XTC_PO4 + +PO4_SELECT = 'name PO4' + + +def po4_universe(gro=GRO_PO4): + """Universe with first frame loaded""" + universe = mda.Universe(gro) + universe.trajectory[0] + return universe -class MembraneCurvatureBenchmark(): - """ - Benchmark for MembraneCurvature class - """ +def po4_frame0_context(gro=GRO_PO4): + """Positions and box ranges for one PO4 frame""" + universe = po4_universe(gro) + atomgroup = universe.select_atoms(PO4_SELECT) + return { + 'positions': atomgroup.positions.copy(), + 'x_range': (0.0, float(universe.dimensions[0])), + 'y_range': (0.0, float(universe.dimensions[1])), + } + +def dummy_height_grid(n_bins=100, seed=0): + """Smooth height field for gradient / Monge curvature micro-benchmarks.""" + rng = np.random.default_rng(seed) + x = np.linspace(0, 2 * np.pi, n_bins) + y = np.linspace(0, 2 * np.pi, n_bins) + xx, yy = np.meshgrid(x, y) + z = np.sin(xx) * np.cos(yy) + 0.1 * rng.standard_normal((n_bins, n_bins)) + return z, 0.1, 0.1 + + +class MembraneCurvatureSmallBenchmark: def setup(self): - self.u = mda.Universe(GRO_PO4_SMALL) - self.sel = mda.select_atoms('name PO4') + self.universe = po4_universe(GRO_PO4_SMALL) + + def time_run_fourier_low_modes(self): + # Nine PO4 atoms support at most fourier_m = fourier_n = 1. + MembraneCurvature(self.universe, select=PO4_SELECT, fourier_m=1, fourier_n=1).run() + + def time_run_binning_default_wrap(self): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=3, + n_y_bins=3, + ).run() + + def time_run_binning_no_wrap(self): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=3, + n_y_bins=3, + wrap=False, + ).run() + + +class MembraneCurvatureBenchmark: + """Benchmark for PO4-only membrane (~900 atoms), regular grid sizes.""" + + params = ([25, 50, 100],) + param_names = ['n_bins'] + + def setup(self, n_bins): + self.universe = po4_universe(GRO_PO4) + + def time_run_fourier(self, n_bins): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + n_x_bins=n_bins, + n_y_bins=n_bins, + ).run() + + def time_run_binning_wrap(self, n_bins): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=n_bins, + n_y_bins=n_bins, + ).run() + + def time_run_binning_no_wrap(self, n_bins): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=n_bins, + n_y_bins=n_bins, + wrap=False, + ).run() + + +class MembraneCurvatureFourierModesBenchmark: + """Benchmarks for Fourier modes with PO4 selection.""" + + params = ([(2, 2), (3, 3), (5, 5)],) + param_names = ['modes'] + + def setup(self, modes): + self.universe = po4_universe(GRO_PO4) + self.fourier_m, self.fourier_n = modes + + def time_run_fourier(self, modes): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + fourier_m=self.fourier_m, + fourier_n=self.fourier_n, + ).run() + + +class MembraneCurvatureTrajectoryBenchmark: + """Multi-frame trajectory""" + + def setup(self): + self.universe = mda.Universe(GRO_PO4, XTC_PO4) + + def time_run_fourier_trajectory(self): + MembraneCurvature(self.universe, select=PO4_SELECT).run() + + def time_run_binning_trajectory(self): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=3, + n_y_bins=3, + ).run() + + def peakmem_run_fourier_trajectory(self): + MembraneCurvature(self.universe, select=PO4_SELECT).run() - def time_surface(self): - MembraneCurvature(self.u, select=self.sel).run() + def peakmem_run_binning_trajectory(self): + MembraneCurvature( + self.universe, + select=PO4_SELECT, + surface_method='binning', + n_x_bins=3, + n_y_bins=3, + ).run() From 57f55950f6f77c5f6366a02c3a6f0999a9e0b4b6 Mon Sep 17 00:00:00 2001 From: ojeda-e Date: Fri, 12 Jun 2026 23:28:22 -0600 Subject: [PATCH 3/5] Add benchmarks avs dep from git --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 69438c44..b308d19d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,11 @@ dev = [ "ty" ] +benchmarks = [ + # Use ASV from github to include the uv plugin + "asv @ git+https://github.com/airspeed-velocity/asv.git", +] + [tool.setuptools.packages.find] include = ["membrane_curvature*"] From 2d4ea2ff038983f31c2309f368eb47f0e5aa5779 Mon Sep 17 00:00:00 2001 From: ojeda-e Date: Fri, 12 Jun 2026 23:38:57 -0600 Subject: [PATCH 4/5] Apply ATX style consistently --- benchmarks/README.md | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 267aed4f..a7995f4c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,5 +1,4 @@ -MembraneCurvature benchmarks -============================= +# MembraneCurvature benchmarks Performance benchmarks for MembraneCurvature using [Airspeed Velocity] (ASV). @@ -7,8 +6,7 @@ ASV creates isolated environments with the [uv plugin] (See [environment_type: u Each benchmark run uses Python **3.11** and **3.14**, the minimum and latest versions declared in [pyproject.toml]. -Prerequisites -------------- +## Prerequisites - [uv] on your `PATH` (`uv --version`) @@ -31,8 +29,7 @@ cd benchmarks uv run asv machine --yes ``` -Usage ------ +## Usage > [!NOTE] > Run all `asv` commands from the `benchmarks/` directory. @@ -104,8 +101,7 @@ uv run asv run --quick --show-stderr # fast smoke check; results not saved uv run asv run --show-stderr -b Small # debug failures (regex on benchmark name) ``` -Clean previous outputs ----------------------- +## Clean previous outputs Delete generated ASV artifacts when environments go stale, results look inconsistent, or you want a fresh local run: @@ -124,9 +120,7 @@ uv run asv machine --yes Note that all of these paths are gitignored. See [Output layout](#output-layout) for what each directory contains. - -Output layout -------------- +## Output layout - `env/`: ASV uv virtualenvs. - `results/`: Raw JSON timing data. @@ -134,10 +128,8 @@ Output layout - `virtualenv/`: Leftover from older ASV configs. - `.asv/`: Local machine metadata. - ## Benchmark modules - - `MembraneCurvatureSmallBenchmark`: `MembraneCurvature.run()` on a nine-atom PO4 system (Fourier and binning). - `MembraneCurvatureBenchmark`: `MembraneCurvature.run()` on ~900 PO4 atoms at grid sizes 25, 50, and 100. - `MembraneCurvatureFourierModesBenchmark`: Fourier mode truncation `(2, 2)`, `(3, 3)`, and `(5, 5)`. @@ -145,8 +137,7 @@ Output layout The file `membranecurvature.py` contains the benchmarks for the `MembraneCurvature.run()` method for different input sizes and for the two surface derivation methods. Peak memory benchmarks are also included to track RAM usage during multi-frame trajectory runs, where per-frame surface and curvature arrays grow with trajectory length and grid resolution. -Pull requests -------------- +## Pull requests ### Before opening or updating a PR @@ -177,7 +168,7 @@ Pull requests uv run asv continuous main HEAD ``` - ASV compares `HEAD` (your branch) against `main` and prints a summary table of benchmarks that are faster, slower, or unchanged. This runs locally and does **not** modify the public `main` history. This is a quick comparison to be included in the PR discussion (optional). + ASV compares `HEAD` (your branch) against `main` and prints a summary table of benchmarks that are faster, slower, or unchanged. This runs locally and does **not** modify the public `main` history. This is a quick comparison to be included in the PR discussion (optional). [uv]:https://docs.astral.sh/uv/ From f197846faf5d9bf8e97fca1bee58329ddf6d31f4 Mon Sep 17 00:00:00 2001 From: ojeda-e Date: Fri, 12 Jun 2026 23:41:22 -0600 Subject: [PATCH 5/5] fix format --- benchmarks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index a7995f4c..fa4ceb40 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,7 +2,7 @@ Performance benchmarks for MembraneCurvature using [Airspeed Velocity] (ASV). -ASV creates isolated environments with the [uv plugin] (See [environment_type: uv] in [asv.conf.json]). +ASV creates isolated environments with the [uv plugin] (See ``[environment_type: uv]`` in [asv.conf.json]). Each benchmark run uses Python **3.11** and **3.14**, the minimum and latest versions declared in [pyproject.toml].