Add InstantaneousPrecipitation microphysics, remove `ZeroMomentClou…
#1752
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: &paths | |
| - ".github/workflows/Benchmarks.yml" | |
| - "benchmarking/**" | |
| - "src/**" | |
| - "Project.toml" | |
| pull_request: | |
| paths: *paths | |
| concurrency: | |
| # Skip intermediate builds: always. | |
| # Cancel intermediate builds: always. | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # Don't make Reactant occupy all the GPU memory | |
| XLA_REACTANT_GPU_PREALLOCATE: 'false' | |
| JULIA_PKG_SERVER_REGISTRY_PREFERENCE: 'eager' | |
| jobs: | |
| run-benchmarks: | |
| name: "Run benchmarks - dynamics ${{ matrix.dynamics }} - microphysics ${{ matrix.microphysics }} - grid ${{ matrix.grid }} - advection ${{ matrix.advection }} - backend ${{ matrix.backend }} - topology ${{ matrix.topology }}" | |
| permissions: | |
| actions: write | |
| contents: write | |
| pull-requests: read | |
| statuses: write | |
| runs-on: aws-linux-nvidia-gpu-l4 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - dynamics: 'anelastic' | |
| microphysics: 'nothing' | |
| grid: '256x256x128, 512x512x256, 768x768x256' | |
| advection: 'WENO5, WENO9' | |
| backend: 'vanilla' | |
| topology: 'PPB' | |
| extra_flags: '--warmup_steps 5 --time_steps 150' | |
| - dynamics: 'compressible_splitexplicit' | |
| microphysics: 'nothing' | |
| grid: '512x512x256' | |
| advection: 'WENO5' | |
| backend: 'vanilla' | |
| topology: 'PPB' | |
| extra_flags: '' | |
| - dynamics: 'anelastic' | |
| microphysics: 'MixedPhaseEquilibrium, 1M_MixedEquilibrium, 1M_MixedNonEquilibrium' | |
| grid: '512x512x256' | |
| advection: 'WENO5' | |
| backend: 'vanilla' | |
| topology: 'PPB' | |
| extra_flags: '' | |
| # Reactant vs vanilla — compressible_explicit + WENO5 + 1M mixed-phase | |
| # non-eq microphysics with ice on a PBB grid. Uses --simplified to | |
| # drop the geostrophic forcing and field-dependent surface drag BCs | |
| # (which do not currently materialize on ReactantState). | |
| - dynamics: 'compressible_explicit' | |
| microphysics: '1M_MixedNonEquilibrium' | |
| grid: '256x256x128' | |
| advection: 'WENO5' | |
| backend: 'vanilla, reactant' | |
| topology: 'PBB' | |
| extra_flags: '--simplified --float_type=Float32' | |
| # AD: forward+backward via Enzyme reverse-mode through the same | |
| # @trace checkpointed loop. Reactant-only by construction. No | |
| # microphysics in v1 (Enzyme correctness through the 1M scheme is | |
| # untested), small grid + small Nsteps to keep the gradient compile | |
| # and per-step memory bounded. | |
| - dynamics: 'compressible_explicit' | |
| microphysics: 'nothing' | |
| grid: '64x64x32' | |
| advection: 'WENO5' | |
| backend: 'reactant' | |
| topology: 'PBB' | |
| extra_flags: '--ad --simplified --float_type=Float32 --warmup_steps 1 --time_steps 1' | |
| defaults: | |
| run: | |
| shell: bash | |
| working-directory: ./benchmarking | |
| container: | |
| image: 'ghcr.io/numericalearth/breeze-docker-images:benchmarking-julia_1.12.6' | |
| options: --gpus=all | |
| volumes: | |
| # Mount host `/usr/local` so that we can delete some stuff afterwards | |
| - /usr/local:/host-usr-local | |
| timeout-minutes: 30 | |
| steps: | |
| - name: "Check out repository" | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: df before cleanup | |
| run: | | |
| df -hT | |
| - name: Clean up old CUDA toolkits | |
| # Save storage by deleting old CUDA toolkits, we'll use v13 | |
| run: | | |
| rm -rf /host-usr-local/cuda-12.* | |
| - name: df after cleanup | |
| run: | | |
| df -hT | |
| - name: Instantiate benchmarking environment | |
| shell: julia --color=yes --project {0} | |
| run: | | |
| using Pkg | |
| Pkg.instantiate() | |
| - name: Run benchmarks | |
| timeout-minutes: 45 | |
| env: | |
| XLA_FLAGS: '--xla_disable_hlo_passes=multi_output_fusion' | |
| run: | | |
| earlyoom -m 3 -s 100 -r 300 --prefer 'julia' & | |
| julia --color=yes --project run_benchmarks.jl --device GPU --dynamics "${{ matrix.dynamics }}" --microphysics "${{ matrix.microphysics }}" --advection "${{ matrix.advection }}" --size "${{ matrix.grid }}" --backend "${{ matrix.backend }}" --topology "${{ matrix.topology }}" --warmup_steps 8 --time_steps 80 ${{ matrix.extra_flags }} | |
| - name: "Upload benchmark results as artifacts" | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: 'benchmarks-dynamics_${{ matrix.dynamics }}-microphysics_${{ matrix.microphysics }}-grid_${{ matrix.grid }}-advection_${{ matrix.advection }}-backend_${{ matrix.backend }}-topology_${{ matrix.topology }}' | |
| path: benchmarking/benchmark_results.* | |
| retention-days: 90 | |
| overwrite: false | |
| - name: "Upload Manifest" | |
| if: always() | |
| timeout-minutes: 2 | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: 'manifest-dynamics_${{ matrix.dynamics }}-microphysics_${{ matrix.microphysics }}-grid_${{ matrix.grid }}-advection_${{ matrix.advection }}-backend_${{ matrix.backend }}-topology_${{ matrix.topology }}' | |
| path: '**/Manifest*.toml' | |
| retention-days: 90 | |
| overwrite: false | |
| - name: Create job summary | |
| timeout-minutes: 1 | |
| run: | |
| cat "benchmark_results.md" >> "${GITHUB_STEP_SUMMARY}" | |
| publish-benchmarks: | |
| name: "Publish benchmarks" | |
| needs: run-benchmarks | |
| permissions: | |
| actions: write | |
| contents: write | |
| pull-requests: write | |
| statuses: write | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: "Download benchmark results" | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: benchmarks-* | |
| - name: Merge results and create JSON for github-action-benchmark | |
| shell: python | |
| run: | | |
| # Merge benchmark_results.json from all artifact directories, | |
| # then create the JSON file for github-action-benchmark. | |
| import json, glob | |
| all_data = [] | |
| for path in sorted(glob.glob('benchmarks-*/benchmark_results.json')): | |
| with open(path) as f: | |
| all_data.extend(json.load(f)) | |
| # Write merged results for archival | |
| with open('benchmark_results.json', 'w') as f: | |
| json.dump(all_data, f, ensure_ascii=False, indent=2) | |
| # Build github-action-benchmark entries | |
| # Name format: <name>/<properties>/<gpu>/<grid_size> | |
| data = [] | |
| for d in all_data: | |
| base_name = d['name'].split('_')[0] | |
| advection = d['advection'] | |
| # Closure is unused for the time being because it's always the | |
| # same everywhere in the current setup. | |
| closure = d['closure'] | |
| dynamics = d['dynamics'] | |
| ft = d['float_type'] | |
| microphysics = d['microphysics'] | |
| backend = d.get('backend', 'vanilla') | |
| mode = d.get('mode', 'forward') | |
| gpu = d['metadata']['gpu_name'] | |
| grid = d['grid_size'] | |
| grid_str = f"{grid[0]}x{grid[1]}x{grid[2]}" | |
| # AD (forward+backward via Enzyme) entries get their own chart. | |
| # Skip the forward-only clauses below so an AD point doesn't | |
| # also land on a forward chart that happens to match its config. | |
| if mode == 'ad': | |
| level_1 = f'{base_name}; AD; Dynamics: {dynamics}; Microphysics: {microphysics} [{ft}]' | |
| level_2 = f'Advection: {advection}' | |
| level_3 = f'{gpu}' | |
| level_4 = f'{grid_str}' | |
| name = f"{level_1}/{level_2}/{level_3}/{level_4}" | |
| data.append({'name': name, 'unit': 'points/s', 'value': d['grid_points_per_second']}) | |
| continue | |
| # We want to group benchmarks in different ways. NOTE: | |
| # the same benchmark may appear in multiple groups | |
| if dynamics == 'anelastic' and grid_str == '512x512x256' and advection == 'WENO5': | |
| level_1 = f'{base_name}; Dynamics: {dynamics}; Grid: {grid_str} [{ft}]' | |
| level_2 = f'Advection: {advection}' | |
| level_3 = f'{gpu}' | |
| level_4 = f'{microphysics}' | |
| name = f"{level_1}/{level_2}/{level_3}/{level_4}" | |
| data.append({'name': name, 'unit': 'points/s', 'value': d['grid_points_per_second']}) | |
| pass | |
| if dynamics == 'anelastic' and microphysics == 'nothing': | |
| level_1 = f'{base_name}; Dynamics: {dynamics}; Microphysics: {microphysics} [{ft}]' | |
| level_2 = f'Compare advections' | |
| level_3 = f'{gpu}' | |
| level_4 = f'{advection} {grid}' | |
| name = f"{level_1}/{level_2}/{level_3}/{level_4}" | |
| data.append({'name': name, 'unit': 'points/s', 'value': d['grid_points_per_second']}) | |
| pass | |
| if microphysics == 'nothing': | |
| level_1 = f'{base_name}; Dynamics: {dynamics}; Microphysics: {microphysics} [{ft}]' | |
| level_2 = f'Advection: {advection}' | |
| level_3 = f'{gpu}' | |
| level_4 = f'{grid_str}' | |
| name = f"{level_1}/{level_2}/{level_3}/{level_4}" | |
| data.append({'name': name, 'unit': 'points/s', 'value': d['grid_points_per_second']}) | |
| # Reactant vs vanilla sweep: compare backends across grid sizes | |
| # for a fixed compressible_explicit + WENO5 + 1M mixed-phase | |
| # non-eq microphysics configuration. | |
| if dynamics == 'compressible_explicit' and microphysics == '1M_MixedNonEquilibrium' and advection == 'WENO5': | |
| level_1 = f'{base_name}; Dynamics: {dynamics}; Microphysics: {microphysics} [{ft}]' | |
| level_2 = f'Compare backends' | |
| level_3 = f'{gpu}' | |
| level_4 = f'{backend} {grid_str}' | |
| name = f"{level_1}/{level_2}/{level_3}/{level_4}" | |
| data.append({'name': name, 'unit': 'points/s', 'value': d['grid_points_per_second']}) | |
| with open('github-action-benchmark.json', 'w') as f: | |
| json.dump(data, f) | |
| - name: "Generate token" | |
| uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 | |
| if: ${{ ! github.event.pull_request.head.repo.fork }} | |
| id: generate_token | |
| with: | |
| client-id: "${{ secrets.NUMTERRA_BOT_ID }}" | |
| private-key: "${{ secrets.NUMTERRA_BOT_PRIVATE_KEY }}" | |
| owner: NumericalEarth | |
| repositories: | | |
| Breeze.jl | |
| BreezeBenchmarks | |
| - name: Push results to BreezeBenchmarks repo | |
| if: ${{ github.event_name == 'push' }} | |
| run: | | |
| # Clone BreezeBenchmarks | |
| git clone https://numterra-bot:${{ steps.generate_token.outputs.token }}@github.com/NumericalEarth/BreezeBenchmarks.git | |
| cd BreezeBenchmarks | |
| DIRNAME="benchmarks/by_branch/${{ github.head_ref || github.ref_name }}/${{ github.sha }}" | |
| mkdir -vp "${DIRNAME}" | |
| # Copy over file | |
| cp -v ../benchmark_results.json "${DIRNAME}/." | |
| # Push | |
| git config --global user.name "numterra-bot[bot]" | |
| git config --global user.email "242463128+numterra-bot[bot]@users.noreply.github.com" | |
| git add . | |
| git commit -m'Save benchmark results for commit ${{ github.sha }}' | |
| git push -u origin main | |
| - name: "Publish benchmark results" | |
| uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1 | |
| with: | |
| name: Breeze.jl Benchmarks | |
| tool: "customBiggerIsBetter" | |
| output-file-path: "github-action-benchmark.json" | |
| # benchmark-data-dir-path: ${{ github.event_name == 'push' && '.' || format('previews/PR{0}', github.event.number) }} | |
| benchmark-data-dir-path: "." | |
| gh-repository: "github.com/NumericalEarth/BreezeBenchmarks" | |
| github-token: ${{ github.event.pull_request.head.repo.fork && github.token || steps.generate_token.outputs.token }} | |
| comment-always: false | |
| summary-always: true | |
| alert-threshold: "110%" | |
| fail-on-alert: false | |
| # auto-push: true | |
| auto-push: ${{ github.event_name == 'push' }} | |
| max-items-in-chart: 1000 | |
| comment-on-alert: true |