Use JuliaGPU Buildkite CI for GPU tests (trixi-framework#723)

* Use JuliaGPU Buildkite CI for GPU tests

* Remove Codecov token

* Add GPU tests

* Print ENV

* Fix merge error in runtests.jl

* Add CI for AMDGPU and oneAPI

* Fix GPU tests

* Fix formatting of pipeline.yml

* Disable `count_allocations` for GPU tests

* Add single and double precision test

* Add proper GPU tests (and fix problems)

* Reformat

* Disable double precision tests for oneAPI

* Reformat

* Reformat

* Improve readability of boundary extrapolation condition

* Fix EDAC and dam break plate gate on GPUs

* Reformat

* Fix open boundary system

* Fix DEMSystem

* Fix EDAC test

* Undo Float32 literals in smoothing kernels

* Fix allocations

* First step to make packing work on GPUs

* Second step to make packing system GPU compatible

* Revert "Second step to make packing system GPU compatible"

This reverts commit e1b86e3.

* Revert "First step to make packing work on GPUs"

This reverts commit c350289.

* Fix particle packing system

* Reformat

* Fix doctests

* Use manual trigger for buildkite

* Revert "Use manual trigger for buildkite"

This reverts commit 3ac3fc1.

* Add trigger buildkite action

* Fix action

* Try action without if condition

* Try again

* Remove action again

* Try more

* Fix GPU tests

* Reformat

* Fix tests

* Disable oneAPI tests

* Fix

* Add GPU support to README

Author: efaulhaber

.buildkite/pipeline.yml

@@ -0,0 +1,58 @@
+steps:
+  - label: "CUDA"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1"
+    agents:
+      queue: "juliagpu"
+      cuda: "*"
+    command: |
+      julia --color=yes --project=test -e 'using Pkg; Pkg.add("CUDA"); Pkg.develop(path="."); Pkg.instantiate()'
+      julia --color=yes --project=test -e 'include("test/runtests.jl")'
+    env:
+      TRIXIPARTICLES_TEST: cuda
+    timeout_in_minutes: 60
+
+  - label: "AMDGPU"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1"
+    agents:
+      queue: "juliagpu"
+      rocm: "*"
+    command: |
+      julia --color=yes --project=test -e 'using Pkg; Pkg.add("AMDGPU"); Pkg.develop(path="."); Pkg.instantiate()'
+      julia --color=yes --project=test -e 'include("test/runtests.jl")'
+    env:
+      TRIXIPARTICLES_TEST: amdgpu
+    timeout_in_minutes: 60
+
+  - label: "Metal"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1"
+    agents:
+      queue: "juliaecosystem"
+      os: "macos"
+      arch: "aarch64"
+    command: |
+      julia --color=yes --project=test -e 'using Pkg; Pkg.add("Metal"); Pkg.develop(path="."); Pkg.instantiate()'
+      julia --color=yes --project=test -e 'include("test/runtests.jl")'
+    env:
+      TRIXIPARTICLES_TEST: metal
+    timeout_in_minutes: 60
+
+  # Doesn't work. Fails with segfault. See https://github.com/trixi-framework/TrixiParticles.jl/issues/484.
+  # - label: "oneAPI"
+  #   plugins:
+  #     - JuliaCI/julia#v1:
+  #         version: "1"
+  #   agents:
+  #     queue: "juliagpu"
+  #     intel: "*"
+  #   command: |
+  #     julia --color=yes --project=test -e 'using Pkg; Pkg.add("oneAPI"); Pkg.develop(path="."); Pkg.instantiate()'
+  #     julia --color=yes --project=test -e 'include("test/runtests.jl")'
+  #   env:
+  #     TRIXIPARTICLES_TEST: oneapi
+  #   timeout_in_minutes: 60

README.md

@@ -36,6 +36,7 @@ It offers intuitive configuration, robust pre- and post-processing, and vendor-a
 - Particle sampling of complex geometries from `.stl` and `.asc` files.
 - Output formats:
   - VTK
+- Support for GPUs by Nvidia, AMD and Apple (experimental)
 
 ## Examples
 We provide several example simulation setups in the `examples` folder (which can be accessed from Julia via `examples_dir()`).

examples/fluid/dam_break_2d_gpu.jl

@@ -9,7 +9,7 @@ using TrixiParticles
 # Load setup from dam break example
 trixi_include(@__MODULE__,
               joinpath(examples_dir(), "fluid", "dam_break_2d.jl"),
-              sol=nothing)
+              sol=nothing, ode=nothing)
 
 # Define a GPU-compatible neighborhood search
 min_corner = minimum(tank.boundary.coordinates, dims=2)
@@ -23,4 +23,7 @@ trixi_include(@__MODULE__,
               neighborhood_search=neighborhood_search,
               fluid_particle_spacing=fluid_particle_spacing,
               tspan=tspan,
+              density_diffusion=density_diffusion,
+              boundary_layers=boundary_layers, spacing_ratio=spacing_ratio,
+              boundary_model=boundary_model,
               data_type=nothing)

examples/fluid/dam_break_3d.jl

@@ -57,7 +57,7 @@ boundary_system = BoundarySPHSystem(tank.boundary, boundary_model)
 # ==========================================================================================
 # ==== Simulation
 semi = Semidiscretization(fluid_system, boundary_system)
-ode = semidiscretize(semi, tspan)
+ode = semidiscretize(semi, tspan, data_type=nothing)
 
 info_callback = InfoCallback(interval=10)
 saving_callback = SolutionSavingCallback(dt=0.02, prefix="")

examples/fluid/hydrostatic_water_column_2d.jl

@@ -62,7 +62,7 @@ boundary_system = BoundarySPHSystem(tank.boundary, boundary_model, movement=noth
 # ==========================================================================================
 # ==== Simulation
 semi = Semidiscretization(fluid_system, boundary_system)
-ode = semidiscretize(semi, tspan)
+ode = semidiscretize(semi, tspan, data_type=nothing)
 
 info_callback = InfoCallback(interval=50)
 saving_callback = SolutionSavingCallback(dt=0.02, prefix="")

examples/fluid/periodic_channel_2d.jl

@@ -60,7 +60,7 @@ periodic_box = PeriodicBox(min_corner=[0.0, -0.25], max_corner=[1.0, 0.75])
 neighborhood_search = GridNeighborhoodSearch{2}(; periodic_box)
 
 semi = Semidiscretization(fluid_system, boundary_system; neighborhood_search)
-ode = semidiscretize(semi, tspan)
+ode = semidiscretize(semi, tspan, data_type=nothing)
 
 info_callback = InfoCallback(interval=100)
 saving_callback = SolutionSavingCallback(dt=0.02, prefix="")

examples/fluid/pipe_flow_2d.jl

@@ -125,7 +125,7 @@ boundary_system = BoundarySPHSystem(pipe.boundary, boundary_model)
 semi = Semidiscretization(fluid_system, open_boundary_in, open_boundary_out,
                           boundary_system)
 
-ode = semidiscretize(semi, tspan)
+ode = semidiscretize(semi, tspan, data_type=nothing)
 
 info_callback = InfoCallback(interval=100)
 saving_callback = SolutionSavingCallback(dt=0.02, prefix="")

examples/fsi/dam_break_gate_2d.jl

@@ -4,6 +4,9 @@
 # "Study of a complex fluid-structure dam-breaking benchmark problem using a multi-phase SPH method with APR".
 # In: Engineering Analysis with Boundary Elements 104 (2019), pages 240-258.
 # https://doi.org/10.1016/j.enganabound.2019.03.033
+#
+# Use a higher resolution and see the comments below regarding plate thickness
+# to reproduce the results from the paper.
 
 using TrixiParticles
 using OrdinaryDiffEq
@@ -14,7 +17,7 @@ using OrdinaryDiffEq
 # since "larger" particles don't fit through the slightly opened gate. Lower fluid
 # resolutions thereforce cause a later and more violent fluid impact against the gate.
 fluid_particle_spacing = 0.02
-n_particles_x = 5
+n_particles_x = 4
 
 # Change spacing ratio to 3 and boundary layers to 1 when using Monaghan-Kajtar boundary model
 boundary_layers = 3
@@ -54,13 +57,15 @@ is_moving(t) = t < 0.1
 
 gate_movement = BoundaryMovement(movement_function, is_moving)
 
-# Elastic plate/beam
+# Elastic plate/beam.
+# The paper is using a thickness of 0.004, which only works properly when a similar fluid
+# resolution is used. Increase resolution and change to 0.004 to reproduce the results.
 length_beam = 0.09
-thickness = 0.004
+thickness = 0.004 * 10
 solid_density = 1161.54
 
 # Young's modulus and Poisson ratio
-E = 3.5e6
+E = 3.5e6 / 10
 nu = 0.45
 
 # The structure starts at the position of the first particle and ends
@@ -123,24 +128,11 @@ solid_smoothing_kernel = WendlandC2Kernel{2}()
 hydrodynamic_densites = fluid_density * ones(size(solid.density))
 hydrodynamic_masses = hydrodynamic_densites * solid_particle_spacing^2
 
-k_solid = gravity * initial_fluid_size[2]
-beta_solid = fluid_particle_spacing / solid_particle_spacing
-boundary_model_solid = BoundaryModelMonaghanKajtar(k_solid, beta_solid,
-                                                   solid_particle_spacing,
-                                                   hydrodynamic_masses)
-
-# `BoundaryModelDummyParticles` usually produces better results, since Monaghan-Kajtar BCs
-# tend to introduce a non-physical gap between fluid and boundary.
-# However, `BoundaryModelDummyParticles` can only be used when the plate thickness is
-# at least two fluid particle spacings, so that the compact support is fully sampled,
-# or fluid particles can penetrate the solid.
-# For higher fluid resolutions, uncomment the code below for better results.
-#
-# boundary_model_solid = BoundaryModelDummyParticles(hydrodynamic_densites,
-#                                                    hydrodynamic_masses,
-#                                                    state_equation=state_equation,
-#                                                    AdamiPressureExtrapolation(),
-#                                                    smoothing_kernel, smoothing_length)
+boundary_model_solid = BoundaryModelDummyParticles(hydrodynamic_densites,
+                                                   hydrodynamic_masses,
+                                                   state_equation=state_equation,
+                                                   AdamiPressureExtrapolation(),
+                                                   smoothing_kernel, smoothing_length)
 
 solid_system = TotalLagrangianSPHSystem(solid,
                                         solid_smoothing_kernel, solid_smoothing_length,
@@ -152,7 +144,7 @@ solid_system = TotalLagrangianSPHSystem(solid,
 # ==== Simulation
 semi = Semidiscretization(fluid_system, boundary_system_tank,
                           boundary_system_gate, solid_system)
-ode = semidiscretize(semi, tspan)
+ode = semidiscretize(semi, tspan, data_type=nothing)
 
 info_callback = InfoCallback(interval=100)
 saving_callback = SolutionSavingCallback(dt=0.02, prefix="")

examples/solid/oscillating_beam_2d.jl

@@ -56,7 +56,7 @@ solid_system = TotalLagrangianSPHSystem(solid, smoothing_kernel, smoothing_lengt
 # ==== Simulation
 semi = Semidiscretization(solid_system,
                           neighborhood_search=PrecomputedNeighborhoodSearch{2}())
-ode = semidiscretize(semi, tspan)
+ode = semidiscretize(semi, tspan, data_type=nothing)
 
 info_callback = InfoCallback(interval=100)
 

src/general/gpu.jl

@@ -10,9 +10,11 @@
 Adapt.@adapt_structure Semidiscretization
 Adapt.@adapt_structure WeaklyCompressibleSPHSystem
 Adapt.@adapt_structure DensityDiffusionAntuono
+Adapt.@adapt_structure EntropicallyDampedSPHSystem
 Adapt.@adapt_structure BoundarySPHSystem
 Adapt.@adapt_structure BoundaryModelDummyParticles
 Adapt.@adapt_structure BoundaryModelMonaghanKajtar
+Adapt.@adapt_structure BoundaryMovement
 Adapt.@adapt_structure TotalLagrangianSPHSystem
 
 # The initial conditions are only used for initialization, which happens before `adapt`ing