Switch to TestItems.jl setup

test/Project.toml

@@ -1,8 +1,8 @@
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
 Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
-AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
@@ -40,6 +40,7 @@ SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
 SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
 TrixiTest = "0a316866-cbd0-4425-8bcb-08103b2c1f26"
 
 [compat]
@@ -84,6 +85,7 @@ SparseConnectivityTracer = "1.0.1"
 SparseMatrixColorings = "0.4.27"
 StableRNGs = "1.0.2"
 Test = "1"
+TestItemRunner = "1.1"
 TrixiTest = "0.2.1"
 
 [extras]

test/runtests.jl

@@ -1,151 +1,138 @@
-using Test
-using MPI: mpiexec
-import Trixi
+using TestItemRunner
 
-# We run tests in parallel with CI jobs setting the `TRIXI_TEST` environment
-# variable to determine the subset of tests to execute.
-# By default, we just run the threaded tests since they are relatively cheap
-# and test a good amount of different functionality.
+# We run subsets of the test suite in parallel CI jobs by setting the `TRIXI_TEST`
+# environment variable. Its value is matched against the `tags` attached to each
+# `@testitem`. By default (`TRIXI_TEST == "threaded"`) we run everything.
 const TRIXI_TEST = get(ENV, "TRIXI_TEST", "threaded")
-# Some GitHub CI runners may have not much RAM and just 3 virtual CPU cores.
+
+# Some GitHub CI runners may not have much RAM and just 3 virtual CPU cores.
 # In this case, we do not want to use all of the cores to speed-up CI.
 const TRIXI_MPI_NPROCS = clamp(Sys.CPU_THREADS - 1, 2, 3)
 const TRIXI_NTHREADS = clamp(Sys.CPU_THREADS, 2, 3)
 
-@time @testset "Trixi.jl tests" begin
-    # This is placed first since tests error out otherwise if `TRIXI_TEST == "all"`,
-    # at least on some systems.
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "mpi"
-        # Do a dummy `@test true`:
-        # If the process errors out the testset would error out as well,
-        # cf. https://github.com/JuliaParallel/MPI.jl/pull/391
-        @test true
-
-        # We provide a `--heap-size-hint` to avoid/reduce out-of-memory errors during CI testing
-        run(`$(mpiexec()) -n $TRIXI_MPI_NPROCS $(Base.julia_cmd()) --threads=1 --check-bounds=yes --heap-size-hint=0.5G $(joinpath(@__DIR__, "test_mpi.jl"))`)
-    end
-
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "threaded" ||
-             TRIXI_TEST == "threaded_legacy"
-        # Do a dummy `@test true`:
-        # If the process errors out the testset would error out as well,
-        # cf. https://github.com/JuliaParallel/MPI.jl/pull/391
-        @test true
-
-        run(`$(Base.julia_cmd()) --threads=$TRIXI_NTHREADS --check-bounds=yes --code-coverage=none $(joinpath(@__DIR__, "test_threaded.jl"))`)
-    end
-
-    # Downgrade CI currently has issues with running julia processes via `run`, see
-    # https://github.com/trixi-framework/Trixi.jl/pull/2507#issuecomment-3990318366
-    # So we run test_threaded.jl serially.
-    # For `TRIXI_TEST = "all"`, test_threaded.jl is already covered by the threaded run, so we don't need to run it again.
-    @time if TRIXI_TEST == "downgrade"
-        include(joinpath(@__DIR__, "test_threaded.jl"))
-    end
-
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "tree_part1"
-        include(joinpath(@__DIR__, "test_tree_1d.jl"))
-        include(joinpath(@__DIR__, "test_tree_2d_part1.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "tree_part2"
-        include(joinpath(@__DIR__, "test_tree_2d_part2.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "tree_part3"
-        include(joinpath(@__DIR__, "test_tree_2d_part3.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "tree_part4"
-        include(joinpath(@__DIR__, "test_tree_3d_part1.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "tree_part5"
-        include(joinpath(@__DIR__, "test_tree_3d_part2.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "tree_part6"
-        include(joinpath(@__DIR__, "test_tree_3d_part3.jl"))
-    end
-
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "structured"
-        include(joinpath(@__DIR__, "test_structured_1d.jl"))
-        include(joinpath(@__DIR__, "test_structured_2d.jl"))
-        include(joinpath(@__DIR__, "test_structured_3d.jl"))
-    end
-
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "p4est_part1"
-        include(joinpath(@__DIR__, "test_p4est_2d.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "p4est_part2"
-        include(joinpath(@__DIR__, "test_p4est_3d.jl"))
-    end
-
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "t8code_part1"
-        include(joinpath(@__DIR__, "test_t8code_2d.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "t8code_part2"
-        include(joinpath(@__DIR__, "test_t8code_3d.jl"))
-    end
+# A few suites cannot run in the ordinary in-process `@run_package_tests` model:
+# they need a specially-launched Julia process - `mpi` (multiple ranks via
+# `mpiexec`), `threaded` (multiple threads via `--threads`), and
+# `kernelabstractions` (a different threading backend, selected via a preference
+# that only takes effect on a fresh Julia start). We handle them by *relaunching*
+# Julia/`mpiexec` on this very file: the launched worker re-enters `runtests.jl`
+# with `TRIXI_TEST_RUN_ITEMS` set and then runs the tag-filtered test items
+# in-process (`TestItemRunner` evaluates items in the current process, so this
+# also works for every MPI rank). The `TRIXI_TEST` value of such a suite selects
+# the items via the equally-named tag; `threaded_legacy` reuses the `threaded`
+# items but is launched on a different Julia version by CI.
+const SPECIAL_PROCESS_SUITES = ("mpi", "threaded", "kernelabstractions")
+const IN_WORKER = haskey(ENV, "TRIXI_TEST_RUN_ITEMS")
+
+target_tag(suite) = suite == "threaded_legacy" ? :threaded : Symbol(suite)
+
+# Special suites this (parent) invocation will dispatch into their own processes.
+const SUITES_TO_DISPATCH = if IN_WORKER
+    String[]
+elseif TRIXI_TEST == "all"
+    collect(SPECIAL_PROCESS_SUITES)
+elseif TRIXI_TEST in SPECIAL_PROCESS_SUITES || TRIXI_TEST == "threaded_legacy"
+    [TRIXI_TEST]
+else
+    String[]
+end
 
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "unstructured_dgmulti"
-        include(joinpath(@__DIR__, "test_unstructured_2d.jl"))
-        include(joinpath(@__DIR__, "test_dgmulti_1d.jl"))
-        include(joinpath(@__DIR__, "test_dgmulti_2d.jl"))
-        include(joinpath(@__DIR__, "test_dgmulti_3d.jl"))
-    end
+# `import` is only allowed at top level, so load here what `dispatch_special_suite`
+# needs: `MPI` for `mpiexec`, and `Trixi` to toggle the threading backend preference.
+if "mpi" in SUITES_TO_DISPATCH
+    import MPI
+end
+if "kernelabstractions" in SUITES_TO_DISPATCH
+    import Trixi
+end
 
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "parabolic_part1"
-        include(joinpath(@__DIR__, "test_parabolic_1d.jl"))
-        include(joinpath(@__DIR__, "test_parabolic_2d.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "parabolic_part2"
-        include(joinpath(@__DIR__, "test_parabolic_3d.jl"))
-    end
+# The GPU suites run their items only when the respective backend is `functional()`,
+# so requesting `CUDA`/`AMDGPU` on a machine without that GPU warns and runs nothing
+# instead of erroring.
+RUN_CUDA = false
+if !IN_WORKER && TRIXI_TEST in ("all", "CUDA")
+    import CUDA
+    RUN_CUDA = CUDA.functional()
+    RUN_CUDA || @warn "Unable to run CUDA tests on this machine"
+end
+RUN_AMDGPU = false
+if !IN_WORKER && TRIXI_TEST in ("all", "AMDGPU")
+    import AMDGPU
+    RUN_AMDGPU = AMDGPU.functional()
+    RUN_AMDGPU || @warn "Unable to run AMDGPU tests on this machine"
+end
 
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "misc_part1"
-        include(joinpath(@__DIR__, "test_unit.jl"))
-        include(joinpath(@__DIR__, "test_type.jl"))
-        include(joinpath(@__DIR__, "test_visualization.jl"))
-    end
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "misc_part2"
-        include(joinpath(@__DIR__, "test_special_elixirs.jl"))
-        include(joinpath(@__DIR__, "test_aqua.jl"))
-    end
+# Relaunch Julia/`mpiexec` for a suite that needs a special process. The worker
+# re-enters this file with `TRIXI_TEST=<suite>` and `TRIXI_TEST_RUN_ITEMS=true`.
+function run_worker(cmd, suite)
+    run(addenv(cmd, "TRIXI_TEST_RUN_ITEMS" => "true", "TRIXI_TEST" => suite))
+end
 
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "performance_specializations"
-        include(joinpath(@__DIR__, "test_performance_specializations_2d.jl"))
-        include(joinpath(@__DIR__, "test_performance_specializations_3d.jl"))
+function dispatch_special_suite(suite)
+    project = dirname(Base.active_project())
+    julia = Base.julia_cmd()
+
+    if suite == "mpi"
+        cmd = `$(MPI.mpiexec()) -n $TRIXI_MPI_NPROCS $julia --threads=1 --check-bounds=yes --heap-size-hint=0.5G --project=$project $(@__FILE__)`
+        run_worker(cmd, suite)
+    elseif suite == "threaded" || suite == "threaded_legacy"
+        cmd = `$julia --threads=$TRIXI_NTHREADS --check-bounds=yes --code-coverage=none --project=$project $(@__FILE__)`
+        run_worker(cmd, suite)
+    elseif suite == "kernelabstractions"
+        # The threading backend is selected via a preference that is read on Julia
+        # startup, so we set it here (in the parent) and restore it afterwards;
+        # the relaunched worker picks it up.
+        previous_backend = Trixi._PREFERENCE_THREADING
+        Trixi.set_threading_backend!(:kernelabstractions)
+        try
+            cmd = `$julia --threads=$TRIXI_NTHREADS --check-bounds=yes --project=$project $(@__FILE__)`
+            run_worker(cmd, suite)
+        finally
+            Trixi.set_threading_backend!(Symbol(previous_backend))
+        end
     end
+end
 
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "paper_self_gravitating_gas_dynamics"
-        include(joinpath(@__DIR__, "test_paper_self_gravitating_gas_dynamics.jl"))
-    end
+if !isempty(SUITES_TO_DISPATCH)
+    # Dispatch each requested special suite into its own process; the worker(s)
+    # run the tagged items. For a single special suite that is all we do; for
+    # `all` we additionally run the remaining (in-process) items below.
+    foreach(dispatch_special_suite, SUITES_TO_DISPATCH)
+end
 
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "CUDA"
-        import CUDA
-        if CUDA.functional()
-            include(joinpath(@__DIR__, "test_cuda_2d.jl"))
-            include(joinpath(@__DIR__, "test_cuda_3d.jl"))
+if !IN_WORKER && (TRIXI_TEST in SPECIAL_PROCESS_SUITES || TRIXI_TEST == "threaded_legacy")
+    # A single special suite was requested and dispatched above; nothing to run
+    # in this process.
+else
+    # In-process run. Either we are inside a launched worker (run just that suite's
+    # tagged items), or this is an ordinary partition / `all` (in which case we
+    # exclude the special suites, which must run in their own processes).
+    CI_ON_WINDOWS = (get(ENV, "GITHUB_ACTIONS", "false") == "true") && Sys.iswindows()
+    special_tags = (:mpi, :threaded, :kernelabstractions)
+    tag = target_tag(TRIXI_TEST)
+
+    @run_package_tests filter = ti -> begin
+        if TRIXI_TEST == "all"
+            # The special suites run in dedicated processes (see above).
+            any(t -> t in ti.tags, special_tags) && return false
         else
-            @warn "Unable to run CUDA tests on this machine"
+            tag in ti.tags || return false
         end
-    end
-
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "AMDGPU"
-        import AMDGPU
-        if AMDGPU.functional()
-            include(joinpath(@__DIR__, "test_amdgpu_2d.jl"))
-            include(joinpath(@__DIR__, "test_amdgpu_3d.jl"))
-        else
-            @warn "Unable to run AMDGPU tests on this machine"
+        # CI with MPI fails often on Windows for some examples; skip those there.
+        if CI_ON_WINDOWS && :mpi_skip_windows in ti.tags
+            return false
         end
+        # GPU items run only when the respective backend is functional.
+        :CUDA in ti.tags && !RUN_CUDA && return false
+        :AMDGPU in ti.tags && !RUN_AMDGPU && return false
+        return true
     end
+end
 
-    @time if TRIXI_TEST == "all" || TRIXI_TEST == "kernelabstractions"
-        previous_backend = Trixi._PREFERENCE_THREADING
-        Trixi.set_threading_backend!(:kernelabstractions)
-        # relaunching julia
-        try
-            run(`$(Base.julia_cmd()) --threads=$TRIXI_NTHREADS --check-bounds=yes $(abspath("test_kernelabstractions.jl"))`)
-        finally
-            # Restore previous threading backend for later tests
-            Trixi.set_threading_backend!(Symbol(previous_backend))
-        end
-    end
+# Common setup shared by all `@testitem`s. Listing `setup=[Setup]` on a test item
+# makes the `@test_trixi_include` helper macro (and the packages `test_trixi.jl`
+# pulls in) available inside it. `using Trixi`/`using Test` are already provided
+# automatically by the default imports of every `@testitem`/`@testsnippet`.
+@testsnippet Setup begin
+    include("test_trixi.jl")
 end

test/test_amdgpu_2d.jl

@@ -1,20 +1,8 @@
-module TestAMDGPU2D
-
-using Test
-using Trixi
-
-include("test_trixi.jl")
-
-EXAMPLES_DIR = joinpath(examples_dir(), "p4est_2d_dgsem")
-
-# Start with a clean environment: remove Trixi.jl output directory if it exists
-outdir = "out"
-isdir(outdir) && rm(outdir, recursive = true)
-
-@testset "AMDGPU 2D" begin
-#! format: noindent
+@testsnippet AMDGPU2DExamples begin
+    EXAMPLES_DIR = joinpath(examples_dir(), "p4est_2d_dgsem")
+end
 
-@trixi_testset "elixir_advection_basic.jl native" begin
+@testitem "AMDGPU 2D: elixir_advection_basic.jl native" setup=[Setup, AMDGPU2DExamples] tags=[:AMDGPU] begin
     @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"),
                         # Expected errors are exactly the same as with TreeMesh!
                         l2=8.311947673061856e-6,
@@ -39,8 +27,11 @@ isdir(outdir) && rm(outdir, recursive = true)
     @test Trixi.storage_type(ode.p.cache.mortars) === Array
 end
 
-@trixi_testset "elixir_advection_basic.jl Float32 / AMDGPU" begin
-    # Using AMDGPU inside the testset since otherwise the bindings are hiddend by the anonymous modules
+@testitem "AMDGPU 2D: elixir_advection_basic.jl Float32 / AMDGPU" setup=[
+    Setup,
+    AMDGPU2DExamples
+] tags=[:AMDGPU] begin
+    # Using AMDGPU inside the testitem since otherwise the bindings are hidden by the anonymous modules
     using AMDGPU
     @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_advection_basic.jl"),
                         # Expected errors are exactly the same as with TreeMesh!
@@ -69,7 +60,7 @@ end
     @test Trixi.storage_type(ode.p.cache.mortars) === ROCArray
 end
 
-@trixi_testset "elixir_euler_source_terms.jl native" begin
+@testitem "AMDGPU 2D: elixir_euler_source_terms.jl native" setup=[Setup, AMDGPU2DExamples] tags=[:AMDGPU] begin
     @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_source_terms.jl"),
                         # Expected errors are exactly the same as with TreeMesh!
                         l2=[9.321181254378498e-7,
@@ -100,8 +91,11 @@ end
     @test Trixi.storage_type(semi.cache.mortars) === Array
 end
 
-@trixi_testset "elixir_euler_source_terms.jl Float32 / AMDGPU" begin
-    # Using AMDGPU inside the testset since otherwise the bindings are hiddend by the anonymous modules
+@testitem "AMDGPU 2D: elixir_euler_source_terms.jl Float32 / AMDGPU" setup=[
+    Setup,
+    AMDGPU2DExamples
+] tags=[:AMDGPU] begin
+    # Using AMDGPU inside the testitem since otherwise the bindings are hidden by the anonymous modules
     using AMDGPU
     @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_source_terms.jl"),
                         l2=Float32[2.4917018095933837e-6,
@@ -135,8 +129,11 @@ end
     @test Trixi.storage_type(semi.cache.mortars) === ROCArray
 end
 
-@trixi_testset "elixir_euler_source_terms.jl Flux Differencing Float32 / AMDGPU" begin
-    # Using AMDGPU inside the testset since otherwise the bindings are hiddend by the anonymous modules
+@testitem "AMDGPU 2D: elixir_euler_source_terms.jl Flux Differencing Float32 / AMDGPU" setup=[
+    Setup,
+    AMDGPU2DExamples
+] tags=[:AMDGPU] begin
+    # Using AMDGPU inside the testitem since otherwise the bindings are hidden by the anonymous modules
     using AMDGPU
     @test_trixi_include(joinpath(EXAMPLES_DIR, "elixir_euler_source_terms.jl"),
                         l2=Float32[2.7905685982444506e-6,
@@ -172,8 +169,3 @@ end
     @test Trixi.storage_type(semi.cache.boundaries) === ROCArray
     @test Trixi.storage_type(semi.cache.mortars) === ROCArray
 end
-
-# Clean up afterwards: delete Trixi.jl output directory
-@test_nowarn isdir(outdir) && rm(outdir, recursive = true)
-end
-end # module