Errors using Metal.jl backend

[habemus-papadum]

Hi!
The following fails using the Metal backend:

using WaterLily,Plots,Metal

cID = "2DCircle"
function circle(n,m;Re=250f0,U=1f0,mem=MtlArray)
    radius, center = Float32(m/8), Float32(m/2)
    body = AutoBody((x,t)->√sum(abs2, x .- center) - radius)
    Simulation((n,m), (U,0f0), radius; ν=U*radius/Re, body, mem)
end

sim = circle(3*2^6,2^7);#mem=CuArray)

WaterLily.logger(cID) # Log the residual of pressure solver
using Logging; disable_logging(Logging.Debug)

sim_gif!(sim,duration=Float32(10),clims=(-5,5),plotbody=true)

The issue is primarily float64's that metal cannot handle.

Is there an easy way to modify this code to use Float32? Is it possible to easily disable fastmath?

Thanks!

Stack Trace

ERROR: LoadError: InvalidIRError: compiling MethodInstance for (::WaterLily.var"#gpu_##kern_#337#exitBC!##1")(::KernelAbstractions.CompilerMetadata{KernelAbstractions.NDIteration.DynamicSize, KernelAbstractions.NDIteration.DynamicCheck, Nothing, CartesianIndices{2, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}}, KernelAbstractions.NDIteration.NDRange{2, KernelAbstractions.NDIteration.DynamicSize, KernelAbstractions.NDIteration.StaticSize{(64, 1)}, CartesianIndices{2, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}}, Nothing}}, ::MtlDeviceArray{Float32, 3, 1}, ::MtlDeviceArray{Float32, 3, 1}, ::Float32, ::Float64, ::CartesianIndex{2}) resulted in invalid LLVM IR
Reason: unsupported use of double value
Reason: unsupported use of double value
Stacktrace:
 [1] Float64
   @ ./float.jl:345
 [2] convert
   @ ./number.jl:7
 [3] _promote
   @ ./promotion.jl:388
 [4] promote
   @ ./promotion.jl:410
 [5] mul_fast
   @ ./fastmath.jl:286
 [6] macro expansion
   @ ~/.julia/packages/WaterLily/O8g3K/src/util.jl:146
 [7] macro expansion
   @ ~/.julia/packages/KernelAbstractions/X5fk1/src/macros.jl:332
 [8] gpu_
   @ ./none:0
Reason: unsupported use of double value
Stacktrace:
 [1] Float64
   @ ./float.jl:345
 [2] convert
   @ ./number.jl:7
 [3] _promote
   @ ./promotion.jl:388
 [4] promote
   @ ./promotion.jl:410
 [5] mul_fast
   @ ./fastmath.jl:286
 [6] macro expansion
   @ ~/.julia/packages/WaterLily/O8g3K/src/util.jl:146
 [7] macro expansion
   @ ~/.julia/packages/KernelAbstractions/X5fk1/src/macros.jl:332
 [8] gpu_
   @ ./none:0
Reason: unsupported use of double value
Stacktrace:
 [1] mul_fast
   @ ./fastmath.jl:169
 [2] mul_fast
   @ ./fastmath.jl:289
 [3] mul_fast
   @ ./fastmath.jl:286
 [4] macro expansion
   @ ~/.julia/packages/WaterLily/O8g3K/src/util.jl:146
 [5] macro expansion
   @ ~/.julia/packages/KernelAbstractions/X5fk1/src/macros.jl:332
 [6] gpu_
   @ ./none:0
Reason: unsupported use of double value
Stacktrace:
 [1] mul_fast
   @ ./fastmath.jl:169
 [2] mul_fast
   @ ./fastmath.jl:289
 [3] mul_fast
   @ ./fastmath.jl:286
 [4] macro expansion
   @ ~/.julia/packages/WaterLily/O8g3K/src/util.jl:146
 [5] macro expansion
   @ ~/.julia/packages/KernelAbstractions/X5fk1/src/macros.jl:332
 [6] gpu_
   @ ./none:0
Reason: unsupported use of double value
Stacktrace:
 [1] Float64
   @ ./float.jl:345
 [2] convert
   @ ./number.jl:7
 [3] _promote
   @ ./promotion.jl:379
 [4] promote
   @ ./promotion.jl:404
 [5] sub_fast
   @ ./fastmath.jl:260
 [6] macro expansion
   @ ~/.julia/packages/WaterLily/O8g3K/src/util.jl:146
 [7] macro expansion
   @ ~/.julia/packages/KernelAbstractions/X5fk1/src/macros.jl:332
 [8] gpu_
   @ ./none:0
Reason: unsupported use of double value
Stacktrace:
 [1] sub_fast
   @ ./fastmath.jl:168
 [2] sub_fast
   @ ./fastmath.jl:260
 [3] macro expansion
   @ ~/.julia/packages/WaterLily/O8g3K/src/util.jl:146
 [4] macro expansion
   @ ~/.julia/packages/KernelAbstractions/X5fk1/src/macros.jl:332
 [5] gpu_
   @ ./none:0
Reason: unsupported use of double value
Stacktrace:
 [1] Float32
   @ ./float.jl:342
 [2] convert
   @ ./number.jl:7
 [3] setindex!
   @ ~/.julia/packages/Metal/EFzAO/src/device/array.jl:105
 [4] setindex!
   @ ~/.julia/packages/Metal/EFzAO/src/device/array.jl:117
 [5] macro expansion
   @ ~/.julia/packages/WaterLily/O8g3K/src/util.jl:146
 [6] macro expansion
   @ ~/.julia/packages/KernelAbstractions/X5fk1/src/macros.jl:332
 [7] gpu_
   @ ./none:0
Hint: catch this exception as `err` and call `code_typed(err; interactive = true)` to introspect the erroneous code with Cthulhu.jl
Stacktrace:
  [1] check_ir(job::GPUCompiler.CompilerJob{GPUCompiler.MetalCompilerTarget, Metal.MetalCompilerParams}, args::LLVM.Module)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/OCZFZ/src/validation.jl:167
  [2] emit_llvm(job::GPUCompiler.CompilerJob; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/OCZFZ/src/driver.jl:420
  [3] emit_llvm
    @ ~/.julia/packages/GPUCompiler/OCZFZ/src/driver.jl:182 [inlined]
  [4] compile_unhooked(output::Symbol, job::GPUCompiler.CompilerJob; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/OCZFZ/src/driver.jl:95
  [5] compile_unhooked
    @ ~/.julia/packages/GPUCompiler/OCZFZ/src/driver.jl:80 [inlined]
  [6] compile(target::Symbol, job::GPUCompiler.CompilerJob; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/OCZFZ/src/driver.jl:67
  [7] compile
    @ ~/.julia/packages/GPUCompiler/OCZFZ/src/driver.jl:55 [inlined]
  [8] (::Metal.var"#compile##0#compile##1"{GPUCompiler.CompilerJob{GPUCompiler.MetalCompilerTarget, Metal.MetalCompilerParams}})(ctx::LLVM.Context)
    @ Metal ~/.julia/packages/Metal/EFzAO/src/compiler/compilation.jl:173
  [9] JuliaContext(f::Metal.var"#compile##0#compile##1"{GPUCompiler.CompilerJob{GPUCompiler.MetalCompilerTarget, Metal.MetalCompilerParams}}; kwargs::@Kwargs{})
    @ GPUCompiler ~/.julia/packages/GPUCompiler/OCZFZ/src/driver.jl:34
 [10] JuliaContext(f::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/OCZFZ/src/driver.jl:25
 [11] macro expansion
    @ ~/.julia/packages/Metal/EFzAO/src/compiler/compilation.jl:172 [inlined]
 [12] macro expansion
    @ ~/.julia/packages/ObjectiveC/UNTzb/src/os.jl:264 [inlined]
 [13] compile(job::GPUCompiler.CompilerJob)
    @ Metal ~/.julia/packages/Metal/EFzAO/src/compiler/compilation.jl:170
 [14] actual_compilation(cache::Dict{Any, Any}, src::Core.MethodInstance, world::UInt64, cfg::GPUCompiler.CompilerConfig{GPUCompiler.MetalCompilerTarget, Metal.MetalCompilerParams}, compiler::typeof(Metal.compile), linker::typeof(Metal.link))
    @ GPUCompiler ~/.julia/packages/GPUCompiler/OCZFZ/src/execution.jl:245
 [15] cached_compilation(cache::Dict{Any, Any}, src::Core.MethodInstance, cfg::GPUCompiler.CompilerConfig{GPUCompiler.MetalCompilerTarget, Metal.MetalCompilerParams}, compiler::Function, linker::Function)
    @ GPUCompiler ~/.julia/packages/GPUCompiler/OCZFZ/src/execution.jl:159
 [16] macro expansion
    @ ~/.julia/packages/Metal/EFzAO/src/compiler/execution.jl:189 [inlined]
 [17] macro expansion
    @ ./lock.jl:376 [inlined]
 [18] mtlfunction(f::WaterLily.var"#gpu_##kern_#337#exitBC!##1", tt::Type{Tuple{KernelAbstractions.CompilerMetadata{KernelAbstractions.NDIteration.DynamicSize, KernelAbstractions.NDIteration.DynamicCheck, Nothing, CartesianIndices{2, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}}, KernelAbstractions.NDIteration.NDRange{2, KernelAbstractions.NDIteration.DynamicSize, KernelAbstractions.NDIteration.StaticSize{(64, 1)}, CartesianIndices{2, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}}, Nothing}}, MtlDeviceArray{Float32, 3, 1}, MtlDeviceArray{Float32, 3, 1}, Float32, Float64, CartesianIndex{2}}}; name::Nothing, kwargs::@Kwargs{})
    @ Metal ~/.julia/packages/Metal/EFzAO/src/compiler/execution.jl:184
 [19] mtlfunction(f::WaterLily.var"#gpu_##kern_#337#exitBC!##1", tt::Type{Tuple{KernelAbstractions.CompilerMetadata{KernelAbstractions.NDIteration.DynamicSize, KernelAbstractions.NDIteration.DynamicCheck, Nothing, CartesianIndices{2, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}}, KernelAbstractions.NDIteration.NDRange{2, KernelAbstractions.NDIteration.DynamicSize, KernelAbstractions.NDIteration.StaticSize{(64, 1)}, CartesianIndices{2, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}}}, Nothing}}, MtlDeviceArray{Float32, 3, 1}, MtlDeviceArray{Float32, 3, 1}, Float32, Float64, CartesianIndex{2}}})
    @ Metal ~/.julia/packages/Metal/EFzAO/src/compiler/execution.jl:182
 [20] macro expansion
    @ ~/.julia/packages/Metal/EFzAO/src/compiler/execution.jl:85 [inlined]
 [21] (::KernelAbstractions.Kernel{MetalBackend, KernelAbstractions.NDIteration.StaticSize{(64,)}, KernelAbstractions.NDIteration.DynamicSize, WaterLily.var"#gpu_##kern_#337#exitBC!##1"})(::MtlArray{Float32, 3, Metal.PrivateStorage}, ::Vararg{Any}; ndrange::Tuple{Int64, Int64}, workgroupsize::Nothing)
    @ Metal.MetalKernels ~/.julia/packages/Metal/EFzAO/src/MetalKernels.jl:115
 [22] (::WaterLily.var"##kern#336#exitBC!##4"{CartesianIndices{2, Tuple{UnitRange{Int64}, UnitRange{Int64}}}})(u::MtlArray{Float32, 3, Metal.PrivateStorage}, u⁰::MtlArray{Float32, 3, Metal.PrivateStorage}, U::Float32, Δt::Float64)
    @ WaterLily ~/.julia/packages/WaterLily/O8g3K/src/util.jl:149
 [23] exitBC!(u::MtlArray{Float32, 3, Metal.PrivateStorage}, u⁰::MtlArray{Float32, 3, Metal.PrivateStorage}, Δt::Float64)
    @ WaterLily ~/.julia/packages/WaterLily/O8g3K/src/util.jl:151
 [24] Flow(N::Tuple{Int64, Int64}, uBC::Tuple{Float32, Float32}; f::Type, Δt::Float64, ν::Float32, g::Nothing, uλ::Nothing, perdir::Tuple{}, exitBC::Bool, T::Type{Float32})
    @ WaterLily ~/.julia/packages/WaterLily/O8g3K/src/Flow.jl:109
 [25] Simulation(dims::Tuple{Int64, Int64}, uBC::Tuple{Float32, Float32}, L::Float32; Δt::Float64, ν::Float32, g::Nothing, U::Nothing, ϵ::Int64, perdir::Tuple{}, uλ::Nothing, exitBC::Bool, body::AutoBody{WaterLily.var"#comp#145"{Bool, var"#3#4"{Float32, Float32}, WaterLily.var"#146#147"}, WaterLily.var"#146#147"}, T::Type, mem::Type)
    @ WaterLily ~/.julia/packages/WaterLily/O8g3K/src/WaterLily.jl:75
 [26] Simulation
    @ ~/.julia/packages/WaterLily/O8g3K/src/WaterLily.jl:68 [inlined]
 [27] #circle#1
    @ ~/src/wl/mwl.jl:7 [inlined]
 [28] circle(n::Int64, m::Int64)
    @ Main ~/src/wl/mwl.jl:4
 [29] top-level scope
    @ ~/src/wl/mwl.jl:10
 [30] include(mod::Module, _path::String)
    @ Base ./Base.jl:306
 [31] exec_options(opts::Base.JLOptions)
    @ Base ./client.jl:317
 [32] _start()
    @ Base ./client.jl:550
in expression starting at /Users/nehal/src/wl/mwl.jl:10

[b-fg]

Hi! It will be hard for us to reproduce the issues because we do not have access to an Apple GPU. To run a Simulation in Float32, you can pass the T=Float32 keyword argument. Try

using WaterLily, Plots, Metal

cID = "2DCircle"
function circle(n, m; T=Float32, Re=250, U=1, mem=Array)
    radius, center = T(m/8), T(m/2)
    body = AutoBody((x,t)->√sum(abs2, x .- center) - radius)
    Simulation((n, m), (U, 0), radius; T, ν=U*radius/Re, body, mem)
end

sim = circle(3*2^6,2^7; T=Float32, mem=MtlArray)

WaterLily.logger(cID) # Log the residual of pressure solver
using Logging; disable_logging(Logging.Debug)

sim_gif!(sim, duration=10, clims=(-5,5), plotbody=true)

To disable fastmath, you can run Julia with the CLA --math-mode=ieee.

[weymouth]

Did this work @habemus-papadum ?