Added profiling of u0 copy.

Author: b-fg

Project.toml

@@ -10,6 +10,7 @@ ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
+NVTX = "5da4648a-3479-48b8-97b9-01cb529c0a1f"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
@@ -19,23 +20,24 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 [weakdeps]
 AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 ReadVTK = "dc215faf-f008-4882-a9f7-a79a826fadc3"
 WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192"
-Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 
 [extensions]
 WaterLilyAMDGPUExt = "AMDGPU"
 WaterLilyCUDAExt = "CUDA"
+WaterLilyPlotsExt = "Plots"
 WaterLilyReadVTKExt = "ReadVTK"
 WaterLilyWriteVTKExt = "WriteVTK"
-WaterLilyPlotsExt = "Plots"
 
 [compat]
 DocStringExtensions = "0.9"
 EllipsisNotation = "1.8"
 ForwardDiff = "^0.10.18"
 KernelAbstractions = "0.9.1"
 LoggingExtras = "1.1"
+NVTX = "0.3.5"
 Reexport = "^1.2.2"
 Requires = "1.3"
 StaticArrays = "^1.1.0"

src/Flow.jl

@@ -151,21 +151,28 @@ Integrate the `Flow` one time step using the [Boundary Data Immersion Method](ht
 and the `AbstractPoisson` pressure solver to project the velocity onto an incompressible flow.
 """
 @fastmath function mom_step!(a::Flow{N},b::AbstractPoisson) where N
-    a.u⁰ .= a.u; scale_u!(a,0); U = BCTuple(a.U,a.Δt,N)
+    NVTX.@range "copy_u0!" begin a.u⁰ .= a.u end
+    NVTX.@range "scale_u!" begin scale_u!(a,0) end
+    NVTX.@range "BCTuple" begin U = BCTuple(a.U,a.Δt,N) end
     # predictor u → u'
     @log "p"
-    conv_diff!(a.f,a.u⁰,a.σ,ν=a.ν,perdir=a.perdir)
-    accelerate!(a.f,@view(a.Δt[1:end-1]),a.g,a.U)
-    BDIM!(a); BC!(a.u,U,a.exitBC,a.perdir)
-    a.exitBC && exitBC!(a.u,a.u⁰,U,a.Δt[end]) # convective exit
-    project!(a,b); BC!(a.u,U,a.exitBC,a.perdir)
+    NVTX.@range "conv_diff!" begin conv_diff!(a.f,a.u⁰,a.σ,ν=a.ν,perdir=a.perdir) end
+    NVTX.@range "accelerate!" begin accelerate!(a.f,@view(a.Δt[1:end-1]),a.g,a.U) end
+    NVTX.@range "BDIM!" begin BDIM!(a) end
+    NVTX.@range "BC!" begin BC!(a.u,U,a.exitBC,a.perdir) end
+    NVTX.@range "exitBC!" begin a.exitBC && exitBC!(a.u,a.u⁰,U,a.Δt[end]) end # convective exit
+    NVTX.@range "project!" begin project!(a,b) end
+    NVTX.@range "BC!" begin BC!(a.u,U,a.exitBC,a.perdir) end
     # corrector u → u¹
     @log "c"
-    conv_diff!(a.f,a.u,a.σ,ν=a.ν,perdir=a.perdir)
-    accelerate!(a.f,a.Δt,a.g,a.U)
-    BDIM!(a); scale_u!(a,0.5); BC!(a.u,U,a.exitBC,a.perdir)
-    project!(a,b,0.5); BC!(a.u,U,a.exitBC,a.perdir)
-    push!(a.Δt,CFL(a))
+    NVTX.@range "conv_diff!" begin conv_diff!(a.f,a.u,a.σ,ν=a.ν,perdir=a.perdir) end
+    NVTX.@range "accelerate!" begin accelerate!(a.f,a.Δt,a.g,a.U) end
+    NVTX.@range "BDIM!" begin BDIM!(a) end
+    NVTX.@range "scale_u!" begin scale_u!(a,0.5) end
+    NVTX.@range "BC!" begin BC!(a.u,U,a.exitBC,a.perdir) end
+    NVTX.@range "project!" begin project!(a,b,0.5) end
+    NVTX.@range "BC!" begin BC!(a.u,U,a.exitBC,a.perdir) end
+    NVTX.@range "CFL!" begin push!(a.Δt,CFL(a)) end
 end
 scale_u!(a,scale) = @loop a.u[Ii] *= scale over Ii ∈ inside_u(size(a.p))
 

src/WaterLily.jl

@@ -3,7 +3,7 @@ $(README)
 """
 module WaterLily
 
-using DocStringExtensions
+using DocStringExtensions, NVTX
 
 include("util.jl")
 export L₂,BC!,@inside,inside,δ,apply!,loc,@log
@@ -104,7 +104,7 @@ function sim_step!(sim::AbstractSimulation,t_end;remeasure=true,max_steps=typema
     end
 end
 function sim_step!(sim::AbstractSimulation;remeasure=true)
-    remeasure && measure!(sim)
+    NVTX.@range "measure!" begin remeasure && measure!(sim) end
     mom_step!(sim.flow,sim.pois)
 end