PINNErrorVsTime Benchmark Updates

benchmarks/PINNErrorsVsTime/Project.toml

@@ -1,29 +1,28 @@
 [deps]
+Cuba = "8a292aeb-7a57-582c-b821-06e4c11590b1"
+Cubature = "667455a9-e2ce-5579-9412-b964f529a492"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Integrals = "de52edbc-65ea-441a-8357-d3a637375a31"
-IntegralsCuba = "e00cd5f1-6337-4131-8b37-28b2fe4cd6cb"
-IntegralsCubature = "c31f79ba-6e32-46d4-a52f-182a8ac42a54"
 Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
+MLDataDevices = "7e8f7934-dd98-4c1a-8fe8-92b47a384d40"
 ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
 NeuralPDE = "315f7962-48a3-4962-8226-d0f33b1235f0"
+Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
 Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba"
-OptimizationFlux = "253f991c-a7b2-45f8-8852-8b9a9df78a86"
 OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e"
+OptimizationOptimisers = "42dfb2eb-d2b4-4451-abcd-913932933ac1"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 QuasiMonteCarlo = "8a4e6c94-4038-4cdc-81c3-7e6ffdb2a71b"
+ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 SciMLBenchmarks = "31c91b34-3c75-11e9-0341-95557aab0344"
 
 [compat]
 DelimitedFiles = "1"
-Integrals = "3, 4"
-IntegralsCuba = "0.2, 0.3"
-IntegralsCubature = "0.2"
-Lux = "0.4, 0.5"
-ModelingToolkit = "8"
+Integrals = "4"
+Lux = "1"
+ModelingToolkit = "9"
 NeuralPDE = "5"
-Optimization = "3"
-OptimizationFlux = "0.1, 0.2"
-OptimizationOptimJL = "0.1, 0.2, 0.3"
+OptimizationOptimJL = "0.4"
 Plots = "1"
-QuasiMonteCarlo = "0.2"
+QuasiMonteCarlo = "0.3"
 SciMLBenchmarks = "0.1"

benchmarks/PINNErrorsVsTime/allen_cahn_et.jmd

@@ -13,8 +13,9 @@ science-guided AI techniques.
 
 ```julia
 using NeuralPDE
-using Integrals, IntegralsCubature, IntegralsCuba
-using OptimizationFlux, ModelingToolkit, Optimization, OptimizationOptimJL
+using Integrals, Cubature, Cuba
+using ModelingToolkit, Optimization, OptimizationOptimJL
+using Optimisers
 using Lux, Plots
 using DelimitedFiles
 using QuasiMonteCarlo
@@ -77,7 +78,7 @@ function allen_cahn(strategy, minimizer, maxIters)
 
     ## NEURAL NETWORK
     n = 10   #neuron number
-    chain = Lux.Chain(Lux.Dense(5, n, tanh), Lux.Dense(n, n, tanh), Lux.Dense(n, 1))   #Neural network from OptimizationFlux library
+    chain = Lux.Chain(Lux.Dense(5, n, Lux.σ), Lux.Dense(n, n, Lux.σ), Lux.Dense(n, 1))   #Neural network from OptimizationFlux library
 
     indvars = [t, x1, x2, x3, x4]   #physically independent variables
     depvars = [u(t, x1, x2, x3, x4)]       #dependent (target) variable
@@ -106,7 +107,7 @@ function allen_cahn(strategy, minimizer, maxIters)
         ctime = time_ns() - startTime - timeCounter #This variable is the time to use for the time benchmark plot
         append!(times, ctime / 10^9) #Conversion nanosec to seconds
         append!(losses, l)
-        loss_ = loss_function_(p, nothing)
+        loss_ = loss_function_(p.u, nothing)
         append!(error, loss_)
         timeCounter = timeCounter + time_ns() - deltaT_s #timeCounter sums all delays due to the callback functions of the previous iterations
 
@@ -143,23 +144,21 @@ end
 maxIters = [(1,1,1,1,1,1,1000),(1,1,1,1,300,300,300)] #iters for ADAM/LBFGS
 # maxIters = [(1,1,1,1,1,1,10),(1,1,1,3,3,3,3)] #iters for ADAM/LBFGS
 
-strategies = [NeuralPDE.QuadratureTraining(quadrature_alg = CubaCuhre(), reltol = 1e-4, abstol = 1e-4, maxiters = 100),
-              NeuralPDE.QuadratureTraining(quadrature_alg = HCubatureJL(), reltol = 1e-4, abstol = 1e-4, maxiters = 100, batch = 0),
-              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLh(), reltol = 1e-4, abstol = 1e-4, maxiters = 100),
-              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLp(), reltol = 1e-4, abstol = 1e-4, maxiters = 100),
+strategies = [NeuralPDE.QuadratureTraining(quadrature_alg = CubaCuhre(), reltol = 1e-4, abstol = 1e-4, maxiters = 1100),
+              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLh(), reltol = 1e-4, abstol = 1e-4, maxiters = 1100),
+              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLp(), reltol = 1e-4, abstol = 1e-4, maxiters = 1100),
               NeuralPDE.GridTraining(0.2),
               NeuralPDE.StochasticTraining(400 ; bcs_points= 50),
               NeuralPDE.QuasiRandomTraining(400 ; bcs_points= 50)]
 
 strategies_short_name = ["CubaCuhre",
-                        "HCubatureJL",
                         "CubatureJLh",
                         "CubatureJLp",
                         "GridTraining",
                         "StochasticTraining",
                         "QuasiRandomTraining"]
 
-minimizers = [ADAM(0.005),BFGS()]
+minimizers = [Optimisers.ADAM(0.005),BFGS()]
 minimizers_short_name = ["ADAM","BFGS"]
 
 # Run models
@@ -176,15 +175,15 @@ losses_res = Dict()
 ## Convergence
 
 for min =1:length(minimizers) # minimizer
-      for strat=1:length(strategies) # strategy
-            # println(string(strategies_short_name[strat], "  ", minimizers_short_name[min]))
-            res = allen_cahn(strategies[strat], minimizers[min], maxIters[min][strat])
-            push!(error_res, string(strat,min)     => res[1])
-            push!(params_res, string(strat,min) => res[2])
-            push!(domains, string(strat,min)        => res[3])
-            push!(times, string(strat,min)        => res[4])
-            push!(losses_res, string(strat,min)        => res[5])
-      end
+    for strat=1:length(strategies) # strategy
+          # println(string(strategies_short_name[strat], "  ", minimizers_short_name[min]))
+          res = allen_cahn(strategies[strat], minimizers[min], maxIters[min][strat])
+          push!(error_res, string(strat,min)     => res[1])
+          push!(params_res, string(strat,min) => res[2])
+          push!(domains, string(strat,min)        => res[3])
+          push!(times, string(strat,min)        => res[4])
+          push!(losses_res, string(strat,min)        => res[5])
+    end
 end
 ```
 
@@ -194,22 +193,20 @@ end
 print("\n Plotting error vs times")
 #Plotting the first strategy with the first minimizer out from the loop to initialize the canvas
 current_label = string(strategies_short_name[1], " + " , minimizers_short_name[1])
-error = Plots.plot(times["11"], error_res["11"], yaxis=:log10, label = current_label)#, xlims = (0,10))#legend = true)#, size=(1200,700))
-plot!(error, times["21"], error_res["21"], yaxis=:log10, label = string(strategies_short_name[2], " + " , minimizers_short_name[1]))
-plot!(error, times["31"], error_res["31"], yaxis=:log10, label = string(strategies_short_name[3], " + " , minimizers_short_name[1]))
-plot!(error, times["41"], error_res["41"], yaxis=:log10, label = string(strategies_short_name[4], " + " , minimizers_short_name[1]))
-plot!(error, times["51"], error_res["51"], yaxis=:log10, label = string(strategies_short_name[5], " + " , minimizers_short_name[1]))
-plot!(error, times["61"], error_res["61"], yaxis=:log10, label = string(strategies_short_name[6], " + " , minimizers_short_name[1]))
-plot!(error, times["71"], error_res["71"], yaxis=:log10, label = string(strategies_short_name[7], " + " , minimizers_short_name[1]))
-
-
-plot!(error, times["12"], error_res["12"], yaxis=:log10, label = string(strategies_short_name[1], " + " , minimizers_short_name[2]))
-plot!(error, times["22"], error_res["22"], yaxis=:log10, label = string(strategies_short_name[2], " + " , minimizers_short_name[2]))
-plot!(error, times["32"], error_res["32"], yaxis=:log10, label = string(strategies_short_name[3], " + " , minimizers_short_name[2]))
-plot!(error, times["42"], error_res["42"], yaxis=:log10, label = string(strategies_short_name[4], " + " , minimizers_short_name[2]))
-plot!(error, times["52"], error_res["52"], yaxis=:log10, label = string(strategies_short_name[5], " + " , minimizers_short_name[2]))
-plot!(error, times["62"], error_res["62"], yaxis=:log10, label = string(strategies_short_name[6], " + " , minimizers_short_name[2]))
-plot!(error, times["72"], error_res["72"], yaxis=:log10, title = string("Allen Cahn convergence ADAM/LBFGS"), ylabel = "log(error)",xlabel = "t", label = string(strategies_short_name[7], " + " , minimizers_short_name[2]))
+error = Plots.plot(times["11"], error_res["11"], xaxis=:log10, yaxis=:log10, label = current_label, legend=:outerright)#, xlims = (0,10))#legend = true)#, size=(1200,700))
+plot!(error, times["21"], error_res["21"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[2], " + " , minimizers_short_name[1]))
+plot!(error, times["31"], error_res["31"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[3], " + " , minimizers_short_name[1]))
+plot!(error, times["41"], error_res["41"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[4], " + " , minimizers_short_name[1]))
+plot!(error, times["51"], error_res["51"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[5], " + " , minimizers_short_name[1]))
+plot!(error, times["61"], error_res["61"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[6], " + " , minimizers_short_name[1]))
+
+
+plot!(error, times["12"], error_res["12"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[1], " + " , minimizers_short_name[2]))
+plot!(error, times["22"], error_res["22"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[2], " + " , minimizers_short_name[2]))
+plot!(error, times["32"], error_res["32"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[3], " + " , minimizers_short_name[2]))
+plot!(error, times["42"], error_res["42"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[4], " + " , minimizers_short_name[2]))
+plot!(error, times["52"], error_res["52"], xaxis=:log10, yaxis=:log10, label = string(strategies_short_name[5], " + " , minimizers_short_name[2]))
+plot!(error, times["62"], error_res["62"], xaxis=:log10, yaxis=:log10, title = string("Allen Cahn convergence ADAM/LBFGS"), ylabel = "log(error)",xlabel="log(t)", label = string(strategies_short_name[6], " + " , minimizers_short_name[2]))
 ```
 
 ```julia, echo = false

benchmarks/PINNErrorsVsTime/diffusion_et.jmd

@@ -13,9 +13,10 @@ science-guided AI techniques.
 
 ```julia
 using NeuralPDE
-using Integrals, IntegralsCubature, IntegralsCuba
-using OptimizationFlux, ModelingToolkit, Optimization, OptimizationOptimJL
+using Integrals, Cubature, Cuba
+using ModelingToolkit, Optimization, OptimizationOptimJL
 using Lux, Plots
+using OptimizationOptimisers
 using DelimitedFiles
 using QuasiMonteCarlo
 import ModelingToolkit: Interval, infimum, supremum
@@ -60,7 +61,8 @@ function diffusion(strategy, minimizer, maxIters)
     prob_ = discretize(pde_system_, discretization_)
 
     function loss_function_(θ, p)
-        return prob_.f.f(θ, nothing)
+        params = θ.u 
+        return prob_.f.f(params, nothing)
     end
 
     cb_ = function (p,l)
@@ -101,19 +103,19 @@ end
 ```
 
 ```julia
-maxIters = [(5000,5000,5000,5000,5000,5000),(300,300,300,300,300,300)] #iters for ADAM/LBFGS
+maxIters = [(5000,5000,5000,5000,5000),(300,300,300,300,300)] #iters for ADAM/LBFGS
 # maxIters = [(5,5,5,5,5,5),(3,3,3,3,3,3)] #iters for ADAM/LBFGS
 
-strategies = [#NeuralPDE.QuadratureTraining(quadrature_alg = CubaCuhre(), reltol = 1e-4, abstol = 1e-3, maxiters = 10, batch = 10),
-              NeuralPDE.QuadratureTraining(quadrature_alg = HCubatureJL(), reltol = 1e-4, abstol=1e-5, maxiters=100, batch = 0),
-              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLh(), reltol = 1e-4, abstol=1e-5, maxiters=100),
-              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLp(), reltol = 1e-4, abstol=1e-5, maxiters=100),
-              NeuralPDE.GridTraining([0.2,0.1]),
-              NeuralPDE.StochasticTraining(400 ; bcs_points= 50),
-              NeuralPDE.QuasiRandomTraining(400 ; bcs_points= 50)]
+strategies = [
+    NeuralPDE.GridTraining([0.2, 0.1]),
+    NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLh(), reltol = 1e-4, abstol=1e-5, maxiters=1100),
+    NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLp(), reltol = 1e-4, abstol=1e-5, maxiters=1100),
+    NeuralPDE.StochasticTraining(400 ; bcs_points= 50),
+    NeuralPDE.QuasiRandomTraining(400 ; bcs_points= 50)
+]
 
 strategies_short_name = [#"CubaCuhre",
-                        "HCubatureJL",
+                        #"HCubatureJL",
                         "CubatureJLh",
                         "CubatureJLp",
                         #"CubaVegas",
@@ -122,7 +124,7 @@ strategies_short_name = [#"CubaCuhre",
                         "StochasticTraining",
                         "QuasiRandomTraining"]
 
-minimizers = [ADAM(0.001),
+minimizers = [Optimisers.ADAM(0.001),
               #BFGS()]
               LBFGS()]
 
@@ -157,7 +159,6 @@ for min =1:length(minimizers) # minimizer
             push!(times, string(strat,min)        => res[4])
             push!(prediction, string(strat,min)        => res[5])
             push!(losses_res, string(strat,min)        => res[6])
-
       end
 end
 ```
@@ -166,20 +167,17 @@ end
 
 ```julia
 current_label = string(strategies_short_name[1], " + " , minimizers_short_name[1])
-error = Plots.plot(times["11"], error_res["11"], yaxis=:log10, label = current_label)#, xlims = (0,100))#legend = true)#, size=(1200,700))
+error = Plots.plot(times["11"], error_res["11"], yaxis=:log10, label = current_label)
 plot!(error, times["21"], error_res["21"], yaxis=:log10, label = string(strategies_short_name[2], " + " , minimizers_short_name[1]))
 plot!(error, times["31"], error_res["31"], yaxis=:log10, label = string(strategies_short_name[3], " + " , minimizers_short_name[1]))
 plot!(error, times["41"], error_res["41"], yaxis=:log10, label = string(strategies_short_name[4], " + " , minimizers_short_name[1]))
 plot!(error, times["51"], error_res["51"], yaxis=:log10, label = string(strategies_short_name[5], " + " , minimizers_short_name[1]))
-plot!(error, times["61"], error_res["61"], yaxis=:log10, label = string(strategies_short_name[6], " + " , minimizers_short_name[1]))
-
 
 plot!(error, times["12"], error_res["12"], yaxis=:log10, label = string(strategies_short_name[1], " + " , minimizers_short_name[2]))
 plot!(error, times["22"], error_res["22"], yaxis=:log10, label = string(strategies_short_name[2], " + " , minimizers_short_name[2]))
 plot!(error, times["32"], error_res["32"], yaxis=:log10, label = string(strategies_short_name[3], " + " , minimizers_short_name[2]))
 plot!(error, times["42"], error_res["42"], yaxis=:log10, label = string(strategies_short_name[4], " + " , minimizers_short_name[2]))
-plot!(error, times["52"], error_res["52"], yaxis=:log10, label = string(strategies_short_name[5], " + " , minimizers_short_name[2]))
-plot!(error, times["62"], error_res["62"], yaxis=:log10, title = string("Diffusion convergence ADAM/LBFGS"), ylabel = "log(error)",xlabel = "t", label = string(strategies_short_name[6], " + " , minimizers_short_name[2]))
+plot!(error, times["52"], error_res["52"], yaxis=:log10, title = string("Diffusion convergence ADAM/LBFGS"), ylabel = "log(error)", xlabel = "t", label = string(strategies_short_name[5], " + " , minimizers_short_name[2]))
 ```
 
 ```julia, echo = false

benchmarks/PINNErrorsVsTime/hamilton_jacobi_et.jmd

@@ -11,9 +11,10 @@ science-guided AI techniques.
 
 ```julia
 using NeuralPDE
-using Integrals, IntegralsCubature, IntegralsCuba
-using OptimizationFlux, ModelingToolkit, Optimization, OptimizationOptimJL
+using Integrals, Cubature, Cuba
+using ModelingToolkit, Optimization, OptimizationOptimJL
 using Lux, Plots
+using OptimizationOptimisers
 using DelimitedFiles
 using QuasiMonteCarlo
 import ModelingToolkit: Interval, infimum, supremum
@@ -105,21 +106,29 @@ function hamilton_jacobi(strategy, minimizer, maxIters)
     prob_ = discretize(pde_system_, discretization_)
 
     function loss_function_(θ, p)
-        return prob_.f.f(θ, nothing)
+        params = θ.u
+        return prob_.f.f(params, nothing)
     end
 
-    cb_ = function (p, l)
-        deltaT_s = time_ns() #Start a clock when the callback begins, this will evaluate questo misurerà anche il calcolo degli uniform error
-
-        ctime = time_ns() - startTime - timeCounter #This variable is the time to use for the time benchmark plot
-        append!(times, ctime / 10^9) #Conversion nanosec to seconds
-        append!(losses, l)
-        loss_ = loss_function_(p, nothing)
-        append!(error, loss_)
-
-        timeCounter = timeCounter + time_ns() - deltaT_s #timeCounter sums all delays due to the callback functions of the previous iterations
-
-        return false
+    function cb_(p, l)
+        try
+            deltaT_s = time_ns()
+            ctime = time_ns() - startTime - timeCounter
+
+            push!(times, ctime / 1e9)
+            push!(losses, l)
+
+            # Extract parameters for loss calculation
+            params = p.u
+            loss_ = loss_function_(p, nothing)
+            push!(error, loss_)
+
+            timeCounter += time_ns() - deltaT_s
+            return false
+        catch e
+            @warn "Callback error: $e"
+            return false
+        end
     end
 
     @named pde_system = PDESystem(eq, bcs, domains, indvars, depvars)
@@ -146,10 +155,11 @@ end
 maxIters = [(1,1,1,1000,1000,1000,1000),(1,1,1,300,300,300,300)] #iters for ADAM/LBFGS
 # maxIters = [(1,1,1,1,1,2,2),(1,1,1,3,3,3,3)] #iters for ADAM/LBFGS
 
-strategies = [NeuralPDE.QuadratureTraining(quadrature_alg = CubaCuhre(), reltol = 1e-4, abstol = 1e-4, maxiters = 100),
-              NeuralPDE.QuadratureTraining(quadrature_alg = HCubatureJL(), reltol = 1e-4, abstol = 1e-4, maxiters = 100, batch = 0),
-              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLh(), reltol = 1e-4, abstol = 1e-4, maxiters = 100),
-              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLp(), reltol = 1e-4, abstol = 1e-4, maxiters = 100),
+strategies = [NeuralPDE.QuadratureTraining(quadrature_alg = CubaCuhre(), reltol = 1e-4, abstol = 1e-4, maxiters = 1100),
+              #NeuralPDE.QuadratureTraining(quadrature_alg = HCubatureJL(), reltol = 1e-4, abstol = 1e-4, maxiters = 1100, batch = 0),
+              NeuralPDE.GridTraining(0.1),
+              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLh(), reltol = 1e-4, abstol = 1e-4, maxiters = 1100),
+              NeuralPDE.QuadratureTraining(quadrature_alg = CubatureJLp(), reltol = 1e-4, abstol = 1e-4, maxiters = 1100),
               NeuralPDE.GridTraining(0.2),
               NeuralPDE.StochasticTraining(400 ; bcs_points= 50),
               NeuralPDE.QuasiRandomTraining(400 ; bcs_points= 50)]
@@ -162,7 +172,7 @@ strategies_short_name = ["CubaCuhre",
                         "StochasticTraining",
                         "QuasiRandomTraining"]
 
-minimizers = [ADAM(0.005),
+minimizers = [Optimisers.ADAM(0.005),
               #BFGS()]
               LBFGS()]