From 09de3826849228f9d9723fd3ec5cf13a3fb872ed Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 13 Apr 2025 17:09:28 -0400 Subject: [PATCH 1/9] allow relaxation of QuadraticReLU --- src/predictors/ReLU.jl | 16 ++++++++++++++-- test/test_predictors.jl | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/predictors/ReLU.jl b/src/predictors/ReLU.jl index eb080770..6273a8e2 100644 --- a/src/predictors/ReLU.jl +++ b/src/predictors/ReLU.jl @@ -300,7 +300,14 @@ ReLUQuadratic() └ moai_ReLU[2]*moai_z[2] = 0 ``` """ -struct ReLUQuadratic <: AbstractPredictor end +struct ReLUQuadratic <: AbstractPredictor + relax_equality::Bool + relaxation_parameter::Float64 + ReLUQuadratic(; + relax_equality::Bool = false, + relaxation_parameter::Float64 = 0.0, + ) = new(relax_equality, relaxation_parameter) +end function add_predictor( model::JuMP.AbstractModel, @@ -314,6 +321,11 @@ function add_predictor( z = JuMP.@variable(model, [1:m], base_name = "moai_z") _set_bounds_if_finite.(Ref(cons), z, 0, max.(0, -first.(bounds))) append!(cons, JuMP.@constraint(model, x .== y - z)) - append!(cons, JuMP.@constraint(model, y .* z .== 0)) + if predictor.relax_equality + ϵ = predictor.relaxation_parameter + append!(cons, JuMP.@constraint(model, y .* z .<= ϵ)) + else + append!(cons, JuMP.@constraint(model, y .* z .== 0)) + end return y, Formulation(predictor, Any[y; z], cons) end diff --git a/test/test_predictors.jl b/test/test_predictors.jl index f94f152a..efb22bc6 100644 --- a/test/test_predictors.jl +++ b/test/test_predictors.jl @@ -237,6 +237,31 @@ function test_ReLU_Quadratic() return end +function test_ReLU_Quadratic_relaxed() + model = Model(Ipopt.Optimizer) + set_silent(model) + @variable(model, x[1:2]) + f = MathOptAI.ReLUQuadratic(; + relax_equality = true, + relaxation_parameter = 1e-4, + ) + y, formulation = MathOptAI.add_predictor(model, f, x) + # Maximize sum of all variables to exercise the ReLU relaxation + @objective(model, Max, sum(formulation.variables)) + @test length(y) == 2 + @test num_variables(model) == 6 + @test num_constraints(model, AffExpr, MOI.EqualTo{Float64}) == 2 + @test num_constraints(model, QuadExpr, MOI.LessThan{Float64}) == 2 + fix.(x, [-1, 2]) + optimize!(model) + @assert is_solved_and_feasible(model) + # We do not satisfy equality to a tight tolerance + @test !isapprox(value.(y), [0.0, 2.0]; atol = 1e-6) + # But we satisfy equality to a loose tolerance + @test isapprox(value.(y), [0.0, 2.0]; atol = 1e-2) + return +end + function test_Sigmoid() model = Model(Ipopt.Optimizer) set_silent(model) From 6120c5236f7491a8e676b9ca9427f49020294dff Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 13 Apr 2025 17:27:49 -0400 Subject: [PATCH 2/9] format --- src/predictors/ReLU.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/predictors/ReLU.jl b/src/predictors/ReLU.jl index 6273a8e2..42e41bb6 100644 --- a/src/predictors/ReLU.jl +++ b/src/predictors/ReLU.jl @@ -303,10 +303,12 @@ ReLUQuadratic() struct ReLUQuadratic <: AbstractPredictor relax_equality::Bool relaxation_parameter::Float64 - ReLUQuadratic(; + function ReLUQuadratic(; relax_equality::Bool = false, relaxation_parameter::Float64 = 0.0, - ) = new(relax_equality, relaxation_parameter) + ) + return new(relax_equality, relaxation_parameter) + end end function add_predictor( From 4f78d783ffea73ea3a870d297b8a24c4359034e3 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 13 Apr 2025 18:19:35 -0400 Subject: [PATCH 3/9] simplify state on ReLUQuadratic --- src/predictors/ReLU.jl | 14 ++++++-------- test/test_predictors.jl | 6 ++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/predictors/ReLU.jl b/src/predictors/ReLU.jl index 42e41bb6..5f2bbe6b 100644 --- a/src/predictors/ReLU.jl +++ b/src/predictors/ReLU.jl @@ -301,13 +301,11 @@ ReLUQuadratic() ``` """ struct ReLUQuadratic <: AbstractPredictor - relax_equality::Bool - relaxation_parameter::Float64 + relaxation_parameter::Union{Nothing,Float64} function ReLUQuadratic(; - relax_equality::Bool = false, - relaxation_parameter::Float64 = 0.0, + relaxation_parameter::Union{Nothing,Float64} = nothing, ) - return new(relax_equality, relaxation_parameter) + return new(relaxation_parameter) end end @@ -323,11 +321,11 @@ function add_predictor( z = JuMP.@variable(model, [1:m], base_name = "moai_z") _set_bounds_if_finite.(Ref(cons), z, 0, max.(0, -first.(bounds))) append!(cons, JuMP.@constraint(model, x .== y - z)) - if predictor.relax_equality + if predictor.relaxation_parameter === nothing + append!(cons, JuMP.@constraint(model, y .* z .== 0)) + else ϵ = predictor.relaxation_parameter append!(cons, JuMP.@constraint(model, y .* z .<= ϵ)) - else - append!(cons, JuMP.@constraint(model, y .* z .== 0)) end return y, Formulation(predictor, Any[y; z], cons) end diff --git a/test/test_predictors.jl b/test/test_predictors.jl index efb22bc6..199c63ce 100644 --- a/test/test_predictors.jl +++ b/test/test_predictors.jl @@ -225,6 +225,7 @@ function test_ReLU_Quadratic() set_silent(model) @variable(model, x[1:2]) f = MathOptAI.ReLUQuadratic() + @test f.relaxation_parameter === nothing y, formulation = MathOptAI.add_predictor(model, f, x) @test length(y) == 2 @test num_variables(model) == 6 @@ -241,10 +242,7 @@ function test_ReLU_Quadratic_relaxed() model = Model(Ipopt.Optimizer) set_silent(model) @variable(model, x[1:2]) - f = MathOptAI.ReLUQuadratic(; - relax_equality = true, - relaxation_parameter = 1e-4, - ) + f = MathOptAI.ReLUQuadratic(; relaxation_parameter = 1e-4) y, formulation = MathOptAI.add_predictor(model, f, x) # Maximize sum of all variables to exercise the ReLU relaxation @objective(model, Max, sum(formulation.variables)) From 5e1a4299aa8feededd12d7e950b6bcb6cfeef708 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 13 Apr 2025 18:28:29 -0400 Subject: [PATCH 4/9] update docstring --- src/predictors/ReLU.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/predictors/ReLU.jl b/src/predictors/ReLU.jl index 5f2bbe6b..8b948965 100644 --- a/src/predictors/ReLU.jl +++ b/src/predictors/ReLU.jl @@ -244,7 +244,7 @@ function add_predictor( end """ - ReLUQuadratic() <: AbstractPredictor + ReLUQuadratic(; relaxation_parameter = nothing) <: AbstractPredictor An [`AbstractPredictor`](@ref) that represents the relationship: ```math @@ -269,7 +269,7 @@ julia> model = Model(); julia> @variable(model, -1 <= x[i in 1:2] <= i); julia> f = MathOptAI.ReLUQuadratic() -ReLUQuadratic() +ReLUQuadratic(nothing) julia> y, formulation = MathOptAI.add_predictor(model, f, x); @@ -279,7 +279,7 @@ julia> y moai_ReLU[2] julia> formulation -ReLUQuadratic() +ReLUQuadratic(nothing) ├ variables [4] │ ├ moai_ReLU[1] │ ├ moai_ReLU[2] From 8000d8dc1f092e6e5228640417bcc2580034fcc1 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 13 Apr 2025 18:40:55 -0400 Subject: [PATCH 5/9] update docstring --- src/predictors/Pipeline.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/predictors/Pipeline.jl b/src/predictors/Pipeline.jl index ef3d1ae6..679bfaec 100644 --- a/src/predictors/Pipeline.jl +++ b/src/predictors/Pipeline.jl @@ -42,7 +42,7 @@ Affine(A, b) [input: 2, output: 1] │ └ moai_Affine[1] └ constraints [1] └ x[1] + 2 x[2] - moai_Affine[1] = 0 -ReLUQuadratic() +ReLUQuadratic(nothing) ├ variables [2] │ ├ moai_ReLU[1] │ └ moai_z[1] From f1bdb50cc839ce2d60be7e6185bd9eda4156fc29 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 13 Apr 2025 20:42:26 -0400 Subject: [PATCH 6/9] update docstring --- src/predictors/Pipeline.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/predictors/Pipeline.jl b/src/predictors/Pipeline.jl index 679bfaec..8baa5028 100644 --- a/src/predictors/Pipeline.jl +++ b/src/predictors/Pipeline.jl @@ -28,7 +28,7 @@ julia> f = MathOptAI.Pipeline( ) Pipeline with layers: * Affine(A, b) [input: 2, output: 1] - * ReLUQuadratic() + * ReLUQuadratic(nothing) julia> y, formulation = MathOptAI.add_predictor(model, f, x); From 02cced5bb24480ac58320b25f129486b9fd32b0b Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 13 Apr 2025 21:03:36 -0400 Subject: [PATCH 7/9] update docstrings --- ext/MathOptAIFluxExt.jl | 2 +- ext/MathOptAILuxExt.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/MathOptAIFluxExt.jl b/ext/MathOptAIFluxExt.jl index 57a2fb4c..f019d7bb 100644 --- a/ext/MathOptAIFluxExt.jl +++ b/ext/MathOptAIFluxExt.jl @@ -179,7 +179,7 @@ julia> MathOptAI.build_predictor( ) Pipeline with layers: * Affine(A, b) [input: 1, output: 16] - * ReLUQuadratic() + * ReLUQuadratic(nothing) * Affine(A, b) [input: 16, output: 1] ``` """ diff --git a/ext/MathOptAILuxExt.jl b/ext/MathOptAILuxExt.jl index 77a41796..fcb0cd47 100644 --- a/ext/MathOptAILuxExt.jl +++ b/ext/MathOptAILuxExt.jl @@ -148,7 +148,7 @@ julia> MathOptAI.build_predictor( ) Pipeline with layers: * Affine(A, b) [input: 1, output: 16] - * ReLUQuadratic() + * ReLUQuadratic(nothing) * Affine(A, b) [input: 16, output: 1] ``` """ From 0e7892a0113d487ec5553d9b295d806380eaceb5 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 13 Apr 2025 21:09:07 -0400 Subject: [PATCH 8/9] update docstring --- src/predictors/ReLU.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/predictors/ReLU.jl b/src/predictors/ReLU.jl index 8b948965..ad0d73f7 100644 --- a/src/predictors/ReLU.jl +++ b/src/predictors/ReLU.jl @@ -258,6 +258,15 @@ y \\cdot z = 0 \\\\ y, z \\ge 0 \\end{aligned} ``` +If `relaxation_parameter` is set to a value `ϵ`, the constraints become: +```math +\\begin{aligned} +x = y - z \\\\ +y \\cdot z \\leq \\epsilon \\\\ +y, z \\ge 0 +\\end{aligned} +``` + ## Example From 7afb0891b18c2c3374ec85e0d8ff8fe56aa4ed0e Mon Sep 17 00:00:00 2001 From: Oscar Dowson Date: Mon, 14 Apr 2025 13:34:05 +1200 Subject: [PATCH 9/9] Update src/predictors/ReLU.jl --- src/predictors/ReLU.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/predictors/ReLU.jl b/src/predictors/ReLU.jl index ad0d73f7..9f900a18 100644 --- a/src/predictors/ReLU.jl +++ b/src/predictors/ReLU.jl @@ -314,6 +314,7 @@ struct ReLUQuadratic <: AbstractPredictor function ReLUQuadratic(; relaxation_parameter::Union{Nothing,Float64} = nothing, ) + @assert something(relaxation_parameter, 0.0) >= 0.0 return new(relaxation_parameter) end end