From 4fef1eae2cd65f4099545982ceca61328481b7be Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 21 Nov 2025 15:50:19 +0100 Subject: [PATCH 001/135] initial code for GCP computation --- src/plans/box_plan.jl | 253 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 src/plans/box_plan.jl diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl new file mode 100644 index 0000000000..fd4ce81f06 --- /dev/null +++ b/src/plans/box_plan.jl @@ -0,0 +1,253 @@ + + +mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ + TDU <: QuasiNewtonLimitedMemoryDirectionUpdate + } <: AbstractQuasiNewtonDirectionUpdate + qn_du::TDU +end + +abstract type AbstractFPFPPUpdater end + +init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, d, ha) = fpfpp_upd + +struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end + +get_default_fpfpp_updater(::MatrixHessianApproximation) = GenericFPFPPUpdater() + +struct LimitedMemoryFPFPPUpdater{TV<:AbstractVector} <: AbstractFPFPPUpdater + p_s::TV + p_y::TV + c_s::TV + c_y::TV +end + +function get_default_fpfpp_updater(ha::LimitedMemoryHessianApproximation) + return LimitedMemoryFPFPPUpdater(similar(ha.ρ), similar(ha.ρ), similar(ha.ρ), similar(ha.ρ)) +end + +function (::GenericFPFPPUpdater)(M::AbstractManifold, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hess_val_eb(ha, b, z)) + f_double_prime = old_f_double_prime + (2 * -db * hess_val_eb(ha, b, d_old)) + db^2 * hess_val_eb(ha, b) + + return f_prime, f_double_prime +end + +function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater, d, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + fpfpp_upd.c_s .= 0 + fpfpp_upd.c_y .= 0 + ii = 1 + for i in eachindex(ha.ρ) + if iszero(ha.ρ[i]) + continue + end + + fpfpp_upd.p_s[ii] = ha.current_scale * inner(M, ha.p, ha.memory_s[i], d) + fpfpp_upd.p_y[ii] = inner(M, ha.p, ha.memory_y[i], d) + ii += 1 + end + return fpfpp_upd +end + +function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, old_f_prime, old_f_double_prime, dt, db, gb, ha::LimitedMemoryHessianApproximation, b, z, d_old) + + m = length(ha.memory_s) + num_nonzero_rho = count(!iszero, ha.ρ) + + iss_eb_z = get_at_bound_index(M, z, b) + iss_eb_d = get_at_bound_index(M, d_old, b) + + ii = 1 + for i in 1:m + if iszero(ha.ρ[i]) + continue + end + # setting _X to w_b from the paper + ha.coords_Yk_X[ii] = get_at_bound_index(M, ha.memory_y[i], b) + ha.coords_Sk_X[ii] = ha.current_scale * get_at_bound_index(M, ha.memory_s[i], b) + + ii += 1 + end + + coords_Yk_eb = view(ha.coords_Yk_X, 1:num_nonzero_rho) + coords_Sk_eb = view(ha.coords_Sk_X, 1:num_nonzero_rho) + + coords_cy = view(fpfpp_upd.c_y, 1:num_nonzero_rho) + coords_cs = view(fpfpp_upd.c_s, 1:num_nonzero_rho) + coords_py = view(fpfpp_upd.p_y, 1:num_nonzero_rho) + coords_ps = view(fpfpp_upd.p_s, 1:num_nonzero_rho) + + coords_cy .+= dt .* coords_py + coords_cs .+= dt .* coords_ps + + eb_B_z = hess_val_from_wmwt_coords(ha, iss_eb_z, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) + + f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + eb_B_z) + eb_B_d = hess_val_from_wmwt_coords(ha, iss_eb_d, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) + + f_double_prime = old_f_double_prime - 2 * db * eb_B_d + db^2 * hess_val_eb(ha, b) + + coords_py .-= db .* coords_Yk_eb + coords_ps .-= db .* coords_Sk_eb + + return f_prime, f_double_prime +end + +""" + get_bounds_index(::HyperrectangleProduct) + +Get the bound indices of manifold `M`. Standard manifolds don't have bounds, so +`Base.OneTo(1)` is returned. +""" +get_bounds_index(M::Hyperrectangle) = eachindex(M.lb) +get_bounds_index(M::ProductManifold) = get_bounds_index(M.manifolds[1]) + +""" + get_bound_t(M::AbstractManifold, x, d, i) + +Get the upper bound on moving in direction `d` from point `p` on manifold `M`, for the +bound index `i`. +""" +function get_bound_t(M::Hyperrectangle, p, d, i) + if d[i] > 0 + return (M.ub[i] - p[i]) / d[i] + elseif d[i] < 0 + return (M.lb[i] - p[i]) / d[i] + else + return Inf + end +end +function get_bound_t(M::ProductManifold, p, d, i) + return get_bound_t(M.manifolds[1], submanifold_component(M, p, Val(1)), submanifold_component(M, d, Val(1)), i) +end +function set_bound_t_at_index!(M::ProductManifold, p_cp, t, d, i) + set_bound_t_at_index!(M.manifolds[1], submanifold_component(M, p_cp, Val(1)), t, d, i) +end + +function set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) + p_cp[i] += t * d[i] +end + +function set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) + p_cp[i] = d[i] > 0 ? M.ub[i] : M.lb[i] + d[i] = 0 +end + +function set_bound_at_index!(M::ProductManifold, p_cp, d, i) + set_bound_at_index!(M.manifolds[1], submanifold_component(M, p_cp, Val(1)), submanifold_component(M, d, Val(1)), i) +end + + +struct GCPFinder{TM<:AbstractManifold,TX,THA,TFU<:AbstractFPFPPUpdater} + M::TM + Y_tmp::TX + d_old::TX + ha::THA + fpfpp_updater::TFU +end + +function GCPFinder(M::AbstractManifold, p, ha; fpfpp_updater=get_default_fpfpp_updater(ha)) + return GCPFinder(M, zero_vector(M, p), zero_vector(M, p), ha, fpfpp_updater) +end + +""" + find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) + +Find generalized Cauchy point looking from point `p` in direction `d` and save it to `p_cp`. +Gradient of the objective at `p` is `X`. +""" +function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) + + M = gcp.M + copyto!(M, p_cp, p) + zero_vector!(M, gcp.Y_tmp, p) + + bounds_indices = get_bounds_index(M) + TInd = eltype(bounds_indices) + + t = Dict{TInd,Float64}((k, Inf) for k in bounds_indices) + + F_list = Tuple{Float64,TInd}[] + sizehint!(F_list, length(bounds_indices)+1) + + for i in bounds_indices + t[i] = get_bound_t(M, p, d, i) + + if t[i] > 0 + push!(F_list, (t[i], i)) + end + + if M isa ProductManifold + # push also `t` corresponding to max_stepsize if it is considered in the manifold + M2 = M.manifolds[2] + p2 = submanifold_component(M, p, Val(2)) + max_step = Manopt.max_stepsize(M2, p2) + if isfinite(max_step) + d2 = submanifold_component(M, d, Val(2)) + tms = max_step / norm(M2, p2, d2) + push!(F_list, (tms, -1)) + end + end + end + + if isempty(F_list) + @warn "We can't go in the selected direction" + return false + end + + F = BinaryHeap(Base.By(first), F_list) + + f_prime = inner(M, p, X, d) + f_double_prime = hess_val(ha, d) + + if iszero(f_prime) || iszero(f_double_prime) + return false + end + + dt_min = -f_prime / f_double_prime + t_old = 0.0 + + t_current, b = pop!(F) + dt = t_current - t_old + + init_updater!(M, gcp.fpfpp_upd, d, ha) + # b can be -1 if it corresponds to the max stepsize limit on the manifold part + while dt_min > dt && b != -1 + gcp.Y_tmp .+= dt .* d + copyto!(M, gcp.d_old, d) + set_bound_at_index!(M, p_cp, d, b) + + gb = get_at_bound_index(M, grad, b) + db = get_at_bound_index(M, gcp.d_old, b) + + f_prime, f_double_prime = gcp.fpfpp_upd(M, f_prime, f_double_prime, dt, db, gb, ha, b, gcp.Y_tmp, gcp.d_old) + t_old = t_current + + # If f_prime is 0, we've found the local minimizer (GCP) + if iszero(f_prime) || iszero(f_double_prime) + # It means that GCP is at the beginning of the t_current, so we want to set dt_min to 0 (stay in the point) + dt_min = 0.0 + break + end + + dt_min = -f_prime / f_double_prime + + if isempty(F) + break + end + + t_current, b = pop!(F) + dt = t_current - t_old + end + + dt_min = max(dt_min, 0.0) + t_old = t_old + dt_min + + for i in bounds_indices + if t[i] >= t_current + set_bound_t_at_index!(M, p_cp, t_old, d, i) + end + end + + return true +end + From 8bdd5b1aaf22978c4e50480451477aa0cde200bf Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 21 Nov 2025 15:53:58 +0100 Subject: [PATCH 002/135] formatting --- src/plans/box_plan.jl | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index fd4ce81f06..5fed0870db 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -1,7 +1,5 @@ - - mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ - TDU <: QuasiNewtonLimitedMemoryDirectionUpdate + TDU <: QuasiNewtonLimitedMemoryDirectionUpdate, } <: AbstractQuasiNewtonDirectionUpdate qn_du::TDU end @@ -14,7 +12,7 @@ struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end get_default_fpfpp_updater(::MatrixHessianApproximation) = GenericFPFPPUpdater() -struct LimitedMemoryFPFPPUpdater{TV<:AbstractVector} <: AbstractFPFPPUpdater +struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater p_s::TV p_y::TV c_s::TV @@ -28,7 +26,7 @@ end function (::GenericFPFPPUpdater)(M::AbstractManifold, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hess_val_eb(ha, b, z)) f_double_prime = old_f_double_prime + (2 * -db * hess_val_eb(ha, b, d_old)) + db^2 * hess_val_eb(ha, b) - + return f_prime, f_double_prime end @@ -88,7 +86,7 @@ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, old_f_prime coords_py .-= db .* coords_Yk_eb coords_ps .-= db .* coords_Sk_eb - + return f_prime, f_double_prime end @@ -121,23 +119,27 @@ function get_bound_t(M::ProductManifold, p, d, i) end function set_bound_t_at_index!(M::ProductManifold, p_cp, t, d, i) set_bound_t_at_index!(M.manifolds[1], submanifold_component(M, p_cp, Val(1)), t, d, i) + return p_cp end function set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) p_cp[i] += t * d[i] + return p_cp end function set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) p_cp[i] = d[i] > 0 ? M.ub[i] : M.lb[i] d[i] = 0 + return p_cp end function set_bound_at_index!(M::ProductManifold, p_cp, d, i) set_bound_at_index!(M.manifolds[1], submanifold_component(M, p_cp, Val(1)), submanifold_component(M, d, Val(1)), i) + return p_cp end -struct GCPFinder{TM<:AbstractManifold,TX,THA,TFU<:AbstractFPFPPUpdater} +struct GCPFinder{TM <: AbstractManifold, TX, THA, TFU <: AbstractFPFPPUpdater} M::TM Y_tmp::TX d_old::TX @@ -145,7 +147,7 @@ struct GCPFinder{TM<:AbstractManifold,TX,THA,TFU<:AbstractFPFPPUpdater} fpfpp_updater::TFU end -function GCPFinder(M::AbstractManifold, p, ha; fpfpp_updater=get_default_fpfpp_updater(ha)) +function GCPFinder(M::AbstractManifold, p, ha; fpfpp_updater = get_default_fpfpp_updater(ha)) return GCPFinder(M, zero_vector(M, p), zero_vector(M, p), ha, fpfpp_updater) end @@ -160,14 +162,14 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) M = gcp.M copyto!(M, p_cp, p) zero_vector!(M, gcp.Y_tmp, p) - + bounds_indices = get_bounds_index(M) TInd = eltype(bounds_indices) - t = Dict{TInd,Float64}((k, Inf) for k in bounds_indices) + t = Dict{TInd, Float64}((k, Inf) for k in bounds_indices) - F_list = Tuple{Float64,TInd}[] - sizehint!(F_list, length(bounds_indices)+1) + F_list = Tuple{Float64, TInd}[] + sizehint!(F_list, length(bounds_indices) + 1) for i in bounds_indices t[i] = get_bound_t(M, p, d, i) @@ -208,7 +210,7 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) t_current, b = pop!(F) dt = t_current - t_old - + init_updater!(M, gcp.fpfpp_upd, d, ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 @@ -218,7 +220,7 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) gb = get_at_bound_index(M, grad, b) db = get_at_bound_index(M, gcp.d_old, b) - + f_prime, f_double_prime = gcp.fpfpp_upd(M, f_prime, f_double_prime, dt, db, gb, ha, b, gcp.Y_tmp, gcp.d_old) t_old = t_current @@ -230,7 +232,7 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) end dt_min = -f_prime / f_double_prime - + if isempty(F) break end @@ -250,4 +252,3 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) return true end - From d4288074f2114d05195333654fb3fc1c8e6bdc09 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 21 Nov 2025 16:20:23 +0100 Subject: [PATCH 003/135] expand a little --- src/plans/box_plan.jl | 79 +++++++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 5fed0870db..88201bb5f9 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -1,17 +1,59 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ TDU <: QuasiNewtonLimitedMemoryDirectionUpdate, + TM <: AbstractManifold, + F <: Real, + THM <: AbstractMatrix, + V <: AbstractVector, } <: AbstractQuasiNewtonDirectionUpdate + # this approximates inverse Hessian qn_du::TDU + + # fields for approximating the Hessian + current_scale::F + M_11::THM + M_21::THM + M_22::THM + # tolerance for detecting zero inner products between y and s + iszero_abstol::F + # buffer for calculating stuff + coords_Sk_X::V + coords_Sk_Y::V + coords_Yk_X::V + coords_Yk_Y::V end +""" + abstract type AbstractFPFPPUpdater end + +Abstract type for methods that calculate f' and f'' in the GCP calculation in subsequent +line segments in `GCPFinder`. +""" abstract type AbstractFPFPPUpdater end +""" + init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, d, ha) + +Method for initialization of `AbstractFPFPPUpdater` `fpfpp_upd` just before the loop +that examines subsequent intervals for GCP. By default it does nothing. +""" init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, d, ha) = fpfpp_upd +""" + struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end + +Generic f' and f'' calculation that only relies on `hess_val_eb` but is relatively slow for +high-dimensional domains. +""" struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end get_default_fpfpp_updater(::MatrixHessianApproximation) = GenericFPFPPUpdater() +""" + struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater + +f' and f'' calculation that is optimized for `QuasiNewtonLimitedMemoryBoxDirectionUpdate`. +It relies on a specific Hessian structure. +""" struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater p_s::TV p_y::TV @@ -152,13 +194,14 @@ function GCPFinder(M::AbstractManifold, p, ha; fpfpp_updater = get_default_fpfpp end """ - find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) + find_gcp!(gcp::GCPFinder, p_cp, p, d, X) Find generalized Cauchy point looking from point `p` in direction `d` and save it to `p_cp`. Gradient of the objective at `p` is `X`. -""" -function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) +The function returns `true` if the point was found and `false` otherwise. +""" +function find_gcp!(gcp::GCPFinder, p_cp, p, d, X) M = gcp.M copyto!(M, p_cp, p) zero_vector!(M, gcp.Y_tmp, p) @@ -177,18 +220,6 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) if t[i] > 0 push!(F_list, (t[i], i)) end - - if M isa ProductManifold - # push also `t` corresponding to max_stepsize if it is considered in the manifold - M2 = M.manifolds[2] - p2 = submanifold_component(M, p, Val(2)) - max_step = Manopt.max_stepsize(M2, p2) - if isfinite(max_step) - d2 = submanifold_component(M, d, Val(2)) - tms = max_step / norm(M2, p2, d2) - push!(F_list, (tms, -1)) - end - end end if isempty(F_list) @@ -196,10 +227,22 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) return false end + if M isa ProductManifold + # push also `t` corresponding to max_stepsize if it is considered in the manifold + M2 = M.manifolds[2] + p2 = submanifold_component(M, p, Val(2)) + max_step = Manopt.max_stepsize(M2, p2) + if isfinite(max_step) + d2 = submanifold_component(M, d, Val(2)) + tms = max_step / norm(M2, p2, d2) + push!(F_list, (tms, -1)) + end + end + F = BinaryHeap(Base.By(first), F_list) f_prime = inner(M, p, X, d) - f_double_prime = hess_val(ha, d) + f_double_prime = hess_val(gcp.ha, d) if iszero(f_prime) || iszero(f_double_prime) return false @@ -211,7 +254,7 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) t_current, b = pop!(F) dt = t_current - t_old - init_updater!(M, gcp.fpfpp_upd, d, ha) + init_updater!(M, gcp.fpfpp_upd, d, gcp.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 gcp.Y_tmp .+= dt .* d @@ -221,7 +264,7 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X, ha) gb = get_at_bound_index(M, grad, b) db = get_at_bound_index(M, gcp.d_old, b) - f_prime, f_double_prime = gcp.fpfpp_upd(M, f_prime, f_double_prime, dt, db, gb, ha, b, gcp.Y_tmp, gcp.d_old) + f_prime, f_double_prime = gcp.fpfpp_upd(M, f_prime, f_double_prime, dt, db, gb, gcp.ha, b, gcp.Y_tmp, gcp.d_old) t_old = t_current # If f_prime is 0, we've found the local minimizer (GCP) From 435f2a6018250fd61a1e9e02612bda7c3da9cf1f Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 27 Nov 2025 16:02:15 +0100 Subject: [PATCH 004/135] add a few more methods --- ext/ManoptManifoldsExt/manifold_functions.jl | 24 ++ src/plans/box_plan.jl | 264 +++++++++++++++++-- src/plans/plan.jl | 2 + 3 files changed, 265 insertions(+), 25 deletions(-) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index dc81073f89..6fa901addd 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -8,6 +8,19 @@ Manopt.default_point_distance(::Euclidean, p) = norm(p, Inf) Manopt.default_vector_norm(::Euclidean, p, X) = norm(p, Inf) + +Manopt.get_bounds_index(M::Hyperrectangle) = eachindex(M.lb) + +function Manopt.get_bound_t(M::Hyperrectangle, p, d, i) + if d[i] > 0 + return (M.ub[i] - p[i]) / d[i] + elseif d[i] < 0 + return (M.lb[i] - p[i]) / d[i] + else + return Inf + end +end + """ max_stepsize(M::TangentBundle, p) @@ -169,3 +182,14 @@ function reflect!( X .*= -1 return retract!(M, q, p, X, retraction_method) end + +function Manopt.set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) + p_cp[i] += t * d[i] + return p_cp +end + +function Manopt.set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) + p_cp[i] = d[i] > 0 ? M.ub[i] : M.lb[i] + d[i] = 0 + return p_cp +end diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 88201bb5f9..d087ed93fd 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -22,6 +22,239 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ coords_Yk_Y::V end +function reset_update!(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + initialize_update!(ha.qn_du) + return ha +end + +@doc raw""" + hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, X) + +Compute $⟨X, B X⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`. +""" +hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, X) = hess_val_single_pass(gh, X) + +@doc raw""" + hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b) + +Compute $⟨X, B X⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`, and `X` is the +unit vector along index `b`. +""" +hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b) = hess_val_single_pass_eb(gh, b) + +@doc raw""" + hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, Y) + +Compute $⟨X, B Y⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`, where `X` is the +unit vector pointing at index `b`. +""" +hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, Y) = hess_val_single_pass_eb(gh, b, Y) + +function set_M_current_scale!(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + M = gh.M + p = gh.p + m = length(gh.qn_du.memory_s) + for i in m:-1:1 + # what if division by zero happened here, setting to zero ignores this in the next step + # pre-compute in case inner is expensive + v = inner(M, p, gh.qn_du.memory_s[i], gh.qn_du.memory_y[i]) + if isnan(v) + println("NaN in memory") + @show gh.qn_du.memory_s[i] + @show gh.qn_du.memory_y[i] + end + if v < gh.iszero_abstol + # The inner products ⟨s_i,y_i⟩ ≈ 0, i=$i, ignoring summand in approximation. + # (s, y) pairs with negative inner product are broken anyway, so we can reject them here + gh.ρ[i] = zero(eltype(gh.ρ)) + else + gh.ρ[i] = 1 / v + # it's so close to zero that we can skip it + if abs(gh.ρ[i]) < gh.iszero_abstol + gh.ρ[i] = zero(eltype(gh.ρ)) + end + end + end + last_safe_index = -1 + for i in eachindex(gh.ρ) + if abs(gh.ρ[i]) > 0 + last_safe_index = i + end + end + + if (last_safe_index == -1) + # All memory yield zero inner products + gh.current_scale = gh.initial_scale + gh.M_11 = fill(0.0, 0, 0) + gh.M_21 = fill(0.0, 0, 0) + gh.M_22 = fill(0.0, 0, 0) + return gh + end + + invA = Diagonal([-ri for ri in gh.ρ if !iszero(ri)]) + num_nonzero_rho = count(!iszero, gh.ρ) + + Lk = LowerTriangular(zeros(num_nonzero_rho, num_nonzero_rho)) + + # total scaling factor for the initial Hessian + gh.current_scale = (gh.ρ[last_safe_index] * norm(M, p, gh.qn_du.memory_y[last_safe_index])^2) / gh.initial_scale + + tsksk = Symmetric(zeros(num_nonzero_rho, num_nonzero_rho)) + ii = 1 + # fill Dk and Lk + for i in 1:m + if iszero(gh.ρ[i]) + continue + end + jj = 1 + for j in 1:m + if iszero(gh.ρ[j]) + continue + end + if jj < ii + Lk[ii, jj] = inner(M, p, gh.qn_du.memory_s[i], gh.qn_du.memory_y[j]) + end + if ii <= jj + tsksk.data[ii, jj] = inner(M, p, gh.qn_du.memory_s[i], gh.qn_du.memory_s[j]) + end + jj += 1 + end + ii += 1 + end + tsksk.data .*= gh.current_scale + + # matrix inversion using the blockwise formula for speed + # Schur complement of -Dk is the only non-diagonal matrix we actually need to inverse in this step + W1 = Lk * invA + W2 = W1 * Lk' + + gh.M_22 = inv(Symmetric(tsksk - W2)) + W3 = gh.M_22 * W1 + W4 = W1' * W3 + + gh.M_11 = invA + W4 + gh.M_21 = -W3 + + return gh +end + +function hess_val_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) + result = gh.current_scale * iss + if length(cy1) == 0 + return result + end + result -= dot(cy1, gh.M_11, cy2) + result -= 2 * dot(cs1, gh.M_21, cy2) + result -= dot(cs1, gh.M_22, cs2) + + return result +end + +function hess_val_single_pass(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, X) + M = gh.M + p = gh.p + m = length(gh.qn_du.memory_s) + num_nonzero_rho = count(!iszero, gh.ρ) + + normX_sqr = norm(M, p, X)^2 + + if m == 0 || num_nonzero_rho == 0 + return gh.initial_scale \ normX_sqr + end + + ii = 1 + for i in 1:m + if iszero(gh.ρ[i]) + continue + end + gh.coords_Yk_X[ii] = inner(M, p, gh.qn_du.memory_y[i], X) + gh.coords_Sk_X[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], X) + + ii += 1 + end + coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) + coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) + + return hess_val_from_wmwt_coords(gh, normX_sqr, coords_Yk, coords_Sk, coords_Yk, coords_Sk) +end + +function hess_val_single_pass_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b) + M = gh.M + p = gh.p + m = length(gh.qn_du.memory_s) + num_nonzero_rho = count(!iszero, gh.ρ) + + if m == 0 || num_nonzero_rho == 0 + return inv(gh.initial_scale) + end + + ii = 1 + for i in 1:m + if iszero(gh.ρ[i]) + continue + end + gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) + gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) + + ii += 1 + end + coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) + coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) + + return hess_val_from_wmwt_coords(gh, one(eltype(gh.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) +end + +function hess_val_single_pass_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, Y) + M = gh.M + p = gh.p + m = length(gh.qn_du.memory_s) + num_nonzero_rho = count(!iszero, gh.ρ) + + Yb = get_at_bound_index(M, Y, b) + if m == 0 || num_nonzero_rho == 0 + return gh.initial_scale * Yb + end + + ii = 1 + for i in 1:m + if iszero(gh.ρ[i]) + continue + end + gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) + gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) + + gh.coords_Yk_Y[ii] = inner(M, p, gh.qn_du.memory_y[i], Y) + gh.coords_Sk_Y[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], Y) + ii += 1 + end + coords_Yk_X = view(gh.coords_Yk_X, 1:num_nonzero_rho) + coords_Yk_Y = view(gh.coords_Yk_Y, 1:num_nonzero_rho) + coords_Sk_X = view(gh.coords_Sk_X, 1:num_nonzero_rho) + coords_Sk_Y = view(gh.coords_Sk_Y, 1:num_nonzero_rho) + + return hess_val_from_wmwt_coords(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) +end + +@doc raw""" + update_hessian!(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, p) + +Update Hessian approximation `gh` by moving it to point `p` and updating the stored `s` and +`y` vectors. +""" +function update_hessian!( + gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, + mp::AbstractManoptProblem, + st::AbstractManoptSolverState, + p_old, + k::Int, + ) + (capacity(gh.qn_du.memory_s) == 0) && return gh + update_hessian!(gh.qn_du, mp, st, p_old, k) + set_M_current_scale!(gh) + return gh +end + + """ abstract type AbstractFPFPPUpdater end @@ -46,7 +279,7 @@ high-dimensional domains. """ struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end -get_default_fpfpp_updater(::MatrixHessianApproximation) = GenericFPFPPUpdater() +get_default_fpfpp_updater(::AbstractQuasiNewtonDirectionUpdate) = GenericFPFPPUpdater() """ struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater @@ -61,7 +294,7 @@ struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater c_y::TV end -function get_default_fpfpp_updater(ha::LimitedMemoryHessianApproximation) +function get_default_fpfpp_updater(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) return LimitedMemoryFPFPPUpdater(similar(ha.ρ), similar(ha.ρ), similar(ha.ρ), similar(ha.ρ)) end @@ -88,7 +321,7 @@ function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater return fpfpp_upd end -function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, old_f_prime, old_f_double_prime, dt, db, gb, ha::LimitedMemoryHessianApproximation, b, z, d_old) +function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, old_f_prime, old_f_double_prime, dt, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old) m = length(ha.memory_s) num_nonzero_rho = count(!iszero, ha.ρ) @@ -133,12 +366,12 @@ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, old_f_prime end """ - get_bounds_index(::HyperrectangleProduct) + get_bounds_index(::AbstractManifold) Get the bound indices of manifold `M`. Standard manifolds don't have bounds, so `Base.OneTo(1)` is returned. """ -get_bounds_index(M::Hyperrectangle) = eachindex(M.lb) +get_bounds_index(M::AbstractManifold) get_bounds_index(M::ProductManifold) = get_bounds_index(M.manifolds[1]) """ @@ -147,15 +380,7 @@ get_bounds_index(M::ProductManifold) = get_bounds_index(M.manifolds[1]) Get the upper bound on moving in direction `d` from point `p` on manifold `M`, for the bound index `i`. """ -function get_bound_t(M::Hyperrectangle, p, d, i) - if d[i] > 0 - return (M.ub[i] - p[i]) / d[i] - elseif d[i] < 0 - return (M.lb[i] - p[i]) / d[i] - else - return Inf - end -end +get_bound_t(M::AbstractManifold, p, d, i) function get_bound_t(M::ProductManifold, p, d, i) return get_bound_t(M.manifolds[1], submanifold_component(M, p, Val(1)), submanifold_component(M, d, Val(1)), i) end @@ -164,17 +389,6 @@ function set_bound_t_at_index!(M::ProductManifold, p_cp, t, d, i) return p_cp end -function set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) - p_cp[i] += t * d[i] - return p_cp -end - -function set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) - p_cp[i] = d[i] > 0 ? M.ub[i] : M.lb[i] - d[i] = 0 - return p_cp -end - function set_bound_at_index!(M::ProductManifold, p_cp, d, i) set_bound_at_index!(M.manifolds[1], submanifold_component(M, p_cp, Val(1)), submanifold_component(M, d, Val(1)), i) return p_cp diff --git a/src/plans/plan.jl b/src/plans/plan.jl index 76d32e8de7..1879940d0a 100644 --- a/src/plans/plan.jl +++ b/src/plans/plan.jl @@ -162,6 +162,8 @@ include("higher_order_primal_dual_plan.jl") include("stochastic_gradient_plan.jl") +include("box_plan.jl") + include("embedded_objective.jl") include("scaled_objective.jl") From 2168761dbed0c3825f976d541e66f985e1b22749 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 28 Nov 2025 13:36:28 +0100 Subject: [PATCH 005/135] Byrd's way of handling non-positive-definite (s, y) pairs in limited memory quasi-Newton --- Changelog.md | 7 +++++ docs/src/references.bib | 14 ++++++++++ src/plans/box_plan.jl | 9 ++++--- src/plans/quasi_newton_plan.jl | 31 +++++++++++++++++++-- src/solvers/quasi_Newton.jl | 45 ++++++++++++++++++++++++------- test/solvers/test_quasi_Newton.jl | 5 ++++ 6 files changed, 95 insertions(+), 16 deletions(-) diff --git a/Changelog.md b/Changelog.md index 617fcbec81..39455dbb09 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,13 @@ The file was started with Version `0.4`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +* `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. +* `GCPFinder` for handling direction selection in the presence of box (Hyperrectangle) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. + ## [0.5.29] November 26, 2025 ### Added diff --git a/docs/src/references.bib b/docs/src/references.bib index 21a807a875..dc1aedaa12 100644 --- a/docs/src/references.bib +++ b/docs/src/references.bib @@ -260,6 +260,20 @@ @book{Boumal:2023 ISBN = {978-1-00-916616-4} } +@article{ByrdLuNocedalZhu:1995, + title = {A {Limited} {Memory} {Algorithm} for {Bound} {Constrained} {Optimization}}, + volume = {16}, + issn = {1064-8275}, + doi = {10.1137/0916069}, + number = {5}, + journal = {SIAM Journal on Scientific Computing}, + author = {Byrd, Richard H. and Lu, Peihuang and Nocedal, Jorge and Zhu, Ciyou}, + month = sep, + year = {1995}, + note = {Publisher: Society for Industrial and Applied Mathematics}, + pages = {1190--1208}, +} + % --- C % % diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index d087ed93fd..458f12e655 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -13,8 +13,6 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ M_11::THM M_21::THM M_22::THM - # tolerance for detecting zero inner products between y and s - iszero_abstol::F # buffer for calculating stuff coords_Sk_X::V coords_Sk_Y::V @@ -22,7 +20,7 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ coords_Yk_Y::V end -function reset_update!(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) +function initialize_update!(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) initialize_update!(ha.qn_du) return ha end @@ -403,7 +401,10 @@ struct GCPFinder{TM <: AbstractManifold, TX, THA, TFU <: AbstractFPFPPUpdater} fpfpp_updater::TFU end -function GCPFinder(M::AbstractManifold, p, ha; fpfpp_updater = get_default_fpfpp_updater(ha)) +function GCPFinder( + M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; + fpfpp_updater::AbstractFPFPPUpdater = get_default_fpfpp_updater(ha) + ) return GCPFinder(M, zero_vector(M, p), zero_vector(M, p), ha, fpfpp_updater) end diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index 733e3267ee..7e85ada749 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -562,6 +562,19 @@ function is always included and the old, probably no longer relevant, informatio $(_var(:Field, :vector_transport_method)) * `message`: a string containing a potential warning that might have appeared * `project!`: a function to stabilize the update by projecting on the tangent space +* `vector_transport_method`: method for transporting stored s and y directions to the new point +* `nonpositive_curvature_behavior`: how non-positive-definite pairs (s, y) are detected and handled in vector transport. + Allowed values are: + - `:ignore` (default): pairs whose inner product is zero are + omitted from the current Hessian approximation but are + retained in memory for further iterations. This may lead + to non-positive-definite Hessians and non-descent directions + being selected and thus needs to be handled elsewhere. + - `:byrd`: pairs such that `inner(M, p, X_s, Y_s) <= iszero_abstol * norm(M, p, Y_s)^2` + are removed from memory (see [ByrdLuNocedalZhu:1995](@cite), + Eq. (3.9) and its discussion). +* `sy_tol`: tolerance for detecting non-positive-definite pairs (X_s, X_y). + The pairs may lose positive-definiteness after vector transport. # Constructor @@ -597,6 +610,8 @@ mutable struct QuasiNewtonLimitedMemoryDirectionUpdate{ initial_scale::G project!::Proj vector_transport_method::VT + nonpositive_curvature_behavior::Symbol + sy_tol::F message::String end function QuasiNewtonLimitedMemoryDirectionUpdate( @@ -608,6 +623,8 @@ function QuasiNewtonLimitedMemoryDirectionUpdate( initial_scale::G = 1.0, (project!)::Proj = (copyto!), vector_transport_method::VTM = default_vector_transport_method(M, typeof(p)), + nonpositive_curvature_behavior::Symbol = :ignore, + sy_tol::Real = 1.0e-8, ) where { NT <: AbstractQuasiNewtonUpdateRule, T, @@ -630,6 +647,8 @@ function QuasiNewtonLimitedMemoryDirectionUpdate( _initial_state, project!, vector_transport_method, + nonpositive_curvature_behavior, + sy_tol, "", ) end @@ -664,7 +683,7 @@ function (d::QuasiNewtonLimitedMemoryDirectionUpdate{InverseBFGS})( # what if division by zero happened here, setting to zero ignores this in the next step # pre-compute in case inner is expensive v = inner(M, p, d.memory_s[i], d.memory_y[i]) - if iszero(v) + if d.nonpositive_curvature_behavior === :ignore && iszero(v) d.ρ[i] = zero(eltype(d.ρ)) if length(d.message) > 0 d.message = replace(d.message, " i=" => " i=$i,") @@ -672,6 +691,14 @@ function (d::QuasiNewtonLimitedMemoryDirectionUpdate{InverseBFGS})( else d.message = "The inner products ⟨s_i,y_i⟩ ≈ 0, i=$i, ignoring summand in approximation." end + elseif d.nonpositive_curvature_behavior === :byrd && v <= d.sy_tol * norm(M, p, d.memory_y[i]) + d.ρ[i] = zero(eltype(d.ρ)) + if length(d.message) > 0 + d.message = replace(d.message, " i=" => " i=$i,") + d.message = replace(d.message, "summand in" => "summands in") + else + d.message = "The inner products ⟨s_i,y_i⟩ <= $(d.sy_tol * norm(M, p, d.memory_y[i])), i=$i, removing summand from approximation." + end else d.ρ[i] = 1 / v end @@ -732,7 +759,7 @@ end These [`AbstractQuasiNewtonDirectionUpdate`](@ref)s represent any quasi-Newton update rule, which are based on the idea of a so-called cautious update. The search direction is calculated as given in [`QuasiNewtonMatrixDirectionUpdate`](@ref) or [`QuasiNewtonLimitedMemoryDirectionUpdate`](@ref), -butut the update then is only executed if +but the update then is only executed if ```math $(_tex(:frac, "g_{x_{k+1}}(y_k,s_k)", "$(_tex(:norm, "s_k"; index = "x_{k+1}"))^{2}")) ≥ θ $(_tex(:norm, "$(_tex(:grad))f(p_k)"; index = "p_k")), diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 040914f86e..5026c0bb5c 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -210,10 +210,10 @@ $(_var(:Argument, :p)) # Keyword arguments -* `basis=`[`DefaultOrthonormalBasis`](@extref ManifoldsBase.DefaultOrthonormalBasis)`()`: +* `basis::AbstractBasis=`[`DefaultOrthonormalBasis`](@extref ManifoldsBase.DefaultOrthonormalBasis)`()`: basis to use within each of the the tangent spaces to represent the Hessian (inverse) for the cases where it is stored in full (matrix) form. -* `cautious_update=false`: +* `cautious_update::Bool=false`: whether or not to use the [`QuasiNewtonCautiousDirectionUpdate`](@ref) which wraps the `direction_upate`. * `cautious_function=(x) -> x * 1e-4`: @@ -229,7 +229,7 @@ $(_var(:Keyword, :evaluation; add = :GradientExample)) See also `initial_scale`. * `initial_scale=1.0`: scale initial `s` to use in with $(_doc_QN_init_scaling) in the computation of the limited memory approach. see also `initial_operator` -* `memory_size=20`: limited memory, number of ``s_k, y_k`` to store. +* `memory_size::Int=min(manifold_dimension(M), 20)`: limited memory, number of ``s_k, y_k`` to store. Set to a negative value to use a full memory (matrix) representation * `nondescent_direction_behavior=:reinitialize_direction_update`: specify how non-descent direction is handled. This can be @@ -331,6 +331,8 @@ function quasi_Newton!( ), stopping_criterion::StoppingCriterion = StopAfterIteration(max(1000, memory_size)) | StopWhenGradientNormLess(1.0e-6), + nonpositive_curvature_behavior::Symbol = :ignore, + sy_tol::Real = 1.0e-8, kwargs..., ) where { E <: AbstractEvaluationType, @@ -346,6 +348,8 @@ function quasi_Newton!( initial_scale = initial_scale, (project!) = (project!), vector_transport_method = vector_transport_method, + nonpositive_curvature_behavior = nonpositive_curvature_behavior, + sy_tol = sy_tol, ) else local_dir_upd = QuasiNewtonMatrixDirectionUpdate( @@ -724,14 +728,35 @@ function update_hessian!( start = length(d.memory_s) == capacity(d.memory_s) ? 2 : 1 M = get_manifold(mp) p = get_iterate(st) + reforming_required = false for i in start:length(d.memory_s) - # transport all stored tangent vectors in the tangent space of the next iterate - vector_transport_to!( - M, d.memory_s[i], p_old, d.memory_s[i], p, d.vector_transport_method - ) - vector_transport_to!( - M, d.memory_y[i], p_old, d.memory_y[i], p, d.vector_transport_method - ) + if d.nonpositive_curvature_behavior === :byrd && iszero(d.ρ[i]) + reforming_required = true + else + # transport all stored tangent vectors in the tangent space of the next iterate + vector_transport_to!( + M, d.memory_s[i], p_old, d.memory_s[i], p, d.vector_transport_method + ) + vector_transport_to!( + M, d.memory_y[i], p_old, d.memory_y[i], p, d.vector_transport_method + ) + end + end + + if reforming_required + # drop elements with zero inner product + T = eltype(d.memory_s) + memory_size = capacity(d.memory_s) + new_scb = CircularBuffer{T}(memory_size) + new_ycb = CircularBuffer{T}(memory_size) + for i in 1:length(d.memory_s) + if !iszero(d.ρ[i]) + push!(new_scb, d.memory_s[i]) + push!(new_ycb, d.memory_y[i]) + end + end + d.memory_s = new_scb + d.memory_y = new_ycb end # add newest diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index 6f3aead5c2..aea8a9582e 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -230,6 +230,11 @@ end ) @test isapprox(M, x_direction, x_solution; atol = rayleigh_atol) end + + @testset "Byrd's nonpositive rule" begin + x1 = quasi_Newton(M, f, grad_f, x; nonpositive_curvature_behavior = :byrd, sy_tol = 1.0e8) + @test isapprox(M, x1, x_solution; atol = rayleigh_atol) + end end @testset "Brocket" begin From 44b310abfcac05d88c2e863f671b708ac433618f Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 28 Nov 2025 17:12:09 +0100 Subject: [PATCH 006/135] start testing GCP --- Changelog.md | 6 +- ext/ManoptManifoldsExt/manifold_functions.jl | 7 + src/Manopt.jl | 2 +- src/plans/box_plan.jl | 262 +++++++++++-------- src/plans/quasi_newton_plan.jl | 14 +- src/solvers/quasi_Newton.jl | 2 +- test/solvers/test_quasi_Newton.jl | 31 +++ 7 files changed, 201 insertions(+), 123 deletions(-) diff --git a/Changelog.md b/Changelog.md index 39455dbb09..12521c9a55 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,8 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -* `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. -* `GCPFinder` for handling direction selection in the presence of box (Hyperrectangle) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. +* `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. (#554) +* `GCPFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) ## [0.5.29] November 26, 2025 @@ -28,7 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Removed `atol` from `DebugFeasibility` and instead use the one newly added `atol` from the `ConstrainedManifoldObjective`. (#546) * Move from CompatHelper to dependabot to keep track of dependency updates in Julia packages. (#547) * moved the `ManoptTestSuite` module to a sub module `Manopt.Test` within `Manopt.jl`, -so it can be easier resused by others as well (#550) + so it can be easier reused by others as well (#550) * moved to using a `Project.toml` for tests and an overall `[Workspace]`. This also allows finally to run single test files without installing all packages manually, but instead just switching to and instantiating the test environment. (#550) * for compatibility, state also `[source]` entries consistently in the sub `Project.toml` files. (#550) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 6fa901addd..8a93d06a83 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -193,3 +193,10 @@ function Manopt.set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) d[i] = 0 return p_cp end + +function Manopt.bound_direction_tweak!(::Hyperrectangle, d_out, d, p, p_cp) + return d_out .= p_cp .- p +end + +Manopt.requires_gcp(::Hyperrectangle) = true +Manopt.get_at_bound_index(::Hyperrectangle, X, b) = X[b] diff --git a/src/Manopt.jl b/src/Manopt.jl index a6ce33fc55..eb649afbdd 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -17,7 +17,7 @@ import LinearAlgebra: cross using ColorSchemes using ColorTypes using Colors -using DataStructures: CircularBuffer, capacity, length, push!, size, isfull +using DataStructures: BinaryHeap, CircularBuffer, capacity, length, push!, size, isfull using Dates: Millisecond, Nanosecond, Period, canonicalize, value using LinearAlgebra: cond, diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 458f12e655..1606a9827d 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -1,3 +1,12 @@ +""" + requires_gcp(M::AbstractManifold) + +Return `true` if `M` is a `Hyperrectangle`-like manifold with corners, or a product of it +with a standard manifold. Otherwise return `false`. +""" +requires_gcp(::AbstractManifold) = false +requires_gcp(M::ProductManifold) = requires_gcp(M.manifolds[1]) + mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ TDU <: QuasiNewtonLimitedMemoryDirectionUpdate, TM <: AbstractManifold, @@ -25,28 +34,105 @@ function initialize_update!(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) return ha end +function get_at_bound_index(M::ProductManifold, X, b) + return get_at_bound_index(M.manifolds[1], submanifold_component(M, X, Val(1)), b) +end + @doc raw""" - hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, X) + hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M, p, X) Compute $⟨X, B X⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`. """ -hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, X) = hess_val_single_pass(gh, X) +function hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M, p, X) + m = length(gh.qn_du.memory_s) + num_nonzero_rho = count(!iszero, gh.ρ) + + normX_sqr = norm(M, p, X)^2 + + if m == 0 || num_nonzero_rho == 0 + return gh.initial_scale \ normX_sqr + end + + ii = 1 + for i in 1:m + if iszero(gh.ρ[i]) + continue + end + gh.coords_Yk_X[ii] = inner(M, p, gh.qn_du.memory_y[i], X) + gh.coords_Sk_X[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], X) + + ii += 1 + end + coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) + coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) + + return hess_val_from_wmwt_coords(gh, normX_sqr, coords_Yk, coords_Sk, coords_Yk, coords_Sk) +end @doc raw""" - hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b) + hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) Compute $⟨X, B X⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`, and `X` is the unit vector along index `b`. """ -hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b) = hess_val_single_pass_eb(gh, b) +function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) + m = length(gh.qn_du.memory_s) + num_nonzero_rho = count(!iszero, gh.ρ) + + if m == 0 || num_nonzero_rho == 0 + return inv(gh.initial_scale) + end + + ii = 1 + for i in 1:m + if iszero(gh.ρ[i]) + continue + end + gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) + gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) + + ii += 1 + end + coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) + coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) + + return hess_val_from_wmwt_coords(gh, one(eltype(gh.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) +end @doc raw""" - hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, Y) + hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) Compute $⟨X, B Y⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`, where `X` is the unit vector pointing at index `b`. """ -hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, Y) = hess_val_single_pass_eb(gh, b, Y) +function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) + m = length(gh.qn_du.memory_s) + num_nonzero_rho = count(!iszero, gh.ρ) + + Yb = get_at_bound_index(M, Y, b) + if m == 0 || num_nonzero_rho == 0 + return gh.initial_scale * Yb + end + + ii = 1 + for i in 1:m + if iszero(gh.ρ[i]) + continue + end + gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) + gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) + + gh.coords_Yk_Y[ii] = inner(M, p, gh.qn_du.memory_y[i], Y) + gh.coords_Sk_Y[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], Y) + ii += 1 + end + coords_Yk_X = view(gh.coords_Yk_X, 1:num_nonzero_rho) + coords_Yk_Y = view(gh.coords_Yk_Y, 1:num_nonzero_rho) + coords_Sk_X = view(gh.coords_Sk_X, 1:num_nonzero_rho) + coords_Sk_Y = view(gh.coords_Sk_Y, 1:num_nonzero_rho) + + return hess_val_from_wmwt_coords(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) +end function set_M_current_scale!(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) M = gh.M @@ -148,90 +234,6 @@ function hess_val_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdat return result end -function hess_val_single_pass(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, X) - M = gh.M - p = gh.p - m = length(gh.qn_du.memory_s) - num_nonzero_rho = count(!iszero, gh.ρ) - - normX_sqr = norm(M, p, X)^2 - - if m == 0 || num_nonzero_rho == 0 - return gh.initial_scale \ normX_sqr - end - - ii = 1 - for i in 1:m - if iszero(gh.ρ[i]) - continue - end - gh.coords_Yk_X[ii] = inner(M, p, gh.qn_du.memory_y[i], X) - gh.coords_Sk_X[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], X) - - ii += 1 - end - coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) - coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) - - return hess_val_from_wmwt_coords(gh, normX_sqr, coords_Yk, coords_Sk, coords_Yk, coords_Sk) -end - -function hess_val_single_pass_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b) - M = gh.M - p = gh.p - m = length(gh.qn_du.memory_s) - num_nonzero_rho = count(!iszero, gh.ρ) - - if m == 0 || num_nonzero_rho == 0 - return inv(gh.initial_scale) - end - - ii = 1 - for i in 1:m - if iszero(gh.ρ[i]) - continue - end - gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) - gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) - - ii += 1 - end - coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) - coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) - - return hess_val_from_wmwt_coords(gh, one(eltype(gh.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) -end - -function hess_val_single_pass_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, Y) - M = gh.M - p = gh.p - m = length(gh.qn_du.memory_s) - num_nonzero_rho = count(!iszero, gh.ρ) - - Yb = get_at_bound_index(M, Y, b) - if m == 0 || num_nonzero_rho == 0 - return gh.initial_scale * Yb - end - - ii = 1 - for i in 1:m - if iszero(gh.ρ[i]) - continue - end - gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) - gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) - - gh.coords_Yk_Y[ii] = inner(M, p, gh.qn_du.memory_y[i], Y) - gh.coords_Sk_Y[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], Y) - ii += 1 - end - coords_Yk_X = view(gh.coords_Yk_X, 1:num_nonzero_rho) - coords_Yk_Y = view(gh.coords_Yk_Y, 1:num_nonzero_rho) - coords_Sk_X = view(gh.coords_Sk_X, 1:num_nonzero_rho) - coords_Sk_Y = view(gh.coords_Sk_Y, 1:num_nonzero_rho) - - return hess_val_from_wmwt_coords(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) -end @doc raw""" update_hessian!(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, p) @@ -296,9 +298,9 @@ function get_default_fpfpp_updater(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdat return LimitedMemoryFPFPPUpdater(similar(ha.ρ), similar(ha.ρ), similar(ha.ρ), similar(ha.ρ)) end -function (::GenericFPFPPUpdater)(M::AbstractManifold, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) - f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hess_val_eb(ha, b, z)) - f_double_prime = old_f_double_prime + (2 * -db * hess_val_eb(ha, b, d_old)) + db^2 * hess_val_eb(ha, b) +function (::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hess_val_eb(ha, M, p, b, z)) + f_double_prime = old_f_double_prime + (2 * -db * hess_val_eb(ha, M, p, b, d_old)) + db^2 * hess_val_eb(ha, M, p, b) return f_prime, f_double_prime end @@ -319,7 +321,7 @@ function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater return fpfpp_upd end -function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, old_f_prime, old_f_double_prime, dt, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old) +function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old) m = length(ha.memory_s) num_nonzero_rho = count(!iszero, ha.ρ) @@ -355,7 +357,7 @@ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, old_f_prime f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + eb_B_z) eb_B_d = hess_val_from_wmwt_coords(ha, iss_eb_d, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) - f_double_prime = old_f_double_prime - 2 * db * eb_B_d + db^2 * hess_val_eb(ha, b) + f_double_prime = old_f_double_prime - 2 * db * eb_B_d + db^2 * hess_val_eb(ha, M, p, b) coords_py .-= db .* coords_Yk_eb coords_ps .-= db .* coords_Sk_eb @@ -392,9 +394,18 @@ function set_bound_at_index!(M::ProductManifold, p_cp, d, i) return p_cp end +function bound_direction_tweak!(::ProductManifold, d_out, d, p, p_cp) + bound_direction_tweak!( + M.manifolds[1], submanifold_component(M, d_out, Val(1)), + submanifold_component(M, d, Val(1)), submanifold_component(M, p, Val(1)), + submanifold_component(M, p_cp, Val(1)) + ) + return d_out +end -struct GCPFinder{TM <: AbstractManifold, TX, THA, TFU <: AbstractFPFPPUpdater} +struct GCPFinder{TM <: AbstractManifold, TP, TX, THA, TFU <: AbstractFPFPPUpdater} M::TM + p_cp::TP Y_tmp::TX d_old::TX ha::THA @@ -405,44 +416,51 @@ function GCPFinder( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; fpfpp_updater::AbstractFPFPPUpdater = get_default_fpfpp_updater(ha) ) - return GCPFinder(M, zero_vector(M, p), zero_vector(M, p), ha, fpfpp_updater) + return GCPFinder(M, copy(M, p), zero_vector(M, p), zero_vector(M, p), ha, fpfpp_updater) end """ - find_gcp!(gcp::GCPFinder, p_cp, p, d, X) + find_gcp!(gcp::GCPFinder, d_out, p, d, X) -Find generalized Cauchy point looking from point `p` in direction `d` and save it to `p_cp`. -Gradient of the objective at `p` is `X`. +Find generalized Cauchy point looking from point `p` in direction `d` and save the tangent +vector pointing at it to `d_out`. Gradient of the objective at `p` is `X`. -The function returns `true` if the point was found and `false` otherwise. +The function returns +* `:found_limited` if the point was found and we can perform a step of length at most 1 + in direction `d_out` afterwards, +* `:found_unlimited` if the point was found and we can perform a step of length at most + `max_stepsize(M, p)` in direction `d_out` afterwards, +* `:not_found` if the search cannot be performed in direction `d`. """ -function find_gcp!(gcp::GCPFinder, p_cp, p, d, X) +function find_gcp_direction!(gcp::GCPFinder, d_out, p, d, X) M = gcp.M - copyto!(M, p_cp, p) + copyto!(M, gcp.p_cp, p) + p_cp = gcp.p_cp zero_vector!(M, gcp.Y_tmp, p) + copyto!(M, d_out, d) bounds_indices = get_bounds_index(M) TInd = eltype(bounds_indices) + TF = number_eltype(d) - t = Dict{TInd, Float64}((k, Inf) for k in bounds_indices) + t = Dict{TInd, TF}((k, Inf) for k in bounds_indices) - F_list = Tuple{Float64, TInd}[] + F_list = Tuple{TF, TInd}[] sizehint!(F_list, length(bounds_indices) + 1) + has_finite_limit = false + for i in bounds_indices t[i] = get_bound_t(M, p, d, i) if t[i] > 0 push!(F_list, (t[i], i)) end - end - - if isempty(F_list) - @warn "We can't go in the selected direction" - return false + has_finite_limit |= isfinite(t[i]) end if M isa ProductManifold + # Hyperrectangle × something else # push also `t` corresponding to max_stepsize if it is considered in the manifold M2 = M.manifolds[2] p2 = submanifold_component(M, p, Val(2)) @@ -452,15 +470,21 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X) tms = max_step / norm(M2, p2, d2) push!(F_list, (tms, -1)) end + else + # Check only when we work on a pure Hyperrectangle + if isempty(F_list) + @warn "We can't go in the selected direction" + return :not_found + end end F = BinaryHeap(Base.By(first), F_list) f_prime = inner(M, p, X, d) - f_double_prime = hess_val(gcp.ha, d) + f_double_prime = hess_val(gcp.ha, M, p, d) if iszero(f_prime) || iszero(f_double_prime) - return false + return :not_found end dt_min = -f_prime / f_double_prime @@ -469,17 +493,17 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X) t_current, b = pop!(F) dt = t_current - t_old - init_updater!(M, gcp.fpfpp_upd, d, gcp.ha) + init_updater!(M, gcp.fpfpp_updater, d, gcp.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 gcp.Y_tmp .+= dt .* d copyto!(M, gcp.d_old, d) set_bound_at_index!(M, p_cp, d, b) - gb = get_at_bound_index(M, grad, b) + gb = get_at_bound_index(M, X, b) db = get_at_bound_index(M, gcp.d_old, b) - f_prime, f_double_prime = gcp.fpfpp_upd(M, f_prime, f_double_prime, dt, db, gb, gcp.ha, b, gcp.Y_tmp, gcp.d_old) + f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, dt, db, gb, gcp.ha, b, gcp.Y_tmp, gcp.d_old) t_old = t_current # If f_prime is 0, we've found the local minimizer (GCP) @@ -508,5 +532,11 @@ function find_gcp!(gcp::GCPFinder, p_cp, p, d, X) end end - return true + bound_direction_tweak!(M, d_out, d, p, p_cp) + + if has_finite_limit + return :found_limited + else + return :found_unlimited + end end diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index 7e85ada749..2881f3d961 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -409,8 +409,8 @@ space ``T_{p_{k+1}} $(_tex(:Cal, "M"))``, preferably with an isometric vector tr # Provided functors -* `(mp::AbstractManoptproblem, st::QuasiNewtonState) -> η` to compute the update direction -* `(η, mp::AbstractManoptproblem, st::QuasiNewtonState) -> η` to compute the update direction in-place of `η` +* `(mp::AbstractManoptProblem, st::QuasiNewtonState) -> η` to compute the update direction +* `(η, mp::AbstractManoptProblem, st::QuasiNewtonState) -> η` to compute the update direction in-place of `η` # Fields @@ -511,6 +511,16 @@ function initialize_update!(d::QuasiNewtonMatrixDirectionUpdate) copyto!(d.matrix, I) return d end +function hess_val(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} + c = get_coordinates(M, p, X) + return dot(c, d.matrix, c) +end +function hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, b) where {T <: Union{BFGS, DFP, SR1, Broyden}} + return d.matrix[b, b] +end +function hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, b, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} + return dot(d.matrix[b, :], get_coordinates(M, p, X)) +end _doc_QN_B = """ ```math diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 5026c0bb5c..70e5816445 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -215,7 +215,7 @@ $(_var(:Argument, :p)) the Hessian (inverse) for the cases where it is stored in full (matrix) form. * `cautious_update::Bool=false`: whether or not to use the [`QuasiNewtonCautiousDirectionUpdate`](@ref) - which wraps the `direction_upate`. + which wraps the `direction_update`. * `cautious_function=(x) -> x * 1e-4`: a monotone increasing function for the cautious update that is zero at ``x=0`` and strictly increasing at ``0`` diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index aea8a9582e..76556ad386 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -500,4 +500,35 @@ end Manopt.update_hessian!(qns.direction_update, mp, qns, p, 1) # But I am not totally sure what to test for afterwards end + + @testset "Hyperrectangle domain" begin + + @testset "GCPFinder" begin + M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) + ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS()) + + p = [0.0, 0.0, 0.0] + gf = Manopt.GCPFinder(M, p, ha) + + X1 = [-5.0, 0.0, 0.0] + + d = -X1 + d_out = similar(d) + + @test Manopt.find_gcp_direction!(gf, d_out, p, d, X1) === :found_limited + @test d_out ≈ [2.0, 0.0, 0.0] + + end + @testset "Pure Hyperrectangle" begin + M = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 .* p + p0 = [0.0, 4.0, 10.0] + p_opt = quasi_Newton(M, f, grad_f, p0) + end + + @testset "Hyperrectangle × Sphere" begin + + end + end end From c3a8a68f46eb4f4708ce2f1bea984d5052f882fa Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 2 Dec 2025 14:39:29 +0100 Subject: [PATCH 007/135] A few more tests --- src/plans/box_plan.jl | 4 +- test/solvers/test_quasi_Newton.jl | 87 ++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 1606a9827d..6fa70fdeaf 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -39,11 +39,11 @@ function get_at_bound_index(M::ProductManifold, X, b) end @doc raw""" - hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M, p, X) + hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) Compute $⟨X, B X⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`. """ -function hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M, p, X) +function hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) m = length(gh.qn_du.memory_s) num_nonzero_rho = count(!iszero, gh.ρ) diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index 76556ad386..b51b9f0335 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -1,5 +1,5 @@ using Manopt, Manifolds, Test -using LinearAlgebra: I, eigvecs, tr, Diagonal +using LinearAlgebra: I, eigvecs, tr, Diagonal, dot mutable struct QuasiNewtonGradientDirectionUpdate{VT <: AbstractVectorTransportMethod} <: AbstractQuasiNewtonDirectionUpdate @@ -502,6 +502,91 @@ end end @testset "Hyperrectangle domain" begin + @testset "get_bound_t - basic" begin + M = Hyperrectangle([0.0, 0.0], [2.0, 2.0]) + + # d[i] > 0 + p = [0.0, 1.0]; d = [1.0, 1.0] + @test Manopt.get_bound_t(M, p, d, 1) ≈ (2.0 - 0.0) / 1.0 # = 2.0 + @test Manopt.get_bound_t(M, p, d, 2) ≈ (2.0 - 1.0) / 1.0 # = 1.0 + + # d[i] < 0 + p = [0.0, 1.0]; d = [-1.0, -1.0] + @test Manopt.get_bound_t(M, p, d, 1) ≈ (0.0 - 0.0) / -1.0 # = 0.0 + @test Manopt.get_bound_t(M, p, d, 2) ≈ (0.0 - 1.0) / -1.0 # = 1.0 + + # d[i] = 0 + p = [0.0, 1.0]; d = [0.0, 0.0] + @test Manopt.get_bound_t(M, p, d, 1) ≈ Inf + @test Manopt.get_bound_t(M, p, d, 2) ≈ Inf + end + + + @testset "update_fp_fpp - basic d = -g" begin + M = Hyperrectangle([0.0, 1.0], [3.0, 3.0]) + + grad = [1.0, 4.0] + d = [-1.0, -4.0] + p = [0.0, 0.0] + + # values taken from loop iteration found in test case: "find_gcp! - with bounds, single variable is held fixed" + old_f_prime = -17.0 + old_f_double_prime = 34.0 + dt = 0.25 + gb = 4.0 + db = -4.0 # in case of d = -g, db = -gb + ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [2.0 0.0; 0.0 2.0]) + b = 2 + z = [-0.25, -1.0] + d_old = [-1.0, -4.0] + + d[2] = 0.0 + + # optimized formula + f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + @test f_prime ≈ -0.5 + @test f_double_prime ≈ 2.0 + + # original formula + f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) + f_original_double_prime = Manopt.hess_val(ha, M, p, d) + + @test f_prime == f_original_prime + @test f_double_prime == f_original_double_prime + end + + + @testset "update_fp_fpp - basic d = [-2.0, -1.0]" begin + M = Hyperrectangle([0.0, 1.0], [3.0, 3.0]) + + grad = [1.0, 4.0] + d = [-2.0, -1.0] + p = [0.0, 0.0] + + old_f_prime = -6.0 + old_f_double_prime = 10.0 + dt = 0.25 + gb = 1.0 + db = -2.0 + ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [2.0 0.0; 0.0 2.0]) + b = 1 + z = [-0.5, -0.25] + d_old = [-2.0, -1.0] + + d[1] = 0.0 + + # optimized formula + f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + @test f_prime == -3.5 + @test f_double_prime == 2 + + # original formula + f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) + f_original_double_prime = Manopt.hess_val(ha, M, p, d) + + @test f_prime == f_original_prime + @test f_double_prime == f_original_double_prime + end @testset "GCPFinder" begin M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) From 75f7f15c6d4b2e60a93d43fdb0521dc4ea1339b2 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 2 Dec 2025 14:59:16 +0100 Subject: [PATCH 008/135] fix "typos" --- src/plans/box_plan.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 6fa70fdeaf..ff1e3d4225 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -11,7 +11,7 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ TDU <: QuasiNewtonLimitedMemoryDirectionUpdate, TM <: AbstractManifold, F <: Real, - THM <: AbstractMatrix, + T_HM <: AbstractMatrix, V <: AbstractVector, } <: AbstractQuasiNewtonDirectionUpdate # this approximates inverse Hessian @@ -19,9 +19,9 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ # fields for approximating the Hessian current_scale::F - M_11::THM - M_21::THM - M_22::THM + M_11::T_HM + M_21::T_HM + M_22::T_HM # buffer for calculating stuff coords_Sk_X::V coords_Sk_Y::V @@ -403,12 +403,12 @@ function bound_direction_tweak!(::ProductManifold, d_out, d, p, p_cp) return d_out end -struct GCPFinder{TM <: AbstractManifold, TP, TX, THA, TFU <: AbstractFPFPPUpdater} +struct GCPFinder{TM <: AbstractManifold, TP, TX, T_HA, TFU <: AbstractFPFPPUpdater} M::TM p_cp::TP Y_tmp::TX d_old::TX - ha::THA + ha::T_HA fpfpp_updater::TFU end From 82ac90a4e79e11c4f7be1acc73f8217f6d0075e7 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 3 Dec 2025 10:16:03 +0100 Subject: [PATCH 009/135] box tests in a separate file --- src/plans/box_plan.jl | 2 +- test/runtests.jl | 1 + test/solvers/test_quasi_Newton.jl | 116 ----------------------- test/solvers/test_quasi_Newton_box.jl | 131 ++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 117 deletions(-) create mode 100644 test/solvers/test_quasi_Newton_box.jl diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index ff1e3d4225..0e64d24cc0 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -403,7 +403,7 @@ function bound_direction_tweak!(::ProductManifold, d_out, d, p, p_cp) return d_out end -struct GCPFinder{TM <: AbstractManifold, TP, TX, T_HA, TFU <: AbstractFPFPPUpdater} +struct GCPFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} M::TM p_cp::TP Y_tmp::TX diff --git a/test/runtests.jl b/test/runtests.jl index 33e9654baa..cc96aead0f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -64,6 +64,7 @@ using Manifolds, ManifoldsBase, Manopt, Test include("solvers/test_proximal_gradient_method.jl") include("solvers/test_proximal_point.jl") include("solvers/test_quasi_Newton.jl") + include("solvers/test_quasi_Newton_box.jl") include("solvers/test_particle_swarm.jl") include("solvers/test_primal_dual_semismooth_Newton.jl") include("solvers/test_stochastic_gradient_descent.jl") diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index b51b9f0335..fd8b0500ad 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -500,120 +500,4 @@ end Manopt.update_hessian!(qns.direction_update, mp, qns, p, 1) # But I am not totally sure what to test for afterwards end - - @testset "Hyperrectangle domain" begin - @testset "get_bound_t - basic" begin - M = Hyperrectangle([0.0, 0.0], [2.0, 2.0]) - - # d[i] > 0 - p = [0.0, 1.0]; d = [1.0, 1.0] - @test Manopt.get_bound_t(M, p, d, 1) ≈ (2.0 - 0.0) / 1.0 # = 2.0 - @test Manopt.get_bound_t(M, p, d, 2) ≈ (2.0 - 1.0) / 1.0 # = 1.0 - - # d[i] < 0 - p = [0.0, 1.0]; d = [-1.0, -1.0] - @test Manopt.get_bound_t(M, p, d, 1) ≈ (0.0 - 0.0) / -1.0 # = 0.0 - @test Manopt.get_bound_t(M, p, d, 2) ≈ (0.0 - 1.0) / -1.0 # = 1.0 - - # d[i] = 0 - p = [0.0, 1.0]; d = [0.0, 0.0] - @test Manopt.get_bound_t(M, p, d, 1) ≈ Inf - @test Manopt.get_bound_t(M, p, d, 2) ≈ Inf - end - - - @testset "update_fp_fpp - basic d = -g" begin - M = Hyperrectangle([0.0, 1.0], [3.0, 3.0]) - - grad = [1.0, 4.0] - d = [-1.0, -4.0] - p = [0.0, 0.0] - - # values taken from loop iteration found in test case: "find_gcp! - with bounds, single variable is held fixed" - old_f_prime = -17.0 - old_f_double_prime = 34.0 - dt = 0.25 - gb = 4.0 - db = -4.0 # in case of d = -g, db = -gb - ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [2.0 0.0; 0.0 2.0]) - b = 2 - z = [-0.25, -1.0] - d_old = [-1.0, -4.0] - - d[2] = 0.0 - - # optimized formula - f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) - @test f_prime ≈ -0.5 - @test f_double_prime ≈ 2.0 - - # original formula - f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) - f_original_double_prime = Manopt.hess_val(ha, M, p, d) - - @test f_prime == f_original_prime - @test f_double_prime == f_original_double_prime - end - - - @testset "update_fp_fpp - basic d = [-2.0, -1.0]" begin - M = Hyperrectangle([0.0, 1.0], [3.0, 3.0]) - - grad = [1.0, 4.0] - d = [-2.0, -1.0] - p = [0.0, 0.0] - - old_f_prime = -6.0 - old_f_double_prime = 10.0 - dt = 0.25 - gb = 1.0 - db = -2.0 - ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [2.0 0.0; 0.0 2.0]) - b = 1 - z = [-0.5, -0.25] - d_old = [-2.0, -1.0] - - d[1] = 0.0 - - # optimized formula - f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) - @test f_prime == -3.5 - @test f_double_prime == 2 - - # original formula - f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) - f_original_double_prime = Manopt.hess_val(ha, M, p, d) - - @test f_prime == f_original_prime - @test f_double_prime == f_original_double_prime - end - - @testset "GCPFinder" begin - M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) - ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS()) - - p = [0.0, 0.0, 0.0] - gf = Manopt.GCPFinder(M, p, ha) - - X1 = [-5.0, 0.0, 0.0] - - d = -X1 - d_out = similar(d) - - @test Manopt.find_gcp_direction!(gf, d_out, p, d, X1) === :found_limited - @test d_out ≈ [2.0, 0.0, 0.0] - - end - @testset "Pure Hyperrectangle" begin - M = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) - f(M, p) = sum(p .^ 2) - grad_f(M, p) = 2 .* p - p0 = [0.0, 4.0, 10.0] - p_opt = quasi_Newton(M, f, grad_f, p0) - end - - @testset "Hyperrectangle × Sphere" begin - - end - end end diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl new file mode 100644 index 0000000000..c90ae0286b --- /dev/null +++ b/test/solvers/test_quasi_Newton_box.jl @@ -0,0 +1,131 @@ +using Manopt, Manifolds, Test +using LinearAlgebra: I, eigvecs, tr, Diagonal, dot + +@testset "Riemannian quasi-Newton Methods with box-like domains" begin + @testset "get_bound_t - basic" begin + M = Hyperrectangle([0.0, 0.0], [2.0, 2.0]) + + # d[i] > 0 + p = [0.0, 1.0]; d = [1.0, 1.0] + @test Manopt.get_bound_t(M, p, d, 1) ≈ (2.0 - 0.0) / 1.0 # = 2.0 + @test Manopt.get_bound_t(M, p, d, 2) ≈ (2.0 - 1.0) / 1.0 # = 1.0 + + # d[i] < 0 + p = [0.0, 1.0]; d = [-1.0, -1.0] + @test Manopt.get_bound_t(M, p, d, 1) ≈ (0.0 - 0.0) / -1.0 # = 0.0 + @test Manopt.get_bound_t(M, p, d, 2) ≈ (0.0 - 1.0) / -1.0 # = 1.0 + + # d[i] = 0 + p = [0.0, 1.0]; d = [0.0, 0.0] + @test Manopt.get_bound_t(M, p, d, 1) ≈ Inf + @test Manopt.get_bound_t(M, p, d, 2) ≈ Inf + end + + + @testset "update_fp_fpp - basic d = -g" begin + M = Hyperrectangle([0.0, 1.0], [3.0, 3.0]) + + grad = [1.0, 4.0] + d = [-1.0, -4.0] + p = [0.0, 0.0] + + # values taken from loop iteration found in test case: "find_gcp! - with bounds, single variable is held fixed" + old_f_prime = -17.0 + old_f_double_prime = 34.0 + dt = 0.25 + gb = 4.0 + db = -4.0 # in case of d = -g, db = -gb + ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [2.0 0.0; 0.0 2.0]) + b = 2 + z = [-0.25, -1.0] + d_old = [-1.0, -4.0] + + d[2] = 0.0 + + # optimized formula + f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + @test f_prime ≈ -0.5 + @test f_double_prime ≈ 2.0 + + # original formula + f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) + f_original_double_prime = Manopt.hess_val(ha, M, p, d) + + @test f_prime == f_original_prime + @test f_double_prime == f_original_double_prime + end + + + @testset "update_fp_fpp - basic d = [-2.0, -1.0]" begin + M = Hyperrectangle([0.0, 1.0], [3.0, 3.0]) + + grad = [1.0, 4.0] + d = [-2.0, -1.0] + p = [0.0, 0.0] + + old_f_prime = -6.0 + old_f_double_prime = 10.0 + dt = 0.25 + gb = 1.0 + db = -2.0 + ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [2.0 0.0; 0.0 2.0]) + b = 1 + z = [-0.5, -0.25] + d_old = [-2.0, -1.0] + + d[1] = 0.0 + + # optimized formula + f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + @test f_prime == -3.5 + @test f_double_prime == 2 + + # original formula + f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) + f_original_double_prime = Manopt.hess_val(ha, M, p, d) + + @test f_prime == f_original_prime + @test f_double_prime == f_original_double_prime + end + + @testset "GCPFinder" begin + M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) + ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS()) + + p = [0.0, 0.0, 0.0] + gf = Manopt.GCPFinder(M, p, ha) + + X1 = [-5.0, 0.0, 0.0] + + d = -X1 + d_out = similar(d) + + @test Manopt.find_gcp_direction!(gf, d_out, p, d, X1) === :found_limited + @test d_out ≈ [2.0, 0.0, 0.0] + + d2 = [0.0, 1.0, 0.0] + + @test Manopt.find_gcp_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === :found_unlimited + @test d_out ≈ d2 + + @test Manopt.find_gcp_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === :found_limited + @test d_out ≈ [2.0, 10.0, 0.0] + end + @testset "Pure Hyperrectangle" begin + M = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 .* p + p0 = [0.0, 4.0, 10.0] + p_opt = quasi_Newton(M, f, grad_f, p0) + end + + @testset "requires_gcp" begin + @test !Manopt.requires_gcp(Sphere(2)) + @test Manopt.requires_gcp(Hyperrectangle([1], [2])) + @test Manopt.requires_gcp(ProductManifold(Hyperrectangle([1], [2]), Sphere(2))) + end + + @testset "Hyperrectangle × Sphere" begin + + end +end From b94b0f11cfafb92812ad37898e004d2d5c01cedf Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 3 Dec 2025 17:38:41 +0100 Subject: [PATCH 010/135] start integration of GCP into quasi_Newton --- docs/src/references.bib | 14 ++++++ src/helpers/LineSearchesTypes.jl | 3 +- src/plans/box_plan.jl | 67 +++++++++++++++++++++++++-- src/plans/stepsize/stepsize.jl | 14 ++++-- src/solvers/quasi_Newton.jl | 5 ++ test/solvers/test_quasi_Newton_box.jl | 2 +- 6 files changed, 94 insertions(+), 11 deletions(-) diff --git a/docs/src/references.bib b/docs/src/references.bib index 933cb7ccf6..bd79ce494d 100644 --- a/docs/src/references.bib +++ b/docs/src/references.bib @@ -260,6 +260,20 @@ @book{Boumal:2023 ISBN = {978-1-00-916616-4} } +@article{ByrdNocedalSchnabel:1994, + title = {Representations of quasi-{Newton} matrices and their use in limited memory methods}, + volume = {63}, + issn = {1436-4646}, + doi = {10.1007/BF01582063}, + number = {1}, + urldate = {2025-09-06}, + journal = {Mathematical Programming}, + author = {Byrd, Richard H. and Nocedal, Jorge and Schnabel, Robert B.}, + month = jan, + year = {1994}, + pages = {129--156}, +} + @article{ByrdLuNocedalZhu:1995, title = {A {Limited} {Memory} {Algorithm} for {Bound} {Constrained} {Optimization}}, volume = {16}, diff --git a/src/helpers/LineSearchesTypes.jl b/src/helpers/LineSearchesTypes.jl index 742d7f21c0..5989755143 100644 --- a/src/helpers/LineSearchesTypes.jl +++ b/src/helpers/LineSearchesTypes.jl @@ -14,7 +14,8 @@ Wrapper for line searches available in the `LineSearches.jl` library. Wrap `linesearch` (for example [`HagerZhang`](https://julianlsolvers.github.io/LineSearches.jl/latest/reference/linesearch.html#LineSearches.HagerZhang) or [`MoreThuente`](https://julianlsolvers.github.io/LineSearches.jl/latest/reference/linesearch.html#LineSearches.MoreThuente)). -The initial step selection from Linesearches.jl is not yet supported and the value 1.0 is used. +The initial step selection from Linesearches.jl is not yet supported and `initial_guess` is +always used (by default [`ConstantInitialGuess`](@ref)). # Keyword Arguments diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 0e64d24cc0..fd0b713b79 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -7,9 +7,24 @@ with a standard manifold. Otherwise return `false`. requires_gcp(::AbstractManifold) = false requires_gcp(M::ProductManifold) = requires_gcp(M.manifolds[1]) +@doc raw""" + mutable struct LimitedMemoryHessianApproximation end + +An approximation of Hessian of a scalar function of the form ``B_0 = θ I``, +``B_{k+1} = B_k - W_k M_k W_k^{\mathrm{T}}``, +where ``\theta > 0`` is an initial scaling guess. +Matrix ``M_k = \begin{psmallmatrix}M_{11} & M_{21}^{\mathrm{T}}\\ M_{21} & M_{22}\end{psmallmatrix}`` +is stored using its blocks. +Blocks ``W_k`` are (implicitly) composed from `memory_y` and `memory_s`. + +Initial scale ``\theta`` is stored in the field `initial_scale` but if the memory isn't empty, +the current scale is set to squared norm of $s_k$ divided by inner product of ``s_k`` and ``y_k`` +where ``k`` is the oldest index for which the denominator is not equal to 0. + +See [ByrdNocedalSchnabel:1994](@cite) for details. +""" mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ TDU <: QuasiNewtonLimitedMemoryDirectionUpdate, - TM <: AbstractManifold, F <: Real, T_HM <: AbstractMatrix, V <: AbstractVector, @@ -22,18 +37,60 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ M_11::T_HM M_21::T_HM M_22::T_HM - # buffer for calculating stuff + # buffer for calculating W_k blocks coords_Sk_X::V coords_Sk_Y::V coords_Yk_X::V coords_Yk_Y::V end +function QuasiNewtonLimitedMemoryBoxDirectionUpdate( + qn_du::QuasiNewtonLimitedMemoryDirectionUpdate{<:AbstractQuasiNewtonUpdateRule, T, F} + ) where {T, F <: Real} + memory_size = capacity(qn_du.memory_s) + M_11 = zeros(F, memory_size, memory_size) + M_21 = zeros(F, memory_size, memory_size) + M_22 = zeros(F, memory_size, memory_size) + coords_Sk_X = zeros(F, memory_size) + coords_Sk_Y = zeros(F, memory_size) + coords_Yk_X = zeros(F, memory_size) + coords_Yk_Y = zeros(F, memory_size) + return QuasiNewtonLimitedMemoryBoxDirectionUpdate{ + typeof(qn_du), F, typeof(M_11), typeof(coords_Sk_X), + }( + qn_du, + qn_du.initial_scale, + M_11, + M_21, + M_22, + coords_Sk_X, + coords_Sk_Y, + coords_Yk_X, + coords_Yk_Y, + ) +end + function initialize_update!(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) initialize_update!(ha.qn_du) return ha end +function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( + mp::AbstractManoptProblem, st + ) + r = zero_vector(get_manifold(mp), get_iterate(st)) + return d(r, mp, st) +end +function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( + r, mp::AbstractManoptProblem, st + ) + d.qn_du(r, mp, st) + # TODO find_gcp_direction! + return r +end + +get_update_vector_transport(u::QuasiNewtonLimitedMemoryBoxDirectionUpdate) = get_update_vector_transport(u.qn_du) + function get_at_bound_index(M::ProductManifold, X, b) return get_at_bound_index(M.manifolds[1], submanifold_component(M, X, Val(1)), b) end @@ -306,8 +363,8 @@ function (::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_doub end function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater, d, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) - fpfpp_upd.c_s .= 0 - fpfpp_upd.c_y .= 0 + fill!(fpfpp_upd.c_s, 0) + fill!(fpfpp_upd.c_y, 0) ii = 1 for i in eachindex(ha.ρ) if iszero(ha.ρ[i]) @@ -420,7 +477,7 @@ function GCPFinder( end """ - find_gcp!(gcp::GCPFinder, d_out, p, d, X) + find_gcp_direction!(gcp::GCPFinder, d_out, p, d, X) Find generalized Cauchy point looking from point `p` in direction `d` and save the tangent vector pointing at it to `d_out`. Gradient of the objective at `p` is `X`. diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index cb998521c9..7288fd781b 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -114,13 +114,19 @@ function (a::ArmijoLinesearchStepsize)( ) p = get_iterate(s) grad = isnothing(gradient) ? get_gradient(mp, get_iterate(s)) : gradient - return a(mp, p, grad, η; initial_guess = a.initial_guess(mp, s, k, a.last_stepsize, η)) + return a(mp, p, grad, η; initial_guess = a.initial_guess(mp, s, k, a.last_stepsize, η), kwargs...) end function (a::ArmijoLinesearchStepsize)( - mp::AbstractManoptProblem, p, X, η; initial_guess = 1.0, kwargs... + mp::AbstractManoptProblem, p, X, η; initial_guess::Real = 1.0, kwargs... ) reset_messages!(a.messages) l = norm(get_manifold(mp), p, η) + local swse + if :stop_when_stepsize_exceeds in keys(kwargs) + swse = kwargs.stop_when_stepsize_exceeds + else + swse = (a.stop_when_stepsize_exceeds / l) + end a.last_stepsize = linesearch_backtrack!( get_manifold(mp), a.candidate_point, @@ -133,7 +139,7 @@ function (a::ArmijoLinesearchStepsize)( gradient = X, retraction_method = a.retraction_method, stop_when_stepsize_less = (a.stop_when_stepsize_less / l), - stop_when_stepsize_exceeds = (a.stop_when_stepsize_exceeds / l), + stop_when_stepsize_exceeds = swse, stop_increasing_at_step = a.stop_increasing_at_step, stop_decreasing_at_step = a.stop_decreasing_at_step, additional_decrease_condition = a.additional_decrease_condition, @@ -1251,7 +1257,7 @@ mutable struct NonmonotoneLinesearchStepsize{ retraction_method::TRM = default_retraction_method(M), stepsize_reduction::R = 0.5, stop_when_stepsize_less::R = 0.0, - stop_when_stepsize_exceeds = real(max_stepsize(M)), + stop_when_stepsize_exceeds::R = real(max_stepsize(M)), stop_increasing_at_step::I = 100, stop_decreasing_at_step::I = 1000, storage::Union{Nothing, StoreStateAction} = StoreStateAction( diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index f551384fbb..eb211f0487 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -339,6 +339,7 @@ function quasi_Newton!( O <: Union{AbstractManifoldFirstOrderObjective{E}, AbstractDecoratedManifoldObjective{E}}, } keywords_accepted(quasi_Newton!; kwargs...) + local local_dir_upd if memory_size >= 0 local_dir_upd = QuasiNewtonLimitedMemoryDirectionUpdate( M, @@ -351,6 +352,9 @@ function quasi_Newton!( nonpositive_curvature_behavior = nonpositive_curvature_behavior, sy_tol = sy_tol, ) + if requires_gcp(M) + local_dir_upd = QuasiNewtonLimitedMemoryBoxDirectionUpdate(local_dir_upd) + end else local_dir_upd = QuasiNewtonMatrixDirectionUpdate( M, @@ -401,6 +405,7 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) M = get_manifold(mp) get_gradient!(mp, qns.X, qns.p) qns.direction_update(qns.η, mp, qns) + # current_max_length = get_parameter(qns.direction_update, Val(:max_length)) if !(qns.nondescent_direction_behavior === :ignore) qns.nondescent_direction_value = real(inner(M, qns.p, qns.η, qns.X)) if qns.nondescent_direction_value > 0 diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index c90ae0286b..a3e2877a7c 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -116,7 +116,7 @@ using LinearAlgebra: I, eigvecs, tr, Diagonal, dot f(M, p) = sum(p .^ 2) grad_f(M, p) = 2 .* p p0 = [0.0, 4.0, 10.0] - p_opt = quasi_Newton(M, f, grad_f, p0) + # p_opt = quasi_Newton(M, f, grad_f, p0) end @testset "requires_gcp" begin From 2049c87f3b38bb54536ef27360559ce7f35fc5d0 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 8 Dec 2025 18:43:27 +0100 Subject: [PATCH 011/135] integrate GCPFinder into direction update (why doesn't this work right?) --- src/Manopt.jl | 2 +- src/plans/box_plan.jl | 101 +++++++++++--------------- src/plans/debug.jl | 2 +- src/plans/quasi_newton_plan.jl | 25 +------ src/solvers/quasi_Newton.jl | 45 ++++++++++-- test/solvers/test_quasi_Newton_box.jl | 4 +- 6 files changed, 85 insertions(+), 94 deletions(-) diff --git a/src/Manopt.jl b/src/Manopt.jl index 660d809514..873ee2f8fc 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -13,7 +13,7 @@ import LinearAlgebra: reflect! import ManifoldsBase: embed!, plot_slope, prepare_check_result, find_best_slope_window import ManifoldsBase: base_manifold, base_point, get_basis import ManifoldsBase: project, project! -import LinearAlgebra: cross +import LinearAlgebra: cross, LowerTriangular using ColorSchemes using ColorTypes using Colors diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index fd0b713b79..88823377b8 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -85,7 +85,11 @@ function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( r, mp::AbstractManoptProblem, st ) d.qn_du(r, mp, st) - # TODO find_gcp_direction! + M = get_manifold(mp) + p = get_iterate(st) + X = get_gradient(st) + gcp = GCPFinder(M, p, d) + find_gcp_direction!(gcp, r, p, r, X) return r end @@ -98,21 +102,21 @@ end @doc raw""" hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) -Compute $⟨X, B X⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`. +Compute ``⟨X, B X⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`. """ function hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) m = length(gh.qn_du.memory_s) - num_nonzero_rho = count(!iszero, gh.ρ) + num_nonzero_rho = count(!iszero, gh.qn_du.ρ) normX_sqr = norm(M, p, X)^2 if m == 0 || num_nonzero_rho == 0 - return gh.initial_scale \ normX_sqr + return gh.qn_du.initial_scale \ normX_sqr end ii = 1 for i in 1:m - if iszero(gh.ρ[i]) + if iszero(gh.qn_du.ρ[i]) continue end gh.coords_Yk_X[ii] = inner(M, p, gh.qn_du.memory_y[i], X) @@ -129,20 +133,20 @@ end @doc raw""" hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) -Compute $⟨X, B X⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`, and `X` is the +Compute ``⟨X, B X⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`, and `X` is the unit vector along index `b`. """ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) m = length(gh.qn_du.memory_s) - num_nonzero_rho = count(!iszero, gh.ρ) + num_nonzero_rho = count(!iszero, gh.qn_du.ρ) if m == 0 || num_nonzero_rho == 0 - return inv(gh.initial_scale) + return inv(gh.qn_du.initial_scale) end ii = 1 for i in 1:m - if iszero(gh.ρ[i]) + if iszero(gh.qn_du.ρ[i]) continue end gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) @@ -159,21 +163,21 @@ end @doc raw""" hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) -Compute $⟨X, B Y⟩$, where $B$ is the (1, 1)-Hessian represented by `gh`, where `X` is the +Compute ``⟨X, B Y⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`, where `X` is the unit vector pointing at index `b`. """ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) m = length(gh.qn_du.memory_s) - num_nonzero_rho = count(!iszero, gh.ρ) + num_nonzero_rho = count(!iszero, gh.qn_du.ρ) Yb = get_at_bound_index(M, Y, b) if m == 0 || num_nonzero_rho == 0 - return gh.initial_scale * Yb + return gh.qn_du.initial_scale * Yb end ii = 1 for i in 1:m - if iszero(gh.ρ[i]) + if iszero(gh.qn_du.ρ[i]) continue end gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) @@ -191,65 +195,42 @@ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstract return hess_val_from_wmwt_coords(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) end -function set_M_current_scale!(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) - M = gh.M - p = gh.p +function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) m = length(gh.qn_du.memory_s) - for i in m:-1:1 - # what if division by zero happened here, setting to zero ignores this in the next step - # pre-compute in case inner is expensive - v = inner(M, p, gh.qn_du.memory_s[i], gh.qn_du.memory_y[i]) - if isnan(v) - println("NaN in memory") - @show gh.qn_du.memory_s[i] - @show gh.qn_du.memory_y[i] - end - if v < gh.iszero_abstol - # The inner products ⟨s_i,y_i⟩ ≈ 0, i=$i, ignoring summand in approximation. - # (s, y) pairs with negative inner product are broken anyway, so we can reject them here - gh.ρ[i] = zero(eltype(gh.ρ)) - else - gh.ρ[i] = 1 / v - # it's so close to zero that we can skip it - if abs(gh.ρ[i]) < gh.iszero_abstol - gh.ρ[i] = zero(eltype(gh.ρ)) - end - end - end last_safe_index = -1 - for i in eachindex(gh.ρ) - if abs(gh.ρ[i]) > 0 + for i in eachindex(gh.qn_du.ρ) + if abs(gh.qn_du.ρ[i]) > 0 last_safe_index = i end end if (last_safe_index == -1) # All memory yield zero inner products - gh.current_scale = gh.initial_scale + gh.current_scale = gh.qn_du.initial_scale gh.M_11 = fill(0.0, 0, 0) gh.M_21 = fill(0.0, 0, 0) gh.M_22 = fill(0.0, 0, 0) return gh end - invA = Diagonal([-ri for ri in gh.ρ if !iszero(ri)]) - num_nonzero_rho = count(!iszero, gh.ρ) + invA = Diagonal([-ri for ri in gh.qn_du.ρ if !iszero(ri)]) + num_nonzero_rho = count(!iszero, gh.qn_du.ρ) Lk = LowerTriangular(zeros(num_nonzero_rho, num_nonzero_rho)) # total scaling factor for the initial Hessian - gh.current_scale = (gh.ρ[last_safe_index] * norm(M, p, gh.qn_du.memory_y[last_safe_index])^2) / gh.initial_scale + gh.current_scale = (gh.qn_du.ρ[last_safe_index] * norm(M, p, gh.qn_du.memory_y[last_safe_index])^2) / gh.qn_du.initial_scale tsksk = Symmetric(zeros(num_nonzero_rho, num_nonzero_rho)) ii = 1 # fill Dk and Lk for i in 1:m - if iszero(gh.ρ[i]) + if iszero(gh.qn_du.ρ[i]) continue end jj = 1 for j in 1:m - if iszero(gh.ρ[j]) + if iszero(gh.qn_du.ρ[j]) continue end if jj < ii @@ -307,7 +288,7 @@ function update_hessian!( ) (capacity(gh.qn_du.memory_s) == 0) && return gh update_hessian!(gh.qn_du, mp, st, p_old, k) - set_M_current_scale!(gh) + set_M_current_scale!(get_manifold(mp), get_iterate(st), gh) return gh end @@ -321,12 +302,12 @@ line segments in `GCPFinder`. abstract type AbstractFPFPPUpdater end """ - init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, d, ha) + init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) Method for initialization of `AbstractFPFPPUpdater` `fpfpp_upd` just before the loop that examines subsequent intervals for GCP. By default it does nothing. """ -init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, d, ha) = fpfpp_upd +init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) = fpfpp_upd """ struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end @@ -352,7 +333,7 @@ struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater end function get_default_fpfpp_updater(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) - return LimitedMemoryFPFPPUpdater(similar(ha.ρ), similar(ha.ρ), similar(ha.ρ), similar(ha.ρ)) + return LimitedMemoryFPFPPUpdater(similar(ha.qn_du.ρ), similar(ha.qn_du.ρ), similar(ha.qn_du.ρ), similar(ha.qn_du.ρ)) end function (::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) @@ -362,17 +343,17 @@ function (::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_doub return f_prime, f_double_prime end -function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater, d, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) +function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater, p, d, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) fill!(fpfpp_upd.c_s, 0) fill!(fpfpp_upd.c_y, 0) ii = 1 - for i in eachindex(ha.ρ) - if iszero(ha.ρ[i]) + for i in eachindex(ha.qn_du.ρ) + if iszero(ha.qn_du.ρ[i]) continue end - fpfpp_upd.p_s[ii] = ha.current_scale * inner(M, ha.p, ha.memory_s[i], d) - fpfpp_upd.p_y[ii] = inner(M, ha.p, ha.memory_y[i], d) + fpfpp_upd.p_s[ii] = ha.current_scale * inner(M, p, ha.qn_du.memory_s[i], d) + fpfpp_upd.p_y[ii] = inner(M, p, ha.qn_du.memory_y[i], d) ii += 1 end return fpfpp_upd @@ -380,20 +361,20 @@ end function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old) - m = length(ha.memory_s) - num_nonzero_rho = count(!iszero, ha.ρ) + m = length(ha.qn_du.memory_s) + num_nonzero_rho = count(!iszero, ha.qn_du.ρ) iss_eb_z = get_at_bound_index(M, z, b) iss_eb_d = get_at_bound_index(M, d_old, b) ii = 1 for i in 1:m - if iszero(ha.ρ[i]) + if iszero(ha.qn_du.ρ[i]) continue end # setting _X to w_b from the paper - ha.coords_Yk_X[ii] = get_at_bound_index(M, ha.memory_y[i], b) - ha.coords_Sk_X[ii] = ha.current_scale * get_at_bound_index(M, ha.memory_s[i], b) + ha.coords_Yk_X[ii] = get_at_bound_index(M, ha.qn_du.memory_y[i], b) + ha.coords_Sk_X[ii] = ha.current_scale * get_at_bound_index(M, ha.qn_du.memory_s[i], b) ii += 1 end @@ -550,7 +531,7 @@ function find_gcp_direction!(gcp::GCPFinder, d_out, p, d, X) t_current, b = pop!(F) dt = t_current - t_old - init_updater!(M, gcp.fpfpp_updater, d, gcp.ha) + init_updater!(M, gcp.fpfpp_updater, p, d, gcp.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 gcp.Y_tmp .+= dt .* d diff --git a/src/plans/debug.jl b/src/plans/debug.jl index 270f630b8c..0a9201f0b2 100644 --- a/src/plans/debug.jl +++ b/src/plans/debug.jl @@ -1175,7 +1175,7 @@ function (d::DebugWarnIfGradientNormTooLarge)( p_inj = d.factor * max_stepsize(M, p) if Xn > p_inj @warn """At iteration #$k - the gradient norm ($Xn) is larger that $(d.factor) times the injectivity radius $(p_inj) at the current iterate. + the gradient norm ($Xn) is larger than $(d.factor) times the injectivity radius $(p_inj) at the current iterate. """ if d.status === :Once @warn "Further warnings will be suppressed, use DebugWarnIfGradientNormTooLarge($(d.factor), :Always) to get all warnings." diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index 2881f3d961..c4b09d9e89 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -690,28 +690,9 @@ function (d::QuasiNewtonLimitedMemoryDirectionUpdate{InverseBFGS})( end # backward pass for i in m:-1:1 - # what if division by zero happened here, setting to zero ignores this in the next step - # pre-compute in case inner is expensive - v = inner(M, p, d.memory_s[i], d.memory_y[i]) - if d.nonpositive_curvature_behavior === :ignore && iszero(v) - d.ρ[i] = zero(eltype(d.ρ)) - if length(d.message) > 0 - d.message = replace(d.message, " i=" => " i=$i,") - d.message = replace(d.message, "summand in" => "summands in") - else - d.message = "The inner products ⟨s_i,y_i⟩ ≈ 0, i=$i, ignoring summand in approximation." - end - elseif d.nonpositive_curvature_behavior === :byrd && v <= d.sy_tol * norm(M, p, d.memory_y[i]) - d.ρ[i] = zero(eltype(d.ρ)) - if length(d.message) > 0 - d.message = replace(d.message, " i=" => " i=$i,") - d.message = replace(d.message, "summand in" => "summands in") - else - d.message = "The inner products ⟨s_i,y_i⟩ <= $(d.sy_tol * norm(M, p, d.memory_y[i])), i=$i, removing summand from approximation." - end - else - d.ρ[i] = 1 / v - end + # d.ρ is precomputed in the Hessian update + fill_rho_i!(M, p, d, i) + d.ξ[i] = inner(M, p, d.memory_s[i], r) * d.ρ[i] r .-= d.ξ[i] .* d.memory_y[i] end diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index eb211f0487..43fb9d1b50 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -720,6 +720,29 @@ function update_hessian!( return d end +function fill_rho_i!(M::AbstractManifold, p, d::QuasiNewtonLimitedMemoryDirectionUpdate, i::Int) + v = inner(M, p, d.memory_s[i], d.memory_y[i]) + if d.nonpositive_curvature_behavior === :ignore && iszero(v) + d.ρ[i] = zero(eltype(d.ρ)) + if length(d.message) > 0 + d.message = replace(d.message, " i=" => " i=$i,") + d.message = replace(d.message, "summand in" => "summands in") + else + d.message = "The inner products ⟨s_i,y_i⟩ ≈ 0, i=$i, ignoring summand in approximation." + end + elseif d.nonpositive_curvature_behavior === :byrd && v <= d.sy_tol * norm(M, p, d.memory_y[i]) + d.ρ[i] = zero(eltype(d.ρ)) + if length(d.message) > 0 + d.message = replace(d.message, " i=" => " i=$i,") + d.message = replace(d.message, "summand in" => "summands in") + else + d.message = "The inner products ⟨s_i,y_i⟩ <= $(d.sy_tol * norm(M, p, d.memory_y[i])), i=$i, removing summand from approximation." + end + else + d.ρ[i] = 1 / v + end +end + # Limited-memory update function update_hessian!( d::QuasiNewtonLimitedMemoryDirectionUpdate{U}, @@ -735,16 +758,19 @@ function update_hessian!( p = get_iterate(st) reforming_required = false for i in start:length(d.memory_s) + # transport all stored tangent vectors in the tangent space of the next iterate + vector_transport_to!( + M, d.memory_s[i], p_old, d.memory_s[i], p, d.vector_transport_method + ) + vector_transport_to!( + M, d.memory_y[i], p_old, d.memory_y[i], p, d.vector_transport_method + ) + + # what if division by zero happened here, setting to zero ignores this in the next step + # pre-compute in case inner is expensive + fill_rho_i!(M, p, d, i) if d.nonpositive_curvature_behavior === :byrd && iszero(d.ρ[i]) reforming_required = true - else - # transport all stored tangent vectors in the tangent space of the next iterate - vector_transport_to!( - M, d.memory_s[i], p_old, d.memory_s[i], p, d.vector_transport_method - ) - vector_transport_to!( - M, d.memory_y[i], p_old, d.memory_y[i], p, d.vector_transport_method - ) end end @@ -780,6 +806,9 @@ function update_hessian!( else push!(d.memory_y, copy(M, st.yk)) end + + fill_rho_i!(M, p, d, 1) + return d end diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index a3e2877a7c..fe85e5d549 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -114,8 +114,8 @@ using LinearAlgebra: I, eigvecs, tr, Diagonal, dot @testset "Pure Hyperrectangle" begin M = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) f(M, p) = sum(p .^ 2) - grad_f(M, p) = 2 .* p - p0 = [0.0, 4.0, 10.0] + grad_f(M, p) = project(M, p, 2 .* p) + p0 = [0.0, 4.0, 1.0] # p_opt = quasi_Newton(M, f, grad_f, p0) end From 34c796b4f62e1b500a4a9ba88dde4636a4aecffd Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 8 Dec 2025 18:53:57 +0100 Subject: [PATCH 012/135] return d from fill_rho_i! --- src/solvers/quasi_Newton.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 43fb9d1b50..e178ebec8c 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -741,6 +741,7 @@ function fill_rho_i!(M::AbstractManifold, p, d::QuasiNewtonLimitedMemoryDirectio else d.ρ[i] = 1 / v end + return d end # Limited-memory update From 672e4671cedcb971e083c7c54a6b213c366c0778 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 9 Dec 2025 10:54:04 +0100 Subject: [PATCH 013/135] fix rho update --- src/plans/quasi_newton_plan.jl | 2 -- src/solvers/quasi_Newton.jl | 10 +++++++++- test/solvers/test_quasi_Newton.jl | 16 ++++++++-------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index c4b09d9e89..c0171e7e99 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -691,8 +691,6 @@ function (d::QuasiNewtonLimitedMemoryDirectionUpdate{InverseBFGS})( # backward pass for i in m:-1:1 # d.ρ is precomputed in the Hessian update - fill_rho_i!(M, p, d, i) - d.ξ[i] = inner(M, p, d.memory_s[i], r) * d.ρ[i] r .-= d.ξ[i] .* d.memory_y[i] end diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index e178ebec8c..4be2266566 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -781,14 +781,20 @@ function update_hessian!( memory_size = capacity(d.memory_s) new_scb = CircularBuffer{T}(memory_size) new_ycb = CircularBuffer{T}(memory_size) + new_ρ = similar(d.ρ) + fill!(new_ρ, 0) + j = 1 for i in 1:length(d.memory_s) if !iszero(d.ρ[i]) push!(new_scb, d.memory_s[i]) push!(new_ycb, d.memory_y[i]) + new_ρ[j] = d.ρ[i] + j += 1 end end d.memory_s = new_scb d.memory_y = new_ycb + d.ρ = new_ρ end # add newest @@ -797,6 +803,7 @@ function update_hessian!( old_sk = popfirst!(d.memory_s) copyto!(M, old_sk, st.sk) push!(d.memory_s, old_sk) + circshift!(d.ρ, -1) else push!(d.memory_s, copy(M, st.sk)) end @@ -808,7 +815,7 @@ function update_hessian!( push!(d.memory_y, copy(M, st.yk)) end - fill_rho_i!(M, p, d, 1) + fill_rho_i!(M, p, d, length(d.memory_s)) return d end @@ -838,6 +845,7 @@ function update_hessian!( vector_transport_to!( M, d.update.memory_y[i], p_old, d.update.memory_y[i], p, d.update.vector_transport_method, ) + fill_rho_i!(M, p, d.update, i) end end return d diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index fd8b0500ad..ebf648ecd2 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -265,7 +265,7 @@ end vector_transport_method = ProjectionTransport(), retraction_method = QRRetraction(), cautious_update = true, - stopping_criterion = StopWhenGradientNormLess(1.0e-6), + stopping_criterion = StopWhenGradientNormLess(1.0e-6) | StopAfterIteration(100), ) x_inverseBFGSHuang = quasi_Newton( @@ -282,7 +282,7 @@ end vector_transport_method = ProjectionTransport(), retraction_method = QRRetraction(), cautious_update = true, - stopping_criterion = StopWhenGradientNormLess(1.0e-6), + stopping_criterion = StopWhenGradientNormLess(1.0e-6) | StopAfterIteration(100), ) @test isapprox(M, x_inverseBFGSCautious, x_inverseBFGSHuang; atol = 2.0e-4) end @@ -310,7 +310,7 @@ end x; basis = get_basis(M, x, DefaultOrthonormalBasis()), memory_size = -1, - stopping_criterion = StopWhenGradientNormLess(1.0e-9), + stopping_criterion = StopWhenGradientNormLess(1.0e-9) | StopAfterIteration(1000), ) @test norm(abs.(x_lrbfgs) - x_solution) ≈ 0 atol = rayleigh_atol end @@ -411,13 +411,13 @@ end mp = DefaultManoptProblem(M, gmp) qns = QuasiNewtonState(M; p = p) # push zeros to memory - push!(qns.direction_update.memory_s, copy(p)) - push!(qns.direction_update.memory_s, copy(p)) - push!(qns.direction_update.memory_y, copy(p)) - push!(qns.direction_update.memory_y, copy(p)) + qns.yk = copy(p) + qns.sk = copy(p) + update_hessian!(qns.direction_update, mp, qns, p, 1) + update_hessian!(qns.direction_update, mp, qns, p, 2) + @test contains(qns.direction_update.message, "i=2,1,1") qns.direction_update(mp, qns) # Update (1) says at i=1 inner products are zero (2) all are zero -> gradient proposal - @test contains(qns.direction_update.message, "i=1,2") @test contains(qns.direction_update.message, "gradient") end From 7b5dfe65f7767f919672b9b20288042754cb4a44 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 9 Dec 2025 18:56:31 +0100 Subject: [PATCH 014/135] first successful run --- src/Manopt.jl | 1 + src/plans/box_plan.jl | 12 ++++- src/plans/stepsize/stepsize.jl | 3 ++ src/plans/stopping_criterion.jl | 73 +++++++++++++++++++++++++-- src/solvers/quasi_Newton.jl | 7 ++- test/solvers/test_quasi_Newton_box.jl | 7 ++- 6 files changed, 94 insertions(+), 9 deletions(-) diff --git a/src/Manopt.jl b/src/Manopt.jl index 873ee2f8fc..992c0a07e1 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -567,6 +567,7 @@ export StopAfter, StopWhenGradientChangeLess, StopWhenGradientMappingNormLess, StopWhenGradientNormLess, + StopWhenProjectedMinusGradientNormLess, StopWhenFirstOrderProgress, StopWhenIterateNaN, StopWhenKKTResidualLess, diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 88823377b8..127cefd14a 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -42,6 +42,15 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ coords_Sk_Y::V coords_Yk_X::V coords_Yk_Y::V + last_gcp_result::Symbol +end + +function get_parameter(d::QuasiNewtonLimitedMemoryBoxDirectionUpdate, ::Val{:max_stepsize}) + if d.last_gcp_result === :found_limited + return 1.0 + else + return Inf + end end function QuasiNewtonLimitedMemoryBoxDirectionUpdate( @@ -67,6 +76,7 @@ function QuasiNewtonLimitedMemoryBoxDirectionUpdate( coords_Sk_Y, coords_Yk_X, coords_Yk_Y, + :not_searched, ) end @@ -89,7 +99,7 @@ function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( p = get_iterate(st) X = get_gradient(st) gcp = GCPFinder(M, p, d) - find_gcp_direction!(gcp, r, p, r, X) + d.last_gcp_result = find_gcp_direction!(gcp, r, p, r, X) return r end diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index 7288fd781b..e1acb58d86 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -1724,6 +1724,9 @@ function (a::WolfePowellLinesearchStepsize)( max_step_increase = ifelse( isfinite(a.max_stepsize), min(1.0e9, a.max_stepsize / grad_norm), 1.0e9 ) + if :stop_when_stepsize_exceeds in keys(kwargs) + max_step_increase = min(max_step_increase, kwargs[:stop_when_stepsize_exceeds]) + end step = ifelse(isfinite(a.max_stepsize), min(1.0, a.max_stepsize / grad_norm), 1.0) s_plus = step s_minus = step diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index bc26c6df62..15274b2247 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -722,7 +722,7 @@ Create a stopping criterion with threshold `ε` for the gradient, that is, this indicates to stop when [`get_gradient`](@ref) returns a gradient vector of norm less than `ε`, where the norm to use can be specified in the `norm=` keyword. """ -mutable struct StopWhenGradientNormLess{F, TF, N <: Union{Missing, Real}} <: StoppingCriterion +mutable struct StopWhenGradientNormLess{F, TF <: Real, N <: Union{Missing, Real}} <: StoppingCriterion norm::F threshold::TF last_change::TF @@ -730,7 +730,7 @@ mutable struct StopWhenGradientNormLess{F, TF, N <: Union{Missing, Real}} <: Sto outer_norm::N function StopWhenGradientNormLess( ε::TF; norm::F = norm, outer_norm::N = missing - ) where {F, TF, N <: Union{Missing, Real}} + ) where {F, TF <: Real, N <: Union{Missing, Real}} return new{F, TF, N}(norm, ε, zero(ε), -1, outer_norm) end end @@ -769,11 +769,76 @@ function show(io::IO, c::StopWhenGradientNormLess) end """ - set_parameter!(c::StopWhenGradientNormLess, :MinGradNorm, v::Float64) + set_parameter!(c::StopWhenGradientNormLess{F,TF}, ::Val{:MinGradNorm}, v::TF) where {F,TF<:Real} Update the minimal gradient norm when an algorithm shall stop """ -function set_parameter!(c::StopWhenGradientNormLess, ::Val{:MinGradNorm}, v::Float64) +function set_parameter!(c::StopWhenGradientNormLess{F, TF}, ::Val{:MinGradNorm}, v::TF) where {F, TF <: Real} + c.threshold = v + return c +end + +""" + StopWhenProjectedMinusGradientNormLess <: StoppingCriterion + +A stopping criterion similar to [`StopWhenGradientNormLess`](@ref), although it checks the +norm of projected minus gradient. It is primarily useful for optimization involving +[`Hyperrectangle`](@extref). +""" +mutable struct StopWhenProjectedMinusGradientNormLess{F, TF <: Real, N <: Union{Missing, Real}} <: StoppingCriterion + norm::F + threshold::TF + last_change::TF + at_iteration::Int + outer_norm::N + function StopWhenProjectedMinusGradientNormLess( + ε::TF; norm::F = norm, outer_norm::N = missing + ) where {F, TF <: Real, N <: Union{Missing, Real}} + return new{F, TF, N}(norm, ε, zero(ε), -1, outer_norm) + end +end + +function (sc::StopWhenProjectedMinusGradientNormLess)( + mp::AbstractManoptProblem, s::AbstractManoptSolverState, k::Int + ) + M = get_manifold(mp) + if k == 0 # reset on init + sc.at_iteration = -1 + end + if (k > 0) + r = (has_components(M) && !ismissing(sc.outer_norm)) ? (sc.outer_norm,) : () + p = get_iterate(s) + mpg = project(M, p, -get_gradient(s)) + sc.last_change = sc.norm(M, p, mpg, r...) + if sc.last_change < sc.threshold + sc.at_iteration = k + return true + end + end + return false +end +function get_reason(c::StopWhenProjectedMinusGradientNormLess) + if (c.last_change < c.threshold) && (c.at_iteration >= 0) + return "The algorithm reached approximately critical point after $(c.at_iteration) iterations; the gradient norm ($(c.last_change)) is less than $(c.threshold).\n" + end + return "" +end +function status_summary(c::StopWhenProjectedMinusGradientNormLess) + has_stopped = (c.at_iteration >= 0) + s = has_stopped ? "reached" : "not reached" + return "|grad f| < $(c.threshold): $s" +end +indicates_convergence(c::StopWhenProjectedMinusGradientNormLess) = true +function show(io::IO, c::StopWhenProjectedMinusGradientNormLess) + return print(io, "StopWhenProjectedMinusGradientNormLess($(c.threshold))\n $(status_summary(c))") +end + +""" + set_parameter!(c::StopWhenProjectedMinusGradientNormLess{F,TF}, ::Val{:MinGradNorm}, v::TF) where {F, TF<:Real} + +Update the minimal gradient norm when an algorithm shall stop. +""" +function set_parameter!(c::StopWhenProjectedMinusGradientNormLess{F, TF}, ::Val{:MinGradNorm}, v::TF) where {F, TF <: Real} c.threshold = v return c end diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 4be2266566..8e9860d184 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -405,7 +405,10 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) M = get_manifold(mp) get_gradient!(mp, qns.X, qns.p) qns.direction_update(qns.η, mp, qns) - # current_max_length = get_parameter(qns.direction_update, Val(:max_length)) + current_max_stepsize = get_parameter(qns.direction_update, Val(:max_stepsize)) + if !isfinite(current_max_stepsize) + current_max_stepsize = max_stepsize(M, qns.p) / norm(qns.η) + end if !(qns.nondescent_direction_behavior === :ignore) qns.nondescent_direction_value = real(inner(M, qns.p, qns.η, qns.X)) if qns.nondescent_direction_value > 0 @@ -419,7 +422,7 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) end end end - α = qns.stepsize(mp, qns, k, qns.η; gradient = qns.X) + α = qns.stepsize(mp, qns, k, qns.η; gradient = qns.X, stop_when_stepsize_exceeds = current_max_stepsize) copyto!(M, qns.p_old, get_iterate(qns)) ManifoldsBase.retract_fused!(M, qns.p, qns.p, qns.η, α, qns.retraction_method) qns.η .*= α diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index fe85e5d549..09112d1a2f 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -114,9 +114,12 @@ using LinearAlgebra: I, eigvecs, tr, Diagonal, dot @testset "Pure Hyperrectangle" begin M = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) f(M, p) = sum(p .^ 2) - grad_f(M, p) = project(M, p, 2 .* p) + function grad_f(M, p) + return project(M, p, 2 .* p) + end p0 = [0.0, 4.0, 1.0] - # p_opt = quasi_Newton(M, f, grad_f, p0) + p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedMinusGradientNormLess(1.0e-6) | StopAfterIteration(10)) + @test p_opt ≈ [0, 2, 0] end @testset "requires_gcp" begin From 99d193d683cc8b1dd74b5b2f3a75ed27d6b5aba8 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 10 Dec 2025 10:27:59 +0100 Subject: [PATCH 015/135] address review, fix bug --- src/Manopt.jl | 2 +- src/plans/stopping_criterion.jl | 32 ++++++++++++++++----------- src/solvers/quasi_Newton.jl | 9 ++++++-- test/solvers/test_quasi_Newton_box.jl | 2 +- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/Manopt.jl b/src/Manopt.jl index 992c0a07e1..daecc47bb5 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -567,7 +567,7 @@ export StopAfter, StopWhenGradientChangeLess, StopWhenGradientMappingNormLess, StopWhenGradientNormLess, - StopWhenProjectedMinusGradientNormLess, + StopWhenProjectedNegativeGradientNormLess, StopWhenFirstOrderProgress, StopWhenIterateNaN, StopWhenKKTResidualLess, diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 15274b2247..1c048c1716 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -779,26 +779,32 @@ function set_parameter!(c::StopWhenGradientNormLess{F, TF}, ::Val{:MinGradNorm}, end """ - StopWhenProjectedMinusGradientNormLess <: StoppingCriterion + StopWhenProjectedNegativeGradientNormLess <: StoppingCriterion A stopping criterion similar to [`StopWhenGradientNormLess`](@ref), although it checks the norm of projected minus gradient. It is primarily useful for optimization involving [`Hyperrectangle`](@extref). + +On manifolds with boundary and manifolds with corners, for a tangent vector ``X``, +``-X`` might not be a valid tangent vector. As an example, consider the objective +``f(x)=x^2`` on the interval ``[1, 2]``. Its gradient at 1 is equal to 2, but because the +point 1 is at the boundary of the interval, the projected negative gradient is equal to 0 +because we can't go in the negative direction. """ -mutable struct StopWhenProjectedMinusGradientNormLess{F, TF <: Real, N <: Union{Missing, Real}} <: StoppingCriterion +mutable struct StopWhenProjectedNegativeGradientNormLess{F, TF <: Real, N <: Union{Missing, Real}} <: StoppingCriterion norm::F threshold::TF last_change::TF at_iteration::Int outer_norm::N - function StopWhenProjectedMinusGradientNormLess( + function StopWhenProjectedNegativeGradientNormLess( ε::TF; norm::F = norm, outer_norm::N = missing ) where {F, TF <: Real, N <: Union{Missing, Real}} return new{F, TF, N}(norm, ε, zero(ε), -1, outer_norm) end end -function (sc::StopWhenProjectedMinusGradientNormLess)( +function (sc::StopWhenProjectedNegativeGradientNormLess)( mp::AbstractManoptProblem, s::AbstractManoptSolverState, k::Int ) M = get_manifold(mp) @@ -808,7 +814,7 @@ function (sc::StopWhenProjectedMinusGradientNormLess)( if (k > 0) r = (has_components(M) && !ismissing(sc.outer_norm)) ? (sc.outer_norm,) : () p = get_iterate(s) - mpg = project(M, p, -get_gradient(s)) + mpg = embed_project(M, p, -get_gradient(s)) sc.last_change = sc.norm(M, p, mpg, r...) if sc.last_change < sc.threshold sc.at_iteration = k @@ -817,28 +823,28 @@ function (sc::StopWhenProjectedMinusGradientNormLess)( end return false end -function get_reason(c::StopWhenProjectedMinusGradientNormLess) +function get_reason(c::StopWhenProjectedNegativeGradientNormLess) if (c.last_change < c.threshold) && (c.at_iteration >= 0) return "The algorithm reached approximately critical point after $(c.at_iteration) iterations; the gradient norm ($(c.last_change)) is less than $(c.threshold).\n" end return "" end -function status_summary(c::StopWhenProjectedMinusGradientNormLess) +function status_summary(c::StopWhenProjectedNegativeGradientNormLess) has_stopped = (c.at_iteration >= 0) s = has_stopped ? "reached" : "not reached" - return "|grad f| < $(c.threshold): $s" + return "|proj (-grad f)| < $(c.threshold): $s" end -indicates_convergence(c::StopWhenProjectedMinusGradientNormLess) = true -function show(io::IO, c::StopWhenProjectedMinusGradientNormLess) - return print(io, "StopWhenProjectedMinusGradientNormLess($(c.threshold))\n $(status_summary(c))") +indicates_convergence(c::StopWhenProjectedNegativeGradientNormLess) = true +function show(io::IO, c::StopWhenProjectedNegativeGradientNormLess) + return print(io, "StopWhenProjectedNegativeGradientNormLess($(c.threshold))\n $(status_summary(c))") end """ - set_parameter!(c::StopWhenProjectedMinusGradientNormLess{F,TF}, ::Val{:MinGradNorm}, v::TF) where {F, TF<:Real} + set_parameter!(c::StopWhenProjectedNegativeGradientNormLess{F,TF}, ::Val{:MinGradNorm}, v::TF) where {F, TF<:Real} Update the minimal gradient norm when an algorithm shall stop. """ -function set_parameter!(c::StopWhenProjectedMinusGradientNormLess{F, TF}, ::Val{:MinGradNorm}, v::TF) where {F, TF <: Real} +function set_parameter!(c::StopWhenProjectedNegativeGradientNormLess{F, TF}, ::Val{:MinGradNorm}, v::TF) where {F, TF <: Real} c.threshold = v return c end diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 8e9860d184..f95667f5d3 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -406,7 +406,7 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) get_gradient!(mp, qns.X, qns.p) qns.direction_update(qns.η, mp, qns) current_max_stepsize = get_parameter(qns.direction_update, Val(:max_stepsize)) - if !isfinite(current_max_stepsize) + if !isnothing(current_max_stepsize) && !isfinite(current_max_stepsize) current_max_stepsize = max_stepsize(M, qns.p) / norm(qns.η) end if !(qns.nondescent_direction_behavior === :ignore) @@ -422,7 +422,12 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) end end end - α = qns.stepsize(mp, qns, k, qns.η; gradient = qns.X, stop_when_stepsize_exceeds = current_max_stepsize) + local α + if isnothing(current_max_stepsize) + α = qns.stepsize(mp, qns, k, qns.η; gradient = qns.X) + else + α = qns.stepsize(mp, qns, k, qns.η; gradient = qns.X, stop_when_stepsize_exceeds = current_max_stepsize) + end copyto!(M, qns.p_old, get_iterate(qns)) ManifoldsBase.retract_fused!(M, qns.p, qns.p, qns.η, α, qns.retraction_method) qns.η .*= α diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 09112d1a2f..501991dd61 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -118,7 +118,7 @@ using LinearAlgebra: I, eigvecs, tr, Diagonal, dot return project(M, p, 2 .* p) end p0 = [0.0, 4.0, 1.0] - p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedMinusGradientNormLess(1.0e-6) | StopAfterIteration(10)) + p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(10)) @test p_opt ≈ [0, 2, 0] end From da8f5007439f643ea3991898d4356a282b72f2d9 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 10 Dec 2025 13:09:15 +0100 Subject: [PATCH 016/135] a little bit of docs --- docs/make.jl | 1 + docs/src/solvers/box_domain.md | 28 ++++++++++++++++++++++++++++ src/Manopt.jl | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 docs/src/solvers/box_domain.md diff --git a/docs/make.jl b/docs/make.jl index c14c32e6a8..b7887aa113 100755 --- a/docs/make.jl +++ b/docs/make.jl @@ -179,6 +179,7 @@ makedocs(; "Adaptive Regularization with Cubics" => "solvers/adaptive-regularization-with-cubics.md", "Alternating Gradient Descent" => "solvers/alternating_gradient_descent.md", "Augmented Lagrangian Method" => "solvers/augmented_Lagrangian_method.md", + "Box domains" => "solvers/box_domain.md", "Chambolle-Pock" => "solvers/ChambollePock.md", "CMA-ES" => "solvers/cma_es.md", "Conjugate gradient descent" => "solvers/conjugate_gradient_descent.md", diff --git a/docs/src/solvers/box_domain.md b/docs/src/solvers/box_domain.md new file mode 100644 index 0000000000..3835a687f7 --- /dev/null +++ b/docs/src/solvers/box_domain.md @@ -0,0 +1,28 @@ +# Optimization with box domains and products of manifolds and boxes + +A [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) is, in general, not a manifold but a manifold with corners, thus handling it as a domain in optimization requires special attention. +For simple methods like gradient descent using projected gradient and a stopping criterion involving [`StopWhenProjectedNegativeGradientNormLess`](@ref) may be sufficient, however methods that approximate the Hessian can benefit from a more advanced approach. +The core idea is considering a piecewise quadratic approximation of the objective along the descent direction, and selecting the generalized Cauchy point -- its minimizer. +The points at which the approximation might not be differentiable correspond to hitting new boundaries along the initially selected descent direction. +Then, we can perform standard line search between the initial iterate and the generalized Cauchy point. + +## Public types and method + +```@docs +QuasiNewtonLimitedMemoryBoxDirectionUpdate +``` + +## Internal types and method + +```@docs +Manopt.init_updater! +Manopt.hess_val +Manopt.AbstractFPFPPUpdater +Manopt.GenericFPFPPUpdater +Manopt.get_bounds_index +Manopt.requires_gcp +Manopt.find_gcp_direction! +Manopt.hess_val_eb +Manopt.LimitedMemoryFPFPPUpdater +Manopt.get_bound_t +``` diff --git a/src/Manopt.jl b/src/Manopt.jl index daecc47bb5..81bd7673aa 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -422,7 +422,7 @@ export CondensedKKTVectorField, CondensedKKTVectorFieldJacobian export SymmetricLinearSystemObjective export ProximalGradientNonsmoothCost, ProximalGradientNonsmoothSubgradient -export QuasiNewtonState, QuasiNewtonLimitedMemoryDirectionUpdate +export QuasiNewtonState, QuasiNewtonLimitedMemoryDirectionUpdate, QuasiNewtonLimitedMemoryBoxDirectionUpdate export QuasiNewtonMatrixDirectionUpdate export QuasiNewtonPreconditioner export QuasiNewtonCautiousDirectionUpdate, From 3c8cc8e71789c93e87dcaa2020b4f9c806b4608a Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 10 Dec 2025 13:15:29 +0100 Subject: [PATCH 017/135] fix extref again? --- src/plans/stopping_criterion.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 1c048c1716..6c6e316a20 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -783,7 +783,7 @@ end A stopping criterion similar to [`StopWhenGradientNormLess`](@ref), although it checks the norm of projected minus gradient. It is primarily useful for optimization involving -[`Hyperrectangle`](@extref). +[`Hyperrectangle`](@extref Manifolds.Hyperrectangle). On manifolds with boundary and manifolds with corners, for a tangent vector ``X``, ``-X`` might not be a valid tangent vector. As an example, consider the objective From 5088950806f0ca8dc4b4f46669efcb7dbaadc956 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 10 Dec 2025 14:27:42 +0100 Subject: [PATCH 018/135] StopWhenRelativeAPosterioriChangeCostLessOrEqual --- Changelog.md | 1 + docs/src/references.bib | 15 +++++- src/Manopt.jl | 1 + src/plans/box_plan.jl | 2 +- src/plans/stopping_criterion.jl | 71 ++++++++++++++++++++++++++++ test/plans/test_stopping_criteria.jl | 14 ++++++ 6 files changed, 102 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index fb1dcf2cdb..ade02b28de 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. (#554) * `GCPFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) +* New stopping criteria: `StopWhenRelativeAPosterioriChangeCostLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. * add keyword argument `is_feasible_error` to `interior_point_Newton` to control how to handle infeasible starting points (#556) * add keyword argument `at_init` to some debug options to control whether they print already at the initialisation and hence before the first iteration (#552) diff --git a/docs/src/references.bib b/docs/src/references.bib index bd79ce494d..36700231e7 100644 --- a/docs/src/references.bib +++ b/docs/src/references.bib @@ -899,4 +899,17 @@ @article{ZhangSra:2018 TITLE = {Towards Riemannian accelerated gradient methods}, URL = {https://arxiv.org/abs/1806.02812}, YEAR = {2018}, -} \ No newline at end of file +} + +@article{ZhuByrdLuNocedal:1997, + title = {Algorithm 778: {L}-{BFGS}-{B}: {Fortran} subroutines for large-scale bound-constrained optimization}, + volume = {23}, + issn = {0098-3500}, + doi = {10.1145/279232.279236}, + number = {4}, + journal = {ACM Trans. Math. Softw.}, + author = {Zhu, Ciyou and Byrd, Richard H. and Lu, Peihuang and Nocedal, Jorge}, + month = dec, + year = {1997}, + pages = {550--560}, +} diff --git a/src/Manopt.jl b/src/Manopt.jl index 81bd7673aa..038a0429fd 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -579,6 +579,7 @@ export StopAfter, StopWhenPopulationDiverges, StopWhenPopulationStronglyConcentrated, StopWhenProjectedGradientStationary, + StopWhenRelativeAPosterioriChangeCostLessOrEqual, StopWhenRelativeResidualLess, StopWhenRepeated, StopWhenSmallerOrEqual, diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 127cefd14a..1683c38f42 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -13,7 +13,7 @@ requires_gcp(M::ProductManifold) = requires_gcp(M.manifolds[1]) An approximation of Hessian of a scalar function of the form ``B_0 = θ I``, ``B_{k+1} = B_k - W_k M_k W_k^{\mathrm{T}}``, where ``\theta > 0`` is an initial scaling guess. -Matrix ``M_k = \begin{psmallmatrix}M_{11} & M_{21}^{\mathrm{T}}\\ M_{21} & M_{22}\end{psmallmatrix}`` +Matrix ``M_k = \left(\begin{smallmatrix}M_{11} & M_{21}^{\mathrm{T}}\\ M_{21} & M_{22}\end{smallmatrix}\right)`` is stored using its blocks. Blocks ``W_k`` are (implicitly) composed from `memory_y` and `memory_s`. diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 6c6e316a20..82a5b7ee0e 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -463,6 +463,77 @@ function set_parameter!(c::StopWhenCostLess, ::Val{:MinCost}, v) return c end +""" + StopWhenRelativeAPosterioriChangeCostLessOrEqual <: StoppingCriterion + +A stopping criterion to stop when + +````math +\\frac{f_k - f_{k+1}}{\\max(\\lvert f_k \\rvert, \\lvert f_{k+1} \\rvert, 1)} \\leq tol, +```` + +based on Eq. (1) in [ZhuByrdLuNocedal:1997](@cite) + +# Fields +$(_var(:Field, :at_iteration)) +$(_var(:Field, :last_change)) +* `last_cost``: the last cost value + +# Constructor + + StopWhenRelativeAPosterioriChangeCostLessOrEqual(tolerance::F) + +Initialize the stopping criterion to a threshold `tolerance` for the change of the cost function. + + StopWhenRelativeAPosterioriChangeCostLessOrEqual(; factr::Real=1.0e7) + +Initialize tolerance to `factr * eps(factr)`, following the convention in [ZhuByrdLuNocedal:1997](@cite). +""" +mutable struct StopWhenRelativeAPosterioriChangeCostLessOrEqual{F <: Real} <: StoppingCriterion + tolerance::F + at_iteration::Int + last_cost::F + last_change::F +end +function StopWhenRelativeAPosterioriChangeCostLessOrEqual(tol::F) where {F <: Real} + return StopWhenRelativeAPosterioriChangeCostLessOrEqual{F}(tol, -1, zero(tol), 2 * tol) +end +StopWhenRelativeAPosterioriChangeCostLessOrEqual(; factr::F = 1.0e7) where {F <: Real} = StopWhenRelativeAPosterioriChangeCostLessOrEqual(factr * eps(factr)) +function (c::StopWhenRelativeAPosterioriChangeCostLessOrEqual)( + problem::AbstractManoptProblem, state::AbstractManoptSolverState, iteration::Int + ) + if iteration <= 0 # reset on init + c.at_iteration = -1 + c.last_cost = Inf + c.last_change = 2 * c.tolerance + end + current_cost = get_cost(problem, get_iterate(state)) + c.last_change = (c.last_cost - current_cost) / max(abs(c.last_cost), abs(current_cost), 1) + c.last_cost = current_cost + if iteration > 1 && c.last_change <= c.tolerance + c.at_iteration = iteration + return true + end + return false +end +function get_reason(c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) + if c.at_iteration >= 0 + return "At iteration $(c.at_iteration) the algorithm performed a step with a relative a posteriori cost change ($(abs(c.last_change))) less than or equal to $(c.tolerance)." + end + return "" +end +function status_summary(c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) + has_stopped = (c.at_iteration >= 0) + s = has_stopped ? "reached" : "not reached" + return "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = $(abs(c.last_change)) ≤ $(c.tolerance):\t$s" +end +function Base.show(io::IO, c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) + return print( + io, + "StopWhenRelativeAPosterioriChangeCostLessOrEqual with threshold $(c.tolerance).\n $(status_summary(c))", + ) +end + @doc """ StopWhenEntryChangeLess diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 91c83e0541..97ce4fe3bf 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -363,6 +363,20 @@ using Manifolds, ManifoldsBase, Manopt, Test, ManifoldsBase, Dates @test length(get_reason(sc)) == 0 end + @testset "StopWhenRelativeAPosterioriChangeCostLessOrEqual" begin + sc = StopWhenRelativeAPosterioriChangeCostLessOrEqual(; factr = 100.0) + prob = DefaultManoptProblem( + Euclidean(), ManifoldGradientObjective((M, x) -> x^2, x -> 2x) + ) + s = GradientDescentState(Euclidean(); p = 1.0) + @test !sc(prob, s, 1) + @test length(get_reason(sc)) == 0 + s.p = 1.0 - 1.0e-14 + + @test sc(prob, s, 2) + @test length(get_reason(sc)) > 0 + end + @testset "has_converged" begin M = Euclidean(1) pr = Manopt.Test.DummyProblem{typeof(M)}() From 5365a08ed93526a22f3fbd81abf6604b4366bf7e Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 12 Dec 2025 13:16:22 +0100 Subject: [PATCH 019/135] improve consistency of stepsize limits --- Changelog.md | 6 ++++++ src/plans/stepsize/linesearch.jl | 14 ++++++------ src/plans/stepsize/stepsize.jl | 37 +++++++++++++++++++++----------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/Changelog.md b/Changelog.md index 472247499d..731a53b72a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -8,10 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + * `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. (#554) * `GCPFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) * New stopping criteria: `StopWhenRelativeAPosterioriChangeCostLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. +### Fixed + +* Line searches consistently respect `stop_when_stepsize_exceeds` keyword argument as a hard limit. + ## [0.5.30] December 10, 2025 ### Added diff --git a/src/plans/stepsize/linesearch.jl b/src/plans/stepsize/linesearch.jl index 1114ebfc17..d504592dc7 100644 --- a/src/plans/stepsize/linesearch.jl +++ b/src/plans/stepsize/linesearch.jl @@ -129,8 +129,8 @@ $(_var(:Keyword, :retraction_method)) * `gradient = nothing`: precomputed gradient at point `p` * `report_messages_in::NamedTuple = (; )`: a named tuple of [`StepsizeMessage`](@ref)s to report messages in. currently supported keywords are `:non_descent_direction`, `:stepsize_exceeds`, `:stepsize_less`, `:stop_increasing`, `:stop_decreasing` -* `stop_when_stepsize_less=0.0`: to avoid numerical underflow -* `stop_when_stepsize_exceeds=`[`max_stepsize`](@ref)`(M, p) / norm(M, p, η)`) to avoid leaving the injectivity radius on a manifold +* `stop_when_stepsize_less::Real=0.0`: to avoid numerical underflow +* `stop_when_stepsize_exceeds::Real=`[`max_stepsize`](@ref)`(M, p) / norm(M, p, η)`) to avoid leaving the injectivity radius on a manifold or exceeding boundaries on a manifold with corners * `stop_increasing_at_step=100`: stop the initial increase of step size after these many steps * `stop_decreasing_at_step=`1000`: stop the decreasing search after these many steps @@ -159,14 +159,14 @@ function linesearch_backtrack!( decrease, contract, η::T; - lf0 = f(M, p), + lf0::Real = f(M, p), gradient = nothing, Dlf0 = isnothing(gradient) ? nothing : real(inner(M, p, gradient, η)), retraction_method::AbstractRetractionMethod = default_retraction_method(M, typeof(p)), additional_increase_condition = (M, p) -> true, additional_decrease_condition = (M, p) -> true, - stop_when_stepsize_less = 0.0, - stop_when_stepsize_exceeds = max_stepsize(M, p) / norm(M, p, η), + stop_when_stepsize_less::Real = 0.0, + stop_when_stepsize_exceeds::Real = max_stepsize(M, p) / norm(M, p, η), stop_increasing_at_step = 100, stop_decreasing_at_step = 1000, report_messages_in::NamedTuple = (;), @@ -182,14 +182,14 @@ function linesearch_backtrack!( while f_q < lf0 + decrease * s * Dlf0 || !additional_increase_condition(M, q) (stop_increasing_at_step == 0) && break i = i + 1 - s = s / contract + s = min(s / contract, stop_when_stepsize_exceeds) ManifoldsBase.retract_fused!(M, q, p, η, s, retraction_method) f_q = f(M, q) if i == stop_increasing_at_step set_message!(report_messages_in, :stop_increasing, at = i, bound = stop_increasing_at_step, value = s) break end - if s > stop_when_stepsize_exceeds + if s >= stop_when_stepsize_exceeds set_message!(report_messages_in, :stepsize_exceeds, at = i, bound = stop_when_stepsize_exceeds, value = s) break end diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index e1acb58d86..0a9e4c01b7 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -41,10 +41,10 @@ with the fields keyword arguments and the retraction is set to the default retra $(_var(:Keyword, :retraction_method)) * `contraction_factor=0.95` * `sufficient_decrease=0.1` -* `last_stepsize=initialstepsize` +* `last_stepsize=initial_stepsize` * `initial_guess=`[`ArmijoInitialGuess`](@ref)`()` * `stop_when_stepsize_less=0.0`: stop when the stepsize decreased below this version. -* `stop_when_stepsize_exceeds=[`max_step`](@ref)`(M)`: provide an absolute maximal step size. +* `stop_when_stepsize_exceeds=[`max_stepsize`](@ref)`(M)`: provide an absolute maximal step size. * `stop_increasing_at_step=100`: for the initial increase test, stop after these many steps * `stop_decreasing_at_step=1000`: in the backtrack, stop after these many steps """ @@ -672,7 +672,6 @@ end """ cubic_stepsize_update_step(a::Real, b::Real, c::Real, τ::Real) - Step function to determine the stepsize update `c` described in [Hager:1989](@cite). @@ -696,6 +695,8 @@ function cubic_stepsize_update_step(a::Real, b::Real, c::Real, τ::Real) end """ + get_univariate_triple!(mp::AbstractManoptProblem, cbls::CubicBracketingLinesearchStepsize, p, η, t::Real) + Get the `UnivariateTriple` of the problem `mp` related to the step with stepsize ``t`` from ``p`` in direction ``η``. @@ -706,7 +707,7 @@ stepsize ``t`` from ``p`` in direction ``η``. * `η`: search direction at `p` * `t::Real`: step size """ -function get_univariate_triple!(mp::AbstractManoptProblem, cbls::CubicBracketingLinesearchStepsize, p, η, t) +function get_univariate_triple!(mp::AbstractManoptProblem, cbls::CubicBracketingLinesearchStepsize, p, η, t::Real) M = get_manifold(mp) cbls.last_stepsize = t ManifoldsBase.retract_fused!(M, cbls.candidate_point, p, η, t, cbls.retraction_method) @@ -730,7 +731,11 @@ function (cbls::CubicBracketingLinesearchStepsize)( check_curvature(c::UnivariateTriple) = abs(c.df) < cbls.sufficient_curvature * abs(init.df) n_iter = 0 - t = cbls.last_stepsize + max_step = cbls.max_stepsize + if :stop_when_stepsize_exceeds in keys(kwargs) + max_step = min(max_step, kwargs.stop_when_stepsize_exceeds) + end + t = min(cbls.last_stepsize, max_step) c_old = init c = get_univariate_triple!(mp, cbls, p, η, t) a, b = nothing, nothing @@ -745,9 +750,9 @@ function (cbls::CubicBracketingLinesearchStepsize)( (a, b) = c, c_old break end - (t == cbls.max_stepsize) && return t + (t == max_step) && return t t *= cbls.stepsize_increase - t = min(t, cbls.max_stepsize) + t = min(t, max_step) c_old = c c = get_univariate_triple!(mp, cbls, p, η, t) end @@ -1414,6 +1419,13 @@ function (a::NonmonotoneLinesearchStepsize)( end #compute the new step size with the help of the Barzilai-Borwein step size + l = norm(M, p, η) + local swse + if :stop_when_stepsize_exceeds in keys(kwargs) + swse = kwargs.stop_when_stepsize_exceeds + else + swse = (a.stop_when_stepsize_exceeds / l) + end a.last_stepsize = linesearch_backtrack!( M, a.candidate_point, @@ -1423,11 +1435,11 @@ function (a::NonmonotoneLinesearchStepsize)( a.sufficient_decrease, a.stepsize_reduction, η; - lf0 = maximum([a.old_costs[j] for j in 1:min(iter, memory_size)]), + lf0 = maximum(view(a.old_costs, 1:min(iter, memory_size))), gradient = X, retraction_method = a.retraction_method, - stop_when_stepsize_less = (a.stop_when_stepsize_less / norm(M, p, η)), - stop_when_stepsize_exceeds = (a.stop_when_stepsize_exceeds / norm(M, p, η)), + stop_when_stepsize_less = (a.stop_when_stepsize_less / l), + stop_when_stepsize_exceeds = swse, stop_increasing_at_step = a.stop_increasing_at_step, stop_decreasing_at_step = a.stop_decreasing_at_step, report_messages_in = a.messages, @@ -1728,6 +1740,7 @@ function (a::WolfePowellLinesearchStepsize)( max_step_increase = min(max_step_increase, kwargs[:stop_when_stepsize_exceeds]) end step = ifelse(isfinite(a.max_stepsize), min(1.0, a.max_stepsize / grad_norm), 1.0) + step = min(step, max_step_increase) s_plus = step s_minus = step # clear messages @@ -1754,14 +1767,14 @@ function (a::WolfePowellLinesearchStepsize)( break end end - s_plus = 2.0 * s_minus + s_plus = min(2.0 * s_minus, max_step_increase) else vector_transport_to!(M, a.candidate_direction, p, η, a.candidate_point, a.vector_transport_method) if get_differential(mp, a.candidate_point, a.candidate_direction; Y = Y) < a.sufficient_curvature * l i = 0 while fNew <= f0 + a.sufficient_decrease * step * l && (s_plus < max_step_increase) # increase - s_plus = s_plus * 2.0 + s_plus = min(s_plus * 2.0, max_step_increase) step = s_plus ManifoldsBase.retract_fused!(M, a.candidate_point, p, η, step, a.retraction_method) fNew = get_cost(mp, a.candidate_point) From 93da29f12e4ab94939aa3e568b980a25c453fc02 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 15 Dec 2025 13:01:07 +0100 Subject: [PATCH 020/135] some fixes and tests around max_stepsize, bound_direction_tweak! and hess_val_eb --- src/Manopt.jl | 1 + src/plans/box_plan.jl | 4 ++-- src/plans/stepsize/linesearch.jl | 23 +++++++++++++++++-- test/helpers/test_manifold_extra_functions.jl | 9 ++++++++ test/solvers/test_quasi_Newton_box.jl | 22 +++++++++++++++++- 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/Manopt.jl b/src/Manopt.jl index 038a0429fd..2a72ad95fd 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -138,6 +138,7 @@ using ManifoldsBase: set_component!, shortest_geodesic, shortest_geodesic!, + submanifold_component, submanifold_components, vector_transport_to, vector_transport_to!, diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 1683c38f42..de1980976c 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -167,7 +167,7 @@ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstract coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) - return hess_val_from_wmwt_coords(gh, one(eltype(gh.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) + return hess_val_from_wmwt_coords(gh, one(eltype(gh.qn_du.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) end @doc raw""" @@ -442,7 +442,7 @@ function set_bound_at_index!(M::ProductManifold, p_cp, d, i) return p_cp end -function bound_direction_tweak!(::ProductManifold, d_out, d, p, p_cp) +function bound_direction_tweak!(M::ProductManifold, d_out, d, p, p_cp) bound_direction_tweak!( M.manifolds[1], submanifold_component(M, d_out, Val(1)), submanifold_component(M, d, Val(1)), submanifold_component(M, p, Val(1)), diff --git a/src/plans/stepsize/linesearch.jl b/src/plans/stepsize/linesearch.jl index d504592dc7..5a5982cc0c 100644 --- a/src/plans/stepsize/linesearch.jl +++ b/src/plans/stepsize/linesearch.jl @@ -46,21 +46,40 @@ function max_stepsize(M::AbstractManifold, p) injectivity_radius(M, p) catch is_tutorial_mode() && - @warn "`max_stepsize was called, but there seems to not be an `injectivity_raidus` available on $M." + @warn "`max_stepsize was called, but there seems to not be an `injectivity_radius` available on $M." Inf end return s end +function max_stepsize(M::ProductManifold, p) + return min(map(max_stepsize, M.manifolds, submanifold_components(M, p))...) +end +function max_stepsize(M::AbstractPowerManifold, p) + stepsize = number_eltype(p)(Inf) + rep_size = representation_size(M.manifold) + for i in get_iterator(M) + cur_stepsize = max_stepsize(M.manifold, _read(M, rep_size, p, i)) + stepsize = min(cur_stepsize, stepsize) + end + return stepsize +end function max_stepsize(M::AbstractManifold) s = try injectivity_radius(M) catch is_tutorial_mode() && - @warn "`max_stepsize was called, but there seems to not be an `injectivity_raidus` available on $M." + @warn "`max_stepsize was called, but there seems to not be an `injectivity_radius` available on $M." Inf end return s end +function max_stepsize(M::ProductManifold) + return min(map(max_stepsize, M.manifolds)...) +end +function max_stepsize(M::AbstractPowerManifold) + return max_stepsize(M.manifold) +end + """ Linesearch <: Stepsize diff --git a/test/helpers/test_manifold_extra_functions.jl b/test/helpers/test_manifold_extra_functions.jl index f7a7540429..8654af3209 100644 --- a/test/helpers/test_manifold_extra_functions.jl +++ b/test/helpers/test_manifold_extra_functions.jl @@ -69,6 +69,7 @@ Random.seed!(42) R3 = Euclidean(3) TR3 = TangentBundle(R3) + p = [0.0, 1.0, 0.0] X = [0.0, 0.0, 0.0] @@ -83,6 +84,14 @@ Random.seed!(42) @test Manopt.max_stepsize(R3, p) == Inf @test Manopt.max_stepsize(TR3, ArrayPartition(p, X)) == Inf + S_R3 = ProductManifold(M, R3) + @test Manopt.max_stepsize(S_R3) ≈ π + @test Manopt.max_stepsize(S_R3, ArrayPartition(p, [0.0, 0.0, 0.0])) ≈ π + + S_pow = PowerManifold(M, NestedPowerRepresentation(), 3) + @test Manopt.max_stepsize(S_pow) ≈ π + @test Manopt.max_stepsize(S_pow, [p, p, p]) ≈ π + Mfr = FixedRankMatrices(5, 4, 2) pfr = SVDMPoint( [ diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 501991dd61..da6d0193ec 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -1,6 +1,8 @@ using Manopt, Manifolds, Test using LinearAlgebra: I, eigvecs, tr, Diagonal, dot +using RecursiveArrayTools + @testset "Riemannian quasi-Newton Methods with box-like domains" begin @testset "get_bound_t - basic" begin M = Hyperrectangle([0.0, 0.0], [2.0, 2.0]) @@ -120,6 +122,15 @@ using LinearAlgebra: I, eigvecs, tr, Diagonal, dot p0 = [0.0, 4.0, 1.0] p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(10)) @test p_opt ≈ [0, 2, 0] + + + f2(M, p) = sum(p .^ 4) + function grad_f2(M, p) + return project(M, p, 4 .* (p .^ 3)) + end + p0 = [0.0, 4.0, 1.0] + p_opt = quasi_Newton(M, f2, grad_f2, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) + @test f2(M, p_opt) < 16.1 end @testset "requires_gcp" begin @@ -129,6 +140,15 @@ using LinearAlgebra: I, eigvecs, tr, Diagonal, dot end @testset "Hyperrectangle × Sphere" begin - + S2 = Sphere(2) + px = [0.0, 1.0, 0.0] + Mbox = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) + M = Mbox × S2 + f(M, p) = sum(p.x[1] .^ 4) + 0.5 * distance(S2, p.x[2], px)^2 + grad_f(M, p) = ArrayPartition(project(Mbox, p.x[1], 4 .* (p.x[1] .^ 3)), -log(S2, p.x[2], px)) + p0 = ArrayPartition([0.0, 4.0, 1.0], [1.0, 0.0, 0.0]) + + p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) + @test distance(M, p_opt, ArrayPartition([0, 2, 0], px)) < 0.1 end end From 82684999cc736c1d6ad88977750ed8558a66f098 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 16 Dec 2025 09:55:20 +0100 Subject: [PATCH 021/135] fix Base.show for StopWhenRelativeAPosterioriChangeCostLessOrEqual --- src/plans/stopping_criterion.jl | 2 +- test/plans/test_stopping_criteria.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 89b4aeb1dc..e6ffcbc1f6 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -527,7 +527,7 @@ function status_summary(c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) s = has_stopped ? "reached" : "not reached" return "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = $(abs(c.last_change)) ≤ $(c.tolerance):\t$s" end -function Base.show(io::IO, c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) +function Base.show(io::IO, ::MIME"text/plain", c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) return print( io, "StopWhenRelativeAPosterioriChangeCostLessOrEqual with threshold $(c.tolerance).\n $(status_summary(c))", diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 97ce4fe3bf..7d73da9a7f 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -1,5 +1,11 @@ using Manifolds, ManifoldsBase, Manopt, Test, ManifoldsBase, Dates +function to_display_string(obj) + buf = IOBuffer() + show(buf, MIME"text/plain"(), obj) + return String(take!(buf)) +end + @testset "StoppingCriteria" begin @testset "Generic Tests" begin @test_throws ErrorException get_stopping_criteria( @@ -369,12 +375,18 @@ using Manifolds, ManifoldsBase, Manopt, Test, ManifoldsBase, Dates Euclidean(), ManifoldGradientObjective((M, x) -> x^2, x -> 2x) ) s = GradientDescentState(Euclidean(); p = 1.0) + @test !sc(prob, s, -1) @test !sc(prob, s, 1) @test length(get_reason(sc)) == 0 s.p = 1.0 - 1.0e-14 @test sc(prob, s, 2) @test length(get_reason(sc)) > 0 + @test startswith( + to_display_string(sc), + "StopWhenRelativeAPosterioriChangeCostLessOrEqual with threshold 1.4210854715202004e-12.\n", + ) + @test startswith(Manopt.status_summary(sc), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") end @testset "has_converged" begin From fe28d2539358ab1cf9928c464b389b6574dfe243 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 16 Dec 2025 10:36:32 +0100 Subject: [PATCH 022/135] more tests and change one name --- Changelog.md | 2 +- src/Manopt.jl | 2 +- src/plans/stopping_criterion.jl | 24 ++++++++++++------------ test/plans/test_stopping_criteria.jl | 28 +++++++++++++++++++++++++--- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/Changelog.md b/Changelog.md index 731a53b72a..be670856d7 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. (#554) * `GCPFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) -* New stopping criteria: `StopWhenRelativeAPosterioriChangeCostLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. +* New stopping criteria: `StopWhenRelativeAPosterioriCostChangeLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. ### Fixed diff --git a/src/Manopt.jl b/src/Manopt.jl index 2a72ad95fd..fba308dad7 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -580,7 +580,7 @@ export StopAfter, StopWhenPopulationDiverges, StopWhenPopulationStronglyConcentrated, StopWhenProjectedGradientStationary, - StopWhenRelativeAPosterioriChangeCostLessOrEqual, + StopWhenRelativeAPosterioriCostChangeLessOrEqual, StopWhenRelativeResidualLess, StopWhenRepeated, StopWhenSmallerOrEqual, diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index e6ffcbc1f6..a64f701295 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -464,7 +464,7 @@ function set_parameter!(c::StopWhenCostLess, ::Val{:MinCost}, v) end """ - StopWhenRelativeAPosterioriChangeCostLessOrEqual <: StoppingCriterion + StopWhenRelativeAPosterioriCostChangeLessOrEqual <: StoppingCriterion A stopping criterion to stop when @@ -481,25 +481,25 @@ $(_var(:Field, :last_change)) # Constructor - StopWhenRelativeAPosterioriChangeCostLessOrEqual(tolerance::F) + StopWhenRelativeAPosterioriCostChangeLessOrEqual(tolerance::F) Initialize the stopping criterion to a threshold `tolerance` for the change of the cost function. - StopWhenRelativeAPosterioriChangeCostLessOrEqual(; factr::Real=1.0e7) + StopWhenRelativeAPosterioriCostChangeLessOrEqual(; factr::Real=1.0e7) Initialize tolerance to `factr * eps(factr)`, following the convention in [ZhuByrdLuNocedal:1997](@cite). """ -mutable struct StopWhenRelativeAPosterioriChangeCostLessOrEqual{F <: Real} <: StoppingCriterion +mutable struct StopWhenRelativeAPosterioriCostChangeLessOrEqual{F <: Real} <: StoppingCriterion tolerance::F at_iteration::Int last_cost::F last_change::F end -function StopWhenRelativeAPosterioriChangeCostLessOrEqual(tol::F) where {F <: Real} - return StopWhenRelativeAPosterioriChangeCostLessOrEqual{F}(tol, -1, zero(tol), 2 * tol) +function StopWhenRelativeAPosterioriCostChangeLessOrEqual(tol::F) where {F <: Real} + return StopWhenRelativeAPosterioriCostChangeLessOrEqual{F}(tol, -1, zero(tol), 2 * tol) end -StopWhenRelativeAPosterioriChangeCostLessOrEqual(; factr::F = 1.0e7) where {F <: Real} = StopWhenRelativeAPosterioriChangeCostLessOrEqual(factr * eps(factr)) -function (c::StopWhenRelativeAPosterioriChangeCostLessOrEqual)( +StopWhenRelativeAPosterioriCostChangeLessOrEqual(; factr::F = 1.0e7) where {F <: Real} = StopWhenRelativeAPosterioriCostChangeLessOrEqual(factr * eps(factr)) +function (c::StopWhenRelativeAPosterioriCostChangeLessOrEqual)( problem::AbstractManoptProblem, state::AbstractManoptSolverState, iteration::Int ) if iteration <= 0 # reset on init @@ -516,21 +516,21 @@ function (c::StopWhenRelativeAPosterioriChangeCostLessOrEqual)( end return false end -function get_reason(c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) +function get_reason(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) if c.at_iteration >= 0 return "At iteration $(c.at_iteration) the algorithm performed a step with a relative a posteriori cost change ($(abs(c.last_change))) less than or equal to $(c.tolerance)." end return "" end -function status_summary(c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) +function status_summary(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) has_stopped = (c.at_iteration >= 0) s = has_stopped ? "reached" : "not reached" return "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = $(abs(c.last_change)) ≤ $(c.tolerance):\t$s" end -function Base.show(io::IO, ::MIME"text/plain", c::StopWhenRelativeAPosterioriChangeCostLessOrEqual) +function Base.show(io::IO, ::MIME"text/plain", c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) return print( io, - "StopWhenRelativeAPosterioriChangeCostLessOrEqual with threshold $(c.tolerance).\n $(status_summary(c))", + "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold $(c.tolerance).\n $(status_summary(c))", ) end diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 7d73da9a7f..962845a51d 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -369,8 +369,8 @@ end @test length(get_reason(sc)) == 0 end - @testset "StopWhenRelativeAPosterioriChangeCostLessOrEqual" begin - sc = StopWhenRelativeAPosterioriChangeCostLessOrEqual(; factr = 100.0) + @testset "StopWhenRelativeAPosterioriCostChangeLessOrEqual" begin + sc = StopWhenRelativeAPosterioriCostChangeLessOrEqual(; factr = 100.0) prob = DefaultManoptProblem( Euclidean(), ManifoldGradientObjective((M, x) -> x^2, x -> 2x) ) @@ -384,11 +384,33 @@ end @test length(get_reason(sc)) > 0 @test startswith( to_display_string(sc), - "StopWhenRelativeAPosterioriChangeCostLessOrEqual with threshold 1.4210854715202004e-12.\n", + "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold 1.4210854715202004e-12.\n", ) @test startswith(Manopt.status_summary(sc), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") end + @testset "StopWhenProjectedNegativeGradientNormLess" begin + sc = StopWhenProjectedNegativeGradientNormLess(1e-10) + M = Hyperrectangle([1.0], [2.0]) + prob = DefaultManoptProblem( + M, ManifoldGradientObjective((M, x) -> x^2, x -> 2x) + ) + s = GradientDescentState(M; p = [1.0], X=[2.0]) + @test !sc(prob, s, -1) + @test length(get_reason(sc)) == 0 + @test sc(prob, s, 1) + @test length(get_reason(sc)) > 0 + + @test startswith( + to_display_string(sc), + "StopWhenProjectedNegativeGradientNormLess(1.0e-10)\n", + ) + @test startswith(Manopt.status_summary(sc), "|proj (-grad f)| < 1.0e-10") + + Manopt.set_parameter!(sc, Val(:MinGradNorm), 1e-5) + @test sc.threshold == 1e-5 + end + @testset "has_converged" begin M = Euclidean(1) pr = Manopt.Test.DummyProblem{typeof(M)}() From 231989e420f0eeb85a81fcdc8cc00819b592b56a Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 16 Dec 2025 10:41:12 +0100 Subject: [PATCH 023/135] formatting --- test/plans/test_stopping_criteria.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 962845a51d..7b5966e0c7 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -390,12 +390,12 @@ end end @testset "StopWhenProjectedNegativeGradientNormLess" begin - sc = StopWhenProjectedNegativeGradientNormLess(1e-10) + sc = StopWhenProjectedNegativeGradientNormLess(1.0e-10) M = Hyperrectangle([1.0], [2.0]) prob = DefaultManoptProblem( M, ManifoldGradientObjective((M, x) -> x^2, x -> 2x) ) - s = GradientDescentState(M; p = [1.0], X=[2.0]) + s = GradientDescentState(M; p = [1.0], X = [2.0]) @test !sc(prob, s, -1) @test length(get_reason(sc)) == 0 @test sc(prob, s, 1) @@ -407,8 +407,8 @@ end ) @test startswith(Manopt.status_summary(sc), "|proj (-grad f)| < 1.0e-10") - Manopt.set_parameter!(sc, Val(:MinGradNorm), 1e-5) - @test sc.threshold == 1e-5 + Manopt.set_parameter!(sc, Val(:MinGradNorm), 1.0e-5) + @test sc.threshold == 1.0e-5 end @testset "has_converged" begin From 5bac94f62c57ec284349bb4919dac680e14552de Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 16 Dec 2025 17:30:29 +0100 Subject: [PATCH 024/135] more testing --- src/plans/stopping_criterion.jl | 1 + test/plans/test_stopping_criteria.jl | 2 ++ test/solvers/test_quasi_Newton_box.jl | 48 +++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index a64f701295..e167c22b3d 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -516,6 +516,7 @@ function (c::StopWhenRelativeAPosterioriCostChangeLessOrEqual)( end return false end +indicates_convergence(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) = true function get_reason(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) if c.at_iteration >= 0 return "At iteration $(c.at_iteration) the algorithm performed a step with a relative a posteriori cost change ($(abs(c.last_change))) less than or equal to $(c.tolerance)." diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 7b5966e0c7..3683a12ed7 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -387,6 +387,7 @@ end "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold 1.4210854715202004e-12.\n", ) @test startswith(Manopt.status_summary(sc), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") + @test Manopt.indicates_convergence(sc) end @testset "StopWhenProjectedNegativeGradientNormLess" begin @@ -409,6 +410,7 @@ end Manopt.set_parameter!(sc, Val(:MinGradNorm), 1.0e-5) @test sc.threshold == 1.0e-5 + @test Manopt.indicates_convergence(sc) end @testset "has_converged" begin diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index da6d0193ec..2232b4b45e 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -90,6 +90,53 @@ using RecursiveArrayTools @test f_double_prime == f_original_double_prime end + @testset "update_fp_fpp - basic d = [-2.0, -1.0] with limited memory update" begin + M = Hyperrectangle([1.0, 4.0], [2.0, 10.0]) + + p = [2.0, 5.0] + ha = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2)) + st = QuasiNewtonState(M) + + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 * p + gmp = ManifoldGradientObjective(f, grad_f) + mp = DefaultManoptProblem(M, gmp) + + st.yk = [2.0, 4.0] + st.sk = [4.0, 2.0] + update_hessian!(ha, mp, st, p, 1) + grad = grad_f(M, p) + + d = similar(grad) + ha(d, mp, st) + + b = 1 + z = [-0.5, -0.25] + d_old = [-2.0, -1.0] + + d[1] = 0.0 + + old_f_prime = -6.0 + old_f_double_prime = 10.0 + dt = 0.25 + db = d[b] + gb = grad[b] + + # compare the generic and limited memory updater + f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + @test f_prime == -3.5 + @test f_double_prime == 10 + + lmupd = Manopt.get_default_fpfpp_updater(ha) + @test lmupd isa Manopt.LimitedMemoryFPFPPUpdater + + Manopt.init_updater!(M, lmupd, p, d, ha) + f_prime_limited, f_double_prime_limited = lmupd(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + + @test f_prime ≈ f_prime_limited + @test f_double_prime ≈ f_double_prime_limited + end + @testset "GCPFinder" begin M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS()) @@ -113,6 +160,7 @@ using RecursiveArrayTools @test Manopt.find_gcp_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === :found_limited @test d_out ≈ [2.0, 10.0, 0.0] end + @testset "Pure Hyperrectangle" begin M = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) f(M, p) = sum(p .^ 2) From 22c93d9b08889b257599d1944249a4eb82488366 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 17 Dec 2025 11:02:08 +0100 Subject: [PATCH 025/135] improve coverage --- src/solvers/quasi_Newton.jl | 42 +++++++++++++++++-------------- test/solvers/test_quasi_Newton.jl | 27 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index f95667f5d3..061d785bf4 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -752,6 +752,28 @@ function fill_rho_i!(M::AbstractManifold, p, d::QuasiNewtonLimitedMemoryDirectio return d end +function _drop_zero_rho_vectors!(d::QuasiNewtonLimitedMemoryDirectionUpdate{U}) where {U <: InverseBFGS} + T = eltype(d.memory_s) + memory_size = capacity(d.memory_s) + new_scb = CircularBuffer{T}(memory_size) + new_ycb = CircularBuffer{T}(memory_size) + new_ρ = similar(d.ρ) + fill!(new_ρ, 0) + j = 1 + for i in 1:length(d.memory_s) + if !iszero(d.ρ[i]) + push!(new_scb, d.memory_s[i]) + push!(new_ycb, d.memory_y[i]) + new_ρ[j] = d.ρ[i] + j += 1 + end + end + d.memory_s = new_scb + d.memory_y = new_ycb + d.ρ = new_ρ + return d +end + # Limited-memory update function update_hessian!( d::QuasiNewtonLimitedMemoryDirectionUpdate{U}, @@ -784,25 +806,7 @@ function update_hessian!( end if reforming_required - # drop elements with zero inner product - T = eltype(d.memory_s) - memory_size = capacity(d.memory_s) - new_scb = CircularBuffer{T}(memory_size) - new_ycb = CircularBuffer{T}(memory_size) - new_ρ = similar(d.ρ) - fill!(new_ρ, 0) - j = 1 - for i in 1:length(d.memory_s) - if !iszero(d.ρ[i]) - push!(new_scb, d.memory_s[i]) - push!(new_ycb, d.memory_y[i]) - new_ρ[j] = d.ρ[i] - j += 1 - end - end - d.memory_s = new_scb - d.memory_y = new_ycb - d.ρ = new_ρ + _drop_zero_rho_vectors!(d) end # add newest diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index ebf648ecd2..54dcb4b978 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -500,4 +500,31 @@ end Manopt.update_hessian!(qns.direction_update, mp, qns, p, 1) # But I am not totally sure what to test for afterwards end + @testset "Removing zero rho vectors" begin + M = Euclidean(2) + p = [0.0, 1.0] + f(M, p) = sum(p .^ 2) + # A wrong gradient + grad_f(M, p) = -2 .* p + gmp = ManifoldGradientObjective(f, grad_f) + mp = DefaultManoptProblem(M, gmp) + qdu = QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 3) + # push three pairs; middle one has zero inner product + push!(qdu.memory_y, [1, 0]) + push!(qdu.memory_s, [1, 0]) + + push!(qdu.memory_y, [1, 0]) + push!(qdu.memory_s, [0, 1]) + + push!(qdu.memory_y, [0, 2]) + push!(qdu.memory_s, [0, 2]) + qdu.ρ = [1.0, 0.0, 4.0] + # delete the zero inner product pair and check that the removal was correct + Manopt._drop_zero_rho_vectors!(qdu) + @test length(qdu.memory_y) == 2 + @test length(qdu.memory_s) == 2 + @test qdu.ρ[[1, 2]] == [1.0, 4.0] + @test qdu.memory_y[1] == [1, 0] + @test qdu.memory_y[2] == [0, 2] + end end From a4f9fb6a9ccc7b2662b93d329b0b0ad71938100f Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 17 Dec 2025 11:26:48 +0100 Subject: [PATCH 026/135] improve coverage a bit more --- src/plans/box_plan.jl | 20 +++++--------------- test/solvers/test_quasi_Newton_box.jl | 8 ++++++++ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index de1980976c..e475be69de 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -126,9 +126,7 @@ function hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractMan ii = 1 for i in 1:m - if iszero(gh.qn_du.ρ[i]) - continue - end + iszero(gh.qn_du.ρ[i]) && continue gh.coords_Yk_X[ii] = inner(M, p, gh.qn_du.memory_y[i], X) gh.coords_Sk_X[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], X) @@ -156,9 +154,7 @@ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstract ii = 1 for i in 1:m - if iszero(gh.qn_du.ρ[i]) - continue - end + iszero(gh.qn_du.ρ[i]) && continue gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) @@ -187,9 +183,7 @@ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstract ii = 1 for i in 1:m - if iszero(gh.qn_du.ρ[i]) - continue - end + iszero(gh.qn_du.ρ[i]) && continue gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) @@ -235,14 +229,10 @@ function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemo ii = 1 # fill Dk and Lk for i in 1:m - if iszero(gh.qn_du.ρ[i]) - continue - end + iszero(gh.qn_du.ρ[i]) && continue jj = 1 for j in 1:m - if iszero(gh.qn_du.ρ[j]) - continue - end + iszero(gh.qn_du.ρ[j]) && continue if jj < ii Lk[ii, jj] = inner(M, p, gh.qn_du.memory_s[i], gh.qn_du.memory_y[j]) end diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 2232b4b45e..6cd5745cd8 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -110,6 +110,9 @@ using RecursiveArrayTools d = similar(grad) ha(d, mp, st) + d2 = ha(mp, st) + @test d ≈ d2 + b = 1 z = [-0.5, -0.25] d_old = [-2.0, -1.0] @@ -159,6 +162,11 @@ using RecursiveArrayTools @test Manopt.find_gcp_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === :found_limited @test d_out ≈ [2.0, 10.0, 0.0] + + p2 = [-1.0, -2.0, 2.0] + gf2 = Manopt.GCPFinder(M, p2, ha) + + @test Manopt.find_gcp_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === :not_found end @testset "Pure Hyperrectangle" begin From 7bd9c3be21666bec91d0229a037382a8819ead62 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 17 Dec 2025 13:46:36 +0100 Subject: [PATCH 027/135] improve coverage; circumvent a Julia documenter bug --- .github/workflows/documenter.yml | 2 +- src/plans/stepsize/stepsize.jl | 4 ++-- test/solvers/test_quasi_Newton_box.jl | 22 ++++++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/documenter.yml b/.github/workflows/documenter.yml index d1f6c44f27..150202d32a 100644 --- a/.github/workflows/documenter.yml +++ b/.github/workflows/documenter.yml @@ -16,7 +16,7 @@ jobs: version: "1.7.29" - uses: julia-actions/setup-julia@latest with: - version: "1.12" + version: "1.12.2" - name: Julia Cache uses: julia-actions/cache@v2 - name: Cache Quarto diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index 0a9e4c01b7..701d8a97eb 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -123,7 +123,7 @@ function (a::ArmijoLinesearchStepsize)( l = norm(get_manifold(mp), p, η) local swse if :stop_when_stepsize_exceeds in keys(kwargs) - swse = kwargs.stop_when_stepsize_exceeds + swse = kwargs[:stop_when_stepsize_exceeds] else swse = (a.stop_when_stepsize_exceeds / l) end @@ -733,7 +733,7 @@ function (cbls::CubicBracketingLinesearchStepsize)( n_iter = 0 max_step = cbls.max_stepsize if :stop_when_stepsize_exceeds in keys(kwargs) - max_step = min(max_step, kwargs.stop_when_stepsize_exceeds) + max_step = min(max_step, kwargs[:stop_when_stepsize_exceeds]) end t = min(cbls.last_stepsize, max_step) c_old = init diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 6cd5745cd8..2bb1561a7f 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -138,6 +138,19 @@ using RecursiveArrayTools @test f_prime ≈ f_prime_limited @test f_double_prime ≈ f_double_prime_limited + + ha.last_gcp_result = :found_unlimited + @test Manopt.get_parameter(ha, Val(:max_stepsize)) == Inf + + @testset "No memory tests" begin + ha2 = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2)) + @test Manopt.hess_val_eb(ha2, M, p, b, grad) ≈ 4.0 + Manopt.set_M_current_scale!(M, p, ha2) + @test ha2.current_scale == ha2.qn_du.initial_scale + @test ha2.M_11 == fill(0.0, 0, 0) + @test ha2.M_21 == fill(0.0, 0, 0) + @test ha2.M_22 == fill(0.0, 0, 0) + end end @testset "GCPFinder" begin @@ -187,6 +200,15 @@ using RecursiveArrayTools p0 = [0.0, 4.0, 1.0] p_opt = quasi_Newton(M, f2, grad_f2, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) @test f2(M, p_opt) < 16.1 + + for stepsize in [ArmijoLinesearch(), CubicBracketingLinesearch()] + p_opt = quasi_Newton( + M, f2, grad_f2, p0; + stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100), + stepsize = stepsize + ) + @test f2(M, p_opt) < 16.1 + end end @testset "requires_gcp" begin From f84e250f20bcdc7ee004678e15c2d2331b464ced Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 7 Jan 2026 11:09:17 +0100 Subject: [PATCH 028/135] improve coverage --- src/plans/box_plan.jl | 10 ++------- src/plans/stepsize/stepsize.jl | 2 +- src/solvers/quasi_Newton.jl | 4 ++-- test/solvers/test_quasi_Newton_box.jl | 29 +++++++++++++++++++++++++-- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index e475be69de..2956639d31 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -369,9 +369,7 @@ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, p, old_f_pr ii = 1 for i in 1:m - if iszero(ha.qn_du.ρ[i]) - continue - end + iszero(ha.qn_du.ρ[i]) && continue # setting _X to w_b from the paper ha.coords_Yk_X[ii] = get_at_bound_index(M, ha.qn_du.memory_y[i], b) ha.coords_Sk_X[ii] = ha.current_scale * get_at_bound_index(M, ha.qn_du.memory_s[i], b) @@ -552,10 +550,7 @@ function find_gcp_direction!(gcp::GCPFinder, d_out, p, d, X) end dt_min = -f_prime / f_double_prime - - if isempty(F) - break - end + isempty(F) && break t_current, b = pop!(F) dt = t_current - t_old @@ -563,7 +558,6 @@ function find_gcp_direction!(gcp::GCPFinder, d_out, p, d, X) dt_min = max(dt_min, 0.0) t_old = t_old + dt_min - for i in bounds_indices if t[i] >= t_current set_bound_t_at_index!(M, p_cp, t_old, d, i) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index 701d8a97eb..8c832b7637 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -1420,7 +1420,7 @@ function (a::NonmonotoneLinesearchStepsize)( #compute the new step size with the help of the Barzilai-Borwein step size l = norm(M, p, η) - local swse + local swse # COV_EXCL_LINE if :stop_when_stepsize_exceeds in keys(kwargs) swse = kwargs.stop_when_stepsize_exceeds else diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 061d785bf4..549b5fd5f8 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -339,7 +339,7 @@ function quasi_Newton!( O <: Union{AbstractManifoldFirstOrderObjective{E}, AbstractDecoratedManifoldObjective{E}}, } keywords_accepted(quasi_Newton!; kwargs...) - local local_dir_upd + local local_dir_upd # COV_EXCL_LINE if memory_size >= 0 local_dir_upd = QuasiNewtonLimitedMemoryDirectionUpdate( M, @@ -422,7 +422,7 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) end end end - local α + local α # COV_EXCL_LINE if isnothing(current_max_stepsize) α = qns.stepsize(mp, qns, k, qns.η; gradient = qns.X) else diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 2bb1561a7f..76f013d2eb 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -180,6 +180,15 @@ using RecursiveArrayTools gf2 = Manopt.GCPFinder(M, p2, ha) @test Manopt.find_gcp_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === :not_found + + M2 = Hyperrectangle([-10.0], [10.0]) + + ha2 = QuasiNewtonMatrixDirectionUpdate(M2, BFGS(), DefaultOrthonormalBasis(), [100.0;;]) + p3 = [1.0] + gf3 = Manopt.GCPFinder(M2, p3, ha2) + + d_out = similar(p3) + @test Manopt.find_gcp_direction!(gf3, d_out, p3, [1.0], [-10.0]) === :found_limited end @testset "Pure Hyperrectangle" begin @@ -201,14 +210,30 @@ using RecursiveArrayTools p_opt = quasi_Newton(M, f2, grad_f2, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) @test f2(M, p_opt) < 16.1 - for stepsize in [ArmijoLinesearch(), CubicBracketingLinesearch()] + for stepsize in [ArmijoLinesearch(), CubicBracketingLinesearch(), NonmonotoneLinesearch()] p_opt = quasi_Newton( M, f2, grad_f2, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100), stepsize = stepsize ) - @test f2(M, p_opt) < 16.1 + @test f2(M, p_opt) < 64.0 end + + MInf = Hyperrectangle([-Inf, -Inf, -Inf], [Inf, Inf, Inf]) + + f3(M, p) = sum(p .^ 4) - sum(p.^2) + function grad_f3(M, p) + return project(MInf, p, 4 .* (p .^ 3) - 2 .* p) + end + p0 = [0.0, 4.0, 1.0] + p_opt = quasi_Newton(MInf, f3, grad_f3, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) + @test f3(MInf, p_opt) < 16.1 + + p_opt = quasi_Newton( + MInf, f3, grad_f3, p0; + stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100), + ) + @test f3(MInf, p_opt) < 64.0 end @testset "requires_gcp" begin From ea18ab82d8d3042b9c4884d716c7ce90c813b0e2 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 7 Jan 2026 11:12:08 +0100 Subject: [PATCH 029/135] formatting --- src/plans/stepsize/stepsize.jl | 2 +- src/solvers/quasi_Newton.jl | 4 ++-- test/solvers/test_quasi_Newton_box.jl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index 8c832b7637..bba023e3d4 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -1420,7 +1420,7 @@ function (a::NonmonotoneLinesearchStepsize)( #compute the new step size with the help of the Barzilai-Borwein step size l = norm(M, p, η) - local swse # COV_EXCL_LINE + local swse # COV_EXCL_LINE if :stop_when_stepsize_exceeds in keys(kwargs) swse = kwargs.stop_when_stepsize_exceeds else diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 549b5fd5f8..d642f19d10 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -339,7 +339,7 @@ function quasi_Newton!( O <: Union{AbstractManifoldFirstOrderObjective{E}, AbstractDecoratedManifoldObjective{E}}, } keywords_accepted(quasi_Newton!; kwargs...) - local local_dir_upd # COV_EXCL_LINE + local local_dir_upd # COV_EXCL_LINE if memory_size >= 0 local_dir_upd = QuasiNewtonLimitedMemoryDirectionUpdate( M, @@ -422,7 +422,7 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) end end end - local α # COV_EXCL_LINE + local α # COV_EXCL_LINE if isnothing(current_max_stepsize) α = qns.stepsize(mp, qns, k, qns.η; gradient = qns.X) else diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 76f013d2eb..299b31bb56 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -221,7 +221,7 @@ using RecursiveArrayTools MInf = Hyperrectangle([-Inf, -Inf, -Inf], [Inf, Inf, Inf]) - f3(M, p) = sum(p .^ 4) - sum(p.^2) + f3(M, p) = sum(p .^ 4) - sum(p .^ 2) function grad_f3(M, p) return project(MInf, p, 4 .* (p .^ 3) - 2 .* p) end From 8966588bc79dc96d42a9ecb8cd60f5e6eb530d42 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 7 Jan 2026 11:59:23 +0100 Subject: [PATCH 030/135] fix a bug --- src/plans/stepsize/stepsize.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index bba023e3d4..c09815ae96 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -121,7 +121,7 @@ function (a::ArmijoLinesearchStepsize)( ) reset_messages!(a.messages) l = norm(get_manifold(mp), p, η) - local swse + local swse # COV_EXCL_LINE if :stop_when_stepsize_exceeds in keys(kwargs) swse = kwargs[:stop_when_stepsize_exceeds] else @@ -1353,7 +1353,8 @@ function (a::NonmonotoneLinesearchStepsize)( η, p_old, X_old, - k, + k; + kwargs..., ) end function (a::NonmonotoneLinesearchStepsize)( @@ -1422,7 +1423,7 @@ function (a::NonmonotoneLinesearchStepsize)( l = norm(M, p, η) local swse # COV_EXCL_LINE if :stop_when_stepsize_exceeds in keys(kwargs) - swse = kwargs.stop_when_stepsize_exceeds + swse = kwargs[:stop_when_stepsize_exceeds] else swse = (a.stop_when_stepsize_exceeds / l) end From ab5d20ef74682d9d1bce9d6e3ebb8502d4f2f2f8 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 7 Jan 2026 19:11:32 +0100 Subject: [PATCH 031/135] add max_stepsize function for ProbabilitySimplex and corresponding tests --- ext/ManoptManifoldsExt/manifold_functions.jl | 3 +++ test/helpers/test_manifold_extra_functions.jl | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 8a93d06a83..19974552e8 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -73,6 +73,9 @@ function max_stepsize(M::Hyperrectangle) end return ms end +function max_stepsize(M::ProbabilitySimplex) + return 1.0 +end """ mid_point(M, p, q, x) diff --git a/test/helpers/test_manifold_extra_functions.jl b/test/helpers/test_manifold_extra_functions.jl index 8654af3209..c775cb1cc9 100644 --- a/test/helpers/test_manifold_extra_functions.jl +++ b/test/helpers/test_manifold_extra_functions.jl @@ -112,6 +112,9 @@ Random.seed!(42) M = Hyperrectangle([-3, -1.5], [3, 1.5]) @test Manopt.max_stepsize(M) ≈ 6.0 @test Manopt.max_stepsize(M, [-1, 0.5]) ≈ 4.0 + + M = ProbabilitySimplex(3) + @test Manopt.max_stepsize(M) == 1.0 end @testset "Vector space default" begin @test Manopt.Rn(Val(:Manopt), 3) isa ManifoldsBase.DefaultManifold From 4ac289a670376b4c8cfa639ecb97d7f2c57eba37 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 8 Jan 2026 13:04:13 +0100 Subject: [PATCH 032/135] improve docs --- docs/Project.toml | 7 +++- docs/src/references.bib | 1 - docs/src/solvers/box_domain.md | 65 +++++++++++++++++++++++++++++++ src/plans/box_plan.jl | 71 +++++++++++++++++++++++++++++++++- src/plans/quasi_newton_plan.jl | 31 +++++++++++++-- 5 files changed, 168 insertions(+), 7 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index 22187ec9e5..0f62827ba3 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,10 +3,13 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244" DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656" FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" @@ -23,8 +26,8 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" RipQP = "1e40b3f8-35eb-4cd8-8edd-3e515bb9de08" -[sources.Manopt] -path = ".." +[sources] +Manopt = {path = ".."} [compat] BenchmarkTools = "1.3" diff --git a/docs/src/references.bib b/docs/src/references.bib index 36700231e7..74807a45ff 100644 --- a/docs/src/references.bib +++ b/docs/src/references.bib @@ -266,7 +266,6 @@ @article{ByrdNocedalSchnabel:1994 issn = {1436-4646}, doi = {10.1007/BF01582063}, number = {1}, - urldate = {2025-09-06}, journal = {Mathematical Programming}, author = {Byrd, Richard H. and Nocedal, Jorge and Schnabel, Robert B.}, month = jan, diff --git a/docs/src/solvers/box_domain.md b/docs/src/solvers/box_domain.md index 3835a687f7..f6b5fb8b0d 100644 --- a/docs/src/solvers/box_domain.md +++ b/docs/src/solvers/box_domain.md @@ -6,6 +6,67 @@ The core idea is considering a piecewise quadratic approximation of the objectiv The points at which the approximation might not be differentiable correspond to hitting new boundaries along the initially selected descent direction. Then, we can perform standard line search between the initial iterate and the generalized Cauchy point. +Currently `Manopt.jl` can handle domains that are either a `Hyperrectangle` or a `ProductManifold` containing a `Hyperrectangle` as its first factor and other manifolds as subsequent factors. + +## Example + +Consider the problem of fitting covariance matrix with box constraints on variance in principal directions. +The objective is log-probability of data under a multivariate normal distribution with zero mean and covariance matrix given by the variable to optimize. +Although there are better ways to solve this problem, expressing it this way allows us to freely extend the objective to more complex scenarios beyond what is possible with closed-form solutions. + +First, we set up the problem by generating synthetic data. +The data is sampled from a multivariate normal distribution with known covariance matrix. + +```@example example-box-domain +using Manopt, Manifolds, LinearAlgebra, Random, Distributions +using ForwardDiff, DifferentiationInterface, RecursiveArrayTools + +N = 5 # dimensionality of data +M_spd = SymmetricPositiveDefinite(N) +M_rot = Rotations(N) +V = rand(M_rot) +cov_matrix = Symmetric(V * Diagonal([0.5; 2.0; 5.0; 10.0; 20.0]) * V') +distr = MvNormal(zeros(N), cov_matrix) +data = Matrix(rand(distr, 200)') # 200 samples +``` + +The objective function is defined as follows, with gradient calculated using automatic differentiation. + +```@example example-box-domain +function logprob_objective(::AbstractManifold, p) + D, R = p.x + logdet = sum(log, D) + invΣ = R * Diagonal(1 ./ D) * R' + ll = - 0.5 * size(data, 1) * logdet + for row in eachrow(data) + ll -= 0.5 * row' * invΣ * row + end + return -ll # We minimize negative log-likelihood +end + +function logprob_gradient(M::AbstractManifold, p) + Y = DifferentiationInterface.gradient(q -> logprob_objective(M, q), AutoForwardDiff(), p) + return riemannian_gradient(M, p, Y) +end +``` + +Finally, we can solve the optimization problem using a quasi-Newton method with box domain support. +We restrict the variances (diagonal elements of the covariance matrix) to be between 1.0 and 100.0. + +```@example example-box-domain +M = ProductManifold(Hyperrectangle(fill(1.0, N), fill(100.0, N)), M_rot) + +p0 = ArrayPartition(fill(10.0, N), Matrix{Float64}(I(5))) +p_mle = quasi_Newton(M, logprob_objective, logprob_gradient, p0; stopping_criterion = StopAfterIteration(100) | StopWhenProjectedNegativeGradientNormLess(1e-6)) +println("Estimated variances: $(p_mle.x[1])") +cov_matrix_mle = p_mle.x[2] * Diagonal(p_mle.x[1]) * p_mle.x[2]' +println("Estimated covariance matrix:") +println(cov_matrix_mle) +nothing +``` + +We see that despite the original covariance matrix having variances ranging from 0.5 to 20.0, the estimated covariance matrix respects the box constraints of variances between 1.0 and 100.0. + ## Public types and method ```@docs @@ -25,4 +86,8 @@ Manopt.find_gcp_direction! Manopt.hess_val_eb Manopt.LimitedMemoryFPFPPUpdater Manopt.get_bound_t +Manopt.set_M_current_scale! +Manopt.hess_val_from_wmwt_coords +Manopt.GCPFinder +Manopt.bound_direction_tweak! ``` diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 2956639d31..c94f7a0ede 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -199,6 +199,21 @@ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstract return hess_val_from_wmwt_coords(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) end +@doc raw""" + set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + +Refresh the scaling factor and blockwise Hessian approximation stored in `gh` using the +nonzero curvature pairs currently in memory. + +- Identifies the most recent index with nonzero ``ρ_i`` to scale the initial Hessian guess + by ``ρ_i‖y_i‖^2 / θ``. +- Builds ``L_k`` and ``S_k^\top S_k`` from the stored ``(s_i, y_i)`` pairs and updates the + block matrices ``M_{11}``, ``M_{21}``, and ``M_{22}`` via the blockwise inverse formula. +- If all ``ρ_i`` vanish, resets `current_scale` to `initial_scale` and clears the block + matrices. + +Returns the mutated `gh`. +""" function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) m = length(gh.qn_du.memory_s) last_safe_index = -1 @@ -260,6 +275,20 @@ function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemo return gh end +@doc raw""" + hess_val_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) + +Evaluate the quadratic form defined by the current blockwise Hessian approximation stored in +`gh`, given precomputed coordinate vectors. + +Arguments: +- `iss`: inner product of original vectors. +- `cy1`, `cy2`: coordinates of ``y``-like vectors in the ``Y_k`` basis. +- `cs1`, `cs2`: coordinates of ``s``-like vectors in the scaled ``S_k`` basis. + +The result is ``θ·iss - cy₁ᵀ M₁₁ cy₂ - 2·cs₁ᵀ M₂₁ cy₂ - cs₁ᵀ M₂₂ cs₂`` using the blocks +``M₁₁``, ``M₂₁``, ``M₂₂`` stored in `gh` and the current scale ``θ``. Returns the scalar value. +""" function hess_val_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) result = gh.current_scale * iss if length(cy1) == 0 @@ -359,7 +388,31 @@ function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater return fpfpp_upd end -function (fpfpp_upd::LimitedMemoryFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old) +@doc raw""" + (fpfpp_upd::LimitedMemoryFPFPPUpdater)( + M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, + dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old + ) + +Update ``f'`` and ``f''`` for the generalized Cauchy point line search using the limited-memory +block Hessian stored in `ha`. + +## Arguments: + +- `old_f_prime`, `old_f_double_prime`: values carried from the previous segment. +- `dt`: step length along the segment direction. +- `db`: direction component at the bound index `b`. +- `gb`: gradient component at the bound index `b`. +- `z`: trial step vector. +- `d_old`: previous search direction. + +The updater reuses cached coordinate projections in `fpfpp_upd` to cheaply evaluate Hessian +quadratic forms via `hess_val_from_wmwt_coords`, then returns the new `(f', f'')` pair. +""" +function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( + M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, + dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old + ) m = length(ha.qn_du.memory_s) num_nonzero_rho = count(!iszero, ha.qn_du.ρ) @@ -430,6 +483,13 @@ function set_bound_at_index!(M::ProductManifold, p_cp, d, i) return p_cp end +@doc raw""" + bound_direction_tweak!(M::ProductManifold, d_out, d, p, p_cp) + +Set `d_out .= p_cp .- p` on the `Hyperrectangle` part of the `ProductManifold` `M`. + +Return the mutated `d_out`. +""" function bound_direction_tweak!(M::ProductManifold, d_out, d, p, p_cp) bound_direction_tweak!( M.manifolds[1], submanifold_component(M, d_out, Val(1)), @@ -439,6 +499,15 @@ function bound_direction_tweak!(M::ProductManifold, d_out, d, p, p_cp) return d_out end + +@doc raw""" + GCPFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} + +Helper container for generalized Cauchy point search. Stores the manifold `M`, cached +workspace (`p_cp`, `Y_tmp`, `d_old`), the quasi-Newton direction update `ha`, and the +``f'``/``f''`` updater `fpfpp_updater`. Instances are reused across segments during +`find_gcp_direction!` to avoid allocations. +""" struct GCPFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} M::TM p_cp::TP diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index d9ebd2c354..9e651a7028 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -511,15 +511,40 @@ function initialize_update!(d::QuasiNewtonMatrixDirectionUpdate) copyto!(d.matrix, I) return d end +""" + hess_val(d::QuasiNewtonMatrixDirectionUpdate, M, p, X) + +Evaluate the quadratic form associated with the stored quasi-Newton matrix. +Returns the scalar ``c^{\top} B c`` where ``c`` are the coordinates of the +tangent vector `X` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. +""" function hess_val(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} - c = get_coordinates(M, p, X) + c = get_coordinates(M, p, X, d.basis) return dot(c, d.matrix, c) end -function hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, b) where {T <: Union{BFGS, DFP, SR1, Broyden}} + +""" + hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate, M, p, b) + +Evaluate the quadratic form associated with the stored quasi-Newton matrix. +Returns the scalar ``c^{\top} B c`` where ``c`` are the coordinates of the +unit tangent vector along direction with index `b` at `p` (in the basis `d.basis`) +and ``B`` is `d.matrix`. +""" +function hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, ::AbstractManifold, p, b) where {T <: Union{BFGS, DFP, SR1, Broyden}} return d.matrix[b, b] end +""" + hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate, M, p, b, X) + +Evaluate the quadratic form associated with the stored quasi-Newton matrix. +Returns the scalar ``c_b^{\top} B c`` where ``c_b`` are the coordinates of the +unit tangent vector along direction with index `b` at `p` (in the basis `d.basis`), +``c`` are the coordinates of the tangent vector `X` at `p` (in the basis `d.basis`) +and ``B`` is `d.matrix`. +""" function hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, b, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} - return dot(d.matrix[b, :], get_coordinates(M, p, X)) + return dot(d.matrix[b, :], get_coordinates(M, p, X, d.basis)) end _doc_QN_B = """ From 5d4f9b8795c6f7aebc6f0a7386356d82fba73615 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 8 Jan 2026 16:33:04 +0100 Subject: [PATCH 033/135] Apply suggestions from code review Co-authored-by: Ronny Bergmann --- .github/workflows/documenter.yml | 2 +- docs/src/solvers/box_domain.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/documenter.yml b/.github/workflows/documenter.yml index 150202d32a..d1f6c44f27 100644 --- a/.github/workflows/documenter.yml +++ b/.github/workflows/documenter.yml @@ -16,7 +16,7 @@ jobs: version: "1.7.29" - uses: julia-actions/setup-julia@latest with: - version: "1.12.2" + version: "1.12" - name: Julia Cache uses: julia-actions/cache@v2 - name: Cache Quarto diff --git a/docs/src/solvers/box_domain.md b/docs/src/solvers/box_domain.md index f6b5fb8b0d..5f16ce2b98 100644 --- a/docs/src/solvers/box_domain.md +++ b/docs/src/solvers/box_domain.md @@ -6,7 +6,7 @@ The core idea is considering a piecewise quadratic approximation of the objectiv The points at which the approximation might not be differentiable correspond to hitting new boundaries along the initially selected descent direction. Then, we can perform standard line search between the initial iterate and the generalized Cauchy point. -Currently `Manopt.jl` can handle domains that are either a `Hyperrectangle` or a `ProductManifold` containing a `Hyperrectangle` as its first factor and other manifolds as subsequent factors. +Currently `Manopt.jl` can handle domains that are either a [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) or a [`ProductManifold`](@extref ManifoldsBase.ProductManifold) containing a [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) as its first factor and other manifolds as subsequent factors. ## Example From cf287f56af3766e2a0f7c6017fa466314aefdca1 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 8 Jan 2026 17:11:31 +0100 Subject: [PATCH 034/135] address some review comments --- Changelog.md | 4 ++-- docs/src/solvers/box_domain.md | 12 +++++++----- src/plans/box_plan.jl | 20 ++++++++++---------- test/solvers/test_quasi_Newton_box.jl | 18 +++++++++--------- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Changelog.md b/Changelog.md index be670856d7..a650230828 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,8 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -* `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. (#554) -* `GCPFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) +* `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. This resolves issue #549. (#554) +* `GeneralizedCauchyPointFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) * New stopping criteria: `StopWhenRelativeAPosterioriCostChangeLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. ### Fixed diff --git a/docs/src/solvers/box_domain.md b/docs/src/solvers/box_domain.md index 5f16ce2b98..84081a3c56 100644 --- a/docs/src/solvers/box_domain.md +++ b/docs/src/solvers/box_domain.md @@ -33,7 +33,7 @@ data = Matrix(rand(distr, 200)') # 200 samples The objective function is defined as follows, with gradient calculated using automatic differentiation. ```@example example-box-domain -function logprob_objective(::AbstractManifold, p) +function logprob_cost(::AbstractManifold, p) D, R = p.x logdet = sum(log, D) invΣ = R * Diagonal(1 ./ D) * R' @@ -45,19 +45,21 @@ function logprob_objective(::AbstractManifold, p) end function logprob_gradient(M::AbstractManifold, p) - Y = DifferentiationInterface.gradient(q -> logprob_objective(M, q), AutoForwardDiff(), p) + Y = DifferentiationInterface.gradient(q -> logprob_cost(M, q), AutoForwardDiff(), p) return riemannian_gradient(M, p, Y) end ``` Finally, we can solve the optimization problem using a quasi-Newton method with box domain support. We restrict the variances (diagonal elements of the covariance matrix) to be between 1.0 and 100.0. +The covariance matrix is represented using its eigendecomposition $\Sigma = R D R^{\top}$, where $D$ is a diagonal matrix of variances and $R$ is an orthogonal matrix of principal directions. +With constraints on variances, the optimization variable belongs to $[1,100]^N \times \mathrm{SO}(N)$. ```@example example-box-domain M = ProductManifold(Hyperrectangle(fill(1.0, N), fill(100.0, N)), M_rot) p0 = ArrayPartition(fill(10.0, N), Matrix{Float64}(I(5))) -p_mle = quasi_Newton(M, logprob_objective, logprob_gradient, p0; stopping_criterion = StopAfterIteration(100) | StopWhenProjectedNegativeGradientNormLess(1e-6)) +p_mle = quasi_Newton(M, logprob_cost, logprob_gradient, p0; stopping_criterion = StopAfterIteration(100) | StopWhenProjectedNegativeGradientNormLess(1e-6)) println("Estimated variances: $(p_mle.x[1])") cov_matrix_mle = p_mle.x[2] * Diagonal(p_mle.x[1]) * p_mle.x[2]' println("Estimated covariance matrix:") @@ -82,12 +84,12 @@ Manopt.AbstractFPFPPUpdater Manopt.GenericFPFPPUpdater Manopt.get_bounds_index Manopt.requires_gcp -Manopt.find_gcp_direction! +Manopt.find_generalized_cauchy_point_direction! Manopt.hess_val_eb Manopt.LimitedMemoryFPFPPUpdater Manopt.get_bound_t Manopt.set_M_current_scale! Manopt.hess_val_from_wmwt_coords -Manopt.GCPFinder +Manopt.GeneralizedCauchyPointFinder Manopt.bound_direction_tweak! ``` diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index c94f7a0ede..73947fb9c9 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -98,8 +98,8 @@ function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( M = get_manifold(mp) p = get_iterate(st) X = get_gradient(st) - gcp = GCPFinder(M, p, d) - d.last_gcp_result = find_gcp_direction!(gcp, r, p, r, X) + gcp = GeneralizedCauchyPointFinder(M, p, d) + d.last_gcp_result = find_generalized_cauchy_point_direction!(gcp, r, p, r, X) return r end @@ -326,7 +326,7 @@ end abstract type AbstractFPFPPUpdater end Abstract type for methods that calculate f' and f'' in the GCP calculation in subsequent -line segments in `GCPFinder`. +line segments in `GeneralizedCauchyPointFinder`. """ abstract type AbstractFPFPPUpdater end @@ -501,14 +501,14 @@ end @doc raw""" - GCPFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} + GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} Helper container for generalized Cauchy point search. Stores the manifold `M`, cached workspace (`p_cp`, `Y_tmp`, `d_old`), the quasi-Newton direction update `ha`, and the ``f'``/``f''`` updater `fpfpp_updater`. Instances are reused across segments during -`find_gcp_direction!` to avoid allocations. +`find_generalized_cauchy_point_direction!` to avoid allocations. """ -struct GCPFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} +struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} M::TM p_cp::TP Y_tmp::TX @@ -517,15 +517,15 @@ struct GCPFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDire fpfpp_updater::TFU end -function GCPFinder( +function GeneralizedCauchyPointFinder( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; fpfpp_updater::AbstractFPFPPUpdater = get_default_fpfpp_updater(ha) ) - return GCPFinder(M, copy(M, p), zero_vector(M, p), zero_vector(M, p), ha, fpfpp_updater) + return GeneralizedCauchyPointFinder(M, copy(M, p), zero_vector(M, p), zero_vector(M, p), ha, fpfpp_updater) end """ - find_gcp_direction!(gcp::GCPFinder, d_out, p, d, X) + find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFinder, d_out, p, d, X) Find generalized Cauchy point looking from point `p` in direction `d` and save the tangent vector pointing at it to `d_out`. Gradient of the objective at `p` is `X`. @@ -537,7 +537,7 @@ The function returns `max_stepsize(M, p)` in direction `d_out` afterwards, * `:not_found` if the search cannot be performed in direction `d`. """ -function find_gcp_direction!(gcp::GCPFinder, d_out, p, d, X) +function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFinder, d_out, p, d, X) M = gcp.M copyto!(M, gcp.p_cp, p) p_cp = gcp.p_cp diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 299b31bb56..a87065dce8 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -153,42 +153,42 @@ using RecursiveArrayTools end end - @testset "GCPFinder" begin + @testset "GeneralizedCauchyPointFinder" begin M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS()) p = [0.0, 0.0, 0.0] - gf = Manopt.GCPFinder(M, p, ha) + gf = Manopt.GeneralizedCauchyPointFinder(M, p, ha) X1 = [-5.0, 0.0, 0.0] d = -X1 d_out = similar(d) - @test Manopt.find_gcp_direction!(gf, d_out, p, d, X1) === :found_limited + @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d, X1) === :found_limited @test d_out ≈ [2.0, 0.0, 0.0] d2 = [0.0, 1.0, 0.0] - @test Manopt.find_gcp_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === :found_unlimited + @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === :found_unlimited @test d_out ≈ d2 - @test Manopt.find_gcp_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === :found_limited + @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === :found_limited @test d_out ≈ [2.0, 10.0, 0.0] p2 = [-1.0, -2.0, 2.0] - gf2 = Manopt.GCPFinder(M, p2, ha) + gf2 = Manopt.GeneralizedCauchyPointFinder(M, p2, ha) - @test Manopt.find_gcp_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === :not_found + @test Manopt.find_generalized_cauchy_point_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === :not_found M2 = Hyperrectangle([-10.0], [10.0]) ha2 = QuasiNewtonMatrixDirectionUpdate(M2, BFGS(), DefaultOrthonormalBasis(), [100.0;;]) p3 = [1.0] - gf3 = Manopt.GCPFinder(M2, p3, ha2) + gf3 = Manopt.GeneralizedCauchyPointFinder(M2, p3, ha2) d_out = similar(p3) - @test Manopt.find_gcp_direction!(gf3, d_out, p3, [1.0], [-10.0]) === :found_limited + @test Manopt.find_generalized_cauchy_point_direction!(gf3, d_out, p3, [1.0], [-10.0]) === :found_limited end @testset "Pure Hyperrectangle" begin From 5f133a0584b4b95ccf38ba0f08cb8746b84e7711 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 8 Jan 2026 17:55:08 +0100 Subject: [PATCH 035/135] address more comments --- docs/src/solvers/box_domain.md | 2 +- ext/ManoptManifoldsExt/manifold_functions.jl | 46 +++++++++++++++++++- src/plans/box_plan.jl | 9 ++-- src/solvers/quasi_Newton.jl | 2 +- test/solvers/test_quasi_Newton_box.jl | 8 ++-- 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/docs/src/solvers/box_domain.md b/docs/src/solvers/box_domain.md index 84081a3c56..f3a3707f96 100644 --- a/docs/src/solvers/box_domain.md +++ b/docs/src/solvers/box_domain.md @@ -83,7 +83,7 @@ Manopt.hess_val Manopt.AbstractFPFPPUpdater Manopt.GenericFPFPPUpdater Manopt.get_bounds_index -Manopt.requires_gcp +Manopt.requires_generalized_cauchy_point_computation Manopt.find_generalized_cauchy_point_direction! Manopt.hess_val_eb Manopt.LimitedMemoryFPFPPUpdater diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 19974552e8..7adff9447b 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -8,9 +8,23 @@ Manopt.default_point_distance(::Euclidean, p) = norm(p, Inf) Manopt.default_vector_norm(::Euclidean, p, X) = norm(p, Inf) +""" + get_bounds_index(::Hyperrectangle) +Get the bound indices of [`Hyperrectangle`](@ref) `M`. They are the same as the indices of the +lower (or upper) bounds. +""" Manopt.get_bounds_index(M::Hyperrectangle) = eachindex(M.lb) +""" + get_bound_t(M::Hyperrectangle, x, d, i) +Get the upper bound on moving in direction `d` from point `p` on [`Hyperrectangle`](@extref) `M`, +for the bound index `i`. There are three cases: + +1. If `d[i] > 0`, the formula reads `(M.ub[i] - p[i]) / d[i]`. +2. If `d[i] < 0`, the formula reads `(M.lb[i] - p[i]) / d[i]`. +3. If `d[i] == 0`, the result is `Inf`. +""" function Manopt.get_bound_t(M::Hyperrectangle, p, d, i) if d[i] > 0 return (M.ub[i] - p[i]) / d[i] @@ -186,20 +200,50 @@ function reflect!( return retract!(M, q, p, X, retraction_method) end +""" + Manopt.set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) + +Advance the point `p_cp` on [`Hyperrectangle`](@extref) along direction `d` by stepsize `t` +only at index `i`. Used while searching for a bound during generalized Cauchy point updates. +""" function Manopt.set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) p_cp[i] += t * d[i] return p_cp end +""" + Manopt.set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) + +Set element of point `p_cp` on [`Hyperrectangle`](@extref) at index `i` to the +corresponding lower or upper bound of `M` depending on the sign of direction `d` and set +that direction entry to 0. +""" function Manopt.set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) p_cp[i] = d[i] > 0 ? M.ub[i] : M.lb[i] d[i] = 0 return p_cp end +""" + Manopt.bound_direction_tweak!(::Hyperrectangle, d_out, d, p, p_cp) + +Set `d_out` to the difference between `p_cp` and `p`. +""" function Manopt.bound_direction_tweak!(::Hyperrectangle, d_out, d, p, p_cp) return d_out .= p_cp .- p end -Manopt.requires_gcp(::Hyperrectangle) = true +""" + Manopt.requires_generalized_cauchy_point_computation(::Hyperrectangle) + +Returns `true`, as `Hyperrectangle` manifold requires generalized Cauchy point computation in solvers. +""" +Manopt.requires_generalized_cauchy_point_computation(::Hyperrectangle) = true + +""" + Manopt.get_at_bound_index(::Hyperrectangle, X, b) + +Extract the element of tangent vector `X` to a point on [`Hyperrectangle`](@extref) +at index `b`. +""" Manopt.get_at_bound_index(::Hyperrectangle, X, b) = X[b] diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 73947fb9c9..1fa06f1f13 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -1,11 +1,11 @@ """ - requires_gcp(M::AbstractManifold) + requires_generalized_cauchy_point_computation(M::AbstractManifold) Return `true` if `M` is a `Hyperrectangle`-like manifold with corners, or a product of it with a standard manifold. Otherwise return `false`. """ -requires_gcp(::AbstractManifold) = false -requires_gcp(M::ProductManifold) = requires_gcp(M.manifolds[1]) +requires_generalized_cauchy_point_computation(::AbstractManifold) = false +requires_generalized_cauchy_point_computation(M::ProductManifold) = requires_generalized_cauchy_point_computation(M.manifolds[1]) @doc raw""" mutable struct LimitedMemoryHessianApproximation end @@ -15,7 +15,8 @@ An approximation of Hessian of a scalar function of the form ``B_0 = θ I``, where ``\theta > 0`` is an initial scaling guess. Matrix ``M_k = \left(\begin{smallmatrix}M_{11} & M_{21}^{\mathrm{T}}\\ M_{21} & M_{22}\end{smallmatrix}\right)`` is stored using its blocks. -Blocks ``W_k`` are (implicitly) composed from `memory_y` and `memory_s`. +Blocks ``W_k`` are (implicitly) composed from `memory_y` and `memory_s` stored in `qn_du` +of type [`QuasiNewtonLimitedMemoryDirectionUpdate`](@ref). Initial scale ``\theta`` is stored in the field `initial_scale` but if the memory isn't empty, the current scale is set to squared norm of $s_k$ divided by inner product of ``s_k`` and ``y_k`` diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index d642f19d10..d8bcecfb19 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -352,7 +352,7 @@ function quasi_Newton!( nonpositive_curvature_behavior = nonpositive_curvature_behavior, sy_tol = sy_tol, ) - if requires_gcp(M) + if requires_generalized_cauchy_point_computation(M) local_dir_upd = QuasiNewtonLimitedMemoryBoxDirectionUpdate(local_dir_upd) end else diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index a87065dce8..063af41506 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -236,10 +236,10 @@ using RecursiveArrayTools @test f3(MInf, p_opt) < 64.0 end - @testset "requires_gcp" begin - @test !Manopt.requires_gcp(Sphere(2)) - @test Manopt.requires_gcp(Hyperrectangle([1], [2])) - @test Manopt.requires_gcp(ProductManifold(Hyperrectangle([1], [2]), Sphere(2))) + @testset "requires_generalized_cauchy_point_computation" begin + @test !Manopt.requires_generalized_cauchy_point_computation(Sphere(2)) + @test Manopt.requires_generalized_cauchy_point_computation(Hyperrectangle([1], [2])) + @test Manopt.requires_generalized_cauchy_point_computation(ProductManifold(Hyperrectangle([1], [2]), Sphere(2))) end @testset "Hyperrectangle × Sphere" begin From feda7cd566da9d3f46cfde334d2d8875e2e4bbeb Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 8 Jan 2026 17:56:33 +0100 Subject: [PATCH 036/135] one more comment --- src/plans/box_plan.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 1fa06f1f13..33bc8578fb 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -22,6 +22,8 @@ Initial scale ``\theta`` is stored in the field `initial_scale` but if the memor the current scale is set to squared norm of $s_k$ divided by inner product of ``s_k`` and ``y_k`` where ``k`` is the oldest index for which the denominator is not equal to 0. +`last_gcp_result` stores the result of the last generalized Cauchy point search. + See [ByrdNocedalSchnabel:1994](@cite) for details. """ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ From 9ef06f3b8fb7a8a143f53a9c68cfba4e388da8aa Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 8 Jan 2026 18:04:54 +0100 Subject: [PATCH 037/135] fix docs --- docs/src/solvers/box_domain.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/solvers/box_domain.md b/docs/src/solvers/box_domain.md index f3a3707f96..8e2ad0ea62 100644 --- a/docs/src/solvers/box_domain.md +++ b/docs/src/solvers/box_domain.md @@ -92,4 +92,7 @@ Manopt.set_M_current_scale! Manopt.hess_val_from_wmwt_coords Manopt.GeneralizedCauchyPointFinder Manopt.bound_direction_tweak! +Manopt.set_bound_t_at_index! +Manopt.get_at_bound_index +Manopt.set_bound_at_index! ``` From e08a588bae7a4382eff6190e42c5b7fcb04b77d1 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 8 Jan 2026 18:06:48 +0100 Subject: [PATCH 038/135] fixing extref --- ext/ManoptManifoldsExt/manifold_functions.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 7adff9447b..785cda950d 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -11,14 +11,14 @@ Manopt.default_vector_norm(::Euclidean, p, X) = norm(p, Inf) """ get_bounds_index(::Hyperrectangle) -Get the bound indices of [`Hyperrectangle`](@ref) `M`. They are the same as the indices of the +Get the bound indices of [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) `M`. They are the same as the indices of the lower (or upper) bounds. """ Manopt.get_bounds_index(M::Hyperrectangle) = eachindex(M.lb) """ get_bound_t(M::Hyperrectangle, x, d, i) -Get the upper bound on moving in direction `d` from point `p` on [`Hyperrectangle`](@extref) `M`, +Get the upper bound on moving in direction `d` from point `p` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) `M`, for the bound index `i`. There are three cases: 1. If `d[i] > 0`, the formula reads `(M.ub[i] - p[i]) / d[i]`. @@ -203,7 +203,7 @@ end """ Manopt.set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) -Advance the point `p_cp` on [`Hyperrectangle`](@extref) along direction `d` by stepsize `t` +Advance the point `p_cp` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) along direction `d` by stepsize `t` only at index `i`. Used while searching for a bound during generalized Cauchy point updates. """ function Manopt.set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) @@ -214,7 +214,7 @@ end """ Manopt.set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) -Set element of point `p_cp` on [`Hyperrectangle`](@extref) at index `i` to the +Set element of point `p_cp` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) at index `i` to the corresponding lower or upper bound of `M` depending on the sign of direction `d` and set that direction entry to 0. """ @@ -243,7 +243,7 @@ Manopt.requires_generalized_cauchy_point_computation(::Hyperrectangle) = true """ Manopt.get_at_bound_index(::Hyperrectangle, X, b) -Extract the element of tangent vector `X` to a point on [`Hyperrectangle`](@extref) +Extract the element of tangent vector `X` to a point on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) at index `b`. """ Manopt.get_at_bound_index(::Hyperrectangle, X, b) = X[b] From d21a60609609f8cc18c8d0e4ef43a6d652a01fa0 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 8 Jan 2026 19:08:07 +0100 Subject: [PATCH 039/135] don't specify arch --- .github/workflows/ci.yml | 1 - .github/workflows/nightly.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ff18d44b4..7aed8fe55e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,6 @@ jobs: - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.julia-version }} - arch: x64 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d1e08c3c23..a435557be3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,7 +15,6 @@ jobs: - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.julia-version }} - arch: x64 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 env: From 051cc58aaf56516485f5e8abbc6d059e1bb29a96 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 9 Jan 2026 17:17:35 +0100 Subject: [PATCH 040/135] expand docs --- docs/src/solvers/box_domain.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/solvers/box_domain.md b/docs/src/solvers/box_domain.md index 8e2ad0ea62..6df5ffa4f6 100644 --- a/docs/src/solvers/box_domain.md +++ b/docs/src/solvers/box_domain.md @@ -1,6 +1,8 @@ # Optimization with box domains and products of manifolds and boxes -A [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) is, in general, not a manifold but a manifold with corners, thus handling it as a domain in optimization requires special attention. +A [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) is, in general, not a manifold but a manifold with corners because locally at the boundary it looks like $\mathbb{R}^{n-k} \times \mathbb{R}^k_{\geq 0}$ for some $k > 0$, instead of $\mathbb{R}^n$ as required by the definition of a manifold. + +Such spaces require special handling when used as domains in optimization. For simple methods like gradient descent using projected gradient and a stopping criterion involving [`StopWhenProjectedNegativeGradientNormLess`](@ref) may be sufficient, however methods that approximate the Hessian can benefit from a more advanced approach. The core idea is considering a piecewise quadratic approximation of the objective along the descent direction, and selecting the generalized Cauchy point -- its minimizer. The points at which the approximation might not be differentiable correspond to hitting new boundaries along the initially selected descent direction. From 77686a74ca3b39ee07915df838cb7824ff6538ea Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 9 Jan 2026 19:34:05 +0100 Subject: [PATCH 041/135] address more review comments --- docs/make.jl | 3 +- .../generalized_cauchy_point_subsolver.md | 60 +++++++++++++++++++ docs/src/solvers/quasi_Newton.md | 1 + src/plans/box_plan.jl | 38 ++++++------ src/plans/quasi_newton_plan.jl | 12 ++-- test/solvers/test_quasi_Newton_box.jl | 6 +- .../box_domain.md => tutorials/BoxDomain.qmd | 43 ++++--------- 7 files changed, 101 insertions(+), 62 deletions(-) create mode 100644 docs/src/solvers/generalized_cauchy_point_subsolver.md rename docs/src/solvers/box_domain.md => tutorials/BoxDomain.qmd (74%) diff --git a/docs/make.jl b/docs/make.jl index b7887aa113..59b2bef7a6 100755 --- a/docs/make.jl +++ b/docs/make.jl @@ -45,6 +45,7 @@ tutorials_menu = "Implement a solver" => "tutorials/ImplementASolver.md", "Optimize on your own manifold" => "tutorials/ImplementOwnManifold.md", "Do constrained optimization" => "tutorials/ConstrainedOptimization.md", + "Do optimization with bounds" => "tutorials/BoxDomain.md", ] # Check whether all tutorials are rendered, issue a warning if not (and quarto if not set) all_tutorials_exist = true @@ -179,7 +180,6 @@ makedocs(; "Adaptive Regularization with Cubics" => "solvers/adaptive-regularization-with-cubics.md", "Alternating Gradient Descent" => "solvers/alternating_gradient_descent.md", "Augmented Lagrangian Method" => "solvers/augmented_Lagrangian_method.md", - "Box domains" => "solvers/box_domain.md", "Chambolle-Pock" => "solvers/ChambollePock.md", "CMA-ES" => "solvers/cma_es.md", "Conjugate gradient descent" => "solvers/conjugate_gradient_descent.md", @@ -190,6 +190,7 @@ makedocs(; "Douglas—Rachford" => "solvers/DouglasRachford.md", "Exact Penalty Method" => "solvers/exact_penalty_method.md", "Frank-Wolfe" => "solvers/FrankWolfe.md", + "Generalized Cauchy point subsolver" => "solvers/generalized_cauchy_point_subsolver.md", "Gradient Descent" => "solvers/gradient_descent.md", "Interior Point Newton" => "solvers/interior_point_Newton.md", "Levenberg–Marquardt" => "solvers/LevenbergMarquardt.md", diff --git a/docs/src/solvers/generalized_cauchy_point_subsolver.md b/docs/src/solvers/generalized_cauchy_point_subsolver.md new file mode 100644 index 0000000000..4ce32204df --- /dev/null +++ b/docs/src/solvers/generalized_cauchy_point_subsolver.md @@ -0,0 +1,60 @@ +# Generalized Cauchy Point subsolver + +The Generalized Cauchy Point (GCP) subsolver is a component in optimization algorithms that handle problems with bound constraints. It solves the following problem + +```math +\begin{align*} +\operatorname*{arg\,min}_{Y ∈ T_p D \times \mathcal{M}}&\ m_p(Y) = f(p) + +⟨\operatorname{grad}f(p), Y⟩_p + \frac{1}{2} ⟨\mathcal{H}_p[Y], Y⟩_p\\ +\text{such that}& \ \exp_p(Y) \in D \times \mathcal{M} +\end{align*} +``` + +where $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, and $\mathcal{H}_p$ is a Hessian-like linear operator at point $p$. + +The solver is currently primarily intended for internal use by optimization algorithms that require bound-constrained subproblem solutions. + +## Internal types and method + +### Symbols related to the GCP computation + +These symbols are directly used by solvers to compute the descent direction corresponding to the Generalized Cauchy point. + +```@docs +Manopt.requires_generalized_cauchy_point_computation +Manopt.find_generalized_cauchy_point_direction! +Manopt.GeneralizedCauchyPointFinder +``` + +### Symbols related to the Hessian approximation + +These symbols are used to evaluate the Hessian approximation at specific tangent vectors during the generalized Cauchy point computation. + +```@docs +Manopt.hessian_value +Manopt.hessian_value_eb +``` + +### Symbols related to bound handling + +These are internal symbols used to manage and manipulate bound constraints during the GCP computation. + +```@docs +Manopt.init_updater! +Manopt.AbstractFPFPPUpdater +Manopt.GenericFPFPPUpdater +Manopt.get_bounds_index +Manopt.get_bound_t +Manopt.bound_direction_tweak! +Manopt.set_bound_t_at_index! +Manopt.get_at_bound_index +Manopt.set_bound_at_index! +``` + +### Symbols related to specific Hessian approximations + +```@docs +Manopt.LimitedMemoryFPFPPUpdater +Manopt.hessian_value_from_wmwt_coords +Manopt.set_M_current_scale! +``` diff --git a/docs/src/solvers/quasi_Newton.md b/docs/src/solvers/quasi_Newton.md index 02c74365a5..2d608a34db 100644 --- a/docs/src/solvers/quasi_Newton.md +++ b/docs/src/solvers/quasi_Newton.md @@ -88,6 +88,7 @@ QuasiNewtonLimitedMemoryDirectionUpdate QuasiNewtonCautiousDirectionUpdate Manopt.initialize_update! QuasiNewtonPreconditioner +QuasiNewtonLimitedMemoryBoxDirectionUpdate ``` ## Hessian update rules diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 33bc8578fb..6551fe4088 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -113,11 +113,11 @@ function get_at_bound_index(M::ProductManifold, X, b) end @doc raw""" - hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) + hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) Compute ``⟨X, B X⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`. """ -function hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) +function hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) m = length(gh.qn_du.memory_s) num_nonzero_rho = count(!iszero, gh.qn_du.ρ) @@ -138,16 +138,16 @@ function hess_val(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractMan coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) - return hess_val_from_wmwt_coords(gh, normX_sqr, coords_Yk, coords_Sk, coords_Yk, coords_Sk) + return hessian_value_from_wmwt_coords(gh, normX_sqr, coords_Yk, coords_Sk, coords_Yk, coords_Sk) end @doc raw""" - hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) + hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) Compute ``⟨X, B X⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`, and `X` is the unit vector along index `b`. """ -function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) +function hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) m = length(gh.qn_du.memory_s) num_nonzero_rho = count(!iszero, gh.qn_du.ρ) @@ -166,16 +166,16 @@ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstract coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) - return hess_val_from_wmwt_coords(gh, one(eltype(gh.qn_du.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) + return hessian_value_from_wmwt_coords(gh, one(eltype(gh.qn_du.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) end @doc raw""" - hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) + hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) Compute ``⟨X, B Y⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`, where `X` is the unit vector pointing at index `b`. """ -function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) +function hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) m = length(gh.qn_du.memory_s) num_nonzero_rho = count(!iszero, gh.qn_du.ρ) @@ -199,7 +199,7 @@ function hess_val_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstract coords_Sk_X = view(gh.coords_Sk_X, 1:num_nonzero_rho) coords_Sk_Y = view(gh.coords_Sk_Y, 1:num_nonzero_rho) - return hess_val_from_wmwt_coords(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) + return hessian_value_from_wmwt_coords(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) end @doc raw""" @@ -279,7 +279,7 @@ function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemo end @doc raw""" - hess_val_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) + hessian_value_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) Evaluate the quadratic form defined by the current blockwise Hessian approximation stored in `gh`, given precomputed coordinate vectors. @@ -292,7 +292,7 @@ Arguments: The result is ``θ·iss - cy₁ᵀ M₁₁ cy₂ - 2·cs₁ᵀ M₂₁ cy₂ - cs₁ᵀ M₂₂ cs₂`` using the blocks ``M₁₁``, ``M₂₁``, ``M₂₂`` stored in `gh` and the current scale ``θ``. Returns the scalar value. """ -function hess_val_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) +function hessian_value_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) result = gh.current_scale * iss if length(cy1) == 0 return result @@ -344,7 +344,7 @@ init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::Abs """ struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end -Generic f' and f'' calculation that only relies on `hess_val_eb` but is relatively slow for +Generic f' and f'' calculation that only relies on `hessian_value_eb` but is relatively slow for high-dimensional domains. """ struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end @@ -369,8 +369,8 @@ function get_default_fpfpp_updater(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdat end function (::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) - f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hess_val_eb(ha, M, p, b, z)) - f_double_prime = old_f_double_prime + (2 * -db * hess_val_eb(ha, M, p, b, d_old)) + db^2 * hess_val_eb(ha, M, p, b) + f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hessian_value_eb(ha, M, p, b, z)) + f_double_prime = old_f_double_prime + (2 * -db * hessian_value_eb(ha, M, p, b, d_old)) + db^2 * hessian_value_eb(ha, M, p, b) return f_prime, f_double_prime end @@ -410,7 +410,7 @@ block Hessian stored in `ha`. - `d_old`: previous search direction. The updater reuses cached coordinate projections in `fpfpp_upd` to cheaply evaluate Hessian -quadratic forms via `hess_val_from_wmwt_coords`, then returns the new `(f', f'')` pair. +quadratic forms via `hessian_value_from_wmwt_coords`, then returns the new `(f', f'')` pair. """ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, @@ -444,12 +444,12 @@ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( coords_cy .+= dt .* coords_py coords_cs .+= dt .* coords_ps - eb_B_z = hess_val_from_wmwt_coords(ha, iss_eb_z, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) + eb_B_z = hessian_value_from_wmwt_coords(ha, iss_eb_z, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + eb_B_z) - eb_B_d = hess_val_from_wmwt_coords(ha, iss_eb_d, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) + eb_B_d = hessian_value_from_wmwt_coords(ha, iss_eb_d, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) - f_double_prime = old_f_double_prime - 2 * db * eb_B_d + db^2 * hess_val_eb(ha, M, p, b) + f_double_prime = old_f_double_prime - 2 * db * eb_B_d + db^2 * hessian_value_eb(ha, M, p, b) coords_py .-= db .* coords_Yk_eb coords_ps .-= db .* coords_Sk_eb @@ -589,7 +589,7 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin F = BinaryHeap(Base.By(first), F_list) f_prime = inner(M, p, X, d) - f_double_prime = hess_val(gcp.ha, M, p, d) + f_double_prime = hessian_value(gcp.ha, M, p, d) if iszero(f_prime) || iszero(f_double_prime) return :not_found diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index 9e651a7028..7fbf603b42 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -512,30 +512,30 @@ function initialize_update!(d::QuasiNewtonMatrixDirectionUpdate) return d end """ - hess_val(d::QuasiNewtonMatrixDirectionUpdate, M, p, X) + hessian_value(d::QuasiNewtonMatrixDirectionUpdate, M, p, X) Evaluate the quadratic form associated with the stored quasi-Newton matrix. Returns the scalar ``c^{\top} B c`` where ``c`` are the coordinates of the tangent vector `X` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. """ -function hess_val(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} +function hessian_value(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} c = get_coordinates(M, p, X, d.basis) return dot(c, d.matrix, c) end """ - hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate, M, p, b) + hessian_value_eb(d::QuasiNewtonMatrixDirectionUpdate, M, p, b) Evaluate the quadratic form associated with the stored quasi-Newton matrix. Returns the scalar ``c^{\top} B c`` where ``c`` are the coordinates of the unit tangent vector along direction with index `b` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. """ -function hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, ::AbstractManifold, p, b) where {T <: Union{BFGS, DFP, SR1, Broyden}} +function hessian_value_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, ::AbstractManifold, p, b) where {T <: Union{BFGS, DFP, SR1, Broyden}} return d.matrix[b, b] end """ - hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate, M, p, b, X) + hessian_value_eb(d::QuasiNewtonMatrixDirectionUpdate, M, p, b, X) Evaluate the quadratic form associated with the stored quasi-Newton matrix. Returns the scalar ``c_b^{\top} B c`` where ``c_b`` are the coordinates of the @@ -543,7 +543,7 @@ unit tangent vector along direction with index `b` at `p` (in the basis `d.basis ``c`` are the coordinates of the tangent vector `X` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. """ -function hess_val_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, b, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} +function hessian_value_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, b, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} return dot(d.matrix[b, :], get_coordinates(M, p, X, d.basis)) end diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 063af41506..315c70a172 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -51,7 +51,7 @@ using RecursiveArrayTools # original formula f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) - f_original_double_prime = Manopt.hess_val(ha, M, p, d) + f_original_double_prime = Manopt.hessian_value(ha, M, p, d) @test f_prime == f_original_prime @test f_double_prime == f_original_double_prime @@ -84,7 +84,7 @@ using RecursiveArrayTools # original formula f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) - f_original_double_prime = Manopt.hess_val(ha, M, p, d) + f_original_double_prime = Manopt.hessian_value(ha, M, p, d) @test f_prime == f_original_prime @test f_double_prime == f_original_double_prime @@ -144,7 +144,7 @@ using RecursiveArrayTools @testset "No memory tests" begin ha2 = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2)) - @test Manopt.hess_val_eb(ha2, M, p, b, grad) ≈ 4.0 + @test Manopt.hessian_value_eb(ha2, M, p, b, grad) ≈ 4.0 Manopt.set_M_current_scale!(M, p, ha2) @test ha2.current_scale == ha2.qn_du.initial_scale @test ha2.M_11 == fill(0.0, 0, 0) diff --git a/docs/src/solvers/box_domain.md b/tutorials/BoxDomain.qmd similarity index 74% rename from docs/src/solvers/box_domain.md rename to tutorials/BoxDomain.qmd index 6df5ffa4f6..e2d3473441 100644 --- a/docs/src/solvers/box_domain.md +++ b/tutorials/BoxDomain.qmd @@ -1,6 +1,11 @@ +--- +title: "How to do optimization on box domains" +author: "Mateusz Baran" +--- + # Optimization with box domains and products of manifolds and boxes -A [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) is, in general, not a manifold but a manifold with corners because locally at the boundary it looks like $\mathbb{R}^{n-k} \times \mathbb{R}^k_{\geq 0}$ for some $k > 0$, instead of $\mathbb{R}^n$ as required by the definition of a manifold. +A [`Hyperrectangle`]([@extref Manifolds.Hyperrectangle](https://juliamanifolds.github.io/Manifolds.jl/latest/manifolds/hyperrectangle/)) is, in general, not a manifold but a manifold with corners because locally at the boundary it looks like $\mathbb{R}^{n-k} \times \mathbb{R}^k_{\geq 0}$ for some $k > 0$, instead of $\mathbb{R}^n$ as required by the definition of a manifold. Such spaces require special handling when used as domains in optimization. For simple methods like gradient descent using projected gradient and a stopping criterion involving [`StopWhenProjectedNegativeGradientNormLess`](@ref) may be sufficient, however methods that approximate the Hessian can benefit from a more advanced approach. @@ -8,7 +13,7 @@ The core idea is considering a piecewise quadratic approximation of the objectiv The points at which the approximation might not be differentiable correspond to hitting new boundaries along the initially selected descent direction. Then, we can perform standard line search between the initial iterate and the generalized Cauchy point. -Currently `Manopt.jl` can handle domains that are either a [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) or a [`ProductManifold`](@extref ManifoldsBase.ProductManifold) containing a [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) as its first factor and other manifolds as subsequent factors. +Currently `Manopt.jl` can handle domains that are either a [`Hyperrectangle`](https://juliamanifolds.github.io/Manifolds.jl/latest/manifolds/hyperrectangle/) or a [`ProductManifold`](@extref ManifoldsBase.ProductManifold) containing a [`Hyperrectangle`](https://juliamanifolds.github.io/Manifolds.jl/latest/manifolds/hyperrectangle/) as its first factor and other manifolds as subsequent factors. ## Example @@ -19,7 +24,7 @@ Although there are better ways to solve this problem, expressing it this way all First, we set up the problem by generating synthetic data. The data is sampled from a multivariate normal distribution with known covariance matrix. -```@example example-box-domain +```{julia} using Manopt, Manifolds, LinearAlgebra, Random, Distributions using ForwardDiff, DifferentiationInterface, RecursiveArrayTools @@ -34,7 +39,7 @@ data = Matrix(rand(distr, 200)') # 200 samples The objective function is defined as follows, with gradient calculated using automatic differentiation. -```@example example-box-domain +```{julia} function logprob_cost(::AbstractManifold, p) D, R = p.x logdet = sum(log, D) @@ -57,7 +62,7 @@ We restrict the variances (diagonal elements of the covariance matrix) to be bet The covariance matrix is represented using its eigendecomposition $\Sigma = R D R^{\top}$, where $D$ is a diagonal matrix of variances and $R$ is an orthogonal matrix of principal directions. With constraints on variances, the optimization variable belongs to $[1,100]^N \times \mathrm{SO}(N)$. -```@example example-box-domain +```{julia} M = ProductManifold(Hyperrectangle(fill(1.0, N), fill(100.0, N)), M_rot) p0 = ArrayPartition(fill(10.0, N), Matrix{Float64}(I(5))) @@ -70,31 +75,3 @@ nothing ``` We see that despite the original covariance matrix having variances ranging from 0.5 to 20.0, the estimated covariance matrix respects the box constraints of variances between 1.0 and 100.0. - -## Public types and method - -```@docs -QuasiNewtonLimitedMemoryBoxDirectionUpdate -``` - -## Internal types and method - -```@docs -Manopt.init_updater! -Manopt.hess_val -Manopt.AbstractFPFPPUpdater -Manopt.GenericFPFPPUpdater -Manopt.get_bounds_index -Manopt.requires_generalized_cauchy_point_computation -Manopt.find_generalized_cauchy_point_direction! -Manopt.hess_val_eb -Manopt.LimitedMemoryFPFPPUpdater -Manopt.get_bound_t -Manopt.set_M_current_scale! -Manopt.hess_val_from_wmwt_coords -Manopt.GeneralizedCauchyPointFinder -Manopt.bound_direction_tweak! -Manopt.set_bound_t_at_index! -Manopt.get_at_bound_index -Manopt.set_bound_at_index! -``` From c3c12b8e3eafeaed33813620a9fd8900f0a9a913 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 9 Jan 2026 19:44:52 +0100 Subject: [PATCH 042/135] update Project.toml --- docs/Project.toml | 3 --- tutorials/Project.toml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index 0f62827ba3..7078a14ee9 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,13 +3,10 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" -Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244" DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656" FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" -ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" diff --git a/tutorials/Project.toml b/tutorials/Project.toml index e95f59e10f..f299a7376c 100644 --- a/tutorials/Project.toml +++ b/tutorials/Project.toml @@ -2,8 +2,10 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" +DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a" LRUCache = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -22,8 +24,10 @@ Manopt = {path = ".."} ADTypes = "1" BenchmarkTools = "1" Colors = "0.12, 0.13" +DifferentiationInterface = "0.7" Distributions = "0.25" FiniteDifferences = "0.12" +ForwardDiff = "1" IJulia = "1" LRUCache = "1.4" ManifoldDiff = "0.4" From 26acefe2bd543d1c87085dfb08b43399851f929f Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 9 Jan 2026 20:38:48 +0100 Subject: [PATCH 043/135] add some details and fix some links --- docs/src/solvers/generalized_cauchy_point_subsolver.md | 7 +++---- tutorials/BoxDomain.qmd | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_point_subsolver.md b/docs/src/solvers/generalized_cauchy_point_subsolver.md index 4ce32204df..493be2e450 100644 --- a/docs/src/solvers/generalized_cauchy_point_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_point_subsolver.md @@ -4,13 +4,12 @@ The Generalized Cauchy Point (GCP) subsolver is a component in optimization algo ```math \begin{align*} -\operatorname*{arg\,min}_{Y ∈ T_p D \times \mathcal{M}}&\ m_p(Y) = f(p) + -⟨\operatorname{grad}f(p), Y⟩_p + \frac{1}{2} ⟨\mathcal{H}_p[Y], Y⟩_p\\ -\text{such that}& \ \exp_p(Y) \in D \times \mathcal{M} +\operatorname*{arg\,min}_{Y ∈ T_p D \times \mathcal{M}}&\ m_p(Y) = ⟨X_g, Y⟩_p + \frac{1}{2} ⟨\mathcal{H}_p[Y], Y⟩_p\\ +\text{such that}& \ \exp_p(Y) = \exp_p(\alpha X) \in D \times \mathcal{M} \text{ for some } \alpha \in \mathbb{R} \end{align*} ``` -where $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, and $\mathcal{H}_p$ is a Hessian-like linear operator at point $p$. +where $X$ is a given direction, the exponential map handles projection of the tangent vector when reaching the boundary, $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, $X_g$ is the gradient of a scalar function $f$ at point $p$ and $\mathcal{H}_p$ is a linear operator that approximates the Hessian of $f$ at $p$. The solver is currently primarily intended for internal use by optimization algorithms that require bound-constrained subproblem solutions. diff --git a/tutorials/BoxDomain.qmd b/tutorials/BoxDomain.qmd index e2d3473441..c0f3526e63 100644 --- a/tutorials/BoxDomain.qmd +++ b/tutorials/BoxDomain.qmd @@ -5,7 +5,7 @@ author: "Mateusz Baran" # Optimization with box domains and products of manifolds and boxes -A [`Hyperrectangle`]([@extref Manifolds.Hyperrectangle](https://juliamanifolds.github.io/Manifolds.jl/latest/manifolds/hyperrectangle/)) is, in general, not a manifold but a manifold with corners because locally at the boundary it looks like $\mathbb{R}^{n-k} \times \mathbb{R}^k_{\geq 0}$ for some $k > 0$, instead of $\mathbb{R}^n$ as required by the definition of a manifold. +A ``[`Hyperrectangle`](@extref `Manifolds.Hyperrectangle`)``{=commonmark} is, in general, not a manifold but a manifold with corners because locally at the boundary it looks like $\mathbb{R}^{n-k} \times \mathbb{R}^k_{\geq 0}$ for some $k > 0$, instead of $\mathbb{R}^n$ as required by the definition of a manifold. Such spaces require special handling when used as domains in optimization. For simple methods like gradient descent using projected gradient and a stopping criterion involving [`StopWhenProjectedNegativeGradientNormLess`](@ref) may be sufficient, however methods that approximate the Hessian can benefit from a more advanced approach. @@ -13,7 +13,7 @@ The core idea is considering a piecewise quadratic approximation of the objectiv The points at which the approximation might not be differentiable correspond to hitting new boundaries along the initially selected descent direction. Then, we can perform standard line search between the initial iterate and the generalized Cauchy point. -Currently `Manopt.jl` can handle domains that are either a [`Hyperrectangle`](https://juliamanifolds.github.io/Manifolds.jl/latest/manifolds/hyperrectangle/) or a [`ProductManifold`](@extref ManifoldsBase.ProductManifold) containing a [`Hyperrectangle`](https://juliamanifolds.github.io/Manifolds.jl/latest/manifolds/hyperrectangle/) as its first factor and other manifolds as subsequent factors. +Currently `Manopt.jl` can handle domains that are either a ``[`Hyperrectangle`](@extref `Manifolds.Hyperrectangle`)``{=commonmark} or a ``[`ProductManifold`](@extref `ManifoldsBase.ProductManifold`)``{=commonmark} containing a ``[`Hyperrectangle`](@extref `Manifolds.Hyperrectangle`)``{=commonmark} as its first factor and other manifolds as subsequent factors. ## Example From 1dd6b9915d00bdf74f9506272cd488aec0bc137f Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 12 Jan 2026 11:40:27 +0100 Subject: [PATCH 044/135] forgot an inversion --- src/plans/box_plan.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 6551fe4088..1d3ee6caf5 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -212,8 +212,8 @@ nonzero curvature pairs currently in memory. by ``ρ_i‖y_i‖^2 / θ``. - Builds ``L_k`` and ``S_k^\top S_k`` from the stored ``(s_i, y_i)`` pairs and updates the block matrices ``M_{11}``, ``M_{21}``, and ``M_{22}`` via the blockwise inverse formula. -- If all ``ρ_i`` vanish, resets `current_scale` to `initial_scale` and clears the block - matrices. +- If all ``ρ_i`` vanish, resets `current_scale` to the inverse of `initial_scale` and + clears the block matrices. Returns the mutated `gh`. """ @@ -228,7 +228,7 @@ function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemo if (last_safe_index == -1) # All memory yield zero inner products - gh.current_scale = gh.qn_du.initial_scale + gh.current_scale = inv(gh.qn_du.initial_scale) gh.M_11 = fill(0.0, 0, 0) gh.M_21 = fill(0.0, 0, 0) gh.M_22 = fill(0.0, 0, 0) From fadb2c5be3cc1415d1b6052423cd9136a1f7bbcc Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 13 Jan 2026 13:21:52 +0100 Subject: [PATCH 045/135] optimize segment search by not requiring d_old; fix one test to have consistent, reasonable input --- src/plans/box_plan.jl | 45 +++++++++++++-------------- test/solvers/test_quasi_Newton_box.jl | 37 ++++++++++++---------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 1d3ee6caf5..061f255559 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -349,7 +349,16 @@ high-dimensional domains. """ struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end -get_default_fpfpp_updater(::AbstractQuasiNewtonDirectionUpdate) = GenericFPFPPUpdater() +function get_default_fpfpp_updater(::AbstractManifold, p, ::AbstractQuasiNewtonDirectionUpdate) + return GenericFPFPPUpdater() +end + +function (upd::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) + f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hessian_value_eb(ha, M, p, b, z)) + f_double_prime = old_f_double_prime + (2 * -db * hessian_value_eb(ha, M, p, b, d)) + db^2 * hessian_value_eb(ha, M, p, b) + + return f_prime, f_double_prime +end """ struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater @@ -364,17 +373,10 @@ struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater c_y::TV end -function get_default_fpfpp_updater(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) +function get_default_fpfpp_updater(::AbstractManifold, p, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) return LimitedMemoryFPFPPUpdater(similar(ha.qn_du.ρ), similar(ha.qn_du.ρ), similar(ha.qn_du.ρ), similar(ha.qn_du.ρ)) end -function (::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) - f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hessian_value_eb(ha, M, p, b, z)) - f_double_prime = old_f_double_prime + (2 * -db * hessian_value_eb(ha, M, p, b, d_old)) + db^2 * hessian_value_eb(ha, M, p, b) - - return f_prime, f_double_prime -end - function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater, p, d, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) fill!(fpfpp_upd.c_s, 0) fill!(fpfpp_upd.c_y, 0) @@ -394,7 +396,7 @@ end @doc raw""" (fpfpp_upd::LimitedMemoryFPFPPUpdater)( M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, - dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old + dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d ) Update ``f'`` and ``f''`` for the generalized Cauchy point line search using the limited-memory @@ -407,21 +409,19 @@ block Hessian stored in `ha`. - `db`: direction component at the bound index `b`. - `gb`: gradient component at the bound index `b`. - `z`: trial step vector. -- `d_old`: previous search direction. The updater reuses cached coordinate projections in `fpfpp_upd` to cheaply evaluate Hessian quadratic forms via `hessian_value_from_wmwt_coords`, then returns the new `(f', f'')` pair. """ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, - dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d_old + dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d ) m = length(ha.qn_du.memory_s) num_nonzero_rho = count(!iszero, ha.qn_du.ρ) iss_eb_z = get_at_bound_index(M, z, b) - iss_eb_d = get_at_bound_index(M, d_old, b) ii = 1 for i in 1:m @@ -447,7 +447,7 @@ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( eb_B_z = hessian_value_from_wmwt_coords(ha, iss_eb_z, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + eb_B_z) - eb_B_d = hessian_value_from_wmwt_coords(ha, iss_eb_d, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) + eb_B_d = hessian_value_from_wmwt_coords(ha, db, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) f_double_prime = old_f_double_prime - 2 * db * eb_B_d + db^2 * hessian_value_eb(ha, M, p, b) @@ -507,7 +507,7 @@ end GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} Helper container for generalized Cauchy point search. Stores the manifold `M`, cached -workspace (`p_cp`, `Y_tmp`, `d_old`), the quasi-Newton direction update `ha`, and the +workspace (`p_cp`, `Y_tmp`), the quasi-Newton direction update `ha`, and the ``f'``/``f''`` updater `fpfpp_updater`. Instances are reused across segments during `find_generalized_cauchy_point_direction!` to avoid allocations. """ @@ -515,16 +515,15 @@ struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, TX, T_HA <: Abst M::TM p_cp::TP Y_tmp::TX - d_old::TX ha::T_HA fpfpp_updater::TFU end function GeneralizedCauchyPointFinder( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; - fpfpp_updater::AbstractFPFPPUpdater = get_default_fpfpp_updater(ha) + fpfpp_updater::AbstractFPFPPUpdater = get_default_fpfpp_updater(M, p, ha) ) - return GeneralizedCauchyPointFinder(M, copy(M, p), zero_vector(M, p), zero_vector(M, p), ha, fpfpp_updater) + return GeneralizedCauchyPointFinder(M, copy(M, p), zero_vector(M, p), ha, fpfpp_updater) end """ @@ -605,13 +604,13 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 gcp.Y_tmp .+= dt .* d - copyto!(M, gcp.d_old, d) - set_bound_at_index!(M, p_cp, d, b) - + db = get_at_bound_index(M, d, b) gb = get_at_bound_index(M, X, b) - db = get_at_bound_index(M, gcp.d_old, b) - f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, dt, db, gb, gcp.ha, b, gcp.Y_tmp, gcp.d_old) + f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, dt, db, gb, gcp.ha, b, gcp.Y_tmp, d) + + set_bound_at_index!(M, p_cp, d, b) + t_old = t_current # If f_prime is 0, we've found the local minimizer (GCP) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 315c70a172..7e0a8df601 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -40,16 +40,15 @@ using RecursiveArrayTools ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [2.0 0.0; 0.0 2.0]) b = 2 z = [-0.25, -1.0] - d_old = [-1.0, -4.0] - - d[2] = 0.0 # optimized formula - f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) @test f_prime ≈ -0.5 @test f_double_prime ≈ 2.0 # original formula + + d[2] = 0.0 f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) f_original_double_prime = Manopt.hessian_value(ha, M, p, d) @@ -73,16 +72,15 @@ using RecursiveArrayTools ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [2.0 0.0; 0.0 2.0]) b = 1 z = [-0.5, -0.25] - d_old = [-2.0, -1.0] - - d[1] = 0.0 # optimized formula - f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) @test f_prime == -3.5 @test f_double_prime == 2 # original formula + + d[1] = 0.0 f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) f_original_double_prime = Manopt.hessian_value(ha, M, p, d) @@ -106,6 +104,8 @@ using RecursiveArrayTools st.sk = [4.0, 2.0] update_hessian!(ha, mp, st, p, 1) grad = grad_f(M, p) + st.p = p + st.X = grad d = similar(grad) ha(d, mp, st) @@ -114,10 +114,6 @@ using RecursiveArrayTools @test d ≈ d2 b = 1 - z = [-0.5, -0.25] - d_old = [-2.0, -1.0] - - d[1] = 0.0 old_f_prime = -6.0 old_f_double_prime = 10.0 @@ -125,16 +121,23 @@ using RecursiveArrayTools db = d[b] gb = grad[b] + z = dt * d + # compare the generic and limited memory updater - f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) - @test f_prime == -3.5 - @test f_double_prime == 10 + gupd = Manopt.GenericFPFPPUpdater() + Manopt.init_updater!(M, gupd, p, d, ha) + f_prime, f_double_prime = gupd(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) + + @test f_prime ≈ 0.375 + @test f_double_prime ≈ 9.5 - lmupd = Manopt.get_default_fpfpp_updater(ha) + lmupd = Manopt.get_default_fpfpp_updater(M, p, ha) @test lmupd isa Manopt.LimitedMemoryFPFPPUpdater Manopt.init_updater!(M, lmupd, p, d, ha) - f_prime_limited, f_double_prime_limited = lmupd(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d_old) + f_prime_limited, f_double_prime_limited = lmupd(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) + + d[1] = 0.0 @test f_prime ≈ f_prime_limited @test f_double_prime ≈ f_double_prime_limited From fb65141ac2d52627b56513c6f64a4ad4e331be8f Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 13 Jan 2026 13:49:37 +0100 Subject: [PATCH 046/135] we only need d_z/Y_tmp in the generic updater, not the limited memory one --- src/plans/box_plan.jl | 40 ++++++++++++++------------- test/solvers/test_quasi_Newton_box.jl | 16 +++++++---- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 061f255559..3108f04938 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -347,14 +347,22 @@ init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::Abs Generic f' and f'' calculation that only relies on `hessian_value_eb` but is relatively slow for high-dimensional domains. """ -struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end +struct GenericFPFPPUpdater{TX} <: AbstractFPFPPUpdater + d_z::TX +end -function get_default_fpfpp_updater(::AbstractManifold, p, ::AbstractQuasiNewtonDirectionUpdate) - return GenericFPFPPUpdater() +function get_default_fpfpp_updater(M::AbstractManifold, p, ::AbstractQuasiNewtonDirectionUpdate) + return GenericFPFPPUpdater(zero_vector(M, p)) end -function (upd::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) - f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hessian_value_eb(ha, M, p, b, z)) +function init_updater!(M::AbstractManifold, fpfpp_upd::GenericFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) + zero_vector!(M, fpfpp_upd.d_z, p) + return fpfpp_upd +end + +function (upd::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, t::Real, dt::Real, db, gb, ha, b, d) + upd.d_z .+= dt .* d + f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hessian_value_eb(ha, M, p, b, upd.d_z)) f_double_prime = old_f_double_prime + (2 * -db * hessian_value_eb(ha, M, p, b, d)) + db^2 * hessian_value_eb(ha, M, p, b) return f_prime, f_double_prime @@ -396,7 +404,7 @@ end @doc raw""" (fpfpp_upd::LimitedMemoryFPFPPUpdater)( M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, - dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d + t::Real, dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, d ) Update ``f'`` and ``f''`` for the generalized Cauchy point line search using the limited-memory @@ -408,21 +416,18 @@ block Hessian stored in `ha`. - `dt`: step length along the segment direction. - `db`: direction component at the bound index `b`. - `gb`: gradient component at the bound index `b`. -- `z`: trial step vector. The updater reuses cached coordinate projections in `fpfpp_upd` to cheaply evaluate Hessian quadratic forms via `hessian_value_from_wmwt_coords`, then returns the new `(f', f'')` pair. """ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, - dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, z, d + t::Real, dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, d ) m = length(ha.qn_du.memory_s) num_nonzero_rho = count(!iszero, ha.qn_du.ρ) - iss_eb_z = get_at_bound_index(M, z, b) - ii = 1 for i in 1:m iszero(ha.qn_du.ρ[i]) && continue @@ -444,7 +449,7 @@ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( coords_cy .+= dt .* coords_py coords_cs .+= dt .* coords_ps - eb_B_z = hessian_value_from_wmwt_coords(ha, iss_eb_z, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) + eb_B_z = hessian_value_from_wmwt_coords(ha, t * db, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + eb_B_z) eb_B_d = hessian_value_from_wmwt_coords(ha, db, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) @@ -504,17 +509,16 @@ end @doc raw""" - GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} + GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} Helper container for generalized Cauchy point search. Stores the manifold `M`, cached -workspace (`p_cp`, `Y_tmp`), the quasi-Newton direction update `ha`, and the +workspace (`p_cp`), the quasi-Newton direction update `ha`, and the ``f'``/``f''`` updater `fpfpp_updater`. Instances are reused across segments during `find_generalized_cauchy_point_direction!` to avoid allocations. """ -struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} +struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} M::TM p_cp::TP - Y_tmp::TX ha::T_HA fpfpp_updater::TFU end @@ -523,7 +527,7 @@ function GeneralizedCauchyPointFinder( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; fpfpp_updater::AbstractFPFPPUpdater = get_default_fpfpp_updater(M, p, ha) ) - return GeneralizedCauchyPointFinder(M, copy(M, p), zero_vector(M, p), ha, fpfpp_updater) + return GeneralizedCauchyPointFinder(M, copy(M, p), ha, fpfpp_updater) end """ @@ -543,7 +547,6 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin M = gcp.M copyto!(M, gcp.p_cp, p) p_cp = gcp.p_cp - zero_vector!(M, gcp.Y_tmp, p) copyto!(M, d_out, d) bounds_indices = get_bounds_index(M) @@ -603,11 +606,10 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin init_updater!(M, gcp.fpfpp_updater, p, d, gcp.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 - gcp.Y_tmp .+= dt .* d db = get_at_bound_index(M, d, b) gb = get_at_bound_index(M, X, b) - f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, dt, db, gb, gcp.ha, b, gcp.Y_tmp, d) + f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, t_current, dt, db, gb, gcp.ha, b, d) set_bound_at_index!(M, p_cp, d, b) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 7e0a8df601..683db1aa90 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -42,7 +42,9 @@ using RecursiveArrayTools z = [-0.25, -1.0] # optimized formula - f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) + upd = Manopt.GenericFPFPPUpdater(similar(d)) + Manopt.init_updater!(M, upd, p, d, ha) + f_prime, f_double_prime = upd(M, p, old_f_prime, old_f_double_prime, 0 + dt, dt, db, gb, ha, b, d) @test f_prime ≈ -0.5 @test f_double_prime ≈ 2.0 @@ -74,7 +76,9 @@ using RecursiveArrayTools z = [-0.5, -0.25] # optimized formula - f_prime, f_double_prime = Manopt.GenericFPFPPUpdater()(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) + upd = Manopt.GenericFPFPPUpdater(similar(d)) + Manopt.init_updater!(M, upd, p, d, ha) + f_prime, f_double_prime = upd(M, p, old_f_prime, old_f_double_prime, 0 + dt, dt, db, gb, ha, b, d) @test f_prime == -3.5 @test f_double_prime == 2 @@ -121,12 +125,12 @@ using RecursiveArrayTools db = d[b] gb = grad[b] - z = dt * d + t_current = 0 + dt # compare the generic and limited memory updater - gupd = Manopt.GenericFPFPPUpdater() + gupd = Manopt.GenericFPFPPUpdater(similar(d)) Manopt.init_updater!(M, gupd, p, d, ha) - f_prime, f_double_prime = gupd(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) + f_prime, f_double_prime = gupd(M, p, old_f_prime, old_f_double_prime, t_current, dt, db, gb, ha, b, d) @test f_prime ≈ 0.375 @test f_double_prime ≈ 9.5 @@ -135,7 +139,7 @@ using RecursiveArrayTools @test lmupd isa Manopt.LimitedMemoryFPFPPUpdater Manopt.init_updater!(M, lmupd, p, d, ha) - f_prime_limited, f_double_prime_limited = lmupd(M, p, old_f_prime, old_f_double_prime, dt, db, gb, ha, b, z, d) + f_prime_limited, f_double_prime_limited = lmupd(M, p, old_f_prime, old_f_double_prime, t_current, dt, db, gb, ha, b, d) d[1] = 0.0 From 0df091d99ab9463db6eaddb33a295a37eda00485 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 13 Jan 2026 14:34:48 +0100 Subject: [PATCH 047/135] adapt to glossaries --- src/plans/stopping_criterion.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index b5ebd4edde..1c229553dd 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -471,8 +471,7 @@ A stopping criterion to stop when based on Eq. (1) in [ZhuByrdLuNocedal:1997](@cite) # Fields -$(_var(:Field, :at_iteration)) -$(_var(:Field, :last_change)) +$(_fields([:at_iteration, :last_change])) * `last_cost``: the last cost value # Constructor From b8e176a89202c34c60ee216054298e3b2c8fa093 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 13 Jan 2026 14:49:56 +0100 Subject: [PATCH 048/135] improve coverage --- src/plans/box_plan.jl | 4 ++-- test/solvers/test_quasi_Newton_box.jl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 3108f04938..c11be8bffb 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -337,9 +337,9 @@ abstract type AbstractFPFPPUpdater end init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) Method for initialization of `AbstractFPFPPUpdater` `fpfpp_upd` just before the loop -that examines subsequent intervals for GCP. By default it does nothing. +that examines subsequent intervals for GCP. """ -init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) = fpfpp_upd +init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) """ struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 683db1aa90..7d779c115b 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -175,6 +175,11 @@ using RecursiveArrayTools @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d, X1) === :found_limited @test d_out ≈ [2.0, 0.0, 0.0] + + d_out = similar(d) + + @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, 0 * d, X1) === :not_found + d2 = [0.0, 1.0, 0.0] @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === :found_unlimited From 441b98a7aab2de522b23076c65301488bbeed22c Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 14 Jan 2026 15:39:17 +0100 Subject: [PATCH 049/135] streamline GCP direction search a bit --- .../generalized_cauchy_point_subsolver.md | 4 +- ext/ManoptManifoldsExt/manifold_functions.jl | 40 +++++------ src/plans/box_plan.jl | 71 +++++++++---------- 3 files changed, 53 insertions(+), 62 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_point_subsolver.md b/docs/src/solvers/generalized_cauchy_point_subsolver.md index 493be2e450..74fbc28754 100644 --- a/docs/src/solvers/generalized_cauchy_point_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_point_subsolver.md @@ -44,10 +44,8 @@ Manopt.AbstractFPFPPUpdater Manopt.GenericFPFPPUpdater Manopt.get_bounds_index Manopt.get_bound_t -Manopt.bound_direction_tweak! -Manopt.set_bound_t_at_index! Manopt.get_at_bound_index -Manopt.set_bound_at_index! +Manopt.set_zero_bound_at_index! ``` ### Symbols related to specific Hessian approximations diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 1dba90bf2f..03007540a8 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -199,37 +199,35 @@ function reflect!( return retract!(M, q, p, X, retraction_method) end -""" - Manopt.set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) - -Advance the point `p_cp` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) along direction `d` by stepsize `t` -only at index `i`. Used while searching for a bound during generalized Cauchy point updates. -""" -function Manopt.set_bound_t_at_index!(::Hyperrectangle, p_cp, t, d, i) - p_cp[i] += t * d[i] - return p_cp -end """ - Manopt.set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) + Manopt.set_zero_bound_at_index!(M::Hyperrectangle, d, i) -Set element of point `p_cp` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) at index `i` to the -corresponding lower or upper bound of `M` depending on the sign of direction `d` and set -that direction entry to 0. +Set element of tangent vector `d` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) +at index `i` to 0. """ -function Manopt.set_bound_at_index!(M::Hyperrectangle, p_cp, d, i) - p_cp[i] = d[i] > 0 ? M.ub[i] : M.lb[i] +function Manopt.set_zero_bound_at_index!(M::Hyperrectangle, d, i) d[i] = 0 - return p_cp + return d end """ - Manopt.bound_direction_tweak!(::Hyperrectangle, d_out, d, p, p_cp) + Manopt.set_bound_for_t!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real, t_old::Real) -Set `d_out` to the difference between `p_cp` and `p`. +For each index `i`, set element of tangent vector `d_out` on +[`Hyperrectangle`](@extref Manifolds.Hyperrectangle) `M` according to the following rule: +- if `t[i] >= t_current`, multiply `d_out[i]` by `t_old`; +- else, set `d_out[i]` to the distance from `p[i]` to the bound in the direction of `d_out[i]`. """ -function Manopt.bound_direction_tweak!(::Hyperrectangle, d_out, d, p, p_cp) - return d_out .= p_cp .- p +function Manopt.set_bound_for_t!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real, t_old::Real) + for i in eachindex(M.lb) + if ts[i] >= t_current + d_out[i] *= t_old + else + d_out[i] = d_out[i] > 0 ? M.ub[i] - p[i] : M.lb[i] - p[i] + end + end + return d_out end """ diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index c11be8bffb..6b41362fc7 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -481,44 +481,44 @@ get_bound_t(M::AbstractManifold, p, d, i) function get_bound_t(M::ProductManifold, p, d, i) return get_bound_t(M.manifolds[1], submanifold_component(M, p, Val(1)), submanifold_component(M, d, Val(1)), i) end -function set_bound_t_at_index!(M::ProductManifold, p_cp, t, d, i) - set_bound_t_at_index!(M.manifolds[1], submanifold_component(M, p_cp, Val(1)), t, d, i) - return p_cp -end -function set_bound_at_index!(M::ProductManifold, p_cp, d, i) - set_bound_at_index!(M.manifolds[1], submanifold_component(M, p_cp, Val(1)), submanifold_component(M, d, Val(1)), i) - return p_cp -end +""" + set_zero_bound_at_index!(M::ProductManifold, d, i) -@doc raw""" - bound_direction_tweak!(M::ProductManifold, d_out, d, p, p_cp) +Set the element of the first component of `d` at bound index `i` to zero. +""" +function set_zero_bound_at_index!(M::ProductManifold, d, i) + set_zero_bound_at_index!(M.manifolds[1], submanifold_component(M, d, Val(1)), i) + return d +end -Set `d_out .= p_cp .- p` on the `Hyperrectangle` part of the `ProductManifold` `M`. +""" + Manopt.set_bound_for_t!(M::ProductManifold, d_out, p, ts::Dict, t_current::Real, t_old::Real) -Return the mutated `d_out`. +Set `d_out` so that it points from `p` to the generalized Cauchy point given times to +bounds `ts`. """ -function bound_direction_tweak!(M::ProductManifold, d_out, d, p, p_cp) - bound_direction_tweak!( +function set_bound_for_t!( + M::ProductManifold, d_out, p, ts::Dict, t_current::Real, t_old::Real + ) + set_bound_for_t!( M.manifolds[1], submanifold_component(M, d_out, Val(1)), - submanifold_component(M, d, Val(1)), submanifold_component(M, p, Val(1)), - submanifold_component(M, p_cp, Val(1)) + submanifold_component(M, p, Val(1)), ts, t_current, t_old ) return d_out end - @doc raw""" GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} Helper container for generalized Cauchy point search. Stores the manifold `M`, cached -workspace (`p_cp`), the quasi-Newton direction update `ha`, and the +workspace (`d_tmp`), the quasi-Newton direction update `ha`, and the ``f'``/``f''`` updater `fpfpp_updater`. Instances are reused across segments during `find_generalized_cauchy_point_direction!` to avoid allocations. """ -struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} +struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} M::TM - p_cp::TP + d_tmp::TX ha::T_HA fpfpp_updater::TFU end @@ -527,7 +527,7 @@ function GeneralizedCauchyPointFinder( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; fpfpp_updater::AbstractFPFPPUpdater = get_default_fpfpp_updater(M, p, ha) ) - return GeneralizedCauchyPointFinder(M, copy(M, p), ha, fpfpp_updater) + return GeneralizedCauchyPointFinder(M, zero_vector(M, p), ha, fpfpp_updater) end """ @@ -545,15 +545,15 @@ The function returns """ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFinder, d_out, p, d, X) M = gcp.M - copyto!(M, gcp.p_cp, p) - p_cp = gcp.p_cp copyto!(M, d_out, d) + d_tmp = gcp.d_tmp + copyto!(M, d_tmp, d) bounds_indices = get_bounds_index(M) TInd = eltype(bounds_indices) TF = number_eltype(d) - t = Dict{TInd, TF}((k, Inf) for k in bounds_indices) + ts = Dict{TInd, TF}((k, Inf) for k in bounds_indices) F_list = Tuple{TF, TInd}[] sizehint!(F_list, length(bounds_indices) + 1) @@ -561,12 +561,12 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin has_finite_limit = false for i in bounds_indices - t[i] = get_bound_t(M, p, d, i) + ts[i] = get_bound_t(M, p, d, i) - if t[i] > 0 - push!(F_list, (t[i], i)) + if ts[i] > 0 + push!(F_list, (ts[i], i)) end - has_finite_limit |= isfinite(t[i]) + has_finite_limit |= isfinite(ts[i]) end if M isa ProductManifold @@ -603,15 +603,15 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin t_current, b = pop!(F) dt = t_current - t_old - init_updater!(M, gcp.fpfpp_updater, p, d, gcp.ha) + init_updater!(M, gcp.fpfpp_updater, p, d_tmp, gcp.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 - db = get_at_bound_index(M, d, b) + db = get_at_bound_index(M, d_tmp, b) gb = get_at_bound_index(M, X, b) - f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, t_current, dt, db, gb, gcp.ha, b, d) + f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, t_current, dt, db, gb, gcp.ha, b, d_tmp) - set_bound_at_index!(M, p_cp, d, b) + set_zero_bound_at_index!(M, d_tmp, b) t_old = t_current @@ -631,13 +631,8 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin dt_min = max(dt_min, 0.0) t_old = t_old + dt_min - for i in bounds_indices - if t[i] >= t_current - set_bound_t_at_index!(M, p_cp, t_old, d, i) - end - end - bound_direction_tweak!(M, d_out, d, p, p_cp) + set_bound_for_t!(M, d_out, p, ts, t_current, t_old) if has_finite_limit return :found_limited From 97f6e8e622b9709d9c775f941e6dfcb90839c0ac Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 14 Jan 2026 15:52:40 +0100 Subject: [PATCH 050/135] rename --- docs/src/solvers/generalized_cauchy_point_subsolver.md | 3 ++- ext/ManoptManifoldsExt/manifold_functions.jl | 4 ++-- src/plans/box_plan.jl | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_point_subsolver.md b/docs/src/solvers/generalized_cauchy_point_subsolver.md index 74fbc28754..1b2fc10186 100644 --- a/docs/src/solvers/generalized_cauchy_point_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_point_subsolver.md @@ -45,7 +45,8 @@ Manopt.GenericFPFPPUpdater Manopt.get_bounds_index Manopt.get_bound_t Manopt.get_at_bound_index -Manopt.set_zero_bound_at_index! +Manopt.set_bound_for_t! +Manopt.set_zero_at_index! ``` ### Symbols related to specific Hessian approximations diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 03007540a8..e55f562652 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -201,12 +201,12 @@ end """ - Manopt.set_zero_bound_at_index!(M::Hyperrectangle, d, i) + Manopt.set_zero_at_index!(M::Hyperrectangle, d, i) Set element of tangent vector `d` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) at index `i` to 0. """ -function Manopt.set_zero_bound_at_index!(M::Hyperrectangle, d, i) +function Manopt.set_zero_at_index!(M::Hyperrectangle, d, i) d[i] = 0 return d end diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 6b41362fc7..4cadd22d4c 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -483,12 +483,12 @@ function get_bound_t(M::ProductManifold, p, d, i) end """ - set_zero_bound_at_index!(M::ProductManifold, d, i) + set_zero_at_index!(M::ProductManifold, d, i) Set the element of the first component of `d` at bound index `i` to zero. """ -function set_zero_bound_at_index!(M::ProductManifold, d, i) - set_zero_bound_at_index!(M.manifolds[1], submanifold_component(M, d, Val(1)), i) +function set_zero_at_index!(M::ProductManifold, d, i) + set_zero_at_index!(M.manifolds[1], submanifold_component(M, d, Val(1)), i) return d end @@ -611,7 +611,7 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, t_current, dt, db, gb, gcp.ha, b, d_tmp) - set_zero_bound_at_index!(M, d_tmp, b) + set_zero_at_index!(M, d_tmp, b) t_old = t_current From fb05f46d2936d2f75c2dc926d1ca42a7f25a70d9 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 14 Jan 2026 17:46:04 +0100 Subject: [PATCH 051/135] refactor segment updater to be more minimal (common logic went to the GCP subsolver) --- .../generalized_cauchy_point_subsolver.md | 6 +- src/plans/box_plan.jl | 126 ++++++++++-------- test/solvers/test_quasi_Newton_box.jl | 54 ++++---- 3 files changed, 96 insertions(+), 90 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_point_subsolver.md b/docs/src/solvers/generalized_cauchy_point_subsolver.md index 1b2fc10186..c3740590c9 100644 --- a/docs/src/solvers/generalized_cauchy_point_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_point_subsolver.md @@ -40,8 +40,8 @@ These are internal symbols used to manage and manipulate bound constraints durin ```@docs Manopt.init_updater! -Manopt.AbstractFPFPPUpdater -Manopt.GenericFPFPPUpdater +Manopt.AbstractSegmentHessianUpdater +Manopt.GenericSegmentHessianUpdater Manopt.get_bounds_index Manopt.get_bound_t Manopt.get_at_bound_index @@ -52,7 +52,7 @@ Manopt.set_zero_at_index! ### Symbols related to specific Hessian approximations ```@docs -Manopt.LimitedMemoryFPFPPUpdater +Manopt.LimitedMemorySegmentHessianUpdater Manopt.hessian_value_from_wmwt_coords Manopt.set_M_current_scale! ``` diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 4cadd22d4c..7bb2b247cb 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -326,103 +326,112 @@ end """ - abstract type AbstractFPFPPUpdater end + abstract type AbstractSegmentHessianUpdater end Abstract type for methods that calculate f' and f'' in the GCP calculation in subsequent line segments in `GeneralizedCauchyPointFinder`. """ -abstract type AbstractFPFPPUpdater end +abstract type AbstractSegmentHessianUpdater end """ - init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) + init_updater!(::AbstractManifold, hessian_segment_updater::AbstractSegmentHessianUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) -Method for initialization of `AbstractFPFPPUpdater` `fpfpp_upd` just before the loop +Method for initialization of `AbstractSegmentHessianUpdater` `hessian_segment_updater` just before the loop that examines subsequent intervals for GCP. """ -init_updater!(::AbstractManifold, fpfpp_upd::AbstractFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) +init_updater!(::AbstractManifold, hessian_segment_updater::AbstractSegmentHessianUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) """ - struct GenericFPFPPUpdater <: AbstractFPFPPUpdater end + struct GenericSegmentHessianUpdater <: AbstractSegmentHessianUpdater end Generic f' and f'' calculation that only relies on `hessian_value_eb` but is relatively slow for high-dimensional domains. """ -struct GenericFPFPPUpdater{TX} <: AbstractFPFPPUpdater +struct GenericSegmentHessianUpdater{TX} <: AbstractSegmentHessianUpdater d_z::TX end -function get_default_fpfpp_updater(M::AbstractManifold, p, ::AbstractQuasiNewtonDirectionUpdate) - return GenericFPFPPUpdater(zero_vector(M, p)) +function get_default_hessian_segment_updater(M::AbstractManifold, p, ::AbstractQuasiNewtonDirectionUpdate) + return GenericSegmentHessianUpdater(zero_vector(M, p)) end -function init_updater!(M::AbstractManifold, fpfpp_upd::GenericFPFPPUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) - zero_vector!(M, fpfpp_upd.d_z, p) - return fpfpp_upd +function init_updater!(M::AbstractManifold, hessian_segment_updater::GenericSegmentHessianUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) + zero_vector!(M, hessian_segment_updater.d_z, p) + return hessian_segment_updater end -function (upd::GenericFPFPPUpdater)(M::AbstractManifold, p, old_f_prime, old_f_double_prime, t::Real, dt::Real, db, gb, ha, b, d) +@doc raw""" + (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, db, ha::AbstractQuasiNewtonDirectionUpdate, b, d) + +Calculate Hessian values ``⟨e_b, B d_z⟩`` and ``⟨e_b, B d⟩`` for the generalized Cauchy +point line search using the generic approach via `hessian_value_eb`. +``d_z`` start with 0 and is updated in-place by adding `dt * d` to it. +""" +function (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, db, ha, b, d) upd.d_z .+= dt .* d - f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + hessian_value_eb(ha, M, p, b, upd.d_z)) - f_double_prime = old_f_double_prime + (2 * -db * hessian_value_eb(ha, M, p, b, d)) + db^2 * hessian_value_eb(ha, M, p, b) + hv_eb_dz = hessian_value_eb(ha, M, p, b, upd.d_z) + hv_eb_d = hessian_value_eb(ha, M, p, b, d) - return f_prime, f_double_prime + return hv_eb_dz, hv_eb_d end """ - struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater + struct LimitedMemorySegmentHessianUpdater{TV <: AbstractVector} <: AbstractSegmentHessianUpdater -f' and f'' calculation that is optimized for `QuasiNewtonLimitedMemoryBoxDirectionUpdate`. -It relies on a specific Hessian structure. +Hessian value calculation for generalized Cauchy point line segments that is optimized for +`QuasiNewtonLimitedMemoryBoxDirectionUpdate`. It relies on a specific Hessian structure. """ -struct LimitedMemoryFPFPPUpdater{TV <: AbstractVector} <: AbstractFPFPPUpdater +struct LimitedMemorySegmentHessianUpdater{TV <: AbstractVector} <: AbstractSegmentHessianUpdater p_s::TV p_y::TV c_s::TV c_y::TV end -function get_default_fpfpp_updater(::AbstractManifold, p, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) - return LimitedMemoryFPFPPUpdater(similar(ha.qn_du.ρ), similar(ha.qn_du.ρ), similar(ha.qn_du.ρ), similar(ha.qn_du.ρ)) +function get_default_hessian_segment_updater(::AbstractManifold, p, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + return LimitedMemorySegmentHessianUpdater(similar(ha.qn_du.ρ), similar(ha.qn_du.ρ), similar(ha.qn_du.ρ), similar(ha.qn_du.ρ)) end -function init_updater!(M::AbstractManifold, fpfpp_upd::LimitedMemoryFPFPPUpdater, p, d, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) - fill!(fpfpp_upd.c_s, 0) - fill!(fpfpp_upd.c_y, 0) +function init_updater!(M::AbstractManifold, hessian_segment_updater::LimitedMemorySegmentHessianUpdater, p, d, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + fill!(hessian_segment_updater.c_s, 0) + fill!(hessian_segment_updater.c_y, 0) ii = 1 for i in eachindex(ha.qn_du.ρ) if iszero(ha.qn_du.ρ[i]) continue end - fpfpp_upd.p_s[ii] = ha.current_scale * inner(M, p, ha.qn_du.memory_s[i], d) - fpfpp_upd.p_y[ii] = inner(M, p, ha.qn_du.memory_y[i], d) + hessian_segment_updater.p_s[ii] = ha.current_scale * inner(M, p, ha.qn_du.memory_s[i], d) + hessian_segment_updater.p_y[ii] = inner(M, p, ha.qn_du.memory_y[i], d) ii += 1 end - return fpfpp_upd + return hessian_segment_updater end @doc raw""" - (fpfpp_upd::LimitedMemoryFPFPPUpdater)( - M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, + (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( + M::AbstractManifold, p, t::Real, dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, d ) -Update ``f'`` and ``f''`` for the generalized Cauchy point line search using the limited-memory -block Hessian stored in `ha`. +Calculate Hessian values ``⟨e_b, B d_z⟩`` and ``⟨e_b, B d⟩`` for the generalized Cauchy +point line search using the limited-memory block Hessian stored in `ha`. +``d_z`` start with 0 and is updated in-place by adding `dt * d` to it. ## Arguments: -- `old_f_prime`, `old_f_double_prime`: values carried from the previous segment. -- `dt`: step length along the segment direction. +- `M`: manifold. +- `p`: current iterate. +- `t`: current step length from `p`. +- `dt`: step length increment from the last step. - `db`: direction component at the bound index `b`. -- `gb`: gradient component at the bound index `b`. -The updater reuses cached coordinate projections in `fpfpp_upd` to cheaply evaluate Hessian -quadratic forms via `hessian_value_from_wmwt_coords`, then returns the new `(f', f'')` pair. +The updater reuses cached coordinate projections in `hessian_segment_updater` to cheaply +evaluate Hessian quadratic forms via `hessian_value_from_wmwt_coords`. """ -function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( - M::AbstractManifold, p, old_f_prime::Real, old_f_double_prime::Real, - t::Real, dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, d +function (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( + M::AbstractManifold, p, + t::Real, dt::Real, db, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, d ) m = length(ha.qn_du.memory_s) @@ -441,25 +450,22 @@ function (fpfpp_upd::LimitedMemoryFPFPPUpdater)( coords_Yk_eb = view(ha.coords_Yk_X, 1:num_nonzero_rho) coords_Sk_eb = view(ha.coords_Sk_X, 1:num_nonzero_rho) - coords_cy = view(fpfpp_upd.c_y, 1:num_nonzero_rho) - coords_cs = view(fpfpp_upd.c_s, 1:num_nonzero_rho) - coords_py = view(fpfpp_upd.p_y, 1:num_nonzero_rho) - coords_ps = view(fpfpp_upd.p_s, 1:num_nonzero_rho) + coords_cy = view(hessian_segment_updater.c_y, 1:num_nonzero_rho) + coords_cs = view(hessian_segment_updater.c_s, 1:num_nonzero_rho) + coords_py = view(hessian_segment_updater.p_y, 1:num_nonzero_rho) + coords_ps = view(hessian_segment_updater.p_s, 1:num_nonzero_rho) coords_cy .+= dt .* coords_py coords_cs .+= dt .* coords_ps eb_B_z = hessian_value_from_wmwt_coords(ha, t * db, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) - f_prime = old_f_prime + dt * old_f_double_prime - db * (gb + eb_B_z) eb_B_d = hessian_value_from_wmwt_coords(ha, db, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) - f_double_prime = old_f_double_prime - 2 * db * eb_B_d + db^2 * hessian_value_eb(ha, M, p, b) - coords_py .-= db .* coords_Yk_eb coords_ps .-= db .* coords_Sk_eb - return f_prime, f_double_prime + return eb_B_z, eb_B_d end """ @@ -509,25 +515,26 @@ function set_bound_for_t!( end @doc raw""" - GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} + GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} Helper container for generalized Cauchy point search. Stores the manifold `M`, cached -workspace (`d_tmp`), the quasi-Newton direction update `ha`, and the -``f'``/``f''`` updater `fpfpp_updater`. Instances are reused across segments during -`find_generalized_cauchy_point_direction!` to avoid allocations. +workspace (`d_tmp`), the quasi-Newton direction update `ha`, and the `hessian_segment_updater`, +which computes certain values of the Hessian while advancing segments. +Instances are reused across segments during `find_generalized_cauchy_point_direction!` to +avoid allocations. """ -struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractFPFPPUpdater} +struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} M::TM d_tmp::TX ha::T_HA - fpfpp_updater::TFU + hessian_segment_updater::TFU end function GeneralizedCauchyPointFinder( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; - fpfpp_updater::AbstractFPFPPUpdater = get_default_fpfpp_updater(M, p, ha) + hessian_segment_updater::AbstractSegmentHessianUpdater = get_default_hessian_segment_updater(M, p, ha) ) - return GeneralizedCauchyPointFinder(M, zero_vector(M, p), ha, fpfpp_updater) + return GeneralizedCauchyPointFinder(M, zero_vector(M, p), ha, hessian_segment_updater) end """ @@ -603,13 +610,16 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin t_current, b = pop!(F) dt = t_current - t_old - init_updater!(M, gcp.fpfpp_updater, p, d_tmp, gcp.ha) + init_updater!(M, gcp.hessian_segment_updater, p, d_tmp, gcp.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 db = get_at_bound_index(M, d_tmp, b) gb = get_at_bound_index(M, X, b) - f_prime, f_double_prime = gcp.fpfpp_updater(M, p, f_prime, f_double_prime, t_current, dt, db, gb, gcp.ha, b, d_tmp) + hv_eb_dz, hv_eb_d = gcp.hessian_segment_updater(M, p, t_current, dt, db, gcp.ha, b, d_tmp) + + f_prime += dt * f_double_prime - db * (gb + hv_eb_dz) + f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_eb(gcp.ha, M, p, b) set_zero_at_index!(M, d_tmp, b) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 7d779c115b..c97f077878 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -42,20 +42,19 @@ using RecursiveArrayTools z = [-0.25, -1.0] # optimized formula - upd = Manopt.GenericFPFPPUpdater(similar(d)) + upd = Manopt.GenericSegmentHessianUpdater(similar(d)) Manopt.init_updater!(M, upd, p, d, ha) - f_prime, f_double_prime = upd(M, p, old_f_prime, old_f_double_prime, 0 + dt, dt, db, gb, ha, b, d) - @test f_prime ≈ -0.5 - @test f_double_prime ≈ 2.0 + hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, db, ha, b, d) + @test hv_eb_dz ≈ -2.0 + @test hv_eb_d ≈ -8.0 # original formula - d[2] = 0.0 - f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) - f_original_double_prime = Manopt.hessian_value(ha, M, p, d) + original_hv_eb_dz = dot([0, 1], ha.matrix, z) + original_hv_eb_d = dot([0, 1], ha.matrix, d) - @test f_prime == f_original_prime - @test f_double_prime == f_original_double_prime + @test hv_eb_dz == original_hv_eb_dz + @test hv_eb_d == original_hv_eb_d end @@ -76,20 +75,19 @@ using RecursiveArrayTools z = [-0.5, -0.25] # optimized formula - upd = Manopt.GenericFPFPPUpdater(similar(d)) + upd = Manopt.GenericSegmentHessianUpdater(similar(d)) Manopt.init_updater!(M, upd, p, d, ha) - f_prime, f_double_prime = upd(M, p, old_f_prime, old_f_double_prime, 0 + dt, dt, db, gb, ha, b, d) - @test f_prime == -3.5 - @test f_double_prime == 2 + hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, db, ha, b, d) + @test hv_eb_dz == -1.0 + @test hv_eb_d == -4.0 # original formula - d[1] = 0.0 - f_original_prime = dot(grad, d) + dot(d, ha.matrix, z) - f_original_double_prime = Manopt.hessian_value(ha, M, p, d) + original_hv_eb_dz = dot([1, 0], ha.matrix, z) + original_hv_eb_d = dot([1, 0], ha.matrix, d) - @test f_prime == f_original_prime - @test f_double_prime == f_original_double_prime + @test hv_eb_dz == original_hv_eb_dz + @test hv_eb_d == original_hv_eb_d end @testset "update_fp_fpp - basic d = [-2.0, -1.0] with limited memory update" begin @@ -128,23 +126,21 @@ using RecursiveArrayTools t_current = 0 + dt # compare the generic and limited memory updater - gupd = Manopt.GenericFPFPPUpdater(similar(d)) + gupd = Manopt.GenericSegmentHessianUpdater(similar(d)) Manopt.init_updater!(M, gupd, p, d, ha) - f_prime, f_double_prime = gupd(M, p, old_f_prime, old_f_double_prime, t_current, dt, db, gb, ha, b, d) + hv_eb_dz, hv_eb_d = gupd(M, p, t_current, dt, db, ha, b, d) - @test f_prime ≈ 0.375 - @test f_double_prime ≈ 9.5 + @test hv_eb_dz ≈ -0.125 + @test hv_eb_d ≈ -0.5 - lmupd = Manopt.get_default_fpfpp_updater(M, p, ha) - @test lmupd isa Manopt.LimitedMemoryFPFPPUpdater + lmupd = Manopt.get_default_hessian_segment_updater(M, p, ha) + @test lmupd isa Manopt.LimitedMemorySegmentHessianUpdater Manopt.init_updater!(M, lmupd, p, d, ha) - f_prime_limited, f_double_prime_limited = lmupd(M, p, old_f_prime, old_f_double_prime, t_current, dt, db, gb, ha, b, d) + hv_eb_dz_limited, hv_eb_d_limited = lmupd(M, p, t_current, dt, db, ha, b, d) - d[1] = 0.0 - - @test f_prime ≈ f_prime_limited - @test f_double_prime ≈ f_double_prime_limited + @test hv_eb_dz ≈ hv_eb_dz_limited + @test hv_eb_d ≈ hv_eb_d_limited ha.last_gcp_result = :found_unlimited @test Manopt.get_parameter(ha, Val(:max_stepsize)) == Inf From 4f52e804d311a800e2e2b13df4df527142bd088d Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 15 Jan 2026 12:35:43 +0100 Subject: [PATCH 052/135] more cleanup in GCP --- ext/ManoptManifoldsExt/manifold_functions.jl | 15 ++++++--------- src/plans/box_plan.jl | 10 +++++----- test/solvers/test_quasi_Newton_box.jl | 1 - 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index e55f562652..f52e4e23d3 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -212,18 +212,15 @@ function Manopt.set_zero_at_index!(M::Hyperrectangle, d, i) end """ - Manopt.set_bound_for_t!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real, t_old::Real) + Manopt.set_bound_for_t!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real) -For each index `i`, set element of tangent vector `d_out` on -[`Hyperrectangle`](@extref Manifolds.Hyperrectangle) `M` according to the following rule: -- if `t[i] >= t_current`, multiply `d_out[i]` by `t_old`; -- else, set `d_out[i]` to the distance from `p[i]` to the bound in the direction of `d_out[i]`. +For each index `i`, `t[i] < t_current`, set element of tangent vector `d_out` on +[`Hyperrectangle`](@extref Manifolds.Hyperrectangle) to the distance from `p[i]` to the +bound in the direction of `d_out[i]`. """ -function Manopt.set_bound_for_t!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real, t_old::Real) +function Manopt.set_bound_for_t!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real) for i in eachindex(M.lb) - if ts[i] >= t_current - d_out[i] *= t_old - else + if ts[i] < t_current d_out[i] = d_out[i] > 0 ? M.ub[i] - p[i] : M.lb[i] - p[i] end end diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 7bb2b247cb..f0cc225132 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -499,17 +499,17 @@ function set_zero_at_index!(M::ProductManifold, d, i) end """ - Manopt.set_bound_for_t!(M::ProductManifold, d_out, p, ts::Dict, t_current::Real, t_old::Real) + Manopt.set_bound_for_t!(M::ProductManifold, d_out, p, ts::Dict, t_current::Real) Set `d_out` so that it points from `p` to the generalized Cauchy point given times to bounds `ts`. """ function set_bound_for_t!( - M::ProductManifold, d_out, p, ts::Dict, t_current::Real, t_old::Real + M::ProductManifold, d_out, p, ts::Dict, t_current::Real ) set_bound_for_t!( M.manifolds[1], submanifold_component(M, d_out, Val(1)), - submanifold_component(M, p, Val(1)), ts, t_current, t_old + submanifold_component(M, p, Val(1)), ts, t_current ) return d_out end @@ -641,8 +641,8 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin dt_min = max(dt_min, 0.0) t_old = t_old + dt_min - - set_bound_for_t!(M, d_out, p, ts, t_current, t_old) + d_out .*= t_old + set_bound_for_t!(M, d_out, p, ts, t_current) if has_finite_limit return :found_limited diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index c97f077878..22de3268b9 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -171,7 +171,6 @@ using RecursiveArrayTools @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d, X1) === :found_limited @test d_out ≈ [2.0, 0.0, 0.0] - d_out = similar(d) @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, 0 * d, X1) === :not_found From cc74add44f1934c4be5d88da3b551f68019e5477 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 15 Jan 2026 14:42:52 +0100 Subject: [PATCH 053/135] rearrange arguments to a more logical order --- src/plans/box_plan.jl | 6 +++--- test/solvers/test_quasi_Newton_box.jl | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index f0cc225132..7434f99314 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -367,7 +367,7 @@ Calculate Hessian values ``⟨e_b, B d_z⟩`` and ``⟨e_b, B d⟩`` for the gen point line search using the generic approach via `hessian_value_eb`. ``d_z`` start with 0 and is updated in-place by adding `dt * d` to it. """ -function (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, db, ha, b, d) +function (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, b, db, ha, d) upd.d_z .+= dt .* d hv_eb_dz = hessian_value_eb(ha, M, p, b, upd.d_z) hv_eb_d = hessian_value_eb(ha, M, p, b, d) @@ -431,7 +431,7 @@ evaluate Hessian quadratic forms via `hessian_value_from_wmwt_coords`. """ function (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( M::AbstractManifold, p, - t::Real, dt::Real, db, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, d + t::Real, dt::Real, b, db, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, d ) m = length(ha.qn_du.memory_s) @@ -616,7 +616,7 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin db = get_at_bound_index(M, d_tmp, b) gb = get_at_bound_index(M, X, b) - hv_eb_dz, hv_eb_d = gcp.hessian_segment_updater(M, p, t_current, dt, db, gcp.ha, b, d_tmp) + hv_eb_dz, hv_eb_d = gcp.hessian_segment_updater(M, p, t_current, dt, b, db, gcp.ha, d_tmp) f_prime += dt * f_double_prime - db * (gb + hv_eb_dz) f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_eb(gcp.ha, M, p, b) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 22de3268b9..3b19c9c494 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -44,7 +44,7 @@ using RecursiveArrayTools # optimized formula upd = Manopt.GenericSegmentHessianUpdater(similar(d)) Manopt.init_updater!(M, upd, p, d, ha) - hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, db, ha, b, d) + hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha, d) @test hv_eb_dz ≈ -2.0 @test hv_eb_d ≈ -8.0 @@ -77,7 +77,7 @@ using RecursiveArrayTools # optimized formula upd = Manopt.GenericSegmentHessianUpdater(similar(d)) Manopt.init_updater!(M, upd, p, d, ha) - hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, db, ha, b, d) + hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha, d) @test hv_eb_dz == -1.0 @test hv_eb_d == -4.0 @@ -128,7 +128,7 @@ using RecursiveArrayTools # compare the generic and limited memory updater gupd = Manopt.GenericSegmentHessianUpdater(similar(d)) Manopt.init_updater!(M, gupd, p, d, ha) - hv_eb_dz, hv_eb_d = gupd(M, p, t_current, dt, db, ha, b, d) + hv_eb_dz, hv_eb_d = gupd(M, p, t_current, dt, b, db, ha, d) @test hv_eb_dz ≈ -0.125 @test hv_eb_d ≈ -0.5 @@ -137,7 +137,7 @@ using RecursiveArrayTools @test lmupd isa Manopt.LimitedMemorySegmentHessianUpdater Manopt.init_updater!(M, lmupd, p, d, ha) - hv_eb_dz_limited, hv_eb_d_limited = lmupd(M, p, t_current, dt, db, ha, b, d) + hv_eb_dz_limited, hv_eb_d_limited = lmupd(M, p, t_current, dt, b, db, ha, d) @test hv_eb_dz ≈ hv_eb_dz_limited @test hv_eb_d ≈ hv_eb_d_limited From 1063eb62a8fd3fff1537f09ba717516c61119495 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 15 Jan 2026 15:32:17 +0100 Subject: [PATCH 054/135] CMA ES should probably use passed RNG for the initial point? --- test/solvers/test_cma_es.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/solvers/test_cma_es.jl b/test/solvers/test_cma_es.jl index 64a10d0563..b18ac96e8d 100644 --- a/test/solvers/test_cma_es.jl +++ b/test/solvers/test_cma_es.jl @@ -34,7 +34,7 @@ flat_example(::AbstractManifold, p) = 0.0 @test griewank(M, p1) < 0.1 p1 = cma_es(M, griewank; σ = 10.0, rng = MersenneTwister(123)) - @test griewank(M, p1) < 0.2 + @test griewank(M, p1) < 0.25 p1 = [10.0, 10.0] cma_es!(M, griewank, p1; σ = 10.0, rng = MersenneTwister(123)) From e8afe8d0dc22dc278eb976416eb6a0741c550666 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 16 Jan 2026 18:39:39 +0100 Subject: [PATCH 055/135] move one temporary to the segment propagator that needs it --- src/plans/box_plan.jl | 33 ++++++++++++++------------- test/solvers/test_quasi_Newton_box.jl | 14 ++++++------ 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 7434f99314..b017a21505 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -349,28 +349,32 @@ high-dimensional domains. """ struct GenericSegmentHessianUpdater{TX} <: AbstractSegmentHessianUpdater d_z::TX + d_tmp::TX end function get_default_hessian_segment_updater(M::AbstractManifold, p, ::AbstractQuasiNewtonDirectionUpdate) - return GenericSegmentHessianUpdater(zero_vector(M, p)) + return GenericSegmentHessianUpdater(zero_vector(M, p), zero_vector(M, p)) end function init_updater!(M::AbstractManifold, hessian_segment_updater::GenericSegmentHessianUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) zero_vector!(M, hessian_segment_updater.d_z, p) + copyto!(M, hessian_segment_updater.d_tmp, d) return hessian_segment_updater end @doc raw""" - (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, db, ha::AbstractQuasiNewtonDirectionUpdate, b, d) + (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, b, db, ha::AbstractQuasiNewtonDirectionUpdate) -Calculate Hessian values ``⟨e_b, B d_z⟩`` and ``⟨e_b, B d⟩`` for the generalized Cauchy +Calculate Hessian values ``⟨e_b, B d_z⟩`` and ``⟨e_b, B d_tmp⟩`` for the generalized Cauchy point line search using the generic approach via `hessian_value_eb`. ``d_z`` start with 0 and is updated in-place by adding `dt * d` to it. """ -function (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, b, db, ha, d) - upd.d_z .+= dt .* d +function (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, b, db, ha) + upd.d_z .+= dt .* upd.d_tmp hv_eb_dz = hessian_value_eb(ha, M, p, b, upd.d_z) - hv_eb_d = hessian_value_eb(ha, M, p, b, d) + hv_eb_d = hessian_value_eb(ha, M, p, b, upd.d_tmp) + + set_zero_at_index!(M, upd.d_tmp, b) return hv_eb_dz, hv_eb_d end @@ -411,7 +415,7 @@ end @doc raw""" (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( M::AbstractManifold, p, - t::Real, dt::Real, db, gb, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, b, d + t::Real, dt::Real, b, db, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate ) Calculate Hessian values ``⟨e_b, B d_z⟩`` and ``⟨e_b, B d⟩`` for the generalized Cauchy @@ -424,14 +428,15 @@ point line search using the limited-memory block Hessian stored in `ha`. - `p`: current iterate. - `t`: current step length from `p`. - `dt`: step length increment from the last step. -- `db`: direction component at the bound index `b`. +- `b`: bound index of the current segment. +- `db`: search direction component at the bound index `b`. The updater reuses cached coordinate projections in `hessian_segment_updater` to cheaply evaluate Hessian quadratic forms via `hessian_value_from_wmwt_coords`. """ function (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( M::AbstractManifold, p, - t::Real, dt::Real, b, db, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate, d + t::Real, dt::Real, b, db, ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate ) m = length(ha.qn_du.memory_s) @@ -553,8 +558,6 @@ The function returns function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFinder, d_out, p, d, X) M = gcp.M copyto!(M, d_out, d) - d_tmp = gcp.d_tmp - copyto!(M, d_tmp, d) bounds_indices = get_bounds_index(M) TInd = eltype(bounds_indices) @@ -610,19 +613,17 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFin t_current, b = pop!(F) dt = t_current - t_old - init_updater!(M, gcp.hessian_segment_updater, p, d_tmp, gcp.ha) + init_updater!(M, gcp.hessian_segment_updater, p, d, gcp.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 - db = get_at_bound_index(M, d_tmp, b) + db = get_at_bound_index(M, d, b) gb = get_at_bound_index(M, X, b) - hv_eb_dz, hv_eb_d = gcp.hessian_segment_updater(M, p, t_current, dt, b, db, gcp.ha, d_tmp) + hv_eb_dz, hv_eb_d = gcp.hessian_segment_updater(M, p, t_current, dt, b, db, gcp.ha) f_prime += dt * f_double_prime - db * (gb + hv_eb_dz) f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_eb(gcp.ha, M, p, b) - set_zero_at_index!(M, d_tmp, b) - t_old = t_current # If f_prime is 0, we've found the local minimizer (GCP) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 3b19c9c494..ce7b49ec30 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -42,9 +42,9 @@ using RecursiveArrayTools z = [-0.25, -1.0] # optimized formula - upd = Manopt.GenericSegmentHessianUpdater(similar(d)) + upd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) Manopt.init_updater!(M, upd, p, d, ha) - hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha, d) + hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha) @test hv_eb_dz ≈ -2.0 @test hv_eb_d ≈ -8.0 @@ -75,9 +75,9 @@ using RecursiveArrayTools z = [-0.5, -0.25] # optimized formula - upd = Manopt.GenericSegmentHessianUpdater(similar(d)) + upd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) Manopt.init_updater!(M, upd, p, d, ha) - hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha, d) + hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha) @test hv_eb_dz == -1.0 @test hv_eb_d == -4.0 @@ -126,9 +126,9 @@ using RecursiveArrayTools t_current = 0 + dt # compare the generic and limited memory updater - gupd = Manopt.GenericSegmentHessianUpdater(similar(d)) + gupd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) Manopt.init_updater!(M, gupd, p, d, ha) - hv_eb_dz, hv_eb_d = gupd(M, p, t_current, dt, b, db, ha, d) + hv_eb_dz, hv_eb_d = gupd(M, p, t_current, dt, b, db, ha) @test hv_eb_dz ≈ -0.125 @test hv_eb_d ≈ -0.5 @@ -137,7 +137,7 @@ using RecursiveArrayTools @test lmupd isa Manopt.LimitedMemorySegmentHessianUpdater Manopt.init_updater!(M, lmupd, p, d, ha) - hv_eb_dz_limited, hv_eb_d_limited = lmupd(M, p, t_current, dt, b, db, ha, d) + hv_eb_dz_limited, hv_eb_d_limited = lmupd(M, p, t_current, dt, b, db, ha) @test hv_eb_dz ≈ hv_eb_dz_limited @test hv_eb_d ≈ hv_eb_d_limited From e854a1e253c3cbd8e57e9f60f450c9069fd37c05 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 17 Jan 2026 12:53:54 +0100 Subject: [PATCH 056/135] improve coverage --- test/solvers/test_quasi_Newton_box.jl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index ce7b49ec30..ef52bc7210 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -258,6 +258,20 @@ using RecursiveArrayTools grad_f(M, p) = ArrayPartition(project(Mbox, p.x[1], 4 .* (p.x[1] .^ 3)), -log(S2, p.x[2], px)) p0 = ArrayPartition([0.0, 4.0, 1.0], [1.0, 0.0, 0.0]) + @testset "Hessian updater" begin + d = -grad_f(M, p0) + ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis()) + gupd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) + Manopt.init_updater!(M, gupd, p0, d, ha) + b = 2 + dt = 0.25 + t_current = 0 + dt + db = d[b] + hv_eb_dz, hv_eb_d = gupd(M, p0, t_current, dt, b, db, ha) + @test hv_eb_dz ≈ -64.0 + @test hv_eb_d ≈ -256.0 + end + p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) @test distance(M, p_opt, ArrayPartition([0, 2, 0], px)) < 0.1 end From 3fc6a7f0bd8361bfffd008b0b2377baea80bcc0e Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 20 Jan 2026 12:17:42 +0100 Subject: [PATCH 057/135] Add test for hitting multiple bounds in Generalized Cauchy Point Finder --- test/solvers/test_quasi_Newton_box.jl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index ef52bc7210..2344b44081 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -198,6 +198,21 @@ using RecursiveArrayTools @test Manopt.find_generalized_cauchy_point_direction!(gf3, d_out, p3, [1.0], [-10.0]) === :found_limited end + @testset "Hitting multiple bounds at the same time in GCD" begin + M = Hyperrectangle([-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]) + ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [1.0 0 0; 0 1 0; 0 0 1]) + + p = [0.0, 0.0, 0.0] + gf = Manopt.GeneralizedCauchyPointFinder(M, p, ha) + + d = [-2.0, -2.0, -1.0] + d_out = similar(d) + X = [10.0, 10.0, 10.0] + + @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d, X) === :found_limited + @test d_out ≈ [-1.0, -1.0, -1.0] + end + @testset "Pure Hyperrectangle" begin M = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) f(M, p) = sum(p .^ 2) From bb4a56034eacd29c685ea5af09428b2bb4e4f795 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 20 Jan 2026 12:21:36 +0100 Subject: [PATCH 058/135] improve naming --- Changelog.md | 2 +- .../generalized_cauchy_point_subsolver.md | 4 +-- src/plans/box_plan.jl | 32 +++++++++---------- test/solvers/test_quasi_Newton_box.jl | 10 +++--- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Changelog.md b/Changelog.md index df6eede107..2bb1f5f9e1 100644 --- a/Changelog.md +++ b/Changelog.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. This resolves issue #549. (#554) -* `GeneralizedCauchyPointFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) +* `GeneralizedCauchyDirectionFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) * New stopping criteria: `StopWhenRelativeAPosterioriCostChangeLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. ### Fixed diff --git a/docs/src/solvers/generalized_cauchy_point_subsolver.md b/docs/src/solvers/generalized_cauchy_point_subsolver.md index c3740590c9..108106c0c6 100644 --- a/docs/src/solvers/generalized_cauchy_point_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_point_subsolver.md @@ -22,7 +22,7 @@ These symbols are directly used by solvers to compute the descent direction corr ```@docs Manopt.requires_generalized_cauchy_point_computation Manopt.find_generalized_cauchy_point_direction! -Manopt.GeneralizedCauchyPointFinder +Manopt.GeneralizedCauchyDirectionFinder ``` ### Symbols related to the Hessian approximation @@ -53,6 +53,6 @@ Manopt.set_zero_at_index! ```@docs Manopt.LimitedMemorySegmentHessianUpdater -Manopt.hessian_value_from_wmwt_coords +Manopt.hessian_value_from_inner_products Manopt.set_M_current_scale! ``` diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index b017a21505..20be6a6c90 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -101,7 +101,7 @@ function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( M = get_manifold(mp) p = get_iterate(st) X = get_gradient(st) - gcp = GeneralizedCauchyPointFinder(M, p, d) + gcp = GeneralizedCauchyDirectionFinder(M, p, d) d.last_gcp_result = find_generalized_cauchy_point_direction!(gcp, r, p, r, X) return r end @@ -138,7 +138,7 @@ function hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstra coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) - return hessian_value_from_wmwt_coords(gh, normX_sqr, coords_Yk, coords_Sk, coords_Yk, coords_Sk) + return hessian_value_from_inner_products(gh, normX_sqr, coords_Yk, coords_Sk, coords_Yk, coords_Sk) end @doc raw""" @@ -166,7 +166,7 @@ function hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abs coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) - return hessian_value_from_wmwt_coords(gh, one(eltype(gh.qn_du.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) + return hessian_value_from_inner_products(gh, one(eltype(gh.qn_du.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) end @doc raw""" @@ -199,7 +199,7 @@ function hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abs coords_Sk_X = view(gh.coords_Sk_X, 1:num_nonzero_rho) coords_Sk_Y = view(gh.coords_Sk_Y, 1:num_nonzero_rho) - return hessian_value_from_wmwt_coords(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) + return hessian_value_from_inner_products(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) end @doc raw""" @@ -279,7 +279,7 @@ function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemo end @doc raw""" - hessian_value_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) + hessian_value_from_inner_products(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) Evaluate the quadratic form defined by the current blockwise Hessian approximation stored in `gh`, given precomputed coordinate vectors. @@ -292,7 +292,7 @@ Arguments: The result is ``θ·iss - cy₁ᵀ M₁₁ cy₂ - 2·cs₁ᵀ M₂₁ cy₂ - cs₁ᵀ M₂₂ cs₂`` using the blocks ``M₁₁``, ``M₂₁``, ``M₂₂`` stored in `gh` and the current scale ``θ``. Returns the scalar value. """ -function hessian_value_from_wmwt_coords(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) +function hessian_value_from_inner_products(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, iss::Real, cy1, cs1, cy2, cs2) result = gh.current_scale * iss if length(cy1) == 0 return result @@ -329,7 +329,7 @@ end abstract type AbstractSegmentHessianUpdater end Abstract type for methods that calculate f' and f'' in the GCP calculation in subsequent -line segments in `GeneralizedCauchyPointFinder`. +line segments in [`GeneralizedCauchyDirectionFinder`](@ref). """ abstract type AbstractSegmentHessianUpdater end @@ -432,7 +432,7 @@ point line search using the limited-memory block Hessian stored in `ha`. - `db`: search direction component at the bound index `b`. The updater reuses cached coordinate projections in `hessian_segment_updater` to cheaply -evaluate Hessian quadratic forms via `hessian_value_from_wmwt_coords`. +evaluate Hessian quadratic forms via `hessian_value_from_inner_products`. """ function (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( M::AbstractManifold, p, @@ -463,9 +463,9 @@ function (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( coords_cy .+= dt .* coords_py coords_cs .+= dt .* coords_ps - eb_B_z = hessian_value_from_wmwt_coords(ha, t * db, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) + eb_B_z = hessian_value_from_inner_products(ha, t * db, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) - eb_B_d = hessian_value_from_wmwt_coords(ha, db, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) + eb_B_d = hessian_value_from_inner_products(ha, db, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) coords_py .-= db .* coords_Yk_eb coords_ps .-= db .* coords_Sk_eb @@ -520,7 +520,7 @@ function set_bound_for_t!( end @doc raw""" - GeneralizedCauchyPointFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} + GeneralizedCauchyDirectionFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} Helper container for generalized Cauchy point search. Stores the manifold `M`, cached workspace (`d_tmp`), the quasi-Newton direction update `ha`, and the `hessian_segment_updater`, @@ -528,22 +528,22 @@ which computes certain values of the Hessian while advancing segments. Instances are reused across segments during `find_generalized_cauchy_point_direction!` to avoid allocations. """ -struct GeneralizedCauchyPointFinder{TM <: AbstractManifold, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} +struct GeneralizedCauchyDirectionFinder{TM <: AbstractManifold, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} M::TM d_tmp::TX ha::T_HA hessian_segment_updater::TFU end -function GeneralizedCauchyPointFinder( +function GeneralizedCauchyDirectionFinder( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; hessian_segment_updater::AbstractSegmentHessianUpdater = get_default_hessian_segment_updater(M, p, ha) ) - return GeneralizedCauchyPointFinder(M, zero_vector(M, p), ha, hessian_segment_updater) + return GeneralizedCauchyDirectionFinder(M, zero_vector(M, p), ha, hessian_segment_updater) end """ - find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFinder, d_out, p, d, X) + find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyDirectionFinder, d_out, p, d, X) Find generalized Cauchy point looking from point `p` in direction `d` and save the tangent vector pointing at it to `d_out`. Gradient of the objective at `p` is `X`. @@ -555,7 +555,7 @@ The function returns `max_stepsize(M, p)` in direction `d_out` afterwards, * `:not_found` if the search cannot be performed in direction `d`. """ -function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyPointFinder, d_out, p, d, X) +function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyDirectionFinder, d_out, p, d, X) M = gcp.M copyto!(M, d_out, d) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 2344b44081..71898f7dc7 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -156,12 +156,12 @@ using RecursiveArrayTools end end - @testset "GeneralizedCauchyPointFinder" begin + @testset "GeneralizedCauchyDirectionFinder" begin M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS()) p = [0.0, 0.0, 0.0] - gf = Manopt.GeneralizedCauchyPointFinder(M, p, ha) + gf = Manopt.GeneralizedCauchyDirectionFinder(M, p, ha) X1 = [-5.0, 0.0, 0.0] @@ -184,7 +184,7 @@ using RecursiveArrayTools @test d_out ≈ [2.0, 10.0, 0.0] p2 = [-1.0, -2.0, 2.0] - gf2 = Manopt.GeneralizedCauchyPointFinder(M, p2, ha) + gf2 = Manopt.GeneralizedCauchyDirectionFinder(M, p2, ha) @test Manopt.find_generalized_cauchy_point_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === :not_found @@ -192,7 +192,7 @@ using RecursiveArrayTools ha2 = QuasiNewtonMatrixDirectionUpdate(M2, BFGS(), DefaultOrthonormalBasis(), [100.0;;]) p3 = [1.0] - gf3 = Manopt.GeneralizedCauchyPointFinder(M2, p3, ha2) + gf3 = Manopt.GeneralizedCauchyDirectionFinder(M2, p3, ha2) d_out = similar(p3) @test Manopt.find_generalized_cauchy_point_direction!(gf3, d_out, p3, [1.0], [-10.0]) === :found_limited @@ -203,7 +203,7 @@ using RecursiveArrayTools ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [1.0 0 0; 0 1 0; 0 0 1]) p = [0.0, 0.0, 0.0] - gf = Manopt.GeneralizedCauchyPointFinder(M, p, ha) + gf = Manopt.GeneralizedCauchyDirectionFinder(M, p, ha) d = [-2.0, -2.0, -1.0] d_out = similar(d) From 2545ee8836c253b4b14a37afcfd429a13ea7bb20 Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Tue, 20 Jan 2026 18:14:21 +0100 Subject: [PATCH 059/135] Remove IJulia from the tutorials project. Maybe that helps. --- tutorials/Project.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tutorials/Project.toml b/tutorials/Project.toml index f299a7376c..cb71050151 100644 --- a/tutorials/Project.toml +++ b/tutorials/Project.toml @@ -6,7 +6,6 @@ DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" -IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a" LRUCache = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" ManifoldDiff = "af67fdf4-a580-4b9f-bbec-742ef357defd" @@ -28,7 +27,6 @@ DifferentiationInterface = "0.7" Distributions = "0.25" FiniteDifferences = "0.12" ForwardDiff = "1" -IJulia = "1" LRUCache = "1.4" ManifoldDiff = "0.4" Manifolds = "0.11" From 4026cd3bb29fc75c7ef063dba30f230dedefa1eb Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 21 Jan 2026 10:10:59 +0100 Subject: [PATCH 060/135] rename t to stepsize in function names --- ext/ManoptManifoldsExt/manifold_functions.jl | 8 +++---- src/plans/box_plan.jl | 22 ++++++++++---------- test/solvers/test_quasi_Newton_box.jl | 14 ++++++------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index f52e4e23d3..91a6e74246 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -16,7 +16,7 @@ lower (or upper) bounds. """ Manopt.get_bounds_index(M::Hyperrectangle) = eachindex(M.lb) """ - get_bound_t(M::Hyperrectangle, x, d, i) + get_stepsize_bound(M::Hyperrectangle, x, d, i) Get the upper bound on moving in direction `d` from point `p` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) `M`, for the bound index `i`. There are three cases: @@ -25,7 +25,7 @@ for the bound index `i`. There are three cases: 2. If `d[i] < 0`, the formula reads `(M.lb[i] - p[i]) / d[i]`. 3. If `d[i] == 0`, the result is `Inf`. """ -function Manopt.get_bound_t(M::Hyperrectangle, p, d, i) +function Manopt.get_stepsize_bound(M::Hyperrectangle, p, d, i) if d[i] > 0 return (M.ub[i] - p[i]) / d[i] elseif d[i] < 0 @@ -212,13 +212,13 @@ function Manopt.set_zero_at_index!(M::Hyperrectangle, d, i) end """ - Manopt.set_bound_for_t!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real) + Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real) For each index `i`, `t[i] < t_current`, set element of tangent vector `d_out` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) to the distance from `p[i]` to the bound in the direction of `d_out[i]`. """ -function Manopt.set_bound_for_t!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real) +function Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real) for i in eachindex(M.lb) if ts[i] < t_current d_out[i] = d_out[i] > 0 ? M.ub[i] - p[i] : M.lb[i] - p[i] diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 20be6a6c90..3843cfa93a 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -483,14 +483,14 @@ get_bounds_index(M::AbstractManifold) get_bounds_index(M::ProductManifold) = get_bounds_index(M.manifolds[1]) """ - get_bound_t(M::AbstractManifold, x, d, i) + get_stepsize_bound(M::AbstractManifold, x, d, i) Get the upper bound on moving in direction `d` from point `p` on manifold `M`, for the bound index `i`. """ -get_bound_t(M::AbstractManifold, p, d, i) -function get_bound_t(M::ProductManifold, p, d, i) - return get_bound_t(M.manifolds[1], submanifold_component(M, p, Val(1)), submanifold_component(M, d, Val(1)), i) +get_stepsize_bound(M::AbstractManifold, p, d, i) +function get_stepsize_bound(M::ProductManifold, p, d, i) + return get_stepsize_bound(M.manifolds[1], submanifold_component(M, p, Val(1)), submanifold_component(M, d, Val(1)), i) end """ @@ -504,15 +504,15 @@ function set_zero_at_index!(M::ProductManifold, d, i) end """ - Manopt.set_bound_for_t!(M::ProductManifold, d_out, p, ts::Dict, t_current::Real) + Manopt.set_stepsize_bound!(M::ProductManifold, d_out, p, ts::Dict, t_current::Real) -Set `d_out` so that it points from `p` to the generalized Cauchy point given times to -bounds `ts`. +Set `d_out` so that it points from `p` to the generalized Cauchy point given step sizes to +bounds in `ts`. """ -function set_bound_for_t!( +function set_stepsize_bound!( M::ProductManifold, d_out, p, ts::Dict, t_current::Real ) - set_bound_for_t!( + set_stepsize_bound!( M.manifolds[1], submanifold_component(M, d_out, Val(1)), submanifold_component(M, p, Val(1)), ts, t_current ) @@ -571,7 +571,7 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyDirectio has_finite_limit = false for i in bounds_indices - ts[i] = get_bound_t(M, p, d, i) + ts[i] = get_stepsize_bound(M, p, d, i) if ts[i] > 0 push!(F_list, (ts[i], i)) @@ -643,7 +643,7 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyDirectio dt_min = max(dt_min, 0.0) t_old = t_old + dt_min d_out .*= t_old - set_bound_for_t!(M, d_out, p, ts, t_current) + set_stepsize_bound!(M, d_out, p, ts, t_current) if has_finite_limit return :found_limited diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 71898f7dc7..0f81b62aed 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -4,23 +4,23 @@ using LinearAlgebra: I, eigvecs, tr, Diagonal, dot using RecursiveArrayTools @testset "Riemannian quasi-Newton Methods with box-like domains" begin - @testset "get_bound_t - basic" begin + @testset "get_stepsize_bound - basic" begin M = Hyperrectangle([0.0, 0.0], [2.0, 2.0]) # d[i] > 0 p = [0.0, 1.0]; d = [1.0, 1.0] - @test Manopt.get_bound_t(M, p, d, 1) ≈ (2.0 - 0.0) / 1.0 # = 2.0 - @test Manopt.get_bound_t(M, p, d, 2) ≈ (2.0 - 1.0) / 1.0 # = 1.0 + @test Manopt.get_stepsize_bound(M, p, d, 1) ≈ (2.0 - 0.0) / 1.0 # = 2.0 + @test Manopt.get_stepsize_bound(M, p, d, 2) ≈ (2.0 - 1.0) / 1.0 # = 1.0 # d[i] < 0 p = [0.0, 1.0]; d = [-1.0, -1.0] - @test Manopt.get_bound_t(M, p, d, 1) ≈ (0.0 - 0.0) / -1.0 # = 0.0 - @test Manopt.get_bound_t(M, p, d, 2) ≈ (0.0 - 1.0) / -1.0 # = 1.0 + @test Manopt.get_stepsize_bound(M, p, d, 1) ≈ (0.0 - 0.0) / -1.0 # = 0.0 + @test Manopt.get_stepsize_bound(M, p, d, 2) ≈ (0.0 - 1.0) / -1.0 # = 1.0 # d[i] = 0 p = [0.0, 1.0]; d = [0.0, 0.0] - @test Manopt.get_bound_t(M, p, d, 1) ≈ Inf - @test Manopt.get_bound_t(M, p, d, 2) ≈ Inf + @test Manopt.get_stepsize_bound(M, p, d, 1) ≈ Inf + @test Manopt.get_stepsize_bound(M, p, d, 2) ≈ Inf end From bc94a85b64942154802c18b26bfef77696258b1d Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 21 Jan 2026 10:17:37 +0100 Subject: [PATCH 061/135] fix convergence indication for StopWhenRelativeAPosterioriCostChangeLessOrEqual --- Changelog.md | 4 ++-- src/plans/stopping_criterion.jl | 2 +- test/plans/test_stopping_criteria.jl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Changelog.md b/Changelog.md index 2bb1f5f9e1..de11cf5d47 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,11 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. This resolves issue #549. (#554) * `GeneralizedCauchyDirectionFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) -* New stopping criteria: `StopWhenRelativeAPosterioriCostChangeLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. +* New stopping criteria: `StopWhenRelativeAPosterioriCostChangeLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. (#554) ### Fixed -* Line searches consistently respect `stop_when_stepsize_exceeds` keyword argument as a hard limit. +* Line searches consistently respect `stop_when_stepsize_exceeds` keyword argument as a hard limit. (#554) ## [0.5.32] January 15, 2026 diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 1c229553dd..407fee431d 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -511,7 +511,7 @@ function (c::StopWhenRelativeAPosterioriCostChangeLessOrEqual)( end return false end -indicates_convergence(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) = true +indicates_convergence(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) = false function get_reason(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) if c.at_iteration >= 0 return "At iteration $(c.at_iteration) the algorithm performed a step with a relative a posteriori cost change ($(abs(c.last_change))) less than or equal to $(c.tolerance)." diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 3683a12ed7..b64c19650e 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -387,7 +387,7 @@ end "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold 1.4210854715202004e-12.\n", ) @test startswith(Manopt.status_summary(sc), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") - @test Manopt.indicates_convergence(sc) + @test !Manopt.indicates_convergence(sc) end @testset "StopWhenProjectedNegativeGradientNormLess" begin From ef47ff2b4ff7ecabbb00de39e4d94302d64cbb7a Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 21 Jan 2026 10:34:26 +0100 Subject: [PATCH 062/135] rename GCP to GCD --- docs/make.jl | 2 +- .../generalized_cauchy_point_subsolver.md | 58 ------------------- ext/ManoptManifoldsExt/manifold_functions.jl | 4 +- src/plans/box_plan.jl | 6 +- src/solvers/quasi_Newton.jl | 2 +- test/solvers/test_quasi_Newton_box.jl | 8 +-- 6 files changed, 11 insertions(+), 69 deletions(-) delete mode 100644 docs/src/solvers/generalized_cauchy_point_subsolver.md diff --git a/docs/make.jl b/docs/make.jl index 59b2bef7a6..ef551cf7d4 100755 --- a/docs/make.jl +++ b/docs/make.jl @@ -190,7 +190,7 @@ makedocs(; "Douglas—Rachford" => "solvers/DouglasRachford.md", "Exact Penalty Method" => "solvers/exact_penalty_method.md", "Frank-Wolfe" => "solvers/FrankWolfe.md", - "Generalized Cauchy point subsolver" => "solvers/generalized_cauchy_point_subsolver.md", + "Generalized Cauchy direction subsolver" => "solvers/generalized_cauchy_direction_subsolver.md", "Gradient Descent" => "solvers/gradient_descent.md", "Interior Point Newton" => "solvers/interior_point_Newton.md", "Levenberg–Marquardt" => "solvers/LevenbergMarquardt.md", diff --git a/docs/src/solvers/generalized_cauchy_point_subsolver.md b/docs/src/solvers/generalized_cauchy_point_subsolver.md deleted file mode 100644 index 108106c0c6..0000000000 --- a/docs/src/solvers/generalized_cauchy_point_subsolver.md +++ /dev/null @@ -1,58 +0,0 @@ -# Generalized Cauchy Point subsolver - -The Generalized Cauchy Point (GCP) subsolver is a component in optimization algorithms that handle problems with bound constraints. It solves the following problem - -```math -\begin{align*} -\operatorname*{arg\,min}_{Y ∈ T_p D \times \mathcal{M}}&\ m_p(Y) = ⟨X_g, Y⟩_p + \frac{1}{2} ⟨\mathcal{H}_p[Y], Y⟩_p\\ -\text{such that}& \ \exp_p(Y) = \exp_p(\alpha X) \in D \times \mathcal{M} \text{ for some } \alpha \in \mathbb{R} -\end{align*} -``` - -where $X$ is a given direction, the exponential map handles projection of the tangent vector when reaching the boundary, $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, $X_g$ is the gradient of a scalar function $f$ at point $p$ and $\mathcal{H}_p$ is a linear operator that approximates the Hessian of $f$ at $p$. - -The solver is currently primarily intended for internal use by optimization algorithms that require bound-constrained subproblem solutions. - -## Internal types and method - -### Symbols related to the GCP computation - -These symbols are directly used by solvers to compute the descent direction corresponding to the Generalized Cauchy point. - -```@docs -Manopt.requires_generalized_cauchy_point_computation -Manopt.find_generalized_cauchy_point_direction! -Manopt.GeneralizedCauchyDirectionFinder -``` - -### Symbols related to the Hessian approximation - -These symbols are used to evaluate the Hessian approximation at specific tangent vectors during the generalized Cauchy point computation. - -```@docs -Manopt.hessian_value -Manopt.hessian_value_eb -``` - -### Symbols related to bound handling - -These are internal symbols used to manage and manipulate bound constraints during the GCP computation. - -```@docs -Manopt.init_updater! -Manopt.AbstractSegmentHessianUpdater -Manopt.GenericSegmentHessianUpdater -Manopt.get_bounds_index -Manopt.get_bound_t -Manopt.get_at_bound_index -Manopt.set_bound_for_t! -Manopt.set_zero_at_index! -``` - -### Symbols related to specific Hessian approximations - -```@docs -Manopt.LimitedMemorySegmentHessianUpdater -Manopt.hessian_value_from_inner_products -Manopt.set_M_current_scale! -``` diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 91a6e74246..138fd9b063 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -228,11 +228,11 @@ function Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, ts::Dict, t_cur end """ - Manopt.requires_generalized_cauchy_point_computation(::Hyperrectangle) + Manopt.requires_generalized_cauchy_direction_computation(::Hyperrectangle) Returns `true`, as `Hyperrectangle` manifold requires generalized Cauchy point computation in solvers. """ -Manopt.requires_generalized_cauchy_point_computation(::Hyperrectangle) = true +Manopt.requires_generalized_cauchy_direction_computation(::Hyperrectangle) = true """ Manopt.get_at_bound_index(::Hyperrectangle, X, b) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 3843cfa93a..6bc695e43d 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -1,11 +1,11 @@ """ - requires_generalized_cauchy_point_computation(M::AbstractManifold) + requires_generalized_cauchy_direction_computation(M::AbstractManifold) Return `true` if `M` is a `Hyperrectangle`-like manifold with corners, or a product of it with a standard manifold. Otherwise return `false`. """ -requires_generalized_cauchy_point_computation(::AbstractManifold) = false -requires_generalized_cauchy_point_computation(M::ProductManifold) = requires_generalized_cauchy_point_computation(M.manifolds[1]) +requires_generalized_cauchy_direction_computation(::AbstractManifold) = false +requires_generalized_cauchy_direction_computation(M::ProductManifold) = requires_generalized_cauchy_direction_computation(M.manifolds[1]) @doc raw""" mutable struct LimitedMemoryHessianApproximation end diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 2e89636dd6..6c33080be9 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -348,7 +348,7 @@ function quasi_Newton!( nonpositive_curvature_behavior = nonpositive_curvature_behavior, sy_tol = sy_tol, ) - if requires_generalized_cauchy_point_computation(M) + if requires_generalized_cauchy_direction_computation(M) local_dir_upd = QuasiNewtonLimitedMemoryBoxDirectionUpdate(local_dir_upd) end else diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 0f81b62aed..e8a2d78ad3 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -258,10 +258,10 @@ using RecursiveArrayTools @test f3(MInf, p_opt) < 64.0 end - @testset "requires_generalized_cauchy_point_computation" begin - @test !Manopt.requires_generalized_cauchy_point_computation(Sphere(2)) - @test Manopt.requires_generalized_cauchy_point_computation(Hyperrectangle([1], [2])) - @test Manopt.requires_generalized_cauchy_point_computation(ProductManifold(Hyperrectangle([1], [2]), Sphere(2))) + @testset "requires_generalized_cauchy_direction_computation" begin + @test !Manopt.requires_generalized_cauchy_direction_computation(Sphere(2)) + @test Manopt.requires_generalized_cauchy_direction_computation(Hyperrectangle([1], [2])) + @test Manopt.requires_generalized_cauchy_direction_computation(ProductManifold(Hyperrectangle([1], [2]), Sphere(2))) end @testset "Hyperrectangle × Sphere" begin From f5f190182daf68618b77d84ee458b618fc5cb54b Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 21 Jan 2026 10:45:21 +0100 Subject: [PATCH 063/135] GCD in tutorial --- tutorials/BoxDomain.qmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorials/BoxDomain.qmd b/tutorials/BoxDomain.qmd index c0f3526e63..2ccb696cae 100644 --- a/tutorials/BoxDomain.qmd +++ b/tutorials/BoxDomain.qmd @@ -9,9 +9,9 @@ A ``[`Hyperrectangle`](@extref `Manifolds.Hyperrectangle`)``{=commonmark} is, in Such spaces require special handling when used as domains in optimization. For simple methods like gradient descent using projected gradient and a stopping criterion involving [`StopWhenProjectedNegativeGradientNormLess`](@ref) may be sufficient, however methods that approximate the Hessian can benefit from a more advanced approach. -The core idea is considering a piecewise quadratic approximation of the objective along the descent direction, and selecting the generalized Cauchy point -- its minimizer. +The core idea is considering a piecewise quadratic approximation of the objective along the descent direction in the tangent space at the current iterate, and selecting the generalized Cauchy direction -- its minimizer. The points at which the approximation might not be differentiable correspond to hitting new boundaries along the initially selected descent direction. -Then, we can perform standard line search between the initial iterate and the generalized Cauchy point. +Then, we can perform standard line search from the initial iterate in the generalized Cauchy direction. Currently `Manopt.jl` can handle domains that are either a ``[`Hyperrectangle`](@extref `Manifolds.Hyperrectangle`)``{=commonmark} or a ``[`ProductManifold`](@extref `ManifoldsBase.ProductManifold`)``{=commonmark} containing a ``[`Hyperrectangle`](@extref `Manifolds.Hyperrectangle`)``{=commonmark} as its first factor and other manifolds as subsequent factors. From 89f25302d584fb6bc04f7be37893ec437d515c41 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 21 Jan 2026 10:57:26 +0100 Subject: [PATCH 064/135] set seed to try avoiding random errors --- tutorials/BoxDomain.qmd | 1 + 1 file changed, 1 insertion(+) diff --git a/tutorials/BoxDomain.qmd b/tutorials/BoxDomain.qmd index 2ccb696cae..eb2d2da574 100644 --- a/tutorials/BoxDomain.qmd +++ b/tutorials/BoxDomain.qmd @@ -27,6 +27,7 @@ The data is sampled from a multivariate normal distribution with known covarianc ```{julia} using Manopt, Manifolds, LinearAlgebra, Random, Distributions using ForwardDiff, DifferentiationInterface, RecursiveArrayTools +Random.seed!(41) N = 5 # dimensionality of data M_spd = SymmetricPositiveDefinite(N) From 8060ae5892de0a3e4339a628ab2074b12c3a9af4 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 21 Jan 2026 11:20:40 +0100 Subject: [PATCH 065/135] add missing page --- .../generalized_cauchy_direction_subsolver.md | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docs/src/solvers/generalized_cauchy_direction_subsolver.md diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md new file mode 100644 index 0000000000..4ca63b4497 --- /dev/null +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -0,0 +1,58 @@ +# Generalized Cauchy Direction subsolver + +The Generalized Cauchy Direction (GCD) subsolver is a component in optimization algorithms that handle problems with bound constraints. It solves the following problem + +```math +\begin{align*} +\operatorname*{arg\,min}_{Y ∈ T_p D \times \mathcal{M}}&\ m_p(Y) = ⟨X_g, Y⟩_p + \frac{1}{2} ⟨\mathcal{H}_p[Y], Y⟩_p\\ +\text{such that}& \ \exp_p(Y) = \exp_p(\alpha X) \in D \times \mathcal{M} \text{ for some } \alpha \in \mathbb{R} +\end{align*} +``` + +where $X$ is a given direction, the exponential map handles projection of the tangent vector when reaching the boundary, $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, $X_g$ is the gradient of a scalar function $f$ at point $p$ and $\mathcal{H}_p$ is a linear operator that approximates the Hessian of $f$ at $p$. + +The solver is currently primarily intended for internal use by optimization algorithms that require bound-constrained subproblem solutions. + +## Internal types and method + +### Symbols related to the GCP computation + +These symbols are directly used by solvers to compute the descent direction corresponding to the Generalized Cauchy point. + +```@docs +Manopt.requires_generalized_cauchy_direction_computation +Manopt.find_generalized_cauchy_point_direction! +Manopt.GeneralizedCauchyDirectionFinder +``` + +### Symbols related to the Hessian approximation + +These symbols are used to evaluate the Hessian approximation at specific tangent vectors during the generalized Cauchy point computation. + +```@docs +Manopt.hessian_value +Manopt.hessian_value_eb +``` + +### Symbols related to bound handling + +These are internal symbols used to manage and manipulate bound constraints during the GCP computation. + +```@docs +Manopt.init_updater! +Manopt.AbstractSegmentHessianUpdater +Manopt.GenericSegmentHessianUpdater +Manopt.get_bounds_index +Manopt.get_stepsize_bound +Manopt.get_at_bound_index +Manopt.set_stepsize_bound! +Manopt.set_zero_at_index! +``` + +### Symbols related to specific Hessian approximations + +```@docs +Manopt.LimitedMemorySegmentHessianUpdater +Manopt.hessian_value_from_inner_products +Manopt.set_M_current_scale! +``` From bebc93ff4d118e63842bab6e62bfeba441a0506a Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Wed, 21 Jan 2026 14:19:21 +0100 Subject: [PATCH 066/135] fix convergence indication of StopWhenChangeLess --- Changelog.md | 1 + src/plans/stopping_criterion.jl | 2 +- test/plans/test_stopping_criteria.jl | 9 +++------ 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Changelog.md b/Changelog.md index de11cf5d47..3f6c4f9ec2 100644 --- a/Changelog.md +++ b/Changelog.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed * Line searches consistently respect `stop_when_stepsize_exceeds` keyword argument as a hard limit. (#554) +* `StopWhenChangeLess` falsely claimed to indicate convergence. This is now fixed. ## [0.5.32] January 15, 2026 diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 407fee431d..c50122db85 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -323,7 +323,7 @@ function status_summary(c::StopWhenChangeLess) s = has_stopped ? "reached" : "not reached" return "|Δp| < $(c.threshold): $s" end -indicates_convergence(c::StopWhenChangeLess) = true +indicates_convergence(c::StopWhenChangeLess) = false function show(io::IO, c::StopWhenChangeLess) s = ismissing(c.outer_norm) ? "" : "and outer norm $(c.outer_norm)" return print( diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index b64c19650e..e34d13ea32 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -8,12 +8,10 @@ end @testset "StoppingCriteria" begin @testset "Generic Tests" begin - @test_throws ErrorException get_stopping_criteria( - Manopt.Test.DummyStoppingCriteriaSet() - ) + @test_throws ErrorException get_stopping_criteria(Manopt.Test.DummyStoppingCriteriaSet()) s = StopWhenAll(StopAfterIteration(10), StopWhenChangeLess(Euclidean(), 0.1)) - @test Manopt.indicates_convergence(s) #due to all and change this is true + @test !Manopt.indicates_convergence(s) # Neither of the two indicates convergence @test startswith(repr(s), "StopWhenAll with the") @test get_reason(s) === "" # Trigger second one manually @@ -21,8 +19,7 @@ end s.criteria[2].at_iteration = 3 @test length(get_reason(s.criteria[2])) > 0 s2 = StopWhenAll([StopAfterIteration(10), StopWhenChangeLess(Euclidean(), 0.1)]) - @test get_stopping_criteria(s)[1].max_iterations == - get_stopping_criteria(s2)[1].max_iterations + @test get_stopping_criteria(s)[1].max_iterations == get_stopping_criteria(s2)[1].max_iterations s3 = StopWhenCostLess(0.1) p = DefaultManoptProblem( From 1522b4b3a7cf165fe2e4eb20057fc57ff53bd52c Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 21 Jan 2026 20:23:18 +0100 Subject: [PATCH 067/135] Update docs/src/solvers/generalized_cauchy_direction_subsolver.md Co-authored-by: Ronny Bergmann --- docs/src/solvers/generalized_cauchy_direction_subsolver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index 4ca63b4497..2677739ad4 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -4,7 +4,7 @@ The Generalized Cauchy Direction (GCD) subsolver is a component in optimization ```math \begin{align*} -\operatorname*{arg\,min}_{Y ∈ T_p D \times \mathcal{M}}&\ m_p(Y) = ⟨X_g, Y⟩_p + \frac{1}{2} ⟨\mathcal{H}_p[Y], Y⟩_p\\ +\operatorname*{arg\,min}_{Y ∈ T_p D \times \mathcal{M}}&\ m_p(Y), \qquad m_p(Y) = ⟨X_g, Y⟩_p + \frac{1}{2} ⟨\mathcal{H}_p[Y], Y⟩_p\\ \text{such that}& \ \exp_p(Y) = \exp_p(\alpha X) \in D \times \mathcal{M} \text{ for some } \alpha \in \mathbb{R} \end{align*} ``` From 909c4642b0f93c52b99f6fba0147a7a9f1e7a537 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 22 Jan 2026 12:04:36 +0100 Subject: [PATCH 068/135] rename a few things, expand docs --- .../generalized_cauchy_direction_subsolver.md | 8 +- src/plans/box_plan.jl | 123 +++++++++--------- src/plans/quasi_newton_plan.jl | 35 +++-- test/solvers/test_quasi_Newton_box.jl | 2 +- 4 files changed, 93 insertions(+), 75 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index 2677739ad4..67793820a1 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -5,11 +5,13 @@ The Generalized Cauchy Direction (GCD) subsolver is a component in optimization ```math \begin{align*} \operatorname*{arg\,min}_{Y ∈ T_p D \times \mathcal{M}}&\ m_p(Y), \qquad m_p(Y) = ⟨X_g, Y⟩_p + \frac{1}{2} ⟨\mathcal{H}_p[Y], Y⟩_p\\ -\text{such that}& \ \exp_p(Y) = \exp_p(\alpha X) \in D \times \mathcal{M} \text{ for some } \alpha \in \mathbb{R} +\text{such that}& \ \exp_p(Y) = \exp_p(\alpha X) \in D \times \mathcal{M} \text{ for some } \alpha \in [0, A] \end{align*} ``` -where $X$ is a given direction, the exponential map handles projection of the tangent vector when reaching the boundary, $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, $X_g$ is the gradient of a scalar function $f$ at point $p$ and $\mathcal{H}_p$ is a linear operator that approximates the Hessian of $f$ at $p$. +where $X=(X_{\mathrm{D}}, X_{\mathcal{M}})$ is a given direction, the exponential map handles projection of the tangent vector when reaching the boundary, $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, $X_g$ is the gradient of a scalar function $f$ at point $p=(p_{\mathrm{D}}, p_{\mathcal{M}})$, $A$ is the maximum allowed step size on $\mathcal{M}$ at point $p=(p_{\mathrm{D}}, p_{\mathcal{M}})$ in direction $X_{\mathcal{M}}$ (infinity is supported) and $\mathcal{H}_p$ is a linear operator that approximates the Hessian of $f$ at $p$. + +Additionally, the subsolver indicates whether the selected direction $Y$ reaches the boundary of $D$ at some point, in which case the subsequent step size selection in direction $Y$ needs to be limited to the interval $[0, 1]$. The solver is currently primarily intended for internal use by optimization algorithms that require bound-constrained subproblem solutions. @@ -31,7 +33,7 @@ These symbols are used to evaluate the Hessian approximation at specific tangent ```@docs Manopt.hessian_value -Manopt.hessian_value_eb +Manopt.hessian_value_diag ``` ### Symbols related to bound handling diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 6bc695e43d..b5fd205131 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -41,10 +41,10 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ M_21::T_HM M_22::T_HM # buffer for calculating W_k blocks - coords_Sk_X::V - coords_Sk_Y::V - coords_Yk_X::V - coords_Yk_Y::V + buffer_inner_Sk_X::V + buffer_inner_Sk_Y::V + buffer_inner_Yk_X::V + buffer_inner_Yk_Y::V last_gcp_result::Symbol end @@ -63,22 +63,22 @@ function QuasiNewtonLimitedMemoryBoxDirectionUpdate( M_11 = zeros(F, memory_size, memory_size) M_21 = zeros(F, memory_size, memory_size) M_22 = zeros(F, memory_size, memory_size) - coords_Sk_X = zeros(F, memory_size) - coords_Sk_Y = zeros(F, memory_size) - coords_Yk_X = zeros(F, memory_size) - coords_Yk_Y = zeros(F, memory_size) + buffer_inner_Sk_X = zeros(F, memory_size) + buffer_inner_Sk_Y = zeros(F, memory_size) + buffer_inner_Yk_X = zeros(F, memory_size) + buffer_inner_Yk_Y = zeros(F, memory_size) return QuasiNewtonLimitedMemoryBoxDirectionUpdate{ - typeof(qn_du), F, typeof(M_11), typeof(coords_Sk_X), + typeof(qn_du), F, typeof(M_11), typeof(buffer_inner_Sk_X), }( qn_du, qn_du.initial_scale, M_11, M_21, M_22, - coords_Sk_X, - coords_Sk_Y, - coords_Yk_X, - coords_Yk_Y, + buffer_inner_Sk_X, + buffer_inner_Sk_Y, + buffer_inner_Yk_X, + buffer_inner_Yk_Y, :not_searched, ) end @@ -113,11 +113,11 @@ function get_at_bound_index(M::ProductManifold, X, b) end @doc raw""" - hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) + hessian_value_diag(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) Compute ``⟨X, B X⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`. """ -function hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) +function hessian_value_diag(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X) m = length(gh.qn_du.memory_s) num_nonzero_rho = count(!iszero, gh.qn_du.ρ) @@ -130,24 +130,25 @@ function hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstra ii = 1 for i in 1:m iszero(gh.qn_du.ρ[i]) && continue - gh.coords_Yk_X[ii] = inner(M, p, gh.qn_du.memory_y[i], X) - gh.coords_Sk_X[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], X) + gh.buffer_inner_Yk_X[ii] = inner(M, p, gh.qn_du.memory_y[i], X) + gh.buffer_inner_Sk_X[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], X) ii += 1 end - coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) - coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) + buffer_inner_Yk = view(gh.buffer_inner_Yk_X, 1:num_nonzero_rho) + buffer_inner_Sk = view(gh.buffer_inner_Sk_X, 1:num_nonzero_rho) - return hessian_value_from_inner_products(gh, normX_sqr, coords_Yk, coords_Sk, coords_Yk, coords_Sk) + return hessian_value_from_inner_products(gh, normX_sqr, buffer_inner_Yk, buffer_inner_Sk, buffer_inner_Yk, buffer_inner_Sk) end @doc raw""" - hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) + hessian_value_diag(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X::UnitVector) Compute ``⟨X, B X⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`, and `X` is the -unit vector along index `b`. +[`UnitVector`](@ref). """ -function hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b) +function hessian_value_diag(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X::UnitVector) + b = X.index m = length(gh.qn_du.memory_s) num_nonzero_rho = count(!iszero, gh.qn_du.ρ) @@ -158,24 +159,26 @@ function hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abs ii = 1 for i in 1:m iszero(gh.qn_du.ρ[i]) && continue - gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) - gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) + gh.buffer_inner_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) + gh.buffer_inner_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) ii += 1 end - coords_Yk = view(gh.coords_Yk_X, 1:num_nonzero_rho) - coords_Sk = view(gh.coords_Sk_X, 1:num_nonzero_rho) + buffer_inner_Yk = view(gh.buffer_inner_Yk_X, 1:num_nonzero_rho) + buffer_inner_Sk = view(gh.buffer_inner_Sk_X, 1:num_nonzero_rho) - return hessian_value_from_inner_products(gh, one(eltype(gh.qn_du.ρ)), coords_Yk, coords_Sk, coords_Yk, coords_Sk) + return hessian_value_from_inner_products(gh, one(eltype(gh.qn_du.ρ)), buffer_inner_Yk, buffer_inner_Sk, buffer_inner_Yk, buffer_inner_Sk) end @doc raw""" - hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) + hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X::UnitVector, Y) Compute ``⟨X, B Y⟩``, where ``B`` is the (1, 1)-Hessian represented by `gh`, where `X` is the -unit vector pointing at index `b`. +[`UnitVector`](@ref). """ -function hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, b, Y) +function hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::AbstractManifold, p, X::UnitVector, Y) + b = X.index + m = length(gh.qn_du.memory_s) num_nonzero_rho = count(!iszero, gh.qn_du.ρ) @@ -187,19 +190,19 @@ function hessian_value_eb(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abs ii = 1 for i in 1:m iszero(gh.qn_du.ρ[i]) && continue - gh.coords_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) - gh.coords_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) + gh.buffer_inner_Yk_X[ii] = get_at_bound_index(M, gh.qn_du.memory_y[i], b) + gh.buffer_inner_Sk_X[ii] = gh.current_scale * get_at_bound_index(M, gh.qn_du.memory_s[i], b) - gh.coords_Yk_Y[ii] = inner(M, p, gh.qn_du.memory_y[i], Y) - gh.coords_Sk_Y[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], Y) + gh.buffer_inner_Yk_Y[ii] = inner(M, p, gh.qn_du.memory_y[i], Y) + gh.buffer_inner_Sk_Y[ii] = gh.current_scale * inner(M, p, gh.qn_du.memory_s[i], Y) ii += 1 end - coords_Yk_X = view(gh.coords_Yk_X, 1:num_nonzero_rho) - coords_Yk_Y = view(gh.coords_Yk_Y, 1:num_nonzero_rho) - coords_Sk_X = view(gh.coords_Sk_X, 1:num_nonzero_rho) - coords_Sk_Y = view(gh.coords_Sk_Y, 1:num_nonzero_rho) + buffer_inner_Yk_X = view(gh.buffer_inner_Yk_X, 1:num_nonzero_rho) + buffer_inner_Yk_Y = view(gh.buffer_inner_Yk_Y, 1:num_nonzero_rho) + buffer_inner_Sk_X = view(gh.buffer_inner_Sk_X, 1:num_nonzero_rho) + buffer_inner_Sk_Y = view(gh.buffer_inner_Sk_Y, 1:num_nonzero_rho) - return hessian_value_from_inner_products(gh, Yb, coords_Yk_X, coords_Sk_X, coords_Yk_Y, coords_Sk_Y) + return hessian_value_from_inner_products(gh, Yb, buffer_inner_Yk_X, buffer_inner_Sk_X, buffer_inner_Yk_Y, buffer_inner_Sk_Y) end @doc raw""" @@ -344,7 +347,7 @@ init_updater!(::AbstractManifold, hessian_segment_updater::AbstractSegmentHessia """ struct GenericSegmentHessianUpdater <: AbstractSegmentHessianUpdater end -Generic f' and f'' calculation that only relies on `hessian_value_eb` but is relatively slow for +Generic f' and f'' calculation that only relies on `hessian_value` but is relatively slow for high-dimensional domains. """ struct GenericSegmentHessianUpdater{TX} <: AbstractSegmentHessianUpdater @@ -366,13 +369,13 @@ end (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, b, db, ha::AbstractQuasiNewtonDirectionUpdate) Calculate Hessian values ``⟨e_b, B d_z⟩`` and ``⟨e_b, B d_tmp⟩`` for the generalized Cauchy -point line search using the generic approach via `hessian_value_eb`. +point line search using the generic approach via `hessian_value` with [`UnitVector`](@ref). ``d_z`` start with 0 and is updated in-place by adding `dt * d` to it. """ function (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, b, db, ha) upd.d_z .+= dt .* upd.d_tmp - hv_eb_dz = hessian_value_eb(ha, M, p, b, upd.d_z) - hv_eb_d = hessian_value_eb(ha, M, p, b, upd.d_tmp) + hv_eb_dz = hessian_value(ha, M, p, UnitVector(b), upd.d_z) + hv_eb_d = hessian_value(ha, M, p, UnitVector(b), upd.d_tmp) set_zero_at_index!(M, upd.d_tmp, b) @@ -446,29 +449,29 @@ function (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( for i in 1:m iszero(ha.qn_du.ρ[i]) && continue # setting _X to w_b from the paper - ha.coords_Yk_X[ii] = get_at_bound_index(M, ha.qn_du.memory_y[i], b) - ha.coords_Sk_X[ii] = ha.current_scale * get_at_bound_index(M, ha.qn_du.memory_s[i], b) + ha.buffer_inner_Yk_X[ii] = get_at_bound_index(M, ha.qn_du.memory_y[i], b) + ha.buffer_inner_Sk_X[ii] = ha.current_scale * get_at_bound_index(M, ha.qn_du.memory_s[i], b) ii += 1 end - coords_Yk_eb = view(ha.coords_Yk_X, 1:num_nonzero_rho) - coords_Sk_eb = view(ha.coords_Sk_X, 1:num_nonzero_rho) + buffer_inner_Yk_eb = view(ha.buffer_inner_Yk_X, 1:num_nonzero_rho) + buffer_inner_Sk_eb = view(ha.buffer_inner_Sk_X, 1:num_nonzero_rho) - coords_cy = view(hessian_segment_updater.c_y, 1:num_nonzero_rho) - coords_cs = view(hessian_segment_updater.c_s, 1:num_nonzero_rho) - coords_py = view(hessian_segment_updater.p_y, 1:num_nonzero_rho) - coords_ps = view(hessian_segment_updater.p_s, 1:num_nonzero_rho) + buffer_inner_cy = view(hessian_segment_updater.c_y, 1:num_nonzero_rho) + buffer_inner_cs = view(hessian_segment_updater.c_s, 1:num_nonzero_rho) + buffer_inner_py = view(hessian_segment_updater.p_y, 1:num_nonzero_rho) + buffer_inner_ps = view(hessian_segment_updater.p_s, 1:num_nonzero_rho) - coords_cy .+= dt .* coords_py - coords_cs .+= dt .* coords_ps + buffer_inner_cy .+= dt .* buffer_inner_py + buffer_inner_cs .+= dt .* buffer_inner_ps - eb_B_z = hessian_value_from_inner_products(ha, t * db, coords_Yk_eb, coords_Sk_eb, coords_cy, coords_cs) + eb_B_z = hessian_value_from_inner_products(ha, t * db, buffer_inner_Yk_eb, buffer_inner_Sk_eb, buffer_inner_cy, buffer_inner_cs) - eb_B_d = hessian_value_from_inner_products(ha, db, coords_Yk_eb, coords_Sk_eb, coords_py, coords_ps) + eb_B_d = hessian_value_from_inner_products(ha, db, buffer_inner_Yk_eb, buffer_inner_Sk_eb, buffer_inner_py, buffer_inner_ps) - coords_py .-= db .* coords_Yk_eb - coords_ps .-= db .* coords_Sk_eb + buffer_inner_py .-= db .* buffer_inner_Yk_eb + buffer_inner_ps .-= db .* buffer_inner_Sk_eb return eb_B_z, eb_B_d end @@ -601,7 +604,7 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyDirectio F = BinaryHeap(Base.By(first), F_list) f_prime = inner(M, p, X, d) - f_double_prime = hessian_value(gcp.ha, M, p, d) + f_double_prime = hessian_value_diag(gcp.ha, M, p, d) if iszero(f_prime) || iszero(f_double_prime) return :not_found @@ -622,7 +625,7 @@ function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyDirectio hv_eb_dz, hv_eb_d = gcp.hessian_segment_updater(M, p, t_current, dt, b, db, gcp.ha) f_prime += dt * f_double_prime - db * (gb + hv_eb_dz) - f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_eb(gcp.ha, M, p, b) + f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_diag(gcp.ha, M, p, UnitVector(b)) t_old = t_current diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index c40c3bd367..89b5dbab0b 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -512,39 +512,52 @@ function initialize_update!(d::QuasiNewtonMatrixDirectionUpdate) return d end """ - hessian_value(d::QuasiNewtonMatrixDirectionUpdate, M, p, X) + hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate, M, p, X) Evaluate the quadratic form associated with the stored quasi-Newton matrix. Returns the scalar ``c^{\top} B c`` where ``c`` are the coordinates of the tangent vector `X` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. """ -function hessian_value(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} +function hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} c = get_coordinates(M, p, X, d.basis) return dot(c, d.matrix, c) end """ - hessian_value_eb(d::QuasiNewtonMatrixDirectionUpdate, M, p, b) + UnitVector{TB} + +A type representing a unit tangent vector on a `Hyperrectangle`-like manifold with corners, +or a product of it with a standard manifold. +The field `index` stores the index of the element equal to 1. +All other elements are equal to 0. +""" +struct UnitVector{TB} + index::TB +end + +""" + hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate, M, p, X::UnitVector) Evaluate the quadratic form associated with the stored quasi-Newton matrix. Returns the scalar ``c^{\top} B c`` where ``c`` are the coordinates of the -unit tangent vector along direction with index `b` at `p` (in the basis `d.basis`) -and ``B`` is `d.matrix`. +[`UnitVector`](@ref) `X` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. """ -function hessian_value_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, ::AbstractManifold, p, b) where {T <: Union{BFGS, DFP, SR1, Broyden}} +function hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate{T}, ::AbstractManifold, p, X::UnitVector) where {T <: Union{BFGS, DFP, SR1, Broyden}} + b = X.index return d.matrix[b, b] end """ - hessian_value_eb(d::QuasiNewtonMatrixDirectionUpdate, M, p, b, X) + hessian_value(d::QuasiNewtonMatrixDirectionUpdate, M, p, X::UnitVector, Y) Evaluate the quadratic form associated with the stored quasi-Newton matrix. Returns the scalar ``c_b^{\top} B c`` where ``c_b`` are the coordinates of the -unit tangent vector along direction with index `b` at `p` (in the basis `d.basis`), -``c`` are the coordinates of the tangent vector `X` at `p` (in the basis `d.basis`) +[`UnitVector`](@ref) `X` at `p` (assumed to correspond to the basis `d.basis`), +``c`` are the coordinates of the tangent vector `Y` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. """ -function hessian_value_eb(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, b, X) where {T <: Union{BFGS, DFP, SR1, Broyden}} - return dot(d.matrix[b, :], get_coordinates(M, p, X, d.basis)) +function hessian_value(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X::UnitVector, Y) where {T <: Union{BFGS, DFP, SR1, Broyden}} + b = X.index + return dot(d.matrix[b, :], get_coordinates(M, p, Y, d.basis)) end _doc_QN_B = """ diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index e8a2d78ad3..974bfb97fd 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -147,7 +147,7 @@ using RecursiveArrayTools @testset "No memory tests" begin ha2 = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2)) - @test Manopt.hessian_value_eb(ha2, M, p, b, grad) ≈ 4.0 + @test Manopt.hessian_value(ha2, M, p, Manopt.UnitVector(b), grad) ≈ 4.0 Manopt.set_M_current_scale!(M, p, ha2) @test ha2.current_scale == ha2.qn_du.initial_scale @test ha2.M_11 == fill(0.0, 0, 0) From 4a79ea60ad37c46fa9c00eb273e68aec442cc127 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 22 Jan 2026 12:19:55 +0100 Subject: [PATCH 069/135] add missing docs entry --- docs/src/solvers/generalized_cauchy_direction_subsolver.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index 67793820a1..dc784e1325 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -42,6 +42,7 @@ These are internal symbols used to manage and manipulate bound constraints durin ```@docs Manopt.init_updater! +Manopt.UnitVector Manopt.AbstractSegmentHessianUpdater Manopt.GenericSegmentHessianUpdater Manopt.get_bounds_index From 7f8ecc449ac5b7cad84b797cfdcb762b93e17c42 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 22 Jan 2026 14:34:30 +0100 Subject: [PATCH 070/135] improve names --- .../generalized_cauchy_direction_subsolver.md | 8 ++++---- src/plans/box_plan.jl | 18 +++++++++--------- test/solvers/test_quasi_Newton_box.jl | 14 +++++++------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index dc784e1325..1d8b26b852 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -17,19 +17,19 @@ The solver is currently primarily intended for internal use by optimization algo ## Internal types and method -### Symbols related to the GCP computation +### Symbols related to the GCD computation -These symbols are directly used by solvers to compute the descent direction corresponding to the Generalized Cauchy point. +These symbols are directly used by solvers to compute the descent direction corresponding to the Generalized Cauchy direction. ```@docs Manopt.requires_generalized_cauchy_direction_computation -Manopt.find_generalized_cauchy_point_direction! +Manopt.find_generalized_cauchy_direction! Manopt.GeneralizedCauchyDirectionFinder ``` ### Symbols related to the Hessian approximation -These symbols are used to evaluate the Hessian approximation at specific tangent vectors during the generalized Cauchy point computation. +These symbols are used to evaluate the Hessian approximation at specific tangent vectors during the generalized Cauchy direction computation. ```@docs Manopt.hessian_value diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index b5fd205131..b30fd1697e 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -102,7 +102,7 @@ function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( p = get_iterate(st) X = get_gradient(st) gcp = GeneralizedCauchyDirectionFinder(M, p, d) - d.last_gcp_result = find_generalized_cauchy_point_direction!(gcp, r, p, r, X) + d.last_gcp_result = find_generalized_cauchy_direction!(gcp, r, p, r, X) return r end @@ -385,8 +385,8 @@ end """ struct LimitedMemorySegmentHessianUpdater{TV <: AbstractVector} <: AbstractSegmentHessianUpdater -Hessian value calculation for generalized Cauchy point line segments that is optimized for -`QuasiNewtonLimitedMemoryBoxDirectionUpdate`. It relies on a specific Hessian structure. +Hessian value calculation for generalized Cauchy direction line segments that is optimized for +[`QuasiNewtonLimitedMemoryBoxDirectionUpdate`](@ref). It relies on a specific Hessian structure. """ struct LimitedMemorySegmentHessianUpdater{TV <: AbstractVector} <: AbstractSegmentHessianUpdater p_s::TV @@ -525,10 +525,10 @@ end @doc raw""" GeneralizedCauchyDirectionFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} -Helper container for generalized Cauchy point search. Stores the manifold `M`, cached +Helper container for generalized Cauchy direction search. Stores the manifold `M`, cached workspace (`d_tmp`), the quasi-Newton direction update `ha`, and the `hessian_segment_updater`, which computes certain values of the Hessian while advancing segments. -Instances are reused across segments during `find_generalized_cauchy_point_direction!` to +Instances are reused across segments during [`find_generalized_cauchy_direction!`](@ref) to avoid allocations. """ struct GeneralizedCauchyDirectionFinder{TM <: AbstractManifold, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} @@ -546,10 +546,10 @@ function GeneralizedCauchyDirectionFinder( end """ - find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyDirectionFinder, d_out, p, d, X) + find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinder, d_out, p, d, X) -Find generalized Cauchy point looking from point `p` in direction `d` and save the tangent -vector pointing at it to `d_out`. Gradient of the objective at `p` is `X`. +Find generalized Cauchy direction looking from point `p` in direction `d` and save it to `d_out`. +Gradient of the objective at `p` is `X`. The function returns * `:found_limited` if the point was found and we can perform a step of length at most 1 @@ -558,7 +558,7 @@ The function returns `max_stepsize(M, p)` in direction `d_out` afterwards, * `:not_found` if the search cannot be performed in direction `d`. """ -function find_generalized_cauchy_point_direction!(gcp::GeneralizedCauchyDirectionFinder, d_out, p, d, X) +function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinder, d_out, p, d, X) M = gcp.M copyto!(M, d_out, d) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 974bfb97fd..c69a7d4ae3 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -168,25 +168,25 @@ using RecursiveArrayTools d = -X1 d_out = similar(d) - @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d, X1) === :found_limited + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d, X1) === :found_limited @test d_out ≈ [2.0, 0.0, 0.0] d_out = similar(d) - @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, 0 * d, X1) === :not_found + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, 0 * d, X1) === :not_found d2 = [0.0, 1.0, 0.0] - @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === :found_unlimited + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === :found_unlimited @test d_out ≈ d2 - @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === :found_limited + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === :found_limited @test d_out ≈ [2.0, 10.0, 0.0] p2 = [-1.0, -2.0, 2.0] gf2 = Manopt.GeneralizedCauchyDirectionFinder(M, p2, ha) - @test Manopt.find_generalized_cauchy_point_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === :not_found + @test Manopt.find_generalized_cauchy_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === :not_found M2 = Hyperrectangle([-10.0], [10.0]) @@ -195,7 +195,7 @@ using RecursiveArrayTools gf3 = Manopt.GeneralizedCauchyDirectionFinder(M2, p3, ha2) d_out = similar(p3) - @test Manopt.find_generalized_cauchy_point_direction!(gf3, d_out, p3, [1.0], [-10.0]) === :found_limited + @test Manopt.find_generalized_cauchy_direction!(gf3, d_out, p3, [1.0], [-10.0]) === :found_limited end @testset "Hitting multiple bounds at the same time in GCD" begin @@ -209,7 +209,7 @@ using RecursiveArrayTools d_out = similar(d) X = [10.0, 10.0, 10.0] - @test Manopt.find_generalized_cauchy_point_direction!(gf, d_out, p, d, X) === :found_limited + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d, X) === :found_limited @test d_out ≈ [-1.0, -1.0, -1.0] end From 1ddca2c1486a1ef36133d7545611abe474668cab Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 23 Jan 2026 14:02:41 +0100 Subject: [PATCH 071/135] remove warning that isn't necessary --- src/plans/box_plan.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index b30fd1697e..5be18f4ca2 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -595,8 +595,12 @@ function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinde end else # Check only when we work on a pure Hyperrectangle + # + # In this case we can't move in the direction `d` at all, though it's usually not + # a problem relevant to the end user because it can be handled by step_solver! that + # uses the GCD subsolver. + if isempty(F_list) - @warn "We can't go in the selected direction" return :not_found end end From 7e61e530e3521b46592b0671a9748dea52b0a962 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 23 Jan 2026 14:09:05 +0100 Subject: [PATCH 072/135] formatting --- src/plans/box_plan.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 5be18f4ca2..bd6061a9ec 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -595,7 +595,7 @@ function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinde end else # Check only when we work on a pure Hyperrectangle - # + # # In this case we can't move in the direction `d` at all, though it's usually not # a problem relevant to the end user because it can be handled by step_solver! that # uses the GCD subsolver. From a21d072daef4ab21a46b5bd398bb617a6d3619ae Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 24 Jan 2026 13:46:00 +0100 Subject: [PATCH 073/135] improve numerical robustness --- src/plans/box_plan.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index bd6061a9ec..e0c114a64b 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -244,7 +244,10 @@ function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemo Lk = LowerTriangular(zeros(num_nonzero_rho, num_nonzero_rho)) # total scaling factor for the initial Hessian - gh.current_scale = (gh.qn_du.ρ[last_safe_index] * norm(M, p, gh.qn_du.memory_y[last_safe_index])^2) / gh.qn_du.initial_scale + # written this way to avoid floating point overflow (when ynorm is finite but ynorm^2 is Inf) + # see CUTEst EXPQUAD problem for an example + ynorm = norm(M, p, gh.qn_du.memory_y[last_safe_index]) + gh.current_scale = ((gh.qn_du.ρ[last_safe_index] * ynorm) * ynorm) / gh.qn_du.initial_scale tsksk = Symmetric(zeros(num_nonzero_rho, num_nonzero_rho)) ii = 1 @@ -270,7 +273,6 @@ function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemo # Schur complement of -Dk is the only non-diagonal matrix we actually need to inverse in this step W1 = Lk * invA W2 = W1 * Lk' - gh.M_22 = inv(Symmetric(tsksk - W2)) W3 = gh.M_22 * W1 W4 = W1' * W3 From 7c9377acc5a14ab58a75159a5fefe5f32b38dfb9 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 24 Jan 2026 15:03:32 +0100 Subject: [PATCH 074/135] also detect nondescent direction at inner product 0 --- src/solvers/quasi_Newton.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 6c33080be9..e82f6eb295 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -407,7 +407,7 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) end if !(qns.nondescent_direction_behavior === :ignore) qns.nondescent_direction_value = real(inner(M, qns.p, qns.η, qns.X)) - if qns.nondescent_direction_value > 0 + if qns.nondescent_direction_value >= 0 if qns.nondescent_direction_behavior === :step_towards_negative_gradient || qns.nondescent_direction_behavior === :reinitialize_direction_update copyto!(M, qns.η, qns.X) From a6982b58cc02221e16e4ed83e73d231e3ef9f9c6 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 24 Jan 2026 15:46:51 +0100 Subject: [PATCH 075/135] Wrong eps was taken there --- src/plans/stopping_criterion.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index c50122db85..b410d85d16 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -493,7 +493,7 @@ end function StopWhenRelativeAPosterioriCostChangeLessOrEqual(tol::F) where {F <: Real} return StopWhenRelativeAPosterioriCostChangeLessOrEqual{F}(tol, -1, zero(tol), 2 * tol) end -StopWhenRelativeAPosterioriCostChangeLessOrEqual(; factr::F = 1.0e7) where {F <: Real} = StopWhenRelativeAPosterioriCostChangeLessOrEqual(factr * eps(factr)) +StopWhenRelativeAPosterioriCostChangeLessOrEqual(; factr::F = 1.0e7) where {F <: Real} = StopWhenRelativeAPosterioriCostChangeLessOrEqual(factr * eps(typeof(factr))) function (c::StopWhenRelativeAPosterioriCostChangeLessOrEqual)( problem::AbstractManoptProblem, state::AbstractManoptSolverState, iteration::Int ) From f0200c1364800ca61287d6a4908f20f337273eb0 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 24 Jan 2026 15:59:35 +0100 Subject: [PATCH 076/135] fix test --- test/plans/test_stopping_criteria.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index e34d13ea32..7989cf263f 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -381,7 +381,7 @@ end @test length(get_reason(sc)) > 0 @test startswith( to_display_string(sc), - "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold 1.4210854715202004e-12.\n", + "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold 2.220446049250313e-14.\n", ) @test startswith(Manopt.status_summary(sc), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") @test !Manopt.indicates_convergence(sc) From c4f1aae6f15f66fd55b79c6f943ee95c32436278 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 24 Jan 2026 20:30:05 +0100 Subject: [PATCH 077/135] Update test/plans/test_stopping_criteria.jl Co-authored-by: Ronny Bergmann --- test/plans/test_stopping_criteria.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 7989cf263f..9cd9505e97 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -381,7 +381,7 @@ end @test length(get_reason(sc)) > 0 @test startswith( to_display_string(sc), - "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold 2.220446049250313e-14.\n", + "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold", ) @test startswith(Manopt.status_summary(sc), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") @test !Manopt.indicates_convergence(sc) From 3a30dd802d50b694e268b7e1f92291e626cc512f Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sun, 25 Jan 2026 13:04:50 +0100 Subject: [PATCH 078/135] support stepsize limiting in LineSearchesStepsize --- docs/src/extensions.md | 4 ++- ext/ManoptLineSearchesExt.jl | 45 ++++++++++++++++++++++++++++++- src/helpers/LineSearchesTypes.jl | 19 +++++++++++++ test/helpers/test_linesearches.jl | 13 +++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) diff --git a/docs/src/extensions.md b/docs/src/extensions.md index 0f95f70977..3ef97e297f 100644 --- a/docs/src/extensions.md +++ b/docs/src/extensions.md @@ -45,10 +45,12 @@ x_opt = quasi_Newton( ) ``` -In general this defines the following new [stepsize](@ref Stepsize) +In general this defines the following new [stepsize](@ref Stepsize) with helper functions for setting and getting the maximum step size: ```@docs Manopt.LineSearchesStepsize +Manopt.linesearches_get_max_alpha +Manopt.linesearches_set_max_alpha ``` ## Manifolds.jl diff --git a/ext/ManoptLineSearchesExt.jl b/ext/ManoptLineSearchesExt.jl index c58734e57c..92ad77db7f 100644 --- a/ext/ManoptLineSearchesExt.jl +++ b/ext/ManoptLineSearchesExt.jl @@ -5,6 +5,37 @@ import Manopt: LineSearchesStepsize using ManifoldsBase using LineSearches +Manopt.linesearches_get_max_alpha(ls::LineSearches.HagerZhang) = ls.alphamax +Manopt.linesearches_get_max_alpha(ls::LineSearches.MoreThuente) = ls.alphamax + +function Manopt.linesearches_set_max_alpha(ls::LineSearches.HagerZhang{T, Tm}, max_alpha::T) where {T, Tm} + return HagerZhang{T, Tm}( + delta = ls.delta, + sigma = ls.sigma, + alphamax = max_alpha, + rho = ls.rho, + epsilon = ls.epsilon, + gamma = ls.gamma, + linesearchmax = ls.linesearchmax, + psi3 = ls.psi3, + display = ls.display, + mayterminate = ls.mayterminate, + cache = ls.cache, + check_flatness = ls.check_flatness, + ) +end +function Manopt.linesearches_set_max_alpha(ls::LineSearches.MoreThuente{T}, max_alpha::T) where {T} + return MoreThuente{T}( + f_tol = ls.f_tol, + gtol = ls.gtol, + x_tol = ls.x_tol, + alphamin = ls.alphamin, + alphamax = max_alpha, + maxfev = ls.maxfev, + cache = ls.cache + ) +end + function (cs::Manopt.LineSearchesStepsize)( mp::AbstractManoptProblem, s::AbstractManoptSolverState, @@ -24,6 +55,18 @@ function (cs::Manopt.LineSearchesStepsize)( # guess initial alpha α0 = cs.initial_guess(mp, s, k, cs.last_stepsize, η; lf0 = fp, Dlf0 = dphi_0) + # handle stepsize limit + local ls + if :stop_when_stepsize_exceeds in keys(kwargs) + new_max_alpha = min( + kwargs[:stop_when_stepsize_exceeds], + linesearches_get_max_alpha(cs.linesearch), + ) + ls = linesearches_set_max_alpha(cs.linesearch, new_max_alpha) + else + ls = cs.linesearch + end + # perform actual line-search function ϕ(α) @@ -41,7 +84,7 @@ function (cs::Manopt.LineSearchesStepsize)( return Manopt.get_cost_and_differential(mp, p_tmp, Y_tmp; Y = X_tmp) end - α, fp = cs.linesearch(ϕ, dϕ, ϕdϕ, α0, fp, dphi_0) + α, fp = ls(ϕ, dϕ, ϕdϕ, α0, fp, dphi_0) cs.last_stepsize = α return α end diff --git a/src/helpers/LineSearchesTypes.jl b/src/helpers/LineSearchesTypes.jl index 633321316a..244d831122 100644 --- a/src/helpers/LineSearchesTypes.jl +++ b/src/helpers/LineSearchesTypes.jl @@ -62,6 +62,25 @@ function LineSearchesStepsize( ) end +""" + linesearches_get_max_alpha(ls) + +Get the maximum step size for `LineSearches.jl` line search `ls`. +""" +linesearches_get_max_alpha(ls) + +function linesearches_get_max_alpha end + +""" + linesearches_set_max_alpha(ls, max_alpha::Real) + +Set the maximum step size for `LineSearches.jl` line search `ls` to `max_alpha`. +Return a new line search object with the updated maximum step size. +""" +linesearches_set_max_alpha(ls, max_alpha::Real) + +function linesearches_set_max_alpha end + function Base.show(io::IO, cs::LineSearchesStepsize) return print( io, diff --git a/test/helpers/test_linesearches.jl b/test/helpers/test_linesearches.jl index 69e0526df7..2f7eb95883 100644 --- a/test/helpers/test_linesearches.jl +++ b/test/helpers/test_linesearches.jl @@ -62,4 +62,17 @@ using Test initialize_solver!(mp, st_qn) ls_mt = Manopt.LineSearchesStepsize(M, LineSearches.MoreThuente()) @test_throws ErrorException ls_mt(mp_throw, st_qn, 1; fp = rosenbrock(M, x0)) + + @testset "max stepsize limit setting" begin + lss = [ + LineSearches.MoreThuente(), + LineSearches.HagerZhang(), + ] + for ls in lss + nls = Manopt.linesearches_set_max_alpha(ls, 0.5) + @test Manopt.linesearches_get_max_alpha(nls) == 0.5 + nls2 = Manopt.linesearches_set_max_alpha(ls, Inf) + @test Manopt.linesearches_get_max_alpha(nls2) == Inf + end + end end From eb9bac4199226b841a52701c2cc6956c90cd082a Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sun, 25 Jan 2026 13:50:45 +0100 Subject: [PATCH 079/135] fix + test --- ext/ManoptLineSearchesExt.jl | 5 +++-- test/helpers/test_linesearches.jl | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ext/ManoptLineSearchesExt.jl b/ext/ManoptLineSearchesExt.jl index 92ad77db7f..5859a82dbe 100644 --- a/ext/ManoptLineSearchesExt.jl +++ b/ext/ManoptLineSearchesExt.jl @@ -60,9 +60,10 @@ function (cs::Manopt.LineSearchesStepsize)( if :stop_when_stepsize_exceeds in keys(kwargs) new_max_alpha = min( kwargs[:stop_when_stepsize_exceeds], - linesearches_get_max_alpha(cs.linesearch), + Manopt.linesearches_get_max_alpha(cs.linesearch), ) - ls = linesearches_set_max_alpha(cs.linesearch, new_max_alpha) + ls = Manopt.linesearches_set_max_alpha(cs.linesearch, new_max_alpha) + α0 = min(α0, new_max_alpha) else ls = cs.linesearch end diff --git a/test/helpers/test_linesearches.jl b/test/helpers/test_linesearches.jl index 2f7eb95883..0b558803f8 100644 --- a/test/helpers/test_linesearches.jl +++ b/test/helpers/test_linesearches.jl @@ -63,6 +63,9 @@ using Test ls_mt = Manopt.LineSearchesStepsize(M, LineSearches.MoreThuente()) @test_throws ErrorException ls_mt(mp_throw, st_qn, 1; fp = rosenbrock(M, x0)) + # test max stepsize limit enforcement + @test ls_hz(mp, st_qn, 1, [1.0, 2.0, 3.0, 4.0, 0.0]; stop_when_stepsize_exceeds = 0.1) == 0.1 + @testset "max stepsize limit setting" begin lss = [ LineSearches.MoreThuente(), From 853f37760595ffb4ec09af8d2eb362e907ed1bcd Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sun, 25 Jan 2026 15:09:40 +0100 Subject: [PATCH 080/135] skip non-coverable line --- ext/ManoptLineSearchesExt.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/ManoptLineSearchesExt.jl b/ext/ManoptLineSearchesExt.jl index 5859a82dbe..5343e05289 100644 --- a/ext/ManoptLineSearchesExt.jl +++ b/ext/ManoptLineSearchesExt.jl @@ -56,7 +56,7 @@ function (cs::Manopt.LineSearchesStepsize)( α0 = cs.initial_guess(mp, s, k, cs.last_stepsize, η; lf0 = fp, Dlf0 = dphi_0) # handle stepsize limit - local ls + local ls # COV_EXCL_LINE if :stop_when_stepsize_exceeds in keys(kwargs) new_max_alpha = min( kwargs[:stop_when_stepsize_exceeds], From e2fc841d35384a39f1d24e3009be701601be52f2 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 29 Jan 2026 13:53:27 +0100 Subject: [PATCH 081/135] fix a few issues with quasi-Newton --- src/plans/box_plan.jl | 4 ++-- src/solvers/quasi_Newton.jl | 31 +++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index e0c114a64b..f0a14ba760 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -85,6 +85,7 @@ end function initialize_update!(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) initialize_update!(ha.qn_du) + ha.last_gcp_result = :not_searched return ha end @@ -652,8 +653,7 @@ function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinde dt_min = max(dt_min, 0.0) t_old = t_old + dt_min d_out .*= t_old - set_stepsize_bound!(M, d_out, p, ts, t_current) - + set_stepsize_bound!(M, d_out, p, ts, t_old) if has_finite_limit return :found_limited else diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index e82f6eb295..41a4872c28 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -389,6 +389,15 @@ function quasi_Newton!( end calls_with_kwargs(::typeof(quasi_Newton!)) = (decorate_objective!, decorate_state!) +function _get_max_stepsize(M::AbstractManifold, qns::QuasiNewtonState) + current_max_stepsize = get_parameter(qns.direction_update, Val(:max_stepsize)) + if !isnothing(current_max_stepsize) && !isfinite(current_max_stepsize) + return max_stepsize(M, qns.p) / norm(qns.η) + else + return current_max_stepsize + end +end + function initialize_solver!(amp::AbstractManoptProblem, qns::QuasiNewtonState) M = get_manifold(amp) get_gradient!(amp, qns.X, qns.p) @@ -401,10 +410,7 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) M = get_manifold(mp) get_gradient!(mp, qns.X, qns.p) qns.direction_update(qns.η, mp, qns) - current_max_stepsize = get_parameter(qns.direction_update, Val(:max_stepsize)) - if !isnothing(current_max_stepsize) && !isfinite(current_max_stepsize) - current_max_stepsize = max_stepsize(M, qns.p) / norm(qns.η) - end + current_max_stepsize = _get_max_stepsize(M, qns) if !(qns.nondescent_direction_behavior === :ignore) qns.nondescent_direction_value = real(inner(M, qns.p, qns.η, qns.X)) if qns.nondescent_direction_value >= 0 @@ -416,6 +422,12 @@ function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) if qns.nondescent_direction_behavior === :reinitialize_direction_update initialize_update!(qns.direction_update) end + # update direction after reinitialization to get a valid one + if qns.nondescent_direction_behavior === :step_towards_negative_gradient || + qns.nondescent_direction_behavior === :reinitialize_direction_update + qns.direction_update(qns.η, mp, qns) + current_max_stepsize = _get_max_stepsize(M, qns) + end end end local α # COV_EXCL_LINE @@ -802,6 +814,17 @@ function update_hessian!( end if reforming_required + # we need to move first vectors in memory too because they most likely won't be + # overwritten by new pairs + if start == 2 + vector_transport_to!( + M, d.memory_s[1], p_old, d.memory_s[1], p, d.vector_transport_method + ) + vector_transport_to!( + M, d.memory_y[1], p_old, d.memory_y[1], p, d.vector_transport_method + ) + fill_rho_i!(M, p, d, 1) + end _drop_zero_rho_vectors!(d) end From 2a03750dce6b4e63adaf394673992115486984de Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 29 Jan 2026 15:48:51 +0100 Subject: [PATCH 082/135] improve coverage --- test/solvers/test_quasi_Newton.jl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index 54dcb4b978..194b425dc1 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -527,4 +527,28 @@ end @test qdu.memory_y[1] == [1, 0] @test qdu.memory_y[2] == [0, 2] end + + @testset "reforming_required + (start == 2)" begin + M = Euclidean(2) + p = [0.0, 0.0] + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 * sum(p) + gmp = ManifoldGradientObjective(f, grad_f) + mp = DefaultManoptProblem(M, gmp) + ha = QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2; nonpositive_curvature_behavior = :byrd) + qns = QuasiNewtonState(M; p = p, nonpositive_curvature_behavior = :byrd, direction_update = ha) + + qns.yk = [1.0, 1.0] + qns.sk = [1.0, 2.0] + update_hessian!(qns.direction_update, mp, qns, p, 1) + + qns.yk = [2.0, 1.0] + qns.sk = [1.0, 2.0] + update_hessian!(qns.direction_update, mp, qns, p, 2) + ha.memory_s[2] = [0.0, 0.0] # force reforming_required in next step + update_hessian!(qns.direction_update, mp, qns, p, 3) + # test that the zeroes out pair was replaced + @test qns.direction_update.memory_s[1] == [1.0, 2.0] + @test qns.direction_update.memory_s[2] == [1.0, 2.0] + end end From fa2ad1c74940f452f843ef5ed61007933f538974 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 30 Jan 2026 16:19:50 +0100 Subject: [PATCH 083/135] allow for larger max stepsizes in post-GCD linesearch when away from boundaries --- src/plans/box_plan.jl | 53 ++++++++++++++++----------- test/solvers/test_quasi_Newton_box.jl | 17 +++++---- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index f0a14ba760..7b4e270979 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -22,7 +22,7 @@ Initial scale ``\theta`` is stored in the field `initial_scale` but if the memor the current scale is set to squared norm of $s_k$ divided by inner product of ``s_k`` and ``y_k`` where ``k`` is the oldest index for which the denominator is not equal to 0. -`last_gcp_result` stores the result of the last generalized Cauchy point search. +`last_gcd_result` stores the result of the last generalized Cauchy direction search. See [ByrdNocedalSchnabel:1994](@cite) for details. """ @@ -45,12 +45,13 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ buffer_inner_Sk_Y::V buffer_inner_Yk_X::V buffer_inner_Yk_Y::V - last_gcp_result::Symbol + last_gcd_result::Symbol + last_gcd_stepsize::F end function get_parameter(d::QuasiNewtonLimitedMemoryBoxDirectionUpdate, ::Val{:max_stepsize}) - if d.last_gcp_result === :found_limited - return 1.0 + if d.last_gcd_result === :found_limited + return d.last_gcd_stepsize else return Inf end @@ -80,12 +81,13 @@ function QuasiNewtonLimitedMemoryBoxDirectionUpdate( buffer_inner_Yk_X, buffer_inner_Yk_Y, :not_searched, + NaN, ) end function initialize_update!(ha::QuasiNewtonLimitedMemoryBoxDirectionUpdate) initialize_update!(ha.qn_du) - ha.last_gcp_result = :not_searched + ha.last_gcd_result = :not_searched return ha end @@ -102,8 +104,8 @@ function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( M = get_manifold(mp) p = get_iterate(st) X = get_gradient(st) - gcp = GeneralizedCauchyDirectionFinder(M, p, d) - d.last_gcp_result = find_generalized_cauchy_direction!(gcp, r, p, r, X) + gcd = GeneralizedCauchyDirectionFinder(M, p, d) + d.last_gcd_result, d.last_gcd_stepsize = find_generalized_cauchy_direction!(gcd, r, p, r, X) return r end @@ -334,7 +336,7 @@ end """ abstract type AbstractSegmentHessianUpdater end -Abstract type for methods that calculate f' and f'' in the GCP calculation in subsequent +Abstract type for methods that calculate f' and f'' in the GCD calculation in subsequent line segments in [`GeneralizedCauchyDirectionFinder`](@ref). """ abstract type AbstractSegmentHessianUpdater end @@ -343,7 +345,7 @@ abstract type AbstractSegmentHessianUpdater end init_updater!(::AbstractManifold, hessian_segment_updater::AbstractSegmentHessianUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) Method for initialization of `AbstractSegmentHessianUpdater` `hessian_segment_updater` just before the loop -that examines subsequent intervals for GCP. +that examines subsequent intervals for GCD. """ init_updater!(::AbstractManifold, hessian_segment_updater::AbstractSegmentHessianUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) @@ -549,7 +551,7 @@ function GeneralizedCauchyDirectionFinder( end """ - find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinder, d_out, p, d, X) + find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinder, d_out, p, d, X) Find generalized Cauchy direction looking from point `p` in direction `d` and save it to `d_out`. Gradient of the objective at `p` is `X`. @@ -561,8 +563,8 @@ The function returns `max_stepsize(M, p)` in direction `d_out` afterwards, * `:not_found` if the search cannot be performed in direction `d`. """ -function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinder, d_out, p, d, X) - M = gcp.M +function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinder, d_out, p, d, X) + M = gcd.M copyto!(M, d_out, d) bounds_indices = get_bounds_index(M) @@ -576,11 +578,14 @@ function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinde has_finite_limit = false + smallest_positive_limit = Inf + for i in bounds_indices ts[i] = get_stepsize_bound(M, p, d, i) if ts[i] > 0 push!(F_list, (ts[i], i)) + smallest_positive_limit = min(smallest_positive_limit, ts[i]) end has_finite_limit |= isfinite(ts[i]) end @@ -604,17 +609,17 @@ function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinde # uses the GCD subsolver. if isempty(F_list) - return :not_found + return (:not_found, NaN) end end F = BinaryHeap(Base.By(first), F_list) f_prime = inner(M, p, X, d) - f_double_prime = hessian_value_diag(gcp.ha, M, p, d) + f_double_prime = hessian_value_diag(gcd.ha, M, p, d) if iszero(f_prime) || iszero(f_double_prime) - return :not_found + return (:not_found, NaN) end dt_min = -f_prime / f_double_prime @@ -623,22 +628,22 @@ function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinde t_current, b = pop!(F) dt = t_current - t_old - init_updater!(M, gcp.hessian_segment_updater, p, d, gcp.ha) + init_updater!(M, gcd.hessian_segment_updater, p, d, gcd.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 db = get_at_bound_index(M, d, b) gb = get_at_bound_index(M, X, b) - hv_eb_dz, hv_eb_d = gcp.hessian_segment_updater(M, p, t_current, dt, b, db, gcp.ha) + hv_eb_dz, hv_eb_d = gcd.hessian_segment_updater(M, p, t_current, dt, b, db, gcd.ha) f_prime += dt * f_double_prime - db * (gb + hv_eb_dz) - f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_diag(gcp.ha, M, p, UnitVector(b)) + f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_diag(gcd.ha, M, p, UnitVector(b)) t_old = t_current - # If f_prime is 0, we've found the local minimizer (GCP) + # If f_prime is 0, we've found the local minimizer (GCD) if iszero(f_prime) || iszero(f_double_prime) - # It means that GCP is at the beginning of the t_current, so we want to set dt_min to 0 (stay in the point) + # It means that GCD is at the beginning of the t_current, so we want to set dt_min to 0 (stay in the point) dt_min = 0.0 break end @@ -653,10 +658,14 @@ function find_generalized_cauchy_direction!(gcp::GeneralizedCauchyDirectionFinde dt_min = max(dt_min, 0.0) t_old = t_old + dt_min d_out .*= t_old + # by construction, there is no bound achievable before stepsize 1.0 in direction d_out + # there first bound after that is achieved at smallest_positive_limit / t_old + max_feasible_stepsize = max(1.0, smallest_positive_limit / t_old) + set_stepsize_bound!(M, d_out, p, ts, t_old) if has_finite_limit - return :found_limited + return (:found_limited, max_feasible_stepsize) else - return :found_unlimited + return (:found_unlimited, Inf) end end diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index c69a7d4ae3..151820dcc8 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -142,7 +142,8 @@ using RecursiveArrayTools @test hv_eb_dz ≈ hv_eb_dz_limited @test hv_eb_d ≈ hv_eb_d_limited - ha.last_gcp_result = :found_unlimited + ha.last_gcd_result = :found_unlimited + ha.last_gcd_stepsize = Inf @test Manopt.get_parameter(ha, Val(:max_stepsize)) == Inf @testset "No memory tests" begin @@ -168,25 +169,25 @@ using RecursiveArrayTools d = -X1 d_out = similar(d) - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d, X1) === :found_limited + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d, X1) === (:found_limited, 1.0) @test d_out ≈ [2.0, 0.0, 0.0] d_out = similar(d) - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, 0 * d, X1) === :not_found + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, 0 * d, X1) === (:not_found, NaN) d2 = [0.0, 1.0, 0.0] - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === :found_unlimited + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === (:found_unlimited, Inf) @test d_out ≈ d2 - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === :found_limited + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === (:found_limited, 1.0) @test d_out ≈ [2.0, 10.0, 0.0] p2 = [-1.0, -2.0, 2.0] gf2 = Manopt.GeneralizedCauchyDirectionFinder(M, p2, ha) - @test Manopt.find_generalized_cauchy_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === :not_found + @test Manopt.find_generalized_cauchy_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === (:not_found, NaN) M2 = Hyperrectangle([-10.0], [10.0]) @@ -195,7 +196,7 @@ using RecursiveArrayTools gf3 = Manopt.GeneralizedCauchyDirectionFinder(M2, p3, ha2) d_out = similar(p3) - @test Manopt.find_generalized_cauchy_direction!(gf3, d_out, p3, [1.0], [-10.0]) === :found_limited + @test Manopt.find_generalized_cauchy_direction!(gf3, d_out, p3, [1.0], [-10.0]) === (:found_limited, 90.0) end @testset "Hitting multiple bounds at the same time in GCD" begin @@ -209,7 +210,7 @@ using RecursiveArrayTools d_out = similar(d) X = [10.0, 10.0, 10.0] - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d, X) === :found_limited + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d, X) === (:found_limited, 1.0) @test d_out ≈ [-1.0, -1.0, -1.0] end From 2245a9da1e48462acc635631a5b77a54a3cde66e Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 2 Feb 2026 16:04:35 +0100 Subject: [PATCH 084/135] Add Hager-Zhang 2006 linesearch --- src/Manopt.jl | 1 + src/plans/stepsize/stepsize.jl | 505 ++++++++++++++++++++++++++++++++- test/plans/test_stepsize.jl | 87 ++++++ 3 files changed, 592 insertions(+), 1 deletion(-) diff --git a/src/Manopt.jl b/src/Manopt.jl index 4994b8effb..8fa5d233e9 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -547,6 +547,7 @@ export get_stepsize, get_initial_stepsize, get_last_stepsize export InteriorPointCentralityCondition export DomainBackTracking, DomainBackTrackingStepsize, NullStepBackTrackingStepsize export ProximalGradientMethodBacktracking +export HagerZhangLinesearch # # Stopping Criteria export StoppingCriterion, StoppingCriterionSet diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index ae5ca21814..73ef049657 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -662,12 +662,15 @@ end Returns the extremal of the quadratic polynomial ``p`` with ``p'(a.t)=a.df``, ``p'(b.t)=b.df``. +The result is algebraically equivalent to `(a.t * b.df - b.t * a.df) / (b.df - a.df)` +but the used formula is more numerically stable. + # Input * `a::UnivariateTriple{R}`: triple of bracket value `a` * `b::UnivariateTriple{R}`: triple bracket value `b` """ function secant(a::UnivariateTriple{R}, b::UnivariateTriple{R}) where {R} - return (a.t * b.df - b.t * a.df) / (b.df - a.df) + return (a.t + b.t) / 2 + (b.t - a.t) * (a.df + b.df) / (2 * (a.df - b.df)) end """ @@ -2130,3 +2133,503 @@ end function get_last_stepsize(step::WolfePowellBinaryLinesearchStepsize, ::Any...) return step.last_stepsize end + + +#### Hager-Zhang Linesearch + + +@doc """ + HagerZhangLinesearchStepsize{P,T,R<:Real} <: Linesearch + +Do a bracketing line search to find a step size ``α`` that finds a +local minimum along the search direction ``X`` starting from ``p``, +utilizing cubic polynomial interpolation using the method described in +[HagerZhang:2006:2](@cite). +See [`HagerZhangLinesearch`](@ref) for the mathematical details. + +# Fields +$(_fields(:p; name = "candidate_point")) + as temporary storage for candidates +* `initial_stepsize::R`: the step size to start the search with +$(_fields(:retraction_method)) +$(_fields(:vector_transport_method)) + +# Constructor + + HagerZhangLinesearchStepsize(M::AbstractManifold; kwargs...) + +## Keyword arguments + +$(_kwargs(:p; name = "candidate_point")) as temporary storage for candidates +$(_kwargs(:retraction_method)) +$(_kwargs(:vector_transport_method)) +""" +mutable struct HagerZhangLinesearchStepsize{ + TF <: Real, + TIG <: AbstractInitialLinesearchGuess, + TRM <: AbstractRetractionMethod, + TVTM <: AbstractVectorTransportMethod, + TP, + TX, + } <: Linesearch + # parameters + initial_guess::TIG + retraction_method::TRM + vector_transport_method::TVTM + stepsize_limit::TF + max_bracket_iterations::Int + wolfe_condition_mode::Symbol # :standard, :approximate, :adaptive + ϵ::TF # approximate Wolfe termination parameter + δ::TF # used in approximate Wolfe condition + σ::TF # used in curvature condition + ω::TF + θ::TF # update rule parameter + γ::TF + η::TF + ρ::TF + Δ::TF + ψ₀::TF + ψ₁::TF + ψ₂::TF + # storage for candidates + candidate_point::TP + candidate_direction::TX + temporary_tangent::TX + # storage for function evaluations + triples::Vector{UnivariateTriple{TF}} + last_evaluation_index::Int + # storage to be kept between outer solver iterations + Qₖ::TF + Cₖ::TF + current_mode::Symbol + # other storage + last_stepsize::TF + last_cost::TF + ϵₖ::TF + function HagerZhangLinesearchStepsize( + M::AbstractManifold; + initial_guess::TIG = HagerZhangInitialGuess(), + retraction_method::TRM = default_retraction_method(M), + vector_transport_method::TVTM = default_vector_transport_method(M), + initial_last_stepsize::TF = NaN, + initial_last_cost::TF = NaN, + stepsize_limit::TF = Inf, + candidate_point = allocate_result(M, rand), + candidate_direction = zero_vector(M, candidate_point), + max_bracket_iterations::Int = 10, + max_function_evaluations::Int = 20, + wolfe_condition_mode::Symbol = :adaptive, + ϵ::TF = 1.0e-6, + δ::TF = 0.1, + σ::TF = 0.9, + ω::TF = 1.0e-3, + θ::TF = 0.5, + γ::TF = 0.66, + η::TF = 0.01, + ρ::TF = 5.0, + Δ::TF = 0.7, + ψ₀::TF = 0.01, + ψ₁::TF = 0.1, + ψ₂::TF = 2.0, + ) where { + TIG <: AbstractInitialLinesearchGuess, TRM <: AbstractRetractionMethod, + TVTM <: AbstractVectorTransportMethod, TF <: Real, + } + + # check parameters + @assert δ > 0 && δ < 0.5 + @assert δ <= σ + @assert σ < 1 + @assert ϵ >= 0 + @assert ω >= 0 && ω <= 1 + @assert Δ >= 0 && Δ <= 1 + @assert θ > 0 && θ < 1 + @assert γ > 0 && γ < 1 + @assert η > 0 + @assert ρ > 1 + @assert ψ₀ > 0 && ψ₀ < 1 + @assert ψ₁ > 0 && ψ₁ < 1 + @assert ψ₂ > 1 + @assert stepsize_limit > 0 + @assert wolfe_condition_mode in (:standard, :approximate, :adaptive) + + # allocate storage + triples = Vector{UnivariateTriple{TF}}(undef, max_function_evaluations) + + initial_wolfe_mode = wolfe_condition_mode == :adaptive ? :standard : wolfe_condition_mode + + return new{TF, TIG, TRM, TVTM, typeof(candidate_point), typeof(candidate_direction)}( + initial_guess, retraction_method, vector_transport_method, stepsize_limit, + max_bracket_iterations, wolfe_condition_mode, + ϵ, δ, σ, ω, θ, γ, η, ρ, Δ, ψ₀, ψ₁, ψ₂, + candidate_point, candidate_direction, zero_vector(M, candidate_point), + triples, 0, + 0.0, 0.0, # Qₖ, Cₖ + initial_wolfe_mode, + initial_last_stepsize, initial_last_cost, ϵ, + ) + end +end + +function initialize_stepsize!(hzls::HagerZhangLinesearchStepsize) + hzls.Qₖ = 0.0 + hzls.Cₖ = 0.0 + hzls.last_stepsize = NaN + hzls.last_cost = NaN + hzls.ϵₖ = hzls.ϵ + hzls.current_mode = hzls.wolfe_condition_mode + if hzls.current_mode === :adaptive + hzls.current_mode = :standard + end + hzls.last_evaluation_index = 0 + return hzls +end + +""" + _hz_evaluate_next_step( + hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, + mp::AbstractManoptProblem, p, η, α::Real + ) + +Evaluate and store the next trial step for the Hager-Zhang linesearch. + +Given the current iterate `p`, search direction `η` (in the tangent space at `p`), and a +candidate step size `α`, this function + +1. Retracts from `p` along `η` by step `α` into `hzls.candidate_point` (using + `hzls.retraction_method`), +2. Vector-transports `η` to the candidate point into `hzls.candidate_direction` (using + `hzls.vector_transport_method`), +3. Evaluates the objective and directional derivative via + `get_cost_and_differential(mp, hzls.candidate_point, hzls.candidate_direction)`, +4. Stores the resulting triple `(α, f, df)` in `hzls.triples` and increments + `hzls.last_evaluation_index`. + +This helper is side-effecting by design; it mutates `hzls`' internal storage. + +# Return value + +By default return a tuple with three values: +- the index `i_k::Int` at which the new evaluation was stored. +- `evaluation_limit_termination`: `true` iff the maximum number of stored evaluations + has been reached. +- `wolfe_termination` is `true` iff the (standard or approximate) Wolfe conditions are + satisfied for the current candidate, according to `hzls.current_mode`. + +# Errors + +Throws an error if called more often than the maximum number of allocated function +evaluations (i.e. if `hzls.triples` would overflow). +""" +function _hz_evaluate_next_step( + hzls::HagerZhangLinesearchStepsize, + M::AbstractManifold, + mp::AbstractManoptProblem, + p, + η, + α::Real + ) + triples = hzls.triples + max_evals = length(triples) + if hzls.last_evaluation_index + 1 > max_evals + # this should never happen if the calling code is correct + error("Hager-Zhang linesearch exceeded maximum number of function evaluations $(length(hzls.triples)).") + end + ManifoldsBase.retract_fused!(M, hzls.candidate_point, p, η, α, hzls.retraction_method) + vector_transport_to!( + M, hzls.candidate_direction, p, η, hzls.candidate_point, hzls.vector_transport_method + ) + f, df = get_cost_and_differential(mp, hzls.candidate_point, hzls.candidate_direction) + hzls.last_evaluation_index += 1 + triples[hzls.last_evaluation_index] = UnivariateTriple(α, f, df) + + wolfe_termination = false + evaluation_limit_termination = hzls.last_evaluation_index == max_evals + i_k = hzls.last_evaluation_index + if hzls.current_mode === :standard + # Eq (22) in HagerZhang:2006:2 + # equivalent to the (T1) condition + wolfe_termination = (α * hzls.δ * triples[1].df >= (triples[i_k].f - triples[1].f)) && + (triples[i_k].df >= hzls.σ * triples[1].df) + elseif hzls.current_mode === :approximate + # Eq (23) in HagerZhang:2006:2 + additional criterion in the (T2) condition + wolfe_termination = ((2 * hzls.δ - 1) * triples[1].df >= triples[i_k].df) && + (triples[i_k].df >= hzls.σ * triples[1].df) && triples[i_k].f <= triples[1].f + hzls.ϵₖ + else + error("Unknown Wolfe condition mode $(hzls.current_mode).") + end + + return hzls.last_evaluation_index, evaluation_limit_termination, wolfe_termination +end + +""" + _hz_bracket(hzls::HagerZhangLinesearchStepsize, c::Real, max_alpha::Real) + +Perform the bracketing phase of the Hager-Zhang linesearch starting from an initial +stepsize `c` and not exceeding `max_alpha`. + +Returns a tuple `(i_a, i_b)` where `i_a` and `i_b` are the indices in the stored function +evaluations such that the minimum is bracketed between `triples[i_a].t` and +`triples[i_b].t`. +""" +function _hz_bracket( + hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, + mp::AbstractManoptProblem, p, η, c::Real, max_alpha::Real + ) + # B0 + current_step = c + local c_index, f_eval, f_wolfe + for j in 1:hzls.max_bracket_iterations + c_index, f_eval, f_wolfe = _hz_evaluate_next_step(hzls, M, mp, p, η, current_step) + if f_eval || f_wolfe + break + end + if hzls.triples[c_index].df >= 0 + # B1 -- detecting a positive slope + # handled after the loop + break + else + if hzls.triples[j].f > hzls.triples[1].f + hzls.ϵₖ + # B2 -- function value gets sufficiently larger than at 0 + # perform main bracketing loop (we can skip U0-U2 checks here) + (i_a_bar, i_b_bar, f_eval, f_wolfe) = _hz_u3(hzls, M, mp, p, η, 1, c_index) + return (i_a_bar, i_b_bar, f_eval, f_wolfe) + else + if current_step == max_alpha + # we've reached maximum alpha so we can't expand anymore + # we handle this case after the loop + break + end + # B3 -- widen the bracket + current_step *= hzls.ρ + if current_step > max_alpha + current_step = max_alpha + end + end + end + end + # we detected positive slope, ran out of iterations or reached max stepsize + # B1 seems to be the best choice for all three cases + i_min = 1 + for i in 2:(hzls.last_evaluation_index - 1) + if hzls.triples[i].f <= hzls.triples[1].f + hzls.ϵₖ + i_min = i + end + end + return (i_min, c_index, f_eval, f_wolfe) +end + +""" + _hz_update( + hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, + mp::AbstractManoptProblem, p, η, i_a::Int, i_b::Int, c::Real + ) + +Perform an update procedure of the Hager-Zhang linesearch given the current bracketing +indices `i_a` and `i_b` and a candidate stepsize `c`. + +Returns indices and termination information `(i_A, i_B, i_c, f_eval, f_wolfe)` where the +minimum is now bracketed between `alpha_values[i_A]` and `alpha_values[i_B]`. Index `i_c` +indicates the position at which evaluation of the candidate `c` was stored. If the +candidate `c` is outside of the current bracket, the last index is returned as `-1`. +`f_eval` is `true` if the maximum number of function evaluations has been reached. +`f_wolfe` is `true` if the Wolfe conditions have been satisfied at the candidate `i_c`. +""" +function _hz_update( + hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, + mp::AbstractManoptProblem, p, η, i_a::Int, i_b::Int, c::Real + ) + # U0 + if c < hzls.triples[i_a].t || c > hzls.triples[i_b].t + return (i_a, i_b, -1, false, false) + end + i_c, f_eval, f_wolfe = _hz_evaluate_next_step(hzls, M, mp, p, η, c) + if hzls.triples[i_c].df >= 0 + # U1 + return (i_a, i_c, i_c, f_eval, f_wolfe) + else + if hzls.triples[i_c].f <= hzls.triples[1].f + hzls.ϵₖ + # U2 + return (i_c, i_b, i_c, f_eval, f_wolfe) + else + if f_eval || f_wolfe + # termination condition met + return (i_a, i_b, i_c, f_eval, f_wolfe) + else + # U3 + i_a_bar, i_b_bar, f_eval, f_wolfe = _hz_u3(hzls, M, mp, p, η, i_a, i_b) + return (i_a_bar, i_b_bar, i_c, f_eval, f_wolfe) + end + end + end +end + +function _hz_u3( + hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, + mp::AbstractManoptProblem, p, η, i_a::Int, i_b::Int + ) + i_a_bar = i_a + i_b_bar = i_b + # the loop should typically terminate before exceeding the number of evaluations + f_eval = false + f_wolfe = false + while hzls.last_evaluation_index < length(hzls.triples) + # U3 (a) + d = (1 - hzls.θ) * hzls.triples[i_a].t + hzls.θ * hzls.triples[i_b].t + i_d, f_eval, f_wolfe = _hz_evaluate_next_step(hzls, M, mp, p, η, d) + if hzls.triples[i_d].df >= 0 || f_eval || f_wolfe + return (i_a_bar, i_d, f_eval, f_wolfe) + else + if hzls.triples[i_d].f <= hzls.triples[1].f + hzls.ϵₖ + # U3 (b) + i_a_bar = i_d + else + # U3 (c) + i_b_bar = i_d + end + end + end + return (i_a_bar, i_b_bar, f_eval, f_wolfe) +end + +function _hz_secant2( + hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, + mp::AbstractManoptProblem, p, η, i_a::Int, i_b::Int + ) + # S1 + c = secant(hzls.triples[i_a], hzls.triples[i_b]) + (i_A, i_B, i_c, f_eval, f_wolfe) = _hz_update(hzls, M, mp, p, η, i_a, i_b, c) + if f_eval || f_wolfe + # not present in the original algorithm, but this seems to be the right way to handle this case + return (i_A, i_B, i_c, f_eval, f_wolfe) + end + if i_c == i_B + # S2 + c_bar = secant(hzls.triples[i_b], hzls.triples[i_B]) + # S4, part 1 + return _hz_update(hzls, M, mp, p, η, i_A, i_B, c_bar) + elseif i_c == i_A + # S3 + c_bar = secant(hzls.triples[i_a], hzls.triples[i_A]) + # S4, part 1 + return _hz_update(hzls, M, mp, p, η, i_A, i_B, c_bar) + else + # S4, part 2 + return (i_A, i_B, i_c, f_eval, f_wolfe) + end +end + +function (hzls::HagerZhangLinesearchStepsize)( + mp::AbstractManoptProblem, + s::AbstractManoptSolverState, + k::Int, + η = (-get_gradient(mp, get_iterate(s))); + fp = get_cost(mp, get_iterate(s)), + kwargs..., + ) + M = get_manifold(mp) + p = get_iterate(s) + + dphi_0 = get_differential(mp, p, η; Y = hzls.temporary_tangent) + hzls.triples[1] = UnivariateTriple(0.0, fp, dphi_0) + hzls.last_evaluation_index = 1 + + # update Qₖ, Cₖ + hzls.Qₖ = 1 + hzls.Qₖ * hzls.Δ + hzls.Cₖ += (abs(fp) - hzls.Cₖ) / hzls.Qₖ + + if hzls.wolfe_condition_mode == :adaptive + # Checking the V3 condition + if abs(hzls.last_cost - fp) <= hzls.ω * hzls.Cₖ + hzls.current_mode = :approximate + end + end + + # L0, initialization + # guess initial alpha + α0 = hzls.initial_guess(mp, s, k, hzls.last_stepsize, η; lf0 = fp, Dlf0 = dphi_0) + + # handle stepsize limit + max_alpha = hzls.stepsize_limit + if :stop_when_stepsize_exceeds in keys(kwargs) + max_alpha = min( + kwargs[:stop_when_stepsize_exceeds], + max_alpha, + ) + end + α0 = min(α0, max_alpha) + + # L0, bracket(c) + (i_a_j, i_b_j, f_eval, f_wolfe) = _hz_bracket(hzls, M, mp, p, η, α0, max_alpha) + while !(f_eval || f_wolfe) + # L1 + (i_a, i_b, _i_c, f_eval, f_wolfe) = _hz_secant2(hzls, M, mp, p, η, i_a_j, i_b_j) + # L2 + # we additionally check that we can continue narrowing the bracket + if !(f_eval || f_wolfe) && hzls.triples[i_b].t - hzls.triples[i_a].t > hzls.γ * (hzls.triples[i_b_j].t - hzls.triples[i_a_j].t) + # secant2 did not reduce the bracket sufficiently + # we need to do bisection + (i_a, i_b, _i_c, f_eval, f_wolfe) = _hz_update( + hzls, M, mp, p, η, + i_a, i_b, + (hzls.triples[i_a].t + hzls.triples[i_b].t) / 2, + ) + end + # L3 + i_a_j, i_b_j = i_a, i_b + + # loop terminates when we generate a point satisfying T1 or T2, or when we run out + # of objective evaluations + end + + hzls.last_stepsize = hzls.triples[hzls.last_evaluation_index].t + hzls.last_cost = hzls.triples[hzls.last_evaluation_index].f + return hzls.last_stepsize +end + +function Base.show(io::IO, cbls::HagerZhangLinesearchStepsize) + return print( + io, + """ + HagerZhangLinesearch(; + initial_guess = $(cbls.initial_guess), + retraction_method = $(cbls.retraction_method), + vector_transport_method = $(cbls.vector_transport_method), + stepsize_limit = $(cbls.stepsize_limit), + max_bracket_iterations = $(cbls.max_bracket_iterations), + wolfe_condition_mode = $(cbls.wolfe_condition_mode), + ϵ = $(cbls.ϵ), + δ = $(cbls.δ), + σ = $(cbls.σ), + ω = $(cbls.ω), + θ = $(cbls.θ), + γ = $(cbls.γ), + η = $(cbls.η), + ρ = $(cbls.ρ), + Δ = $(cbls.Δ), + ψ₀ = $(cbls.ψ₀), + ψ₁ = $(cbls.ψ₁), + ψ₂ = $(cbls.ψ₂), + )""", + ) +end +function status_summary(cbls::HagerZhangLinesearchStepsize) + return "$(cbls)\nand a computed last stepsize of $(cbls.last_stepsize)" +end + +@doc """ + HagerZhangLinesearch(; kwargs...) + HagerZhangLinesearch(M::AbstractManifold; kwargs...) + +A functor representing the curvature minimizing cubic bracketing scheme introduced +in [HagerZhang:2006:2](@cite). + +# Keyword arguments + +$(_kwargs(:p)) to store an interim result + +$(_note(:ManifoldDefaultFactory, "HagerZhangLinesearch")) +""" +function HagerZhangLinesearch(args...; kwargs...) + return ManifoldDefaultsFactory(HagerZhangLinesearchStepsize, args...; kwargs...) +end diff --git a/test/plans/test_stepsize.jl b/test/plans/test_stepsize.jl index a5c46dd7e9..b763c3eba6 100644 --- a/test/plans/test_stepsize.jl +++ b/test/plans/test_stepsize.jl @@ -255,6 +255,93 @@ end clbs = CubicBracketingLinesearch(; sufficient_curvature = 1.0e-16, min_bracket_width = 0.0, initial_stepsize = 0.5)(M) @test clbs(dmp, gs, 1) ≈ 1 / 6 atol = 5.0e-4 end + @testset "secant numerical stability" begin + # Large offset, small interval + a = 1.0e7 + b = a + 1.0e-6 + + # Choose derivatives that differ slightly + ga = 1.0 + gb = nextfloat(ga) # smallest representable difference + + # minimizer using affine formula + x_ref = a - ga * (b - a) / (gb - ga) + + err_secant = abs( + Manopt.secant( + Manopt.UnivariateTriple(a, 0.0, ga), + Manopt.UnivariateTriple(b, 0.0, gb) + ) - x_ref + ) + @test err_secant < 1.0e-6 + end + @testset "HagerZhang Linesearch Stepsize" begin + M = Euclidean(2) + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 .* p + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) + p = [1.0, 2.0] + η = -grad_f(M, p) + gs = GradientDescentState(M; p = p) + + hzls = HagerZhangLinesearch()(M) + @test startswith(repr(hzls), "HagerZhangLinesearch(;") + @test startswith(Manopt.status_summary(hzls), "HagerZhangLinesearch(;") + @test Manopt.get_message(hzls) == "" + + α = hzls(dmp, gs, 1, η) + @test isfinite(α) + @test α > 0 + @test hzls.last_stepsize == α + @test hzls.last_cost <= f(M, p) + 1.0e-12 + + hzls_limit = Manopt.HagerZhangLinesearchStepsize(M; stepsize_limit = 0.05) + α_limit = hzls_limit(dmp, gs, 1, η) + @test α_limit <= 0.05 + eps(0.05) + @test hzls_limit.last_stepsize == α_limit + α_limit_kw = hzls_limit(dmp, gs, 2, η; stop_when_stepsize_exceeds = 0.01) + @test α_limit_kw <= 0.01 + eps(0.01) + + hzls_approx = Manopt.HagerZhangLinesearchStepsize( + M; wolfe_condition_mode = :approximate, stepsize_limit = 0.2 + ) + α_approx = hzls_approx(dmp, gs, 1, η) + @test α_approx > 0 + + @testset "termination modes" begin + hzls_std = Manopt.HagerZhangLinesearchStepsize( + M; + wolfe_condition_mode = :standard, + initial_guess = Manopt.ConstantInitialGuess(0.5), + max_function_evaluations = 5, + ) + α_std = hzls_std(dmp, gs, 1, η) + @test isapprox(α_std, 0.5; rtol = 1.0e-12, atol = 0.0) + @test hzls_std.current_mode == :standard + + hzls_adapt = Manopt.HagerZhangLinesearchStepsize( + M; + wolfe_condition_mode = :adaptive, + initial_guess = Manopt.ConstantInitialGuess(0.5), + initial_last_cost = f(M, p), + ω = 1.0, + max_function_evaluations = 5, + ) + α_adapt = hzls_adapt(dmp, gs, 1, η) + @test α_adapt > 0 + @test hzls_adapt.current_mode == :approximate + + hzls_eval = Manopt.HagerZhangLinesearchStepsize( + M; + wolfe_condition_mode = :standard, + initial_guess = Manopt.ConstantInitialGuess(1.0), + max_function_evaluations = 2, + ) + α_eval = hzls_eval(dmp, gs, 1, η) + @test α_eval > 0 + @test hzls_eval.last_evaluation_index == length(hzls_eval.triples) + end + end @testset "Distance over Gradients Stepsize" begin @testset "does not use sectional cuvature (Eucludian)" begin M = Euclidean(2) From 030d5882c0d7e46033ca5c92ed15fd7eeca4cdcc Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 2 Feb 2026 18:37:54 +0100 Subject: [PATCH 085/135] remove initial stepsize parameters; add some docs --- src/plans/stepsize/stepsize.jl | 61 ++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index 73ef049657..8fc7d831a2 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -2163,6 +2163,31 @@ $(_fields(:vector_transport_method)) $(_kwargs(:p; name = "candidate_point")) as temporary storage for candidates $(_kwargs(:retraction_method)) $(_kwargs(:vector_transport_method)) +* `initial_guess::AbstractInitialLinesearchGuess=HagerZhangInitialGuess()`: initial linesearch guess strategy +* `initial_last_stepsize::Real = NaN`: initial value for the stored last stepsize +* `initial_last_cost::Real = NaN`: initial value for the stored last cost +* `stepsize_limit::Real = Inf`: upper bound for trial stepsizes during bracketing +* `candidate_point = allocate_result(M, rand)`: storage for trial points +* `candidate_direction = zero_vector(M, candidate_point)`: storage for transported directions +* `max_bracket_iterations::Int = 10`: maximum number of bracketing iterations +* `start_enforcing_wolfe_conditions_at_bracketing_iteration::Int = 2`: bracketing iteration + number at which Wolfe conditions are started to be enforced; setting to 1 may cause no + bracketing to occur when the initial guess satisfies the Wolfe conditions +* `max_function_evaluations::Int = 20`: maximum number of function evaluations per linesearch +* `wolfe_condition_mode::Symbol = :adaptive`: one of `:standard`, `:approximate`, or `:adaptive`. + Selects between (T1) and (T2) conditions in [HagerZhang:2006:2](@cite). +* `ϵ::Real = 1.0e-6`: initial allowed increase in function value in termination condition (T2). + Allowed range: `ϵ >= 0`. +* `δ::Real = 0.1`: parameter for approximate Wolfe condition. + Allowed range: `0 < δ < 0.5` and `δ <= σ`. +* `σ::Real = 0.9`: curvature condition parameter. Allowed range: `δ <= σ < 1`. +* `ω::Real = 1.0e-3`: interpolation safeguard parameter. Allowed range: `0 <= ω <= 1`. +* `θ::Real = 0.5`: bisection update parameter. Allowed range: `0 < θ < 1`. +* `γ::Real = 0.66`: determines when a bisection step is performed instead of secant. + Allowed range: `0 < γ < 1`. +* `ρ::Real = 5.0`: bracketing expansion factor. Allowed range: `ρ > 1`. +* `Δ::Real = 0.7`: Parameter controlling the rate of change of Qₖ. + Allowed range: `0 <= Δ <= 1`. """ mutable struct HagerZhangLinesearchStepsize{ TF <: Real, @@ -2178,6 +2203,7 @@ mutable struct HagerZhangLinesearchStepsize{ vector_transport_method::TVTM stepsize_limit::TF max_bracket_iterations::Int + start_enforcing_wolfe_conditions_at_bracketing_iteration::Int wolfe_condition_mode::Symbol # :standard, :approximate, :adaptive ϵ::TF # approximate Wolfe termination parameter δ::TF # used in approximate Wolfe condition @@ -2185,12 +2211,8 @@ mutable struct HagerZhangLinesearchStepsize{ ω::TF θ::TF # update rule parameter γ::TF - η::TF ρ::TF Δ::TF - ψ₀::TF - ψ₁::TF - ψ₂::TF # storage for candidates candidate_point::TP candidate_direction::TX @@ -2217,6 +2239,7 @@ mutable struct HagerZhangLinesearchStepsize{ candidate_point = allocate_result(M, rand), candidate_direction = zero_vector(M, candidate_point), max_bracket_iterations::Int = 10, + start_enforcing_wolfe_conditions_at_bracketing_iteration::Int = initial_guess isa ConstantStepsize ? 2 : 1, max_function_evaluations::Int = 20, wolfe_condition_mode::Symbol = :adaptive, ϵ::TF = 1.0e-6, @@ -2225,12 +2248,8 @@ mutable struct HagerZhangLinesearchStepsize{ ω::TF = 1.0e-3, θ::TF = 0.5, γ::TF = 0.66, - η::TF = 0.01, ρ::TF = 5.0, Δ::TF = 0.7, - ψ₀::TF = 0.01, - ψ₁::TF = 0.1, - ψ₂::TF = 2.0, ) where { TIG <: AbstractInitialLinesearchGuess, TRM <: AbstractRetractionMethod, TVTM <: AbstractVectorTransportMethod, TF <: Real, @@ -2245,11 +2264,7 @@ mutable struct HagerZhangLinesearchStepsize{ @assert Δ >= 0 && Δ <= 1 @assert θ > 0 && θ < 1 @assert γ > 0 && γ < 1 - @assert η > 0 @assert ρ > 1 - @assert ψ₀ > 0 && ψ₀ < 1 - @assert ψ₁ > 0 && ψ₁ < 1 - @assert ψ₂ > 1 @assert stepsize_limit > 0 @assert wolfe_condition_mode in (:standard, :approximate, :adaptive) @@ -2260,8 +2275,8 @@ mutable struct HagerZhangLinesearchStepsize{ return new{TF, TIG, TRM, TVTM, typeof(candidate_point), typeof(candidate_direction)}( initial_guess, retraction_method, vector_transport_method, stepsize_limit, - max_bracket_iterations, wolfe_condition_mode, - ϵ, δ, σ, ω, θ, γ, η, ρ, Δ, ψ₀, ψ₁, ψ₂, + max_bracket_iterations, start_enforcing_wolfe_conditions_at_bracketing_iteration, wolfe_condition_mode, + ϵ, δ, σ, ω, θ, γ, ρ, Δ, candidate_point, candidate_direction, zero_vector(M, candidate_point), triples, 0, 0.0, 0.0, # Qₖ, Cₖ @@ -2381,7 +2396,7 @@ function _hz_bracket( local c_index, f_eval, f_wolfe for j in 1:hzls.max_bracket_iterations c_index, f_eval, f_wolfe = _hz_evaluate_next_step(hzls, M, mp, p, η, current_step) - if f_eval || f_wolfe + if f_eval || (f_wolfe && j >= hzls.start_enforcing_wolfe_conditions_at_bracketing_iteration) break end if hzls.triples[c_index].df >= 0 @@ -2563,10 +2578,18 @@ function (hzls::HagerZhangLinesearchStepsize)( (i_a_j, i_b_j, f_eval, f_wolfe) = _hz_bracket(hzls, M, mp, p, η, α0, max_alpha) while !(f_eval || f_wolfe) # L1 - (i_a, i_b, _i_c, f_eval, f_wolfe) = _hz_secant2(hzls, M, mp, p, η, i_a_j, i_b_j) + finite_at_b = isfinite(hzls.triples[i_b_j].f) + if finite_at_b + # _hz_secant2 only makes sense if we have finite function values at both ends + # but _hz_update may still work + (i_a, i_b, _i_c, f_eval, f_wolfe) = _hz_secant2(hzls, M, mp, p, η, i_a_j, i_b_j) + else + (i_a, i_b) = (i_a_j, i_b_j) + end # L2 # we additionally check that we can continue narrowing the bracket - if !(f_eval || f_wolfe) && hzls.triples[i_b].t - hzls.triples[i_a].t > hzls.γ * (hzls.triples[i_b_j].t - hzls.triples[i_a_j].t) + if !(f_eval || f_wolfe) && + (!finite_at_b || hzls.triples[i_b].t - hzls.triples[i_a].t > hzls.γ * (hzls.triples[i_b_j].t - hzls.triples[i_a_j].t)) # secant2 did not reduce the bracket sufficiently # we need to do bisection (i_a, i_b, _i_c, f_eval, f_wolfe) = _hz_update( @@ -2604,12 +2627,8 @@ function Base.show(io::IO, cbls::HagerZhangLinesearchStepsize) ω = $(cbls.ω), θ = $(cbls.θ), γ = $(cbls.γ), - η = $(cbls.η), ρ = $(cbls.ρ), Δ = $(cbls.Δ), - ψ₀ = $(cbls.ψ₀), - ψ₁ = $(cbls.ψ₁), - ψ₂ = $(cbls.ψ₂), )""", ) end From 9c1d35829718dcd5baafb156cef1404feb26f486 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 3 Feb 2026 13:25:26 +0100 Subject: [PATCH 086/135] fix edge case issues of HZ line search; expand docs on GCD; use line search restart in QN; improve coverage; respect stepsize limit in HagerZhangInitialGuess --- .../generalized_cauchy_direction_subsolver.md | 3 +- src/plans/stepsize/initial_guess.jl | 8 + src/plans/stepsize/linesearch.jl | 12 + src/plans/stepsize/stepsize.jl | 31 ++- src/solvers/quasi_Newton.jl | 1 + test/plans/test_stepsize.jl | 218 ++++++++++++++++++ 6 files changed, 266 insertions(+), 7 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index 1d8b26b852..d620c96e80 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -11,7 +11,8 @@ The Generalized Cauchy Direction (GCD) subsolver is a component in optimization where $X=(X_{\mathrm{D}}, X_{\mathcal{M}})$ is a given direction, the exponential map handles projection of the tangent vector when reaching the boundary, $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, $X_g$ is the gradient of a scalar function $f$ at point $p=(p_{\mathrm{D}}, p_{\mathcal{M}})$, $A$ is the maximum allowed step size on $\mathcal{M}$ at point $p=(p_{\mathrm{D}}, p_{\mathcal{M}})$ in direction $X_{\mathcal{M}}$ (infinity is supported) and $\mathcal{H}_p$ is a linear operator that approximates the Hessian of $f$ at $p$. -Additionally, the subsolver indicates whether the selected direction $Y$ reaches the boundary of $D$ at some point, in which case the subsequent step size selection in direction $Y$ needs to be limited to the interval $[0, 1]$. +Additionally, the subsolver indicates whether the selected direction $Y$ reaches the boundary of $D$ at some point, in which case the subsequent step size selection in direction $Y$ needs to be limited to the interval $[0, s_{\max}]$, where the number $1 \leq s_{\max} \leq \infty$ is also returned by the subsolver. +Note that the value $s_{\max}=1$ is obtained when the minimum lies at the boundary of $D$, while larger values indicate that we are further away from the boundary along the selected direction $X$. The solver is currently primarily intended for internal use by optimization algorithms that require bound-constrained subproblem solutions. diff --git a/src/plans/stepsize/initial_guess.jl b/src/plans/stepsize/initial_guess.jl index 4c9788b6a8..be6d67b7d1 100644 --- a/src/plans/stepsize/initial_guess.jl +++ b/src/plans/stepsize/initial_guess.jl @@ -145,6 +145,7 @@ function (hzi::HagerZhangInitialGuess{TF})( k::Int, last_stepsize::Real, η; lf0 = get_cost(mp, get_iterate(s)), Dlf0 = get_differential(mp, get_iterate(s), η), + kwargs... ) where {TF <: Real} M = get_manifold(mp) p = get_iterate(s) @@ -152,6 +153,13 @@ function (hzi::HagerZhangInitialGuess{TF})( alphamax = min(hzi.alphamax, max_stepsize(M, p)) + if :stop_when_stepsize_exceeds in keys(kwargs) + alphamax = min( + kwargs[:stop_when_stepsize_exceeds], + alphamax, + ) + end + if k == 1 point_d = hzi.point_distance(M, p) # Step I0 diff --git a/src/plans/stepsize/linesearch.jl b/src/plans/stepsize/linesearch.jl index ff516af04e..b48e4d72ed 100644 --- a/src/plans/stepsize/linesearch.jl +++ b/src/plans/stepsize/linesearch.jl @@ -23,6 +23,18 @@ abstract type Stepsize end get_message(::S) where {S <: Stepsize} = "" +""" + initialize_stepsize!(sm::Stepsize) + +Initialize the state of a stepsize functor. This is called at the beginning of a solver run, +and can be used to set up internal state of the stepsize functor that is preserved between +line searches in the same optimization, for example adaptive thresholds for Wolfe criteria +in Hager-Zhang line search. + +By default it does nothing. +""" +initialize_stepsize!(sm::Stepsize) = sm + """ default_stepsize(M::AbstractManifold, ams::AbstractManoptSolverState) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index 8fc7d831a2..7890b6dc90 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -2213,6 +2213,7 @@ mutable struct HagerZhangLinesearchStepsize{ γ::TF ρ::TF Δ::TF + secant_acceptance_ratio::TF # storage for candidates candidate_point::TP candidate_direction::TX @@ -2250,6 +2251,7 @@ mutable struct HagerZhangLinesearchStepsize{ γ::TF = 0.66, ρ::TF = 5.0, Δ::TF = 0.7, + secant_acceptance_ratio::TF = 1.0e-8, ) where { TIG <: AbstractInitialLinesearchGuess, TRM <: AbstractRetractionMethod, TVTM <: AbstractVectorTransportMethod, TF <: Real, @@ -2276,7 +2278,7 @@ mutable struct HagerZhangLinesearchStepsize{ return new{TF, TIG, TRM, TVTM, typeof(candidate_point), typeof(candidate_direction)}( initial_guess, retraction_method, vector_transport_method, stepsize_limit, max_bracket_iterations, start_enforcing_wolfe_conditions_at_bracketing_iteration, wolfe_condition_mode, - ϵ, δ, σ, ω, θ, γ, ρ, Δ, + ϵ, δ, σ, ω, θ, γ, ρ, Δ, secant_acceptance_ratio, candidate_point, candidate_direction, zero_vector(M, candidate_point), triples, 0, 0.0, 0.0, # Qₖ, Cₖ @@ -2346,6 +2348,12 @@ function _hz_evaluate_next_step( ) triples = hzls.triples max_evals = length(triples) + for ti in 1:hzls.last_evaluation_index + t = triples[ti] + if t.t == α + error("Hager-Zhang linesearch attempted to evaluate at previously evaluated stepsize $α (index $ti).") + end + end if hzls.last_evaluation_index + 1 > max_evals # this should never happen if the calling code is correct error("Hager-Zhang linesearch exceeded maximum number of function evaluations $(length(hzls.triples)).") @@ -2393,7 +2401,7 @@ function _hz_bracket( ) # B0 current_step = c - local c_index, f_eval, f_wolfe + local c_index, f_eval, f_wolfe # COV_EXCL_LINE for j in 1:hzls.max_bracket_iterations c_index, f_eval, f_wolfe = _hz_evaluate_next_step(hzls, M, mp, p, η, current_step) if f_eval || (f_wolfe && j >= hzls.start_enforcing_wolfe_conditions_at_bracketing_iteration) @@ -2472,7 +2480,7 @@ function _hz_update( return (i_a, i_b, i_c, f_eval, f_wolfe) else # U3 - i_a_bar, i_b_bar, f_eval, f_wolfe = _hz_u3(hzls, M, mp, p, η, i_a, i_b) + i_a_bar, i_b_bar, f_eval, f_wolfe = _hz_u3(hzls, M, mp, p, η, i_a, i_c) return (i_a_bar, i_b_bar, i_c, f_eval, f_wolfe) end end @@ -2490,7 +2498,7 @@ function _hz_u3( f_wolfe = false while hzls.last_evaluation_index < length(hzls.triples) # U3 (a) - d = (1 - hzls.θ) * hzls.triples[i_a].t + hzls.θ * hzls.triples[i_b].t + d = (1 - hzls.θ) * hzls.triples[i_a_bar].t + hzls.θ * hzls.triples[i_b_bar].t i_d, f_eval, f_wolfe = _hz_evaluate_next_step(hzls, M, mp, p, η, d) if hzls.triples[i_d].df >= 0 || f_eval || f_wolfe return (i_a_bar, i_d, f_eval, f_wolfe) @@ -2513,6 +2521,14 @@ function _hz_secant2( ) # S1 c = secant(hzls.triples[i_a], hzls.triples[i_b]) + width = hzls.triples[i_b].t - hzls.triples[i_a].t + if abs(c - hzls.triples[i_a].t) < hzls.secant_acceptance_ratio * width || + abs(c - hzls.triples[i_b].t) < hzls.secant_acceptance_ratio * width + # secant too close to an endpoint, use bisection instead + # this case is not present in the original algorithm, but the following steps don't make much sense in this case + c = (hzls.triples[i_a].t + hzls.triples[i_b].t) / 2 + return _hz_update(hzls, M, mp, p, η, i_a, i_b, c) + end (i_A, i_B, i_c, f_eval, f_wolfe) = _hz_update(hzls, M, mp, p, η, i_a, i_b, c) if f_eval || f_wolfe # not present in the original algorithm, but this seems to be the right way to handle this case @@ -2561,8 +2577,6 @@ function (hzls::HagerZhangLinesearchStepsize)( end # L0, initialization - # guess initial alpha - α0 = hzls.initial_guess(mp, s, k, hzls.last_stepsize, η; lf0 = fp, Dlf0 = dphi_0) # handle stepsize limit max_alpha = hzls.stepsize_limit @@ -2572,9 +2586,14 @@ function (hzls::HagerZhangLinesearchStepsize)( max_alpha, ) end + # guess initial alpha + α0 = hzls.initial_guess(mp, s, k, hzls.last_stepsize, η; lf0 = fp, Dlf0 = dphi_0, stop_when_stepsize_exceeds = max_alpha) + + # in case initial_guess does not take into account the stepsize limit, we enforce it here α0 = min(α0, max_alpha) # L0, bracket(c) + local i_a_j, i_b_j, f_eval, f_wolfe # COV_EXCL_LINE (i_a_j, i_b_j, f_eval, f_wolfe) = _hz_bracket(hzls, M, mp, p, η, α0, max_alpha) while !(f_eval || f_wolfe) # L1 diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 41a4872c28..385ec7d2e1 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -404,6 +404,7 @@ function initialize_solver!(amp::AbstractManoptProblem, qns::QuasiNewtonState) copyto!(M, qns.sk, qns.p, qns.X) copyto!(M, qns.yk, qns.p, qns.X) initialize_update!(qns.direction_update) + initialize_stepsize!(qns.stepsize) return qns end function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) diff --git a/test/plans/test_stepsize.jl b/test/plans/test_stepsize.jl index b763c3eba6..b70edd3805 100644 --- a/test/plans/test_stepsize.jl +++ b/test/plans/test_stepsize.jl @@ -301,6 +301,19 @@ end @test hzls_limit.last_stepsize == α_limit α_limit_kw = hzls_limit(dmp, gs, 2, η; stop_when_stepsize_exceeds = 0.01) @test α_limit_kw <= 0.01 + eps(0.01) + @testset "Running out of evaluations in _hz_evaluate_next_step" begin + N = length(hzls_limit.triples) - hzls_limit.last_evaluation_index + for i in 1:N + Manopt._hz_evaluate_next_step(hzls_limit, M, dmp, p, η, 0.1) + end + @test_throws ErrorException Manopt._hz_evaluate_next_step(hzls_limit, M, dmp, p, η, 0.1) + end + @testset "Wolfe condition modes" begin + hzls_default = Manopt.HagerZhangLinesearchStepsize(M) + hzls.current_mode = :invalid_mode + @test_throws ErrorException hzls(dmp, gs, 1, η) + end + hzls_approx = Manopt.HagerZhangLinesearchStepsize( M; wolfe_condition_mode = :approximate, stepsize_limit = 0.2 @@ -341,6 +354,211 @@ end @test α_eval > 0 @test hzls_eval.last_evaluation_index == length(hzls_eval.triples) end + @testset "B1 bracketing test" begin + M = Euclidean(1) + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 .* p + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) + p = [1.0] + η = -grad_f(M, p) + gs = GradientDescentState(M; p = p) + hzls_b1 = Manopt.HagerZhangLinesearchStepsize( + M; + initial_guess = Manopt.ConstantInitialGuess(0.75), + start_enforcing_wolfe_conditions_at_bracketing_iteration = 2, + max_bracket_iterations = 1, + ) + α_b1 = hzls_b1(dmp, gs, 1, η) + @test α_b1 > 0 + end + @testset "B2 bracketing test" begin + M = Euclidean(1) + # f(x) = -22 x^3 + 33 x^2 - x + # grad_f(x) = -66 x^2 + 66 x - 1 + f(M, p) = -22 * p[1]^3 + 33 * p[1]^2 - p[1] + grad_f(M, p) = [-66 * p[1]^2 + 66 * p[1] - 1] + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) + p = [0.0] + η = [1.0] # Descent direction + gs = GradientDescentState(M; p = p) + hzls_b2 = Manopt.HagerZhangLinesearchStepsize( + M; + initial_guess = Manopt.ConstantInitialGuess(1.0), + start_enforcing_wolfe_conditions_at_bracketing_iteration = 2, + max_bracket_iterations = 2, + ) + α = hzls_b2(dmp, gs, 1, η) + @test α > 0 + end + @testset "B3 bracketing test" begin + M = Euclidean(1) + # f(x) = -x + # grad_f(x) = -1 + f(M, p) = -p[1] + grad_f(M, p) = [-1.0] + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) + p = [0.0] + η = [1.0] # Descent direction + gs = GradientDescentState(M; p = p) + hzls_b3 = Manopt.HagerZhangLinesearchStepsize( + M; + initial_guess = Manopt.ConstantInitialGuess(1.0), + stepsize_limit = 2.0, + max_bracket_iterations = 2, + ) + α = hzls_b3(dmp, gs, 1, η) + @test α > 0 + end + @testset "U1 trigger test" begin + M = Euclidean(1) + # f(x) = x^2 / 2 + # grad_f(x) = x + f(M, p) = p[1]^2 / 2 + grad_f(M, p) = [p[1]] + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) + p = [1.0] + η = [-1.0] # Descent direction + gs = GradientDescentState(M; p = p) + hzls_u1 = Manopt.HagerZhangLinesearchStepsize( + M; + initial_guess = Manopt.ConstantInitialGuess(2.0), + ) + # We expect U1 to be triggered during the update (secant is exact, slope 0 >= 0) + α = hzls_u1(dmp, gs, 1, η) + @test α > 0 + end + @testset "U2 trigger test" begin + M = Euclidean(1) + # We mock f and grad_f to trigger U2 termination + # We need: + # 1. Starting at p=0 with descent direction (df < 0) + # 2. Bracketing finds a point with df > 0 (to finish bracketing) -> p=1.0, df=1.0 + # 3. Refinement hits max evaluations at a point with df < 0 and f > f(0)+eps -> p=0.5, f=10.0, df=-0.1 + + function f_u2(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-9) + return 0.0 + elseif isapprox(v, 1.0; atol = 1.0e-9) + return 0.0 + elseif isapprox(v, 0.5; atol = 1.0e-9) + return 10.0 + end + return 0.0 + end + + function grad_f_u2(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-9) + return [-1.0] + elseif isapprox(v, 1.0; atol = 1.0e-9) + return [1.0] + elseif isapprox(v, 0.5; atol = 1.0e-9) + return [-0.1] + end + return [0.0] + end + + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f_u2, grad_f_u2)) + p = [0.0] + η = [1.0] + gs = GradientDescentState(M; p = p) + hzls_u2 = Manopt.HagerZhangLinesearchStepsize( + M; initial_guess = Manopt.ConstantInitialGuess(1.0), max_function_evaluations = 3 + ) + α = hzls_u2(dmp, gs, 1, η) + @test α > 0 + end + @testset "U3 trigger test" begin + M = Euclidean(1) + # Trigger U3 by having a point that satisfies conditions for U2 but f_eval is false. + # Same landscape as U2: + # p=0, df=-1 (start) + # p=1, df=1 (end of bracket) + # p=0.5, f=10, df=-0.1 (high function value, negative slope) + + function f_u3(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-9) + return 0.0 + elseif isapprox(v, 1.0; atol = 1.0e-9) + return 0.0 + elseif isapprox(v, 0.5; atol = 1.0e-9) + return 10.0 + end + return 0.0 + end + + function grad_f_u3(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-9) + return [-1.0] + elseif isapprox(v, 1.0; atol = 1.0e-9) + return [1.0] + elseif isapprox(v, 0.5; atol = 1.0e-9) + return [-0.1] + end + return [0.0] + end + + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f_u3, grad_f_u3)) + p = [0.0] + η = [1.0] # Descent direction + gs = GradientDescentState(M; p = p) + # Set max_function_evaluations > 3 so we don't hit U2 termination (f_eval=true) + hzls_u3 = Manopt.HagerZhangLinesearchStepsize( + M; initial_guess = Manopt.ConstantInitialGuess(1.0), max_function_evaluations = 5 + ) + α = hzls_u3(dmp, gs, 1, η) + @test α > 0 + end + @testset "S2 trigger test" begin + M = Euclidean(1) + # S2 is triggered within _hz_secant2 when the updated bracket point i_c is the new upper bound i_B + # This happens if slope at c is positive (U1 case in _hz_update). + # Sequence: + # 1. Start p=0, df=-1. + # 2. Initial bracket p=1, df=4 (df > 0 -> bracket found). + # 3. _hz_secant2 calls secant(0, 1) -> c = (0*4 - 1*(-1))/(4 - (-1)) = 0.2. + # 4. At c=0.2, we set df=0.1 (positive slope -> U1 -> i_c = i_B). + # 5. We also need f(0.2) high enough to fail Armijo so we don't return early with f_wolfe=true. + # f(0)=0. f(0.2)=0.5. Armijo check: 0.5 <= 0 + 0.1*0.2*(-1) = -0.02 (False). + + function f_s2(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-9) + return 0.0 + elseif isapprox(v, 1.0; atol = 1.0e-9) + return 2.0 # Arbitrary high value + elseif isapprox(v, 0.2; atol = 1.0e-9) + return 0.5 # Fail Armijo + end + return 0.0 # Fallback (e.g. for c_bar in S2) + end + + function grad_f_s2(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-9) + return [-1.0] + elseif isapprox(v, 1.0; atol = 1.0e-9) + return [4.0] + elseif isapprox(v, 0.2; atol = 1.0e-9) + return [0.1] # Positive slope triggers U1 -> i_c = i_B + end + return [0.0] + end + + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f_s2, grad_f_s2)) + p = [0.0] + η = [1.0] + gs = GradientDescentState(M; p = p) + hzls_s2 = Manopt.HagerZhangLinesearchStepsize( + M; initial_guess = Manopt.ConstantInitialGuess(1.0) + ) + # We expect the S2 log + α = hzls_s2(dmp, gs, 1, η) + @test α > 0 + end end @testset "Distance over Gradients Stepsize" begin @testset "does not use sectional cuvature (Eucludian)" begin From b97b85176336bce599cf508c71a6c0f4836f7349 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 3 Feb 2026 13:37:07 +0100 Subject: [PATCH 087/135] test one more edge case --- test/plans/test_stepsize.jl | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/plans/test_stepsize.jl b/test/plans/test_stepsize.jl index b70edd3805..6386aa11cd 100644 --- a/test/plans/test_stepsize.jl +++ b/test/plans/test_stepsize.jl @@ -559,6 +559,50 @@ end α = hzls_s2(dmp, gs, 1, η) @test α > 0 end + + @testset "Hager-Zhang infinite at b" begin + # A function that is finite for small steps but infinite for larger ones + # and has positive slope where it is infinite to trigger the bracket condition. + + M = Euclidean(1) + + # f(x) = x^2 - x for x < 1.0 + # f(x) = Inf for x >= 1.0 + # Min at x = 0.5, f(0.5) = -0.25 + function f_inf(M, p) + x = p[1] + if x < 1.0 + return x^2 - x + else + return Inf + end + end + + function grad_f_inf(M, p) + x = p[1] + if x < 1.0 + return [2 * x - 1] + else + # Return a positive slope to satisfy _hz_bracket exit condition + return [1.0] + end + end + + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f_inf, grad_f_inf)) + + # Start at 0. f(0)=0. grad(0)=-1. Search direction +1. + s = GradientDescentState(M; p = [0.0]) + + # Force initial guess to be 2.0 (in the infinite region) + hzls = HagerZhangLinesearch(; initial_guess = Manopt.ConstantInitialGuess(2.0))(M) + + # Because initial bracket will be [0, 2] with f(2)=Inf. + # Then bisection will eventually find 0.5. + + step = hzls(dmp, s, 1, [1.0]) + @test abs(step - 0.5) < 1.0e-1 + end + end @testset "Distance over Gradients Stepsize" begin @testset "does not use sectional cuvature (Eucludian)" begin From 89ee8d060f57670ce9fb3d4a69afbbb94d71f279 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 3 Feb 2026 13:42:21 +0100 Subject: [PATCH 088/135] test initialize_stepsize! for HZ --- test/plans/test_stepsize.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/plans/test_stepsize.jl b/test/plans/test_stepsize.jl index 6386aa11cd..b518262c78 100644 --- a/test/plans/test_stepsize.jl +++ b/test/plans/test_stepsize.jl @@ -603,6 +603,19 @@ end @test abs(step - 0.5) < 1.0e-1 end + @testset "Hager-Zhang initialize_stepsize!" begin + hzls = HagerZhangLinesearch()(M) + hzls.last_evaluation_index = 5 + hzls.Qₖ = 2.0 + hzls.Cₖ = 2.0 + hzls.current_mode = :approximate + Manopt.initialize_stepsize!(hzls) + @test hzls.last_evaluation_index == 0 + @test hzls.Qₖ == 0.0 + @test hzls.Cₖ == 0.0 + @test hzls.current_mode == :standard + end + end @testset "Distance over Gradients Stepsize" begin @testset "does not use sectional cuvature (Eucludian)" begin From aebaadb3ccda3d1f80b8868c4646534640f263e0 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 3 Feb 2026 16:37:27 +0100 Subject: [PATCH 089/135] improve coverage --- src/plans/stepsize/stepsize.jl | 6 -- test/plans/test_stepsize.jl | 124 +++++++++++++++++++++++++++++++-- 2 files changed, 118 insertions(+), 12 deletions(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index 7890b6dc90..c7c8e1d502 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -2348,12 +2348,6 @@ function _hz_evaluate_next_step( ) triples = hzls.triples max_evals = length(triples) - for ti in 1:hzls.last_evaluation_index - t = triples[ti] - if t.t == α - error("Hager-Zhang linesearch attempted to evaluate at previously evaluated stepsize $α (index $ti).") - end - end if hzls.last_evaluation_index + 1 > max_evals # this should never happen if the calling code is correct error("Hager-Zhang linesearch exceeded maximum number of function evaluations $(length(hzls.triples)).") diff --git a/test/plans/test_stepsize.jl b/test/plans/test_stepsize.jl index b518262c78..a59a0ca61b 100644 --- a/test/plans/test_stepsize.jl +++ b/test/plans/test_stepsize.jl @@ -277,11 +277,11 @@ end end @testset "HagerZhang Linesearch Stepsize" begin M = Euclidean(2) - f(M, p) = sum(p .^ 2) - grad_f(M, p) = 2 .* p - dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) + f_sum_sq(M, p) = sum(p .^ 2) + grad_f_sum_sq(M, p) = 2 .* p + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f_sum_sq, grad_f_sum_sq)) p = [1.0, 2.0] - η = -grad_f(M, p) + η = -grad_f_sum_sq(M, p) gs = GradientDescentState(M; p = p) hzls = HagerZhangLinesearch()(M) @@ -293,7 +293,7 @@ end @test isfinite(α) @test α > 0 @test hzls.last_stepsize == α - @test hzls.last_cost <= f(M, p) + 1.0e-12 + @test hzls.last_cost <= f_sum_sq(M, p) + 1.0e-12 hzls_limit = Manopt.HagerZhangLinesearchStepsize(M; stepsize_limit = 0.05) α_limit = hzls_limit(dmp, gs, 1, η) @@ -336,7 +336,7 @@ end M; wolfe_condition_mode = :adaptive, initial_guess = Manopt.ConstantInitialGuess(0.5), - initial_last_cost = f(M, p), + initial_last_cost = f_sum_sq(M, p), ω = 1.0, max_function_evaluations = 5, ) @@ -512,6 +512,67 @@ end α = hzls_u3(dmp, gs, 1, η) @test α > 0 end + @testset "U3 (c) info trigger test" begin + M = Euclidean(1) + # Force U3 (c) inside _hz_u3 by making the mid-point have + # negative slope but too large function value. + function f_u3c(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-12) + return 0.0 + elseif isapprox(v, 1.0; atol = 1.0e-12) + return 0.0 + elseif isapprox(v, 0.5; atol = 1.0e-12) + return 1.0 + elseif isapprox(v, 0.25; atol = 1.0e-12) + return 0.0 + end + return 0.0 + end + + function grad_f_u3c(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-12) + return [-1.0] + elseif isapprox(v, 1.0; atol = 1.0e-12) + return [1.0] + elseif isapprox(v, 0.5; atol = 1.0e-12) + return [-0.1] + elseif isapprox(v, 0.25; atol = 1.0e-12) + return [0.1] + end + return [0.0] + end + + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f_u3c, grad_f_u3c)) + p = [0.0] + η = [1.0] + hzls_u3c = Manopt.HagerZhangLinesearchStepsize(M; max_function_evaluations = 4) + Manopt.initialize_stepsize!(hzls_u3c) + Manopt._hz_evaluate_next_step(hzls_u3c, M, dmp, p, η, 0.0) + Manopt._hz_evaluate_next_step(hzls_u3c, M, dmp, p, η, 1.0) + @test (1, 4, true, false) == Manopt._hz_u3(hzls_u3c, M, dmp, p, η, 1, 2) + end + @testset "U3 max evaluations termination" begin + M = Euclidean(1) + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 .* p + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) + p = [0.0] + η = [1.0] + + hzls_u3_max = Manopt.HagerZhangLinesearchStepsize(M; max_function_evaluations = 2) + Manopt.initialize_stepsize!(hzls_u3_max) + Manopt._hz_evaluate_next_step(hzls_u3_max, M, dmp, p, η, 0.0) + Manopt._hz_evaluate_next_step(hzls_u3_max, M, dmp, p, η, 1.0) + @test hzls_u3_max.last_evaluation_index == length(hzls_u3_max.triples) + + (i_a, i_b, f_eval, f_wolfe) = Manopt._hz_u3(hzls_u3_max, M, dmp, p, η, 1, 2) + @test (i_a, i_b) == (1, 2) + @test !f_eval + @test !f_wolfe + end + @testset "S2 trigger test" begin M = Euclidean(1) # S2 is triggered within _hz_secant2 when the updated bracket point i_c is the new upper bound i_B @@ -560,6 +621,57 @@ end @test α > 0 end + @testset "S3 trigger test" begin + M = Euclidean(1) + # S3 is triggered within _hz_secant2 when the updated bracket point i_c is the new lower bound i_A + # (U2 case in _hz_update). We set up: + # 1. Start p=0, df=-1 (descent). + # 2. Bracket at p=1, df=4 (positive slope). + # 3. Secant gives c=0.2. At c, df=-0.1 and f=0 -> U2. + + function f_s3(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-12) + return 0.0 + elseif isapprox(v, 1.0; atol = 1.0e-12) + return 0.0 + elseif isapprox(v, 0.2; atol = 1.0e-12) + return 0.0 + end + return 0.0 + end + + function grad_f_s3(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-12) + return [-1.0] + elseif isapprox(v, 1.0; atol = 1.0e-12) + return [4.0] + elseif isapprox(v, 0.2; atol = 1.0e-12) + return [-0.1] + end + return [0.0] + end + + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f_s3, grad_f_s3)) + p = [0.0] + η = [1.0] + hzls_s3 = Manopt.HagerZhangLinesearchStepsize(M; max_function_evaluations = 5) + Manopt.initialize_stepsize!(hzls_s3) + Manopt._hz_evaluate_next_step(hzls_s3, M, dmp, p, η, 0.0) + Manopt._hz_evaluate_next_step(hzls_s3, M, dmp, p, η, 1.0) + + c = Manopt.secant(hzls_s3.triples[1], hzls_s3.triples[2]) + (i_A, i_B, i_c, f_eval, f_wolfe) = Manopt._hz_secant2(hzls_s3, M, dmp, p, η, 1, 2) + @test !f_eval + @test !f_wolfe + @test hzls_s3.triples[i_A].t ≈ c atol = 1.0e-12 + + c_bar = Manopt.secant(hzls_s3.triples[1], hzls_s3.triples[i_A]) + @test hzls_s3.triples[i_c].t ≈ c_bar atol = 1.0e-12 + @test i_A != i_B + end + @testset "Hager-Zhang infinite at b" begin # A function that is finite for small steps but infinite for larger ones # and has positive slope where it is infinite to trigger the bracket condition. From 8ba964cefd0dbb92f64f10c7d45a7532e673bb17 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 3 Feb 2026 20:37:32 +0100 Subject: [PATCH 090/135] optimize quasi_Newton step --- src/solvers/quasi_Newton.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 385ec7d2e1..606eab0b6a 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -409,7 +409,8 @@ function initialize_solver!(amp::AbstractManoptProblem, qns::QuasiNewtonState) end function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) M = get_manifold(mp) - get_gradient!(mp, qns.X, qns.p) + # qns.X should be the correct gradient at qns.p from initialization or the previous step + # get_gradient!(mp, qns.X, qns.p) qns.direction_update(qns.η, mp, qns) current_max_stepsize = _get_max_stepsize(M, qns) if !(qns.nondescent_direction_behavior === :ignore) From 8d9540a121d903c69d40437266bb8356bd38e8d5 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 5 Feb 2026 11:19:53 +0100 Subject: [PATCH 091/135] more docs for HZ, a new test, removed commented out line --- src/plans/stepsize/stepsize.jl | 45 +++++++++++++++++++++++++++++++--- src/solvers/quasi_Newton.jl | 1 - test/plans/test_stepsize.jl | 1 + 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index c7c8e1d502..b17272f895 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -2170,9 +2170,10 @@ $(_kwargs(:vector_transport_method)) * `candidate_point = allocate_result(M, rand)`: storage for trial points * `candidate_direction = zero_vector(M, candidate_point)`: storage for transported directions * `max_bracket_iterations::Int = 10`: maximum number of bracketing iterations -* `start_enforcing_wolfe_conditions_at_bracketing_iteration::Int = 2`: bracketing iteration - number at which Wolfe conditions are started to be enforced; setting to 1 may cause no - bracketing to occur when the initial guess satisfies the Wolfe conditions +* `start_enforcing_wolfe_conditions_at_bracketing_iteration::Int = initial_guess isa ConstantStepsize ? 2 : 1`: + bracketing iteration number at which Wolfe conditions are started to be enforced; + setting to 1 may cause no bracketing to occur when the initial guess satisfies the Wolfe + conditions. * `max_function_evaluations::Int = 20`: maximum number of function evaluations per linesearch * `wolfe_condition_mode::Symbol = :adaptive`: one of `:standard`, `:approximate`, or `:adaptive`. Selects between (T1) and (T2) conditions in [HagerZhang:2006:2](@cite). @@ -2188,6 +2189,9 @@ $(_kwargs(:vector_transport_method)) * `ρ::Real = 5.0`: bracketing expansion factor. Allowed range: `ρ > 1`. * `Δ::Real = 0.7`: Parameter controlling the rate of change of Qₖ. Allowed range: `0 <= Δ <= 1`. +* `secant_acceptance_ratio::Real = 1.0e-8`: minimum relative interval length + for accepting secant step. Allowed range: `secant_acceptance_ratio >= 0`. + In case of rejection, a bisection step is performed instead. """ mutable struct HagerZhangLinesearchStepsize{ TF <: Real, @@ -2269,6 +2273,7 @@ mutable struct HagerZhangLinesearchStepsize{ @assert ρ > 1 @assert stepsize_limit > 0 @assert wolfe_condition_mode in (:standard, :approximate, :adaptive) + @assert secant_acceptance_ratio >= 0 # allocate storage triples = Vector{UnivariateTriple{TF}}(undef, max_function_evaluations) @@ -2509,6 +2514,40 @@ function _hz_u3( return (i_a_bar, i_b_bar, f_eval, f_wolfe) end +""" + _hz_secant2( + hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, + mp::AbstractManoptProblem, p, η, i_a::Int, i_b::Int + ) + +Perform the secant-based update in the Hager-Zhang linesearch. + +Computes a trial step using a secant interpolation of the bracketing +endpoints. If the trial step is too close to an endpoint, falls back to a +bisection step. Returns the updated bracketing indices and termination flags +from the internal update routine. + +# Arguments +- `hzls`: linesearch state and storage. +- `M`: manifold for retractions and transports. +- `mp`: optimization problem providing cost and differential. +- `p`: current iterate. +- `η`: search direction in the tangent space at `p`. +- `i_a`, `i_b`: indices of the current bracketing interval in `hzls.triples`. + +# Return value +Returns `(i_A, i_B, i_c, f_eval, f_wolfe)` where +- `i_A`, `i_B`: indices bracketing the minimum after the update, +- `i_c`: index of the most recent evaluation (or `-1` if the candidate was out of range), +- `f_eval`: `true` iff the evaluation limit has been reached, +- `f_wolfe`: `true` iff the Wolfe conditions are satisfied. + +# Steps (S1-S4) +- S1: compute a secant trial `c` from the current bracket and accept it unless too close to + an endpoint (otherwise use a bisection step). +- S2/S3: if the trial becomes a new endpoint, perform an update from that side. +- S4: return the updated bracket and termination flags. +""" function _hz_secant2( hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, mp::AbstractManoptProblem, p, η, i_a::Int, i_b::Int diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 606eab0b6a..29f2e178be 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -410,7 +410,6 @@ end function step_solver!(mp::AbstractManoptProblem, qns::QuasiNewtonState, k) M = get_manifold(mp) # qns.X should be the correct gradient at qns.p from initialization or the previous step - # get_gradient!(mp, qns.X, qns.p) qns.direction_update(qns.η, mp, qns) current_max_stepsize = _get_max_stepsize(M, qns) if !(qns.nondescent_direction_behavior === :ignore) diff --git a/test/plans/test_stepsize.jl b/test/plans/test_stepsize.jl index a59a0ca61b..18cef6a46d 100644 --- a/test/plans/test_stepsize.jl +++ b/test/plans/test_stepsize.jl @@ -82,6 +82,7 @@ end s3 = WolfePowellBinaryLinesearch()(M) @test Manopt.get_message(s3) == "" @test startswith(repr(s3), "WolfePowellBinaryLinesearch(;") + @test get_last_stepsize(s3) == 0.0 # no stepsize yet so `repr` and summary are the same @test repr(s3) == Manopt.status_summary(s3) s4 = WolfePowellLinesearch()(M) From 4216ef8b4efbb88fee78641b5b2195c818a08b8e Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 5 Feb 2026 12:25:12 +0100 Subject: [PATCH 092/135] use provided gradient for dphi_0 in HZ --- src/plans/stepsize/stepsize.jl | 7 ++++++- test/plans/test_stepsize.jl | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index b17272f895..7d8ddaa5c2 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -2594,7 +2594,12 @@ function (hzls::HagerZhangLinesearchStepsize)( M = get_manifold(mp) p = get_iterate(s) - dphi_0 = get_differential(mp, p, η; Y = hzls.temporary_tangent) + local dphi_0 # COV_EXCL_LINE + if :gradient in keys(kwargs) + dphi_0 = real(inner(M, p, η, kwargs[:gradient])) + else + dphi_0 = get_differential(mp, p, η; Y = hzls.temporary_tangent) + end hzls.triples[1] = UnivariateTriple(0.0, fp, dphi_0) hzls.last_evaluation_index = 1 diff --git a/test/plans/test_stepsize.jl b/test/plans/test_stepsize.jl index 18cef6a46d..0f5f4b68ec 100644 --- a/test/plans/test_stepsize.jl +++ b/test/plans/test_stepsize.jl @@ -293,6 +293,8 @@ end α = hzls(dmp, gs, 1, η) @test isfinite(α) @test α > 0 + α2 = hzls(dmp, gs, 1, η; gradient = grad_f_sum_sq(M, p)) + @test α2 ≈ α @test hzls.last_stepsize == α @test hzls.last_cost <= f_sum_sq(M, p) + 1.0e-12 From 3180c3075204c923e3b7951e9867823fcf4c1dfd Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Thu, 5 Feb 2026 14:20:50 +0100 Subject: [PATCH 093/135] avoid cost reevaluation for stopping criteria in certain cases --- src/plans/stopping_criterion.jl | 23 +++++++++++++---------- src/solvers/quasi_Newton.jl | 19 +++++++++++++++++++ src/solvers/solver.jl | 11 +++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index b410d85d16..2448636ddf 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -375,7 +375,7 @@ function (c::StopWhenCostChangeLess)( c.last_change = 2 * c.tolerance end c.last_change = c.last_cost - c.last_cost = get_cost(problem, get_iterate(state)) + c.last_cost = get_cost(problem, state) c.last_change = c.last_change - c.last_cost if abs(c.last_change) < c.tolerance c.at_iteration = iteration @@ -405,11 +405,11 @@ end StopWhenCostLess <: StoppingCriterion store a threshold when to stop looking at the cost function of the -optimization problem from within a [`AbstractManoptProblem`](@ref), i.e `get_cost(p,get_iterate(o))`. +optimization problem from within a [`AbstractManoptProblem`](@ref), i.e `get_cost(p, s)`. # Constructor - StopWhenCostLess(ε) + StopWhenCostLess(ε::Real) initialize the stopping criterion to a threshold `ε`. """ @@ -427,7 +427,7 @@ function (c::StopWhenCostLess)( if k == 0 # reset on init c.at_iteration = -1 end - c.last_cost = get_cost(p, get_iterate(s)) + c.last_cost = get_cost(p, s) if c.last_cost < c.threshold c.at_iteration = k return true @@ -502,7 +502,7 @@ function (c::StopWhenRelativeAPosterioriCostChangeLessOrEqual)( c.last_cost = Inf c.last_change = 2 * c.tolerance end - current_cost = get_cost(problem, get_iterate(state)) + current_cost = get_cost(problem, state) c.last_change = (c.last_cost - current_cost) / max(abs(c.last_cost), abs(current_cost), 1) c.last_cost = current_cost if iteration > 1 && c.last_change <= c.tolerance @@ -973,13 +973,14 @@ end """ StopWhenCostNaN <: StoppingCriterion -stop looking at the cost function of the optimization problem from within a [`AbstractManoptProblem`](@ref), i.e `get_cost(p,get_iterate(o))`. +Stop the solver when the cost function of the optimization problem +[`AbstractManoptProblem`](@ref) is `NaN`. The value is obtained using `get_cost(p, s)`. # Constructor StopWhenCostNaN() -initialize the stopping criterion to NaN. +initialize the stopping criterion with `at_iteration` equal to -1. """ mutable struct StopWhenCostNaN <: StoppingCriterion at_iteration::Int @@ -992,7 +993,7 @@ function (c::StopWhenCostNaN)( c.at_iteration = -1 end # but still verify whether it yields NaN - if isnan(get_cost(p, get_iterate(s))) + if isnan(get_cost(p, s)) c.at_iteration = k return true end @@ -1016,13 +1017,15 @@ end """ StopWhenIterateNaN <: StoppingCriterion -stop looking at the cost function of the optimization problem from within a [`AbstractManoptProblem`](@ref), i.e `get_cost(p,get_iterate(o))`. +Stop the solver when the iterate of the optimization problem from within an +[`AbstractManoptProblem`](@ref) contains `NaN` values. +The value is obtained using `get_iterate(s)`. # Constructor StopWhenIterateNaN() -initialize the stopping criterion to NaN. +Initialize `at_iteration` to `-1`. """ mutable struct StopWhenIterateNaN <: StoppingCriterion at_iteration::Int diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index 29f2e178be..e6a16a00ff 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -882,3 +882,22 @@ function update_hessian!( end return d end + +function get_cost( + mp::AbstractManoptProblem, s::QuasiNewtonState{ + P, T, + <:AbstractQuasiNewtonDirectionUpdate, + <:StoppingCriterion, + <:HagerZhangLinesearchStepsize, + } + ) where {P, T} + + hzls = s.stepsize + if hzls.last_evaluation_index === 0 + # if no evaluation was performed, we need to compute the cost + return get_cost(mp, s.p) + else + # we can reuse the stored function value from the linesearch + return hzls.triples[hzls.last_evaluation_index].f + end +end diff --git a/src/solvers/solver.jl b/src/solvers/solver.jl index bcaab743aa..a38faaef3f 100644 --- a/src/solvers/solver.jl +++ b/src/solvers/solver.jl @@ -189,3 +189,14 @@ end function stop_solver!(p::AbstractManoptProblem, s::ReturnSolverState, k) return stop_solver!(p, s.state, k) end + +""" + get_cost(p::AbstractManoptProblem, s::AbstractManoptSolverState) + +Get cost at the current iterate of the solver state `s` for the problem `p`. +The method may be implemented by particular solvers if they store the cost at the current +iterate in the state, but by default it is obtained by calling `get_cost(p, get_iterate(s))`. +""" +function get_cost(p::AbstractManoptProblem, s::AbstractManoptSolverState) + return get_cost(p, get_iterate(s)) +end From f169224a7fd88241ba998e7209ebc0d2bbc8021d Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 6 Feb 2026 15:05:00 +0100 Subject: [PATCH 094/135] avoid tangent allocation --- src/plans/stepsize/stepsize.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index 7d8ddaa5c2..dcfec34240 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -2361,7 +2361,7 @@ function _hz_evaluate_next_step( vector_transport_to!( M, hzls.candidate_direction, p, η, hzls.candidate_point, hzls.vector_transport_method ) - f, df = get_cost_and_differential(mp, hzls.candidate_point, hzls.candidate_direction) + f, df = get_cost_and_differential(mp, hzls.candidate_point, hzls.candidate_direction; Y = hzls.temporary_tangent) hzls.last_evaluation_index += 1 triples[hzls.last_evaluation_index] = UnivariateTriple(α, f, df) From e03cb9cf6280fe2ced984be879701570a89c1cdc Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 6 Feb 2026 18:17:21 +0100 Subject: [PATCH 095/135] optimize a few things --- ext/ManoptManifoldsExt/manifold_functions.jl | 47 +++++++++++-------- src/plans/box_plan.jl | 49 ++++++++++++-------- src/plans/stopping_criterion.jl | 3 +- 3 files changed, 58 insertions(+), 41 deletions(-) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 138fd9b063..637d9dfb45 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -67,16 +67,15 @@ The default maximum stepsize for `Hyperrectangle` manifold with corners is maxim of distances from `p` to each boundary. """ function max_stepsize(M::Hyperrectangle, p) - ms = 0.0 - for i in eachindex(M.lb, p) - dist_ub = M.ub[i] - p[i] - if dist_ub > 0 - ms = max(ms, dist_ub) - end - dist_lb = p[i] - M.lb[i] - if dist_lb > 0 - ms = max(ms, dist_lb) - end + lb = M.lb + ub = M.ub + ms = zero(eltype(p)) + @inbounds @simd for i in eachindex(lb, ub, p) + dist_ub = ub[i] - p[i] + dist_lb = p[i] - lb[i] + cand_ub = ifelse(dist_ub > 0, dist_ub, zero(dist_ub)) + cand_lb = ifelse(dist_lb > 0, dist_lb, zero(dist_lb)) + ms = max(ms, max(cand_ub, cand_lb)) end return ms end @@ -212,16 +211,24 @@ function Manopt.set_zero_at_index!(M::Hyperrectangle, d, i) end """ - Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real) - -For each index `i`, `t[i] < t_current`, set element of tangent vector `d_out` on -[`Hyperrectangle`](@extref Manifolds.Hyperrectangle) to the distance from `p[i]` to the -bound in the direction of `d_out[i]`. -""" -function Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, ts::Dict, t_current::Real) - for i in eachindex(M.lb) - if ts[i] < t_current - d_out[i] = d_out[i] > 0 ? M.ub[i] - p[i] : M.lb[i] - p[i] + Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, F_list::Vector{<:Tuple}, t_current::Real) + +For each pair `(t_i, i)` with index `i` in `F_list`, if `t_i < t_current`, set element of tangent +vector `d_out` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) to the distance from +`p[i]` to the bound in the direction of `d_out[i]`. +""" +function Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, F_list::Vector{<:Tuple}, t_current::Real) + f_idx = 1 + f_len = length(F_list) + for j in eachindex(d_out) + if f_idx <= f_len && F_list[f_idx][2] == j + t_i, i = F_list[f_idx] + if t_i < t_current + d_out[i] = d_out[i] > 0 ? M.ub[i] - p[i] : M.lb[i] - p[i] + end + f_idx += 1 + else + d_out[j] = 0 end end return d_out diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 7b4e270979..5f7d0dc9dd 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -512,17 +512,19 @@ function set_zero_at_index!(M::ProductManifold, d, i) end """ - Manopt.set_stepsize_bound!(M::ProductManifold, d_out, p, ts::Dict, t_current::Real) + set_stepsize_bound!(M::ProductManifold, d_out, p, F_list::Vector{<:Tuple}, t_current::Real) Set `d_out` so that it points from `p` to the generalized Cauchy point given step sizes to -bounds in `ts`. +bounds in `F_list` for coordinates achievable at step size less than `t_current`. +If an index is not in `F_list`, it is assumed that the corresponding coordinate of `d_out` +needs to be set to 0. """ function set_stepsize_bound!( - M::ProductManifold, d_out, p, ts::Dict, t_current::Real + M::ProductManifold, d_out, p, F_list::Vector{<:Tuple}, t_current::Real ) set_stepsize_bound!( M.manifolds[1], submanifold_component(M, d_out, Val(1)), - submanifold_component(M, p, Val(1)), ts, t_current + submanifold_component(M, p, Val(1)), F_list, t_current ) return d_out end @@ -536,18 +538,28 @@ which computes certain values of the Hessian while advancing segments. Instances are reused across segments during [`find_generalized_cauchy_direction!`](@ref) to avoid allocations. """ -struct GeneralizedCauchyDirectionFinder{TM <: AbstractManifold, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} +struct GeneralizedCauchyDirectionFinder{ + TM <: AbstractManifold, TX, + T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater, TFT <: Tuple, TBI, + } M::TM d_tmp::TX ha::T_HA hessian_segment_updater::TFU + F_list::Vector{TFT} + bounds_indices::TBI end function GeneralizedCauchyDirectionFinder( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; hessian_segment_updater::AbstractSegmentHessianUpdater = get_default_hessian_segment_updater(M, p, ha) ) - return GeneralizedCauchyDirectionFinder(M, zero_vector(M, p), ha, hessian_segment_updater) + bounds_indices = get_bounds_index(M) + TInd = eltype(bounds_indices) + TF = number_eltype(p) + F_list = Tuple{TF, TInd}[] + sizehint!(F_list, length(bounds_indices) + 1) + return GeneralizedCauchyDirectionFinder(M, zero_vector(M, p), ha, hessian_segment_updater, F_list, bounds_indices) end """ @@ -567,27 +579,24 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde M = gcd.M copyto!(M, d_out, d) - bounds_indices = get_bounds_index(M) - TInd = eltype(bounds_indices) - TF = number_eltype(d) + F_list = gcd.F_list + empty!(F_list) - ts = Dict{TInd, TF}((k, Inf) for k in bounds_indices) - - F_list = Tuple{TF, TInd}[] - sizehint!(F_list, length(bounds_indices) + 1) + bounds_indices = gcd.bounds_indices has_finite_limit = false smallest_positive_limit = Inf - for i in bounds_indices - ts[i] = get_stepsize_bound(M, p, d, i) + sbi = get_stepsize_bound(M, p, d, i) - if ts[i] > 0 - push!(F_list, (ts[i], i)) - smallest_positive_limit = min(smallest_positive_limit, ts[i]) + if sbi > 0 + push!(F_list, (sbi, i)) + if sbi < smallest_positive_limit + smallest_positive_limit = sbi + end end - has_finite_limit |= isfinite(ts[i]) + has_finite_limit |= isfinite(sbi) end if M isa ProductManifold @@ -662,7 +671,7 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde # there first bound after that is achieved at smallest_positive_limit / t_old max_feasible_stepsize = max(1.0, smallest_positive_limit / t_old) - set_stepsize_bound!(M, d_out, p, ts, t_old) + set_stepsize_bound!(M, d_out, p, F_list, t_old) if has_finite_limit return (:found_limited, max_feasible_stepsize) else diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 2448636ddf..da1bb13a98 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -878,7 +878,8 @@ function (sc::StopWhenProjectedNegativeGradientNormLess)( if (k > 0) r = (has_components(M) && !ismissing(sc.outer_norm)) ? (sc.outer_norm,) : () p = get_iterate(s) - mpg = embed_project(M, p, -get_gradient(s)) + mpg = -get_gradient(s) + embed_project!(M, mpg, p, mpg) sc.last_change = sc.norm(M, p, mpg, r...) if sc.last_change < sc.threshold sc.at_iteration = k From a9859ecedb4ceb5e9948de5c41d2a2614bdc49c7 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 6 Feb 2026 19:37:41 +0100 Subject: [PATCH 096/135] improve coverage --- ext/ManoptManifoldsExt/manifold_functions.jl | 2 +- test/solvers/test_quasi_Newton.jl | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 637d9dfb45..d2d39bdd68 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -76,7 +76,7 @@ function max_stepsize(M::Hyperrectangle, p) cand_ub = ifelse(dist_ub > 0, dist_ub, zero(dist_ub)) cand_lb = ifelse(dist_lb > 0, dist_lb, zero(dist_lb)) ms = max(ms, max(cand_ub, cand_lb)) - end + end # COV_EXCL_LINE return ms end function max_stepsize(M::Hyperrectangle) diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index 194b425dc1..e6fe302dff 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -551,4 +551,24 @@ end @test qns.direction_update.memory_s[1] == [1.0, 2.0] @test qns.direction_update.memory_s[2] == [1.0, 2.0] end + @testset "get_cost specialization" begin + M = Euclidean(2) + p = [0.0, 1.0] + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 .* p + gmp = ManifoldGradientObjective(f, grad_f) + mp = DefaultManoptProblem(M, gmp) + ha = QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2; nonpositive_curvature_behavior = :byrd) + qns = QuasiNewtonState( + M; + p = copy(M, p), + direction_update = ha, + nondescent_direction_behavior = :step_towards_negative_gradient, + stepsize = HagerZhangLinesearch()(M), + ) + @test get_cost(mp, qns) == f(M, get_iterate(qns)) + solve!(mp, qns) + @test get_cost(mp, qns) == f(M, get_iterate(qns)) + + end end From 1c3d2d37baf95e2021c89c6d1693e37b19ad1f80 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 7 Feb 2026 13:51:32 +0100 Subject: [PATCH 097/135] streamline set_stepsize_bound! logic --- ext/ManoptManifoldsExt/ManoptManifoldsExt.jl | 2 +- ext/ManoptManifoldsExt/manifold_functions.jl | 31 +++++------ src/Manopt.jl | 2 +- src/plans/box_plan.jl | 58 ++++++++++++-------- 4 files changed, 53 insertions(+), 40 deletions(-) diff --git a/ext/ManoptManifoldsExt/ManoptManifoldsExt.jl b/ext/ManoptManifoldsExt/ManoptManifoldsExt.jl index b1e027d713..a502802751 100644 --- a/ext/ManoptManifoldsExt/ManoptManifoldsExt.jl +++ b/ext/ManoptManifoldsExt/ManoptManifoldsExt.jl @@ -2,7 +2,7 @@ module ManoptManifoldsExt using ManifoldsBase: exp, log, ParallelTransport, vector_transport_to using Manopt -using Manopt: _math, _tex, ManifoldDefaultsFactory, _produce_type +using Manopt: _math, _tex, ManifoldDefaultsFactory, _produce_type, get_stepsize_bound import Manopt: max_stepsize, get_gradient, diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index d2d39bdd68..62cff8c4c1 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -211,24 +211,23 @@ function Manopt.set_zero_at_index!(M::Hyperrectangle, d, i) end """ - Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, F_list::Vector{<:Tuple}, t_current::Real) - -For each pair `(t_i, i)` with index `i` in `F_list`, if `t_i < t_current`, set element of tangent -vector `d_out` on [`Hyperrectangle`](@extref Manifolds.Hyperrectangle) to the distance from -`p[i]` to the bound in the direction of `d_out[i]`. -""" -function Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, F_list::Vector{<:Tuple}, t_current::Real) - f_idx = 1 - f_len = length(F_list) - for j in eachindex(d_out) - if f_idx <= f_len && F_list[f_idx][2] == j - t_i, i = F_list[f_idx] - if t_i < t_current - d_out[i] = d_out[i] > 0 ? M.ub[i] - p[i] : M.lb[i] - p[i] + Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, d, t_current::Real) + +For each element `i` in the tangent vector `d_out`, if the stepsize bound in direction `d` +for that element is less than `t_current`, set the element of `d_out` to the distance from +`p[i]` to the bound in the direction of `d[i]`. If the stepsize bound is non-positive, +set the element to 0. +""" +function Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, d, t_current::Real) + + for i in eachindex(d_out, d) + bound = get_stepsize_bound(M, p, d, i) + if bound > 0 + if bound < t_current && d_out[i] != 0 + d_out[i] = d[i] > 0 ? M.ub[i] - p[i] : M.lb[i] - p[i] end - f_idx += 1 else - d_out[j] = 0 + d_out[i] = 0 end end return d_out diff --git a/src/Manopt.jl b/src/Manopt.jl index 8fa5d233e9..98d03cd5cb 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -17,7 +17,7 @@ import LinearAlgebra: cross, LowerTriangular using ColorSchemes using ColorTypes using Colors -using DataStructures: BinaryHeap, CircularBuffer, capacity, length, push!, size, isfull +using DataStructures: CircularBuffer, capacity, length, push!, size, isfull, heapify!, heappop! using Dates: Millisecond, Nanosecond, Period, canonicalize, value using Glossaries using LinearAlgebra: diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 5f7d0dc9dd..a4acd14638 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -512,19 +512,27 @@ function set_zero_at_index!(M::ProductManifold, d, i) end """ - set_stepsize_bound!(M::ProductManifold, d_out, p, F_list::Vector{<:Tuple}, t_current::Real) + set_stepsize_bound!(M::AbstractManifold, d_out, p, d, t_current::Real) -Set `d_out` so that it points from `p` to the generalized Cauchy point given step sizes to -bounds in `F_list` for coordinates achievable at step size less than `t_current`. -If an index is not in `F_list`, it is assumed that the corresponding coordinate of `d_out` -needs to be set to 0. +For each component at index `i` in the tangent vector `d_out`, if the stepsize bound in +direction `d` for that component is less than `t_current`, set the element of `d_out` to +the distance from `p[i]` to the bound in the direction of `d[i]`. If the stepsize bound is +non-positive, set the element to 0. + +By default it does not modify `d_out` because most manifolds don't have direction-specific +stepsize bounds, and general anisotropic bounds are handled differently. """ -function set_stepsize_bound!( - M::ProductManifold, d_out, p, F_list::Vector{<:Tuple}, t_current::Real - ) - set_stepsize_bound!( - M.manifolds[1], submanifold_component(M, d_out, Val(1)), - submanifold_component(M, p, Val(1)), F_list, t_current +function set_stepsize_bound!(::AbstractManifold, d_out, p, d, t_current::Real) + return d_out +end + +function set_stepsize_bound!(M::ProductManifold, d_out, p, d, t_current::Real) + map( + (N, d_out_c, p_c, d_c) -> set_stepsize_bound!(N, d_out_c, p_c, d_c, t_current), + M.manifolds, + submanifold_components(M, d_out), + submanifold_components(M, p), + submanifold_components(M, d), ) return d_out end @@ -533,21 +541,23 @@ end GeneralizedCauchyDirectionFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} Helper container for generalized Cauchy direction search. Stores the manifold `M`, cached -workspace (`d_tmp`), the quasi-Newton direction update `ha`, and the `hessian_segment_updater`, -which computes certain values of the Hessian while advancing segments. +original descent direction (`d_original`), the quasi-Newton direction update `ha`, and the +`hessian_segment_updater`, which computes certain values of the Hessian while advancing segments. Instances are reused across segments during [`find_generalized_cauchy_direction!`](@ref) to avoid allocations. """ struct GeneralizedCauchyDirectionFinder{ TM <: AbstractManifold, TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater, TFT <: Tuple, TBI, + TO <: Base.Order.Ordering, } M::TM - d_tmp::TX + d_original::TX ha::T_HA hessian_segment_updater::TFU F_list::Vector{TFT} bounds_indices::TBI + ordering::TO end function GeneralizedCauchyDirectionFinder( @@ -559,7 +569,11 @@ function GeneralizedCauchyDirectionFinder( TF = number_eltype(p) F_list = Tuple{TF, TInd}[] sizehint!(F_list, length(bounds_indices) + 1) - return GeneralizedCauchyDirectionFinder(M, zero_vector(M, p), ha, hessian_segment_updater, F_list, bounds_indices) + ordering = Base.By(first) + return GeneralizedCauchyDirectionFinder( + M, zero_vector(M, p), ha, + hessian_segment_updater, F_list, bounds_indices, ordering + ) end """ @@ -577,8 +591,10 @@ The function returns """ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinder, d_out, p, d, X) M = gcd.M + copyto!(M, gcd.d_original, d) copyto!(M, d_out, d) + ordering = gcd.ordering F_list = gcd.F_list empty!(F_list) @@ -598,7 +614,6 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde end has_finite_limit |= isfinite(sbi) end - if M isa ProductManifold # Hyperrectangle × something else # push also `t` corresponding to max_stepsize if it is considered in the manifold @@ -621,8 +636,7 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde return (:not_found, NaN) end end - - F = BinaryHeap(Base.By(first), F_list) + heapify!(F_list, ordering) f_prime = inner(M, p, X, d) f_double_prime = hessian_value_diag(gcd.ha, M, p, d) @@ -634,7 +648,7 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde dt_min = -f_prime / f_double_prime t_old = 0.0 - t_current, b = pop!(F) + t_current, b = heappop!(F_list, ordering) dt = t_current - t_old init_updater!(M, gcd.hessian_segment_updater, p, d, gcd.ha) @@ -658,9 +672,9 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde end dt_min = -f_prime / f_double_prime - isempty(F) && break + isempty(F_list) && break - t_current, b = pop!(F) + t_current, b = heappop!(F_list, ordering) dt = t_current - t_old end @@ -671,7 +685,7 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde # there first bound after that is achieved at smallest_positive_limit / t_old max_feasible_stepsize = max(1.0, smallest_positive_limit / t_old) - set_stepsize_bound!(M, d_out, p, F_list, t_old) + set_stepsize_bound!(M, d_out, p, gcd.d_original, t_old) if has_finite_limit return (:found_limited, max_feasible_stepsize) else From b5214a30cec5461ca97a394544c13d8377f03a31 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 7 Feb 2026 15:16:45 +0100 Subject: [PATCH 098/135] generalize to Hyperrectangle at any position --- .../generalized_cauchy_direction_subsolver.md | 2 +- ext/ManoptManifoldsExt/manifold_functions.jl | 4 +- src/plans/box_plan.jl | 147 +++++++++++++----- src/plans/quasi_newton_plan.jl | 10 +- src/solvers/quasi_Newton.jl | 2 +- test/solvers/test_quasi_Newton_box.jl | 36 +++-- 6 files changed, 139 insertions(+), 62 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index d620c96e80..aa01a17348 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -23,7 +23,7 @@ The solver is currently primarily intended for internal use by optimization algo These symbols are directly used by solvers to compute the descent direction corresponding to the Generalized Cauchy direction. ```@docs -Manopt.requires_generalized_cauchy_direction_computation +Manopt.has_anisotropic_max_stepsize Manopt.find_generalized_cauchy_direction! Manopt.GeneralizedCauchyDirectionFinder ``` diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 62cff8c4c1..44b5b2b317 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -234,11 +234,11 @@ function Manopt.set_stepsize_bound!(M::Hyperrectangle, d_out, p, d, t_current::R end """ - Manopt.requires_generalized_cauchy_direction_computation(::Hyperrectangle) + Manopt.has_anisotropic_max_stepsize(::Hyperrectangle) Returns `true`, as `Hyperrectangle` manifold requires generalized Cauchy point computation in solvers. """ -Manopt.requires_generalized_cauchy_direction_computation(::Hyperrectangle) = true +Manopt.has_anisotropic_max_stepsize(::Hyperrectangle) = true """ Manopt.get_at_bound_index(::Hyperrectangle, X, b) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index a4acd14638..3481e060ea 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -1,11 +1,12 @@ """ - requires_generalized_cauchy_direction_computation(M::AbstractManifold) + has_anisotropic_max_stepsize(M::AbstractManifold) -Return `true` if `M` is a `Hyperrectangle`-like manifold with corners, or a product of it +Return `true` if `M` has `max_stepsize` that depends on the direction. +For example, if `M` is a `Hyperrectangle`-like manifold with corners, or a product of it with a standard manifold. Otherwise return `false`. """ -requires_generalized_cauchy_direction_computation(::AbstractManifold) = false -requires_generalized_cauchy_direction_computation(M::ProductManifold) = requires_generalized_cauchy_direction_computation(M.manifolds[1]) +has_anisotropic_max_stepsize(::AbstractManifold) = false +has_anisotropic_max_stepsize(M::ProductManifold) = any(has_anisotropic_max_stepsize, M.manifolds) @doc raw""" mutable struct LimitedMemoryHessianApproximation end @@ -111,8 +112,8 @@ end get_update_vector_transport(u::QuasiNewtonLimitedMemoryBoxDirectionUpdate) = get_update_vector_transport(u.qn_du) -function get_at_bound_index(M::ProductManifold, X, b) - return get_at_bound_index(M.manifolds[1], submanifold_component(M, X, Val(1)), b) +function get_at_bound_index(M::ProductManifold, X, b::Tuple{Int, Any}) + return get_at_bound_index(M.manifolds[b[1]], submanifold_component(M, X, Val(1)), b[2]) end @doc raw""" @@ -355,13 +356,15 @@ init_updater!(::AbstractManifold, hessian_segment_updater::AbstractSegmentHessia Generic f' and f'' calculation that only relies on `hessian_value` but is relatively slow for high-dimensional domains. """ -struct GenericSegmentHessianUpdater{TX} <: AbstractSegmentHessianUpdater +struct GenericSegmentHessianUpdater{TITR, TX} <: AbstractSegmentHessianUpdater + itr::TITR d_z::TX d_tmp::TX end function get_default_hessian_segment_updater(M::AbstractManifold, p, ::AbstractQuasiNewtonDirectionUpdate) - return GenericSegmentHessianUpdater(zero_vector(M, p), zero_vector(M, p)) + itr = get_bounds_index(M) + return GenericSegmentHessianUpdater(itr, zero_vector(M, p), zero_vector(M, p)) end function init_updater!(M::AbstractManifold, hessian_segment_updater::GenericSegmentHessianUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) @@ -379,8 +382,8 @@ point line search using the generic approach via `hessian_value` with [`UnitVect """ function (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, b, db, ha) upd.d_z .+= dt .* upd.d_tmp - hv_eb_dz = hessian_value(ha, M, p, UnitVector(b), upd.d_z) - hv_eb_d = hessian_value(ha, M, p, UnitVector(b), upd.d_tmp) + hv_eb_dz = hessian_value(ha, M, p, UnitVector(upd.itr, b), upd.d_z) + hv_eb_d = hessian_value(ha, M, p, UnitVector(upd.itr, b), upd.d_tmp) set_zero_at_index!(M, upd.d_tmp, b) @@ -481,14 +484,61 @@ function (hessian_segment_updater::LimitedMemorySegmentHessianUpdater)( return eb_B_z, eb_B_d end +struct ProductIndex{T <: Tuple} + ranges::T +end + +Base.iterate(itr::ProductIndex) = _iterate(itr.ranges, 1, nothing) +Base.iterate(itr::ProductIndex, state) = _iterate(itr.ranges, state...) + +function _iterate(ranges, i, st) + i > length(ranges) && return nothing + if st === nothing + it = iterate(ranges[i]) + it === nothing && return _iterate(ranges, i + 1, nothing) + (j, st2) = it + return ((i, j), (i, st2)) + else + it = iterate(ranges[i], st) + if it === nothing + return _iterate(ranges, i + 1, nothing) + else + (j, st2) = it + return ((i, j), (i, st2)) + end + end +end + +""" + _to_linear_index(itr::ProductIndex, b) + +Convert element `b` of the iteration of `itr` to a linear index. For example, if `itr` is +an iteration over `(1:4, 1:3)`, then `_to_linear_index(itr, (2, 3))` returns `6`, which +is the linear index of the element `(2, 3)` in the concatenated ranges `1:4, 1:3`. +""" +function _to_linear_index(itr::ProductIndex, b) + i, j = b + r = itr.ranges[i] + offset = sum(k -> length(itr.ranges[k]), 1:(i - 1); init = 0) + return offset + (j - first(r) + 1) +end +_to_linear_index(::Base.OneTo, b::Int) = b + +Base.length(itr::ProductIndex) = sum(length, itr.ranges) + + """ get_bounds_index(::AbstractManifold) Get the bound indices of manifold `M`. Standard manifolds don't have bounds, so `Base.OneTo(1)` is returned. """ -get_bounds_index(M::AbstractManifold) -get_bounds_index(M::ProductManifold) = get_bounds_index(M.manifolds[1]) +get_bounds_index(M::AbstractManifold) = Base.OneTo(0) +function get_bounds_index(M::ProductManifold) + ranges = map(get_bounds_index, M.manifolds) + iter = ProductIndex(ranges) + return iter +end """ get_stepsize_bound(M::AbstractManifold, x, d, i) @@ -497,17 +547,19 @@ Get the upper bound on moving in direction `d` from point `p` on manifold `M`, f bound index `i`. """ get_stepsize_bound(M::AbstractManifold, p, d, i) -function get_stepsize_bound(M::ProductManifold, p, d, i) - return get_stepsize_bound(M.manifolds[1], submanifold_component(M, p, Val(1)), submanifold_component(M, d, Val(1)), i) +function get_stepsize_bound(M::ProductManifold, p, d, i::Tuple{Int, Any}) + i1, i2 = i + return get_stepsize_bound(M.manifolds[i1], submanifold_component(M, p, i1), submanifold_component(M, d, i1), i2) end """ - set_zero_at_index!(M::ProductManifold, d, i) + set_zero_at_index!(M::ProductManifold, d, i::Tuple{Int,Any}) -Set the element of the first component of `d` at bound index `i` to zero. +Set the element of the `i[1]`th component of `d` at bound index `i[2]` to zero. """ -function set_zero_at_index!(M::ProductManifold, d, i) - set_zero_at_index!(M.manifolds[1], submanifold_component(M, d, Val(1)), i) +function set_zero_at_index!(M::ProductManifold, d, i::Tuple{Int, Any}) + i1, i2 = i + set_zero_at_index!(M.manifolds[i1], submanifold_component(M, d, i1), i2) return d end @@ -576,6 +628,29 @@ function GeneralizedCauchyDirectionFinder( ) end +function collect_isotropic_limits!(::AbstractManifold, ::Vector{<:Tuple{TF, Any}}, p, d) where {TF <: Real} + return false, convert(TF, Inf) +end + +function collect_isotropic_limits!(M::ProductManifold, F_list::Vector{<:Tuple{TF, Any}}, p, d) where {TF <: Real} + has_finite_limit = false + smallest_positive_limit = Inf + map(M.manifolds, submanifold_components(M, p), submanifold_components(M, d)) do Mi, p_i, d_i + if !has_anisotropic_max_stepsize(Mi) + max_step = Manopt.max_stepsize(Mi, p_i) + if isfinite(max_step) + tms = max_step / norm(Mi, p_i, d_i) + push!(F_list, (tms, -1)) + has_finite_limit = true + if tms < smallest_positive_limit + smallest_positive_limit = tms + end + end + end + end + return has_finite_limit, smallest_positive_limit +end + """ find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinder, d_out, p, d, X) @@ -600,9 +675,9 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde bounds_indices = gcd.bounds_indices - has_finite_limit = false - - smallest_positive_limit = Inf + # isotropic limits + has_finite_limit, smallest_positive_limit = collect_isotropic_limits!(M, F_list, p, d) + # anisotropic limits for i in bounds_indices sbi = get_stepsize_bound(M, p, d, i) @@ -614,27 +689,12 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde end has_finite_limit |= isfinite(sbi) end - if M isa ProductManifold - # Hyperrectangle × something else - # push also `t` corresponding to max_stepsize if it is considered in the manifold - M2 = M.manifolds[2] - p2 = submanifold_component(M, p, Val(2)) - max_step = Manopt.max_stepsize(M2, p2) - if isfinite(max_step) - d2 = submanifold_component(M, d, Val(2)) - tms = max_step / norm(M2, p2, d2) - push!(F_list, (tms, -1)) - end - else - # Check only when we work on a pure Hyperrectangle - # - # In this case we can't move in the direction `d` at all, though it's usually not - # a problem relevant to the end user because it can be handled by step_solver! that - # uses the GCD subsolver. - - if isempty(F_list) - return (:not_found, NaN) - end + + # In this case we can't move in the direction `d` at all, though it's usually not + # a problem relevant to the end user because it can be handled by step_solver! that + # uses the GCD subsolver. + if isempty(F_list) + return (:not_found, NaN) end heapify!(F_list, ordering) @@ -660,7 +720,8 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde hv_eb_dz, hv_eb_d = gcd.hessian_segment_updater(M, p, t_current, dt, b, db, gcd.ha) f_prime += dt * f_double_prime - db * (gb + hv_eb_dz) - f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_diag(gcd.ha, M, p, UnitVector(b)) + Xb = UnitVector(gcd.bounds_indices, b) + f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_diag(gcd.ha, M, p, Xb) t_old = t_current diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index 89b5dbab0b..53f90b16cb 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -524,14 +524,16 @@ function hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractM end """ - UnitVector{TB} + UnitVector{TR,TB} A type representing a unit tangent vector on a `Hyperrectangle`-like manifold with corners, or a product of it with a standard manifold. The field `index` stores the index of the element equal to 1. All other elements are equal to 0. +`its` stores the overall iterator over all bounds. """ -struct UnitVector{TB} +struct UnitVector{TI, TB} + its::TI index::TB end @@ -543,7 +545,7 @@ Returns the scalar ``c^{\top} B c`` where ``c`` are the coordinates of the [`UnitVector`](@ref) `X` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. """ function hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate{T}, ::AbstractManifold, p, X::UnitVector) where {T <: Union{BFGS, DFP, SR1, Broyden}} - b = X.index + b = _to_linear_index(X.its, X.index) return d.matrix[b, b] end """ @@ -556,7 +558,7 @@ Returns the scalar ``c_b^{\top} B c`` where ``c_b`` are the coordinates of the and ``B`` is `d.matrix`. """ function hessian_value(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X::UnitVector, Y) where {T <: Union{BFGS, DFP, SR1, Broyden}} - b = X.index + b = _to_linear_index(X.its, X.index) return dot(d.matrix[b, :], get_coordinates(M, p, Y, d.basis)) end diff --git a/src/solvers/quasi_Newton.jl b/src/solvers/quasi_Newton.jl index e6a16a00ff..ef81322106 100644 --- a/src/solvers/quasi_Newton.jl +++ b/src/solvers/quasi_Newton.jl @@ -348,7 +348,7 @@ function quasi_Newton!( nonpositive_curvature_behavior = nonpositive_curvature_behavior, sy_tol = sy_tol, ) - if requires_generalized_cauchy_direction_computation(M) + if has_anisotropic_max_stepsize(M) local_dir_upd = QuasiNewtonLimitedMemoryBoxDirectionUpdate(local_dir_upd) end else diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 151820dcc8..eac1868d8e 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -42,7 +42,7 @@ using RecursiveArrayTools z = [-0.25, -1.0] # optimized formula - upd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) + upd = Manopt.GenericSegmentHessianUpdater(Manopt.get_bounds_index(M), similar(d), similar(d)) Manopt.init_updater!(M, upd, p, d, ha) hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha) @test hv_eb_dz ≈ -2.0 @@ -75,7 +75,7 @@ using RecursiveArrayTools z = [-0.5, -0.25] # optimized formula - upd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) + upd = Manopt.GenericSegmentHessianUpdater(Manopt.get_bounds_index(M), similar(d), similar(d)) Manopt.init_updater!(M, upd, p, d, ha) hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha) @test hv_eb_dz == -1.0 @@ -126,7 +126,7 @@ using RecursiveArrayTools t_current = 0 + dt # compare the generic and limited memory updater - gupd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) + gupd = Manopt.GenericSegmentHessianUpdater(Manopt.get_bounds_index(M), similar(d), similar(d)) Manopt.init_updater!(M, gupd, p, d, ha) hv_eb_dz, hv_eb_d = gupd(M, p, t_current, dt, b, db, ha) @@ -148,7 +148,8 @@ using RecursiveArrayTools @testset "No memory tests" begin ha2 = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2)) - @test Manopt.hessian_value(ha2, M, p, Manopt.UnitVector(b), grad) ≈ 4.0 + idx = Manopt.get_bounds_index(M) + @test Manopt.hessian_value(ha2, M, p, Manopt.UnitVector(idx, b), grad) ≈ 4.0 Manopt.set_M_current_scale!(M, p, ha2) @test ha2.current_scale == ha2.qn_du.initial_scale @test ha2.M_11 == fill(0.0, 0, 0) @@ -259,10 +260,10 @@ using RecursiveArrayTools @test f3(MInf, p_opt) < 64.0 end - @testset "requires_generalized_cauchy_direction_computation" begin - @test !Manopt.requires_generalized_cauchy_direction_computation(Sphere(2)) - @test Manopt.requires_generalized_cauchy_direction_computation(Hyperrectangle([1], [2])) - @test Manopt.requires_generalized_cauchy_direction_computation(ProductManifold(Hyperrectangle([1], [2]), Sphere(2))) + @testset "has_anisotropic_max_stepsize" begin + @test !Manopt.has_anisotropic_max_stepsize(Sphere(2)) + @test Manopt.has_anisotropic_max_stepsize(Hyperrectangle([1], [2])) + @test Manopt.has_anisotropic_max_stepsize(ProductManifold(Hyperrectangle([1], [2]), Sphere(2))) end @testset "Hyperrectangle × Sphere" begin @@ -277,12 +278,12 @@ using RecursiveArrayTools @testset "Hessian updater" begin d = -grad_f(M, p0) ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis()) - gupd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) + gupd = Manopt.GenericSegmentHessianUpdater(Manopt.get_bounds_index(M), similar(d), similar(d)) Manopt.init_updater!(M, gupd, p0, d, ha) - b = 2 + b = (1, 2) dt = 0.25 t_current = 0 + dt - db = d[b] + db = d.x[b[1]][b[2]] hv_eb_dz, hv_eb_d = gupd(M, p0, t_current, dt, b, db, ha) @test hv_eb_dz ≈ -64.0 @test hv_eb_d ≈ -256.0 @@ -291,4 +292,17 @@ using RecursiveArrayTools p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) @test distance(M, p_opt, ArrayPartition([0, 2, 0], px)) < 0.1 end + + @testset "Sphere × Hyperrectangle" begin + S2 = Sphere(2) + px = [0.0, 1.0, 0.0] + Mbox = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) + M = S2 × Mbox + f(M, p) = sum(p.x[2] .^ 4) + 0.5 * distance(S2, p.x[1], px)^2 + grad_f(M, p) = ArrayPartition(-log(S2, p.x[1], px), project(Mbox, p.x[2], 4 .* (p.x[2] .^ 3))) + p0 = ArrayPartition([1.0, 0.0, 0.0], [0.0, 4.0, 1.0]) + + p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) + @test distance(M, p_opt, ArrayPartition(px, [0, 2, 0])) < 0.1 + end end From 30a497b1296c7d97605a6dbebd19b75facb12ae7 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 7 Feb 2026 15:41:54 +0100 Subject: [PATCH 099/135] cleanup --- .../generalized_cauchy_direction_subsolver.md | 1 + src/plans/box_plan.jl | 38 ++++++++++--------- src/plans/quasi_newton_plan.jl | 11 +++--- test/solvers/test_quasi_Newton_box.jl | 10 ++--- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index aa01a17348..b007180efe 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -44,6 +44,7 @@ These are internal symbols used to manage and manipulate bound constraints durin ```@docs Manopt.init_updater! Manopt.UnitVector +Manopt.to_coordinate_index Manopt.AbstractSegmentHessianUpdater Manopt.GenericSegmentHessianUpdater Manopt.get_bounds_index diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 3481e060ea..80d844eeff 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -356,15 +356,13 @@ init_updater!(::AbstractManifold, hessian_segment_updater::AbstractSegmentHessia Generic f' and f'' calculation that only relies on `hessian_value` but is relatively slow for high-dimensional domains. """ -struct GenericSegmentHessianUpdater{TITR, TX} <: AbstractSegmentHessianUpdater - itr::TITR +struct GenericSegmentHessianUpdater{TX} <: AbstractSegmentHessianUpdater d_z::TX d_tmp::TX end function get_default_hessian_segment_updater(M::AbstractManifold, p, ::AbstractQuasiNewtonDirectionUpdate) - itr = get_bounds_index(M) - return GenericSegmentHessianUpdater(itr, zero_vector(M, p), zero_vector(M, p)) + return GenericSegmentHessianUpdater(zero_vector(M, p), zero_vector(M, p)) end function init_updater!(M::AbstractManifold, hessian_segment_updater::GenericSegmentHessianUpdater, p, d, ha::AbstractQuasiNewtonDirectionUpdate) @@ -382,8 +380,8 @@ point line search using the generic approach via `hessian_value` with [`UnitVect """ function (upd::GenericSegmentHessianUpdater)(M::AbstractManifold, p, t::Real, dt::Real, b, db, ha) upd.d_z .+= dt .* upd.d_tmp - hv_eb_dz = hessian_value(ha, M, p, UnitVector(upd.itr, b), upd.d_z) - hv_eb_d = hessian_value(ha, M, p, UnitVector(upd.itr, b), upd.d_tmp) + hv_eb_dz = hessian_value(ha, M, p, UnitVector(b), upd.d_z) + hv_eb_d = hessian_value(ha, M, p, UnitVector(b), upd.d_tmp) set_zero_at_index!(M, upd.d_tmp, b) @@ -509,20 +507,25 @@ function _iterate(ranges, i, st) end end + +""" + to_coordinate_index(M::ProductManifold, b::UnitVector, B::AbstractBasis) + +Get the index of coordinate equal to 1 of [`UnitVector`](@ref) `b` with respect to +`AbstractBasis` `B`. +""" +to_coordinate_index(::AbstractManifold, b::UnitVector{Int}, ::AbstractBasis) = b.index """ - _to_linear_index(itr::ProductIndex, b) + to_coordinate_index(M::ProductManifold, b::UnitVector, B::AbstractBasis) -Convert element `b` of the iteration of `itr` to a linear index. For example, if `itr` is -an iteration over `(1:4, 1:3)`, then `_to_linear_index(itr, (2, 3))` returns `6`, which -is the linear index of the element `(2, 3)` in the concatenated ranges `1:4, 1:3`. +Get the index of coordinate equal to 1 of [`UnitVector`](@ref) `b` with respect to +`AbstractBasis` `B`. """ -function _to_linear_index(itr::ProductIndex, b) - i, j = b - r = itr.ranges[i] - offset = sum(k -> length(itr.ranges[k]), 1:(i - 1); init = 0) - return offset + (j - first(r) + 1) +function to_coordinate_index(M::ProductManifold, b::UnitVector, B::AbstractBasis) + i, j = b.index + offset = sum(k -> number_of_coordinates(M.manifolds[k], B), 1:(i - 1); init = 0) + return offset + j end -_to_linear_index(::Base.OneTo, b::Int) = b Base.length(itr::ProductIndex) = sum(length, itr.ranges) @@ -720,8 +723,7 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde hv_eb_dz, hv_eb_d = gcd.hessian_segment_updater(M, p, t_current, dt, b, db, gcd.ha) f_prime += dt * f_double_prime - db * (gb + hv_eb_dz) - Xb = UnitVector(gcd.bounds_indices, b) - f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_diag(gcd.ha, M, p, Xb) + f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_diag(gcd.ha, M, p, UnitVector(b)) t_old = t_current diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index 53f90b16cb..e6e5e1e5d4 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -524,7 +524,7 @@ function hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractM end """ - UnitVector{TR,TB} + UnitVector{TB} A type representing a unit tangent vector on a `Hyperrectangle`-like manifold with corners, or a product of it with a standard manifold. @@ -532,8 +532,7 @@ The field `index` stores the index of the element equal to 1. All other elements are equal to 0. `its` stores the overall iterator over all bounds. """ -struct UnitVector{TI, TB} - its::TI +struct UnitVector{TB} index::TB end @@ -544,8 +543,8 @@ Evaluate the quadratic form associated with the stored quasi-Newton matrix. Returns the scalar ``c^{\top} B c`` where ``c`` are the coordinates of the [`UnitVector`](@ref) `X` at `p` (in the basis `d.basis`) and ``B`` is `d.matrix`. """ -function hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate{T}, ::AbstractManifold, p, X::UnitVector) where {T <: Union{BFGS, DFP, SR1, Broyden}} - b = _to_linear_index(X.its, X.index) +function hessian_value_diag(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X::UnitVector) where {T <: Union{BFGS, DFP, SR1, Broyden}} + b = to_coordinate_index(M, X, d.basis) return d.matrix[b, b] end """ @@ -558,7 +557,7 @@ Returns the scalar ``c_b^{\top} B c`` where ``c_b`` are the coordinates of the and ``B`` is `d.matrix`. """ function hessian_value(d::QuasiNewtonMatrixDirectionUpdate{T}, M::AbstractManifold, p, X::UnitVector, Y) where {T <: Union{BFGS, DFP, SR1, Broyden}} - b = _to_linear_index(X.its, X.index) + b = to_coordinate_index(M, X, d.basis) return dot(d.matrix[b, :], get_coordinates(M, p, Y, d.basis)) end diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index eac1868d8e..fe2c2c3981 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -42,7 +42,7 @@ using RecursiveArrayTools z = [-0.25, -1.0] # optimized formula - upd = Manopt.GenericSegmentHessianUpdater(Manopt.get_bounds_index(M), similar(d), similar(d)) + upd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) Manopt.init_updater!(M, upd, p, d, ha) hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha) @test hv_eb_dz ≈ -2.0 @@ -75,7 +75,7 @@ using RecursiveArrayTools z = [-0.5, -0.25] # optimized formula - upd = Manopt.GenericSegmentHessianUpdater(Manopt.get_bounds_index(M), similar(d), similar(d)) + upd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) Manopt.init_updater!(M, upd, p, d, ha) hv_eb_dz, hv_eb_d = upd(M, p, 0 + dt, dt, b, db, ha) @test hv_eb_dz == -1.0 @@ -126,7 +126,7 @@ using RecursiveArrayTools t_current = 0 + dt # compare the generic and limited memory updater - gupd = Manopt.GenericSegmentHessianUpdater(Manopt.get_bounds_index(M), similar(d), similar(d)) + gupd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) Manopt.init_updater!(M, gupd, p, d, ha) hv_eb_dz, hv_eb_d = gupd(M, p, t_current, dt, b, db, ha) @@ -149,7 +149,7 @@ using RecursiveArrayTools @testset "No memory tests" begin ha2 = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2)) idx = Manopt.get_bounds_index(M) - @test Manopt.hessian_value(ha2, M, p, Manopt.UnitVector(idx, b), grad) ≈ 4.0 + @test Manopt.hessian_value(ha2, M, p, Manopt.UnitVector(b), grad) ≈ 4.0 Manopt.set_M_current_scale!(M, p, ha2) @test ha2.current_scale == ha2.qn_du.initial_scale @test ha2.M_11 == fill(0.0, 0, 0) @@ -278,7 +278,7 @@ using RecursiveArrayTools @testset "Hessian updater" begin d = -grad_f(M, p0) ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis()) - gupd = Manopt.GenericSegmentHessianUpdater(Manopt.get_bounds_index(M), similar(d), similar(d)) + gupd = Manopt.GenericSegmentHessianUpdater(similar(d), similar(d)) Manopt.init_updater!(M, gupd, p0, d, ha) b = (1, 2) dt = 0.25 From 31243d4d4eb2a0555d51692bcc232cf39e94f0d7 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Sat, 7 Feb 2026 16:08:40 +0100 Subject: [PATCH 100/135] fix ambiguity, improve test --- src/plans/box_plan.jl | 2 +- test/solvers/test_quasi_Newton_box.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 80d844eeff..9ed75be47f 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -521,7 +521,7 @@ to_coordinate_index(::AbstractManifold, b::UnitVector{Int}, ::AbstractBasis) = b Get the index of coordinate equal to 1 of [`UnitVector`](@ref) `b` with respect to `AbstractBasis` `B`. """ -function to_coordinate_index(M::ProductManifold, b::UnitVector, B::AbstractBasis) +function to_coordinate_index(M::ProductManifold, b::UnitVector{Tuple{Int, Int}}, B::AbstractBasis) i, j = b.index offset = sum(k -> number_of_coordinates(M.manifolds[k], B), 1:(i - 1); init = 0) return offset + j diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index fe2c2c3981..9c10c1db4b 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -296,13 +296,13 @@ using RecursiveArrayTools @testset "Sphere × Hyperrectangle" begin S2 = Sphere(2) px = [0.0, 1.0, 0.0] - Mbox = Hyperrectangle([-1.0, 2.0, -Inf], [2.0, Inf, 2.0]) + Mbox = Hyperrectangle([-1.0 2.0; -Inf -Inf], [2.0 Inf; 2.0 Inf]) M = S2 × Mbox f(M, p) = sum(p.x[2] .^ 4) + 0.5 * distance(S2, p.x[1], px)^2 grad_f(M, p) = ArrayPartition(-log(S2, p.x[1], px), project(Mbox, p.x[2], 4 .* (p.x[2] .^ 3))) - p0 = ArrayPartition([1.0, 0.0, 0.0], [0.0, 4.0, 1.0]) + p0 = ArrayPartition([1.0, 0.0, 0.0], [0.0 4.0; 1.0 1.0]) p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) - @test distance(M, p_opt, ArrayPartition(px, [0, 2, 0])) < 0.1 + @test distance(M, p_opt, ArrayPartition(px, [0 2; 0 0])) < 0.1 end end From 4d792ed44ac49600cdc10e20724d20d6aa897bdc Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 9 Feb 2026 14:25:23 +0100 Subject: [PATCH 101/135] address review regarding HZ stepsize --- src/plans/stepsize/stepsize.jl | 156 +++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 64 deletions(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index dcfec34240..cb997b2559 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -117,15 +117,15 @@ function (a::ArmijoLinesearchStepsize)( return a(mp, p, grad, η; initial_guess = a.initial_guess(mp, s, k, a.last_stepsize, η), kwargs...) end function (a::ArmijoLinesearchStepsize)( - mp::AbstractManoptProblem, p, X, η; initial_guess::Real = 1.0, kwargs... + mp::AbstractManoptProblem, p, X, η; initial_guess::Real = 1.0, + stop_when_stepsize_exceeds = nothing, kwargs... ) reset_messages!(a.messages) l = norm(get_manifold(mp), p, η) - local swse # COV_EXCL_LINE - if :stop_when_stepsize_exceeds in keys(kwargs) - swse = kwargs[:stop_when_stepsize_exceeds] + swse = if isnothing(stop_when_stepsize_exceeds) + (a.stop_when_stepsize_exceeds / l) else - swse = (a.stop_when_stepsize_exceeds / l) + stop_when_stepsize_exceeds end a.last_stepsize = linesearch_backtrack!( get_manifold(mp), @@ -2144,54 +2144,39 @@ end Do a bracketing line search to find a step size ``α`` that finds a local minimum along the search direction ``X`` starting from ``p``, utilizing cubic polynomial interpolation using the method described in -[HagerZhang:2006:2](@cite). +[HagerZhang:2006:2](@cite). Function [`secant`](@ref) is used to find the minimum of the +cubic polynomial fitted to values of the cost function and its derivative at the endpoints +of the current interval. See [`HagerZhangLinesearch`](@ref) for the mathematical details. # Fields + $(_fields(:p; name = "candidate_point")) as temporary storage for candidates * `initial_stepsize::R`: the step size to start the search with $(_fields(:retraction_method)) $(_fields(:vector_transport_method)) +* `initial_guess`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. +* `stepsize_limit`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. +* `max_bracket_iterations`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. +* `start_enforcing_wolfe_conditions_at_bracketing_iteration`: see keyword arguments of + [`HagerZhangLinesearch`](@ref) for details. +* `wolfe_condition_mode`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. +* `ϵ`, `δ`, `σ`, `ω`, `θ`, `γ`, `ρ`, `Δ`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. +* `secant_acceptance_ratio`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. +* `candidate_direction`, `temporary_tangent`: as temporary storage for tangent vectors +* `triples`: temporary storage for function and derivative evaluations +* `last_evaluation_index`: to keep track of the number of evaluations performed so far; + points at the last filled entry of `triples`. +* `Qₖ`, `Cₖ`: to keep track of the parameters of the Wolfe condition when in adaptive mode +* `current_mode`: to keep track of the current Wolfe condition mode when in adaptive mode +* `last_stepsize`: last stepsize computed since reset +* `last_cost`: last cost value computed since reset +* `ϵₖ`: the current ϵ parameter used in the approximate Wolfe condition and bracketing # Constructor HagerZhangLinesearchStepsize(M::AbstractManifold; kwargs...) - -## Keyword arguments - -$(_kwargs(:p; name = "candidate_point")) as temporary storage for candidates -$(_kwargs(:retraction_method)) -$(_kwargs(:vector_transport_method)) -* `initial_guess::AbstractInitialLinesearchGuess=HagerZhangInitialGuess()`: initial linesearch guess strategy -* `initial_last_stepsize::Real = NaN`: initial value for the stored last stepsize -* `initial_last_cost::Real = NaN`: initial value for the stored last cost -* `stepsize_limit::Real = Inf`: upper bound for trial stepsizes during bracketing -* `candidate_point = allocate_result(M, rand)`: storage for trial points -* `candidate_direction = zero_vector(M, candidate_point)`: storage for transported directions -* `max_bracket_iterations::Int = 10`: maximum number of bracketing iterations -* `start_enforcing_wolfe_conditions_at_bracketing_iteration::Int = initial_guess isa ConstantStepsize ? 2 : 1`: - bracketing iteration number at which Wolfe conditions are started to be enforced; - setting to 1 may cause no bracketing to occur when the initial guess satisfies the Wolfe - conditions. -* `max_function_evaluations::Int = 20`: maximum number of function evaluations per linesearch -* `wolfe_condition_mode::Symbol = :adaptive`: one of `:standard`, `:approximate`, or `:adaptive`. - Selects between (T1) and (T2) conditions in [HagerZhang:2006:2](@cite). -* `ϵ::Real = 1.0e-6`: initial allowed increase in function value in termination condition (T2). - Allowed range: `ϵ >= 0`. -* `δ::Real = 0.1`: parameter for approximate Wolfe condition. - Allowed range: `0 < δ < 0.5` and `δ <= σ`. -* `σ::Real = 0.9`: curvature condition parameter. Allowed range: `δ <= σ < 1`. -* `ω::Real = 1.0e-3`: interpolation safeguard parameter. Allowed range: `0 <= ω <= 1`. -* `θ::Real = 0.5`: bisection update parameter. Allowed range: `0 < θ < 1`. -* `γ::Real = 0.66`: determines when a bisection step is performed instead of secant. - Allowed range: `0 < γ < 1`. -* `ρ::Real = 5.0`: bracketing expansion factor. Allowed range: `ρ > 1`. -* `Δ::Real = 0.7`: Parameter controlling the rate of change of Qₖ. - Allowed range: `0 <= Δ <= 1`. -* `secant_acceptance_ratio::Real = 1.0e-8`: minimum relative interval length - for accepting secant step. Allowed range: `secant_acceptance_ratio >= 0`. - In case of rejection, a bisection step is performed instead. """ mutable struct HagerZhangLinesearchStepsize{ TF <: Real, @@ -2589,16 +2574,16 @@ function (hzls::HagerZhangLinesearchStepsize)( k::Int, η = (-get_gradient(mp, get_iterate(s))); fp = get_cost(mp, get_iterate(s)), + gradient = nothing, kwargs..., ) M = get_manifold(mp) p = get_iterate(s) - local dphi_0 # COV_EXCL_LINE - if :gradient in keys(kwargs) - dphi_0 = real(inner(M, p, η, kwargs[:gradient])) + dphi_0 = if !isnothing(gradient) + real(inner(M, p, η, gradient)) else - dphi_0 = get_differential(mp, p, η; Y = hzls.temporary_tangent) + get_differential(mp, p, η; Y = hzls.temporary_tangent) end hzls.triples[1] = UnivariateTriple(0.0, fp, dphi_0) hzls.last_evaluation_index = 1 @@ -2667,30 +2652,29 @@ function (hzls::HagerZhangLinesearchStepsize)( return hzls.last_stepsize end -function Base.show(io::IO, cbls::HagerZhangLinesearchStepsize) +function Base.show(io::IO, hzls::HagerZhangLinesearchStepsize) return print( io, """ HagerZhangLinesearch(; - initial_guess = $(cbls.initial_guess), - retraction_method = $(cbls.retraction_method), - vector_transport_method = $(cbls.vector_transport_method), - stepsize_limit = $(cbls.stepsize_limit), - max_bracket_iterations = $(cbls.max_bracket_iterations), - wolfe_condition_mode = $(cbls.wolfe_condition_mode), - ϵ = $(cbls.ϵ), - δ = $(cbls.δ), - σ = $(cbls.σ), - ω = $(cbls.ω), - θ = $(cbls.θ), - γ = $(cbls.γ), - ρ = $(cbls.ρ), - Δ = $(cbls.Δ), + initial_guess = $(hzls.initial_guess), + retraction_method = $(hzls.retraction_method), + vector_transport_method = $(hzls.vector_transport_method), + stepsize_limit = $(hzls.stepsize_limit), + max_bracket_iterations = $(hzls.max_bracket_iterations), + start_enforcing_wolfe_conditions_at_bracketing_iteration = $(hzls.start_enforcing_wolfe_conditions_at_bracketing_iteration), + max_function_evaluations = $(length(hzls.triples)), + wolfe_condition_mode = $(hzls.wolfe_condition_mode), + ϵ = $(hzls.ϵ), δ = $(hzls.δ), σ = $(hzls.σ), + ω = $(hzls.ω), + θ = $(hzls.θ), γ = $(hzls.γ), secant_acceptance_ratio = $(hzls.secant_acceptance_ratio), + ρ = $(hzls.ρ), + Δ = $(hzls.Δ), )""", ) end -function status_summary(cbls::HagerZhangLinesearchStepsize) - return "$(cbls)\nand a computed last stepsize of $(cbls.last_stepsize)" +function status_summary(hzls::HagerZhangLinesearchStepsize) + return "$(hzls)\nand a computed last stepsize of $(hzls.last_stepsize)" end @doc """ @@ -2700,9 +2684,53 @@ end A functor representing the curvature minimizing cubic bracketing scheme introduced in [HagerZhang:2006:2](@cite). -# Keyword arguments +The following changes were made to the original algorithm from the paper: +1. The algorithm bails out early out of a secant update that is too close to one of the end + points and switches to bisection. Original algorithm performs a similar check at a later + stage. This precaution prevents a non-productive evaluation of the objective. +2. Added `start_enforcing_wolfe_conditions_at_bracketing_iteration`, since with a very low + stepsize initialization that satisfies Wolfe conditions we might accept the initial + stepsize and not notice that bracketing could help us reach the minimum earlier. + Setting `start_enforcing_wolfe_conditions_at_bracketing_iteration`` to 1 reproduces the + behavior of the original paper. For example a static initial stepsize equal to 1.0 could + benefit from having this parameter increased. +3. The paper isn't entirely clear on what the final stepsize to return is. This + implementation returns the last evaluated stepsize. -$(_kwargs(:p)) to store an interim result +## Keyword arguments + +$(_kwargs(:p; name = "candidate_point")) as temporary storage for candidates +$(_kwargs(:retraction_method)) +$(_kwargs(:vector_transport_method)) +* `initial_guess::AbstractInitialLinesearchGuess=HagerZhangInitialGuess()`: initial linesearch guess strategy +* `initial_last_stepsize::Real = NaN`: initial value for the stored last stepsize +* `initial_last_cost::Real = NaN`: initial value for the stored last cost +* `stepsize_limit::Real = Inf`: upper bound for trial stepsizes during bracketing +* `candidate_point = allocate_result(M, rand)`: storage for trial points +* `candidate_direction = zero_vector(M, candidate_point)`: storage for transported directions +* `max_bracket_iterations::Int = 10`: maximum number of bracketing iterations +* `start_enforcing_wolfe_conditions_at_bracketing_iteration::Int = initial_guess isa ConstantStepsize ? 2 : 1`: + bracketing iteration number at which Wolfe conditions are started to be enforced; + setting to 1 may cause no bracketing to occur when the initial guess satisfies the Wolfe + conditions. +* `max_function_evaluations::Int = 20`: maximum number of function evaluations per linesearch +* `wolfe_condition_mode::Symbol = :adaptive`: one of `:standard`, `:approximate`, or `:adaptive`. + Selects between (T1) and (T2) conditions in [HagerZhang:2006:2](@cite). +* `ϵ::Real = 1.0e-6`: initial allowed increase in function value in termination condition (T2). + Allowed range: `ϵ >= 0`. +* `δ::Real = 0.1`: parameter for approximate Wolfe condition. + Allowed range: `0 < δ < 0.5` and `δ <= σ`. +* `σ::Real = 0.9`: curvature condition parameter. Allowed range: `δ <= σ < 1`. +* `ω::Real = 1.0e-3`: interpolation safeguard parameter. Allowed range: `0 <= ω <= 1`. +* `θ::Real = 0.5`: bisection update parameter. Allowed range: `0 < θ < 1`. +* `γ::Real = 0.66`: determines when a bisection step is performed instead of secant. + Allowed range: `0 < γ < 1`. +* `ρ::Real = 5.0`: bracketing expansion factor. Allowed range: `ρ > 1`. +* `Δ::Real = 0.7`: Parameter controlling the rate of change of Qₖ. + Allowed range: `0 <= Δ <= 1`. +* `secant_acceptance_ratio::Real = 1.0e-8`: minimum relative interval length + for accepting secant step. Allowed range: `secant_acceptance_ratio >= 0`. + In case of rejection, a bisection step is performed instead. $(_note(:ManifoldDefaultFactory, "HagerZhangLinesearch")) """ From 9c2d4afee44ccb29d7fe89b1831255fff6edca29 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 9 Feb 2026 19:00:28 +0100 Subject: [PATCH 102/135] expand changelog, stepsize initialization --- Changelog.md | 6 ++++-- src/solvers/FrankWolfe.jl | 1 + src/solvers/alternating_gradient_descent.jl | 1 + src/solvers/conjugate_gradient_descent.jl | 1 + src/solvers/gradient_descent.jl | 1 + src/solvers/projected_gradient_method.jl | 1 + src/solvers/proximal_gradient_method.jl | 1 + src/solvers/stochastic_gradient_descent.jl | 1 + src/solvers/subgradient.jl | 1 + 9 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index 3f6c4f9ec2..4bc3e46410 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,12 +12,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. This resolves issue #549. (#554) * `GeneralizedCauchyDirectionFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) -* New stopping criteria: `StopWhenRelativeAPosterioriCostChangeLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. (#554) +* New stopping criteria: `StopWhenRelativeAPosterioriCostChangeLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. (#554). +* `HagerZhangLinesearch` stepsize, a state-of-the-art line search for smooth objectives with cubic interpolation and adaptive Wolfe condition checking. (#554) +* Stopping criteria can now be initialized using `initialize_stepsize!`, similar to solvers. (#554) ### Fixed * Line searches consistently respect `stop_when_stepsize_exceeds` keyword argument as a hard limit. (#554) -* `StopWhenChangeLess` falsely claimed to indicate convergence. This is now fixed. +* `StopWhenChangeLess` falsely claimed to indicate convergence. This is now fixed. (#554) ## [0.5.32] January 15, 2026 diff --git a/src/solvers/FrankWolfe.jl b/src/solvers/FrankWolfe.jl index d57b784993..32c211ee75 100644 --- a/src/solvers/FrankWolfe.jl +++ b/src/solvers/FrankWolfe.jl @@ -320,6 +320,7 @@ calls_with_kwargs(::typeof(Frank_Wolfe_method!)) = (decorate_objective!, decorat function initialize_solver!(amp::AbstractManoptProblem, fws::FrankWolfeState) get_gradient!(amp, fws.X, fws.p) + initialize_stepsize!(fws.stepsize) return fws end function step_solver!(amp::AbstractManoptProblem, fws::FrankWolfeState, k) diff --git a/src/solvers/alternating_gradient_descent.jl b/src/solvers/alternating_gradient_descent.jl index d6b09568fd..84f1def1fa 100644 --- a/src/solvers/alternating_gradient_descent.jl +++ b/src/solvers/alternating_gradient_descent.jl @@ -255,6 +255,7 @@ function initialize_solver!( get_gradient!(amp, agds.X, agds.p) (agds.order_type == :FixedRandom || agds.order_type == :Random) && (shuffle!(agds.order)) + initialize_stepsize!(agds.stepsize) return agds end function step_solver!(amp::AbstractManoptProblem, agds::AlternatingGradientDescentState, k) diff --git a/src/solvers/conjugate_gradient_descent.jl b/src/solvers/conjugate_gradient_descent.jl index 62dd86338b..67809b9a51 100644 --- a/src/solvers/conjugate_gradient_descent.jl +++ b/src/solvers/conjugate_gradient_descent.jl @@ -178,6 +178,7 @@ function initialize_solver!(amp::AbstractManoptProblem, cgs::ConjugateGradientDe cgs.δ = -copy(get_manifold(amp), cgs.p, cgs.X) # remember the first gradient in coefficient calculation cgs.coefficient(amp, cgs, 0) + initialize_stepsize!(cgs.stepsize) cgs.β = 0.0 return cgs end diff --git a/src/solvers/gradient_descent.jl b/src/solvers/gradient_descent.jl index 2951d8e03b..575cce2829 100644 --- a/src/solvers/gradient_descent.jl +++ b/src/solvers/gradient_descent.jl @@ -252,6 +252,7 @@ calls_with_kwargs(::typeof(gradient_descent!)) = (decorate_objective!, decorate_ # function initialize_solver!(mp::AbstractManoptProblem, s::GradientDescentState) get_gradient!(mp, s.X, s.p) + initialize_stepsize!(s.stepsize) return s end function step_solver!(p::AbstractManoptProblem, s::GradientDescentState, k) diff --git a/src/solvers/projected_gradient_method.jl b/src/solvers/projected_gradient_method.jl index 8902b10711..542cd42dce 100644 --- a/src/solvers/projected_gradient_method.jl +++ b/src/solvers/projected_gradient_method.jl @@ -262,6 +262,7 @@ calls_with_kwargs(::typeof(projected_gradient_method!)) = (decorate_objective!, function initialize_solver!(amp::AbstractManoptProblem, pgms::ProjectedGradientMethodState) get_gradient!(amp, pgms.X, pgms.p) + initialize_stepsize!(pgms.stepsize) return pgms end diff --git a/src/solvers/proximal_gradient_method.jl b/src/solvers/proximal_gradient_method.jl index 4aa6fec14b..bdf84ee732 100644 --- a/src/solvers/proximal_gradient_method.jl +++ b/src/solvers/proximal_gradient_method.jl @@ -182,6 +182,7 @@ function initialize_solver!(amp::AbstractManoptProblem, pgms::ProximalGradientMe M = get_manifold(amp) zero_vector!(M, pgms.X, pgms.p) copyto!(M, pgms.a, pgms.p) + initialize_stepsize!(pgms.stepsize) return pgms end diff --git a/src/solvers/stochastic_gradient_descent.jl b/src/solvers/stochastic_gradient_descent.jl index 6d371cdade..06bc64aab0 100644 --- a/src/solvers/stochastic_gradient_descent.jl +++ b/src/solvers/stochastic_gradient_descent.jl @@ -297,6 +297,7 @@ calls_with_kwargs(::typeof(stochastic_gradient_descent!)) = (decorate_objective! function initialize_solver!(::AbstractManoptProblem, s::StochasticGradientDescentState) s.k = 1 (s.order_type == :FixedRandom) && (shuffle!(s.order)) + initialize_stepsize!(s.stepsize) return s end function step_solver!(mp::AbstractManoptProblem, s::StochasticGradientDescentState, iter) diff --git a/src/solvers/subgradient.jl b/src/solvers/subgradient.jl index 7c954ce4a1..d511199a45 100644 --- a/src/solvers/subgradient.jl +++ b/src/solvers/subgradient.jl @@ -193,6 +193,7 @@ function initialize_solver!(mp::AbstractManoptProblem, sgs::SubGradientMethodSta M = get_manifold(mp) copyto!(M, sgs.p_star, sgs.p) sgs.X = zero_vector(M, sgs.p) + initialize_stepsize!(sgs.stepsize) return sgs end function step_solver!(mp::AbstractManoptProblem, sgs::SubGradientMethodState, k) From 56ec7a03d83f3d870125c287b467e91ff46faec8 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 10 Feb 2026 09:56:15 +0100 Subject: [PATCH 103/135] Update src/plans/stepsize/stepsize.jl Co-authored-by: Ronny Bergmann --- src/plans/stepsize/stepsize.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index cb997b2559..c8e90ac283 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -2144,7 +2144,7 @@ end Do a bracketing line search to find a step size ``α`` that finds a local minimum along the search direction ``X`` starting from ``p``, utilizing cubic polynomial interpolation using the method described in -[HagerZhang:2006:2](@cite). Function [`secant`](@ref) is used to find the minimum of the +[HagerZhang:2006:2](@cite). The function [`secant`](@ref) is used to find the minimum of the cubic polynomial fitted to values of the cost function and its derivative at the endpoints of the current interval. See [`HagerZhangLinesearch`](@ref) for the mathematical details. From 59d2468153643298f288045d3084526136549bf1 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 13 Feb 2026 12:23:19 +0100 Subject: [PATCH 104/135] Make GCD more type stable --- src/plans/box_plan.jl | 28 ++++++++++++++++++--------- test/solvers/test_quasi_Newton_box.jl | 9 +++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 9ed75be47f..6857a4b11a 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -603,7 +603,7 @@ avoid allocations. """ struct GeneralizedCauchyDirectionFinder{ TM <: AbstractManifold, TX, - T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater, TFT <: Tuple, TBI, + T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater, TFT <: Tuple{<:Real, Any}, TBI, TO <: Base.Order.Ordering, } M::TM @@ -631,11 +631,11 @@ function GeneralizedCauchyDirectionFinder( ) end -function collect_isotropic_limits!(::AbstractManifold, ::Vector{<:Tuple{TF, Any}}, p, d) where {TF <: Real} +function collect_isotropic_limits!(::AbstractManifold, ::Vector{<:Tuple{TF, Any}}, p, d)::Tuple{Bool, TF} where {TF <: Real} return false, convert(TF, Inf) end -function collect_isotropic_limits!(M::ProductManifold, F_list::Vector{<:Tuple{TF, Any}}, p, d) where {TF <: Real} +function collect_isotropic_limits!(M::ProductManifold, F_list::Vector{<:Tuple{TF, Any}}, p, d)::Tuple{Bool, TF} where {TF <: Real} has_finite_limit = false smallest_positive_limit = Inf map(M.manifolds, submanifold_components(M, p), submanifold_components(M, d)) do Mi, p_i, d_i @@ -660,14 +660,24 @@ end Find generalized Cauchy direction looking from point `p` in direction `d` and save it to `d_out`. Gradient of the objective at `p` is `X`. -The function returns +The function returns a pair (status, max_stepsize) where `status` is a symbol describing +the result of the search, and `max_stepsize` is the maximum stepsize that can be taken in +the direction `d_out`. + +The `status` can be one of the following: * `:found_limited` if the point was found and we can perform a step of length at most 1 in direction `d_out` afterwards, * `:found_unlimited` if the point was found and we can perform a step of length at most `max_stepsize(M, p)` in direction `d_out` afterwards, * `:not_found` if the search cannot be performed in direction `d`. """ -function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinder, d_out, p, d, X) +function find_generalized_cauchy_direction!( + gcd::GeneralizedCauchyDirectionFinder{ + <:AbstractManifold, <:Any, + <:AbstractQuasiNewtonDirectionUpdate, <:AbstractSegmentHessianUpdater, <:Tuple{TF, Any}, + }, + d_out, p, d, X + ) where {TF <: Real} M = gcd.M copyto!(M, gcd.d_original, d) copyto!(M, d_out, d) @@ -682,7 +692,7 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde has_finite_limit, smallest_positive_limit = collect_isotropic_limits!(M, F_list, p, d) # anisotropic limits for i in bounds_indices - sbi = get_stepsize_bound(M, p, d, i) + sbi = get_stepsize_bound(M, p, d, i)::TF if sbi > 0 push!(F_list, (sbi, i)) @@ -717,10 +727,10 @@ function find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinde init_updater!(M, gcd.hessian_segment_updater, p, d, gcd.ha) # b can be -1 if it corresponds to the max stepsize limit on the manifold part while dt_min > dt && b != -1 - db = get_at_bound_index(M, d, b) - gb = get_at_bound_index(M, X, b) + db = get_at_bound_index(M, d, b)::TF + gb = get_at_bound_index(M, X, b)::TF - hv_eb_dz, hv_eb_d = gcd.hessian_segment_updater(M, p, t_current, dt, b, db, gcd.ha) + hv_eb_dz, hv_eb_d = gcd.hessian_segment_updater(M, p, t_current, dt, b, db, gcd.ha)::Tuple{TF, TF} f_prime += dt * f_double_prime - db * (gb + hv_eb_dz) f_double_prime += (2 * -db * hv_eb_d) + db^2 * hessian_value_diag(gcd.ha, M, p, UnitVector(b)) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 9c10c1db4b..552c877448 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -289,6 +289,15 @@ using RecursiveArrayTools @test hv_eb_d ≈ -256.0 end + @testset "GCD check" begin + d = -grad_f(M, p0) + ha = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p0, InverseBFGS(), 2)) + gf = Manopt.GeneralizedCauchyDirectionFinder(M, p0, ha) + d_out = similar(d) + X = grad_f(M, p0) + @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p0, d, X) === (:found_limited, 1.0) + end + p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) @test distance(M, p_opt, ArrayPartition([0, 2, 0], px)) < 0.1 end From 0018cd516c4aee321056297f8dca503d35fb63c7 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 13 Feb 2026 13:03:22 +0100 Subject: [PATCH 105/135] Add simple directional stepsize limiting for completeness --- .../generalized_cauchy_direction_subsolver.md | 10 +++ src/plans/box_plan.jl | 84 +++++++++++++++++++ test/solvers/test_quasi_Newton_box.jl | 35 ++++++++ 3 files changed, 129 insertions(+) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index b007180efe..f43ced1efd 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -16,6 +16,16 @@ Note that the value $s_{\max}=1$ is obtained when the minimum lies at the bounda The solver is currently primarily intended for internal use by optimization algorithms that require bound-constrained subproblem solutions. +## Simple stepsize limiting + +In case there is no Hessian approximation available, a simple stepsize limiting procedure is can be used to limit the stepsize in direction $X$ to the maximum allowed by the boundary of $D$ and the maximum allowed stepsize on $\mathcal{M}$. +This procedure is available using the following: + +```@docs +MaxStepsizeInDirectionFinder +find_max_stepsize_in_direction +``` + ## Internal types and method ### Symbols related to the GCD computation diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 6857a4b11a..a0c3f74689 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -765,3 +765,87 @@ function find_generalized_cauchy_direction!( return (:found_unlimited, Inf) end end + +""" + struct MaxStepsizeInDirectionFinder end + +Helper container for finding the maximum stepsize in a direction. Stores the manifold `M`, +container for the list of bounds `F_list`, and the bound indices. + +## Constructor + + MaxStepsizeInDirectionFinder(M::AbstractManifold, p) + +Initialize the `MaxStepsizeInDirectionFinder` for manifold `M` and point `p`. The `F_list` +is initialized to be empty and will be populated during the search for the maximum stepsize +in a direction. Floating point type of the elements bounds in `F_list` is determined by the +number type of `p`. + +The `MaxStepsizeInDirectionFinder` can be reused for multiple different points and +directions on the same manifold, but it is not thread-safe. +""" +struct MaxStepsizeInDirectionFinder{TM <: AbstractManifold, TFT <: Tuple{<:Real, Any}, TBI} + M::TM + F_list::Vector{TFT} + bounds_indices::TBI +end +function MaxStepsizeInDirectionFinder(M::AbstractManifold, p) + bounds_indices = get_bounds_index(M) + TInd = eltype(bounds_indices) + TF = number_eltype(p) + F_list = Tuple{TF, TInd}[] + sizehint!(F_list, length(bounds_indices) + 1) + return MaxStepsizeInDirectionFinder{typeof(M), Tuple{TF, TInd}, typeof(bounds_indices)}(M, F_list, bounds_indices) +end + +""" + find_max_stepsize_in_direction(gcd::MaxStepsizeInDirectionFinder, p, d) + +Find the maximum stepsize that can be performed from point `p` in direction `d`. + +The function returns a pair (status, max_stepsize) where `status` is a symbol describing +the result of the search, and `max_stepsize` is the maximum stepsize that can be taken in +the direction `d_out`. + +The `status` can be one of the following: +* `:found_limited` if the point was found and we can perform a step of length at most 1 + in direction `d_out` afterwards, +* `:found_unlimited` if the point was found and we can perform a step of length at most + `max_stepsize(M, p)` in direction `d_out` afterwards, +* `:not_found` if the search cannot be performed in direction `d`. +""" +function find_max_stepsize_in_direction( + sdf::MaxStepsizeInDirectionFinder{<:AbstractManifold, <:Tuple{TF, Any}}, + p, d + ) where {TF <: Real} + + M = sdf.M + F_list = sdf.F_list + empty!(F_list) + bounds_indices = sdf.bounds_indices + + # isotropic limits + has_finite_limit, smallest_positive_limit = collect_isotropic_limits!(M, F_list, p, d) + # anisotropic limits + for i in bounds_indices + sbi = get_stepsize_bound(M, p, d, i)::TF + + if sbi > 0 + push!(F_list, (sbi, i)) + if sbi < smallest_positive_limit + smallest_positive_limit = sbi + end + end + has_finite_limit |= isfinite(sbi) + end + + if isempty(F_list) + return (:not_found, NaN) + end + if has_finite_limit + return (:found_limited, smallest_positive_limit) + else + return (:found_unlimited, Inf) + end + +end diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 552c877448..5437cd2fcb 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -315,3 +315,38 @@ using RecursiveArrayTools @test distance(M, p_opt, ArrayPartition(px, [0 2; 0 0])) < 0.1 end end + +@testset "MaxStepsizeInDirection" begin + @testset "found_limited" begin + M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) + p = [0.0, 0.0, 0.0] + d = [2.0, 1.0, 1.0] + d_before = copy(d) + + sdf = Manopt.MaxStepsizeInDirectionFinder(M, p) + @test Manopt.find_max_stepsize_in_direction(sdf, p, d) === (:found_limited, 1.0) + @test d == d_before + end + + @testset "found_unlimited" begin + M = Hyperrectangle([-Inf], [Inf]) + p = [0.0] + d = [1.0] + d_before = copy(d) + + sdf = Manopt.MaxStepsizeInDirectionFinder(M, p) + @test Manopt.find_max_stepsize_in_direction(sdf, p, d) === (:found_unlimited, Inf) + @test d == d_before + end + + @testset "not_found" begin + M = Hyperrectangle([0.0], [1.0]) + p = [0.0] + d = [-1.0] + d_before = copy(d) + + sdf = Manopt.MaxStepsizeInDirectionFinder(M, p) + @test Manopt.find_max_stepsize_in_direction(sdf, p, d) === (:not_found, NaN) + @test d == d_before + end +end From a192dcc4a19921ddaed312e7dc88a6e114171224 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 13 Feb 2026 13:07:52 +0100 Subject: [PATCH 106/135] fix docs --- docs/src/solvers/generalized_cauchy_direction_subsolver.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index f43ced1efd..71e0f24089 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -22,8 +22,8 @@ In case there is no Hessian approximation available, a simple stepsize limiting This procedure is available using the following: ```@docs -MaxStepsizeInDirectionFinder -find_max_stepsize_in_direction +Manopt.MaxStepsizeInDirectionFinder +Manopt.find_max_stepsize_in_direction ``` ## Internal types and method From 7308ea027dbd6d85a3478189bc8bc40b209f08a1 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 16 Feb 2026 14:11:17 +0100 Subject: [PATCH 107/135] add an early termination condition for HZ linesearch; add show methods to two types --- src/plans/box_plan.jl | 5 ++++ src/plans/quasi_newton_plan.jl | 4 +++ src/plans/stepsize/stepsize.jl | 45 ++++++++++++++++++++++++++-------- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index a0c3f74689..e026fc9c86 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -849,3 +849,8 @@ function find_max_stepsize_in_direction( end end + +function show(io::IO, qns::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + print(io, "QuasiNewtonLimitedMemoryBoxDirectionUpdate with internal state:\n") + return print(io, qns.qn_du) +end diff --git a/src/plans/quasi_newton_plan.jl b/src/plans/quasi_newton_plan.jl index e6e5e1e5d4..db4686a74f 100644 --- a/src/plans/quasi_newton_plan.jl +++ b/src/plans/quasi_newton_plan.jl @@ -781,6 +781,10 @@ function initialize_update!(d::QuasiNewtonLimitedMemoryDirectionUpdate) return d end +function show(io::IO, qns::QuasiNewtonLimitedMemoryDirectionUpdate) + return print(io, "QuasiNewtonLimitedMemoryDirectionUpdate with memory size $(length(qns.memory_s)) and $(qns.vector_transport_method) as vector transport.") +end + @doc """ QuasiNewtonCautiousDirectionUpdate <: AbstractQuasiNewtonDirectionUpdate diff --git a/src/plans/stepsize/stepsize.jl b/src/plans/stepsize/stepsize.jl index c8e90ac283..4f1b0e5213 100644 --- a/src/plans/stepsize/stepsize.jl +++ b/src/plans/stepsize/stepsize.jl @@ -2161,6 +2161,7 @@ $(_fields(:vector_transport_method)) * `max_bracket_iterations`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. * `start_enforcing_wolfe_conditions_at_bracketing_iteration`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. +* `allow_early_maxstep_termination`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. * `wolfe_condition_mode`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. * `ϵ`, `δ`, `σ`, `ω`, `θ`, `γ`, `ρ`, `Δ`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. * `secant_acceptance_ratio`: see keyword arguments of [`HagerZhangLinesearch`](@ref) for details. @@ -2193,6 +2194,7 @@ mutable struct HagerZhangLinesearchStepsize{ stepsize_limit::TF max_bracket_iterations::Int start_enforcing_wolfe_conditions_at_bracketing_iteration::Int + allow_early_maxstep_termination::Bool wolfe_condition_mode::Symbol # :standard, :approximate, :adaptive ϵ::TF # approximate Wolfe termination parameter δ::TF # used in approximate Wolfe condition @@ -2232,6 +2234,7 @@ mutable struct HagerZhangLinesearchStepsize{ start_enforcing_wolfe_conditions_at_bracketing_iteration::Int = initial_guess isa ConstantStepsize ? 2 : 1, max_function_evaluations::Int = 20, wolfe_condition_mode::Symbol = :adaptive, + allow_early_maxstep_termination::Bool = true, ϵ::TF = 1.0e-6, δ::TF = 0.1, σ::TF = 0.9, @@ -2267,7 +2270,8 @@ mutable struct HagerZhangLinesearchStepsize{ return new{TF, TIG, TRM, TVTM, typeof(candidate_point), typeof(candidate_direction)}( initial_guess, retraction_method, vector_transport_method, stepsize_limit, - max_bracket_iterations, start_enforcing_wolfe_conditions_at_bracketing_iteration, wolfe_condition_mode, + max_bracket_iterations, start_enforcing_wolfe_conditions_at_bracketing_iteration, + allow_early_maxstep_termination, wolfe_condition_mode, ϵ, δ, σ, ω, θ, γ, ρ, Δ, secant_acceptance_ratio, candidate_point, candidate_direction, zero_vector(M, candidate_point), triples, 0, @@ -2375,9 +2379,12 @@ end Perform the bracketing phase of the Hager-Zhang linesearch starting from an initial stepsize `c` and not exceeding `max_alpha`. -Returns a tuple `(i_a, i_b)` where `i_a` and `i_b` are the indices in the stored function -evaluations such that the minimum is bracketed between `triples[i_a].t` and -`triples[i_b].t`. +Returns a tuple `(i_a, i_b, f_eval, f_wolfe, f_early_maxstep)` where `i_a` and `i_b` are +the indices in the stored function evaluations such that the minimum is bracketed between +`triples[i_a].t` and `triples[i_b].t`. `f_eval` is `true` if the maximum number of function +evaluations has been reached during the bracketing phase. `f_wolfe` is `true` if the Wolfe +conditions have been satisfied. `f_early_maxstep` is `true` if the maximum stepsize was +reached early with negative slope and an improvement over the initial point. """ function _hz_bracket( hzls::HagerZhangLinesearchStepsize, M::AbstractManifold, @@ -2386,6 +2393,7 @@ function _hz_bracket( # B0 current_step = c local c_index, f_eval, f_wolfe # COV_EXCL_LINE + ls_early_exit = false for j in 1:hzls.max_bracket_iterations c_index, f_eval, f_wolfe = _hz_evaluate_next_step(hzls, M, mp, p, η, current_step) if f_eval || (f_wolfe && j >= hzls.start_enforcing_wolfe_conditions_at_bracketing_iteration) @@ -2396,15 +2404,16 @@ function _hz_bracket( # handled after the loop break else - if hzls.triples[j].f > hzls.triples[1].f + hzls.ϵₖ + if hzls.triples[c_index].f > hzls.triples[1].f + hzls.ϵₖ # B2 -- function value gets sufficiently larger than at 0 # perform main bracketing loop (we can skip U0-U2 checks here) (i_a_bar, i_b_bar, f_eval, f_wolfe) = _hz_u3(hzls, M, mp, p, η, 1, c_index) - return (i_a_bar, i_b_bar, f_eval, f_wolfe) + return (i_a_bar, i_b_bar, f_eval, f_wolfe, false) else if current_step == max_alpha # we've reached maximum alpha so we can't expand anymore # we handle this case after the loop + ls_early_exit = hzls.allow_early_maxstep_termination break end # B3 -- widen the bracket @@ -2417,13 +2426,20 @@ function _hz_bracket( end # we detected positive slope, ran out of iterations or reached max stepsize # B1 seems to be the best choice for all three cases + + if ls_early_exit + # additional termination condition: we reached the maximum stepsize with negative + # slope and an improvement over the initial point, so we can exit early with this step + return (1, c_index, f_eval, f_wolfe, true) + end + i_min = 1 for i in 2:(hzls.last_evaluation_index - 1) if hzls.triples[i].f <= hzls.triples[1].f + hzls.ϵₖ i_min = i end end - return (i_min, c_index, f_eval, f_wolfe) + return (i_min, c_index, f_eval, f_wolfe, false) end """ @@ -2617,8 +2633,8 @@ function (hzls::HagerZhangLinesearchStepsize)( # L0, bracket(c) local i_a_j, i_b_j, f_eval, f_wolfe # COV_EXCL_LINE - (i_a_j, i_b_j, f_eval, f_wolfe) = _hz_bracket(hzls, M, mp, p, η, α0, max_alpha) - while !(f_eval || f_wolfe) + (i_a_j, i_b_j, f_eval, f_wolfe, f_early_maxstep) = _hz_bracket(hzls, M, mp, p, η, α0, max_alpha) + !f_early_maxstep && while !(f_eval || f_wolfe) # L1 finite_at_b = isfinite(hzls.triples[i_b_j].f) if finite_at_b @@ -2631,7 +2647,7 @@ function (hzls::HagerZhangLinesearchStepsize)( # L2 # we additionally check that we can continue narrowing the bracket if !(f_eval || f_wolfe) && - (!finite_at_b || hzls.triples[i_b].t - hzls.triples[i_a].t > hzls.γ * (hzls.triples[i_b_j].t - hzls.triples[i_a_j].t)) + (!finite_at_b || (hzls.triples[i_b].t - hzls.triples[i_a].t) > hzls.γ * (hzls.triples[i_b_j].t - hzls.triples[i_a_j].t)) # secant2 did not reduce the bracket sufficiently # we need to do bisection (i_a, i_b, _i_c, f_eval, f_wolfe) = _hz_update( @@ -2696,6 +2712,13 @@ The following changes were made to the original algorithm from the paper: benefit from having this parameter increased. 3. The paper isn't entirely clear on what the final stepsize to return is. This implementation returns the last evaluated stepsize. +4. The original algorithm doesn't specify what to do when the maximum stepsize is reached + during the bracketing phase with a negative slope and an improvement over the initial + point. This implementation allows for an early termination in this case, which seems + reasonable since we can't expand the bracket anymore and this point is likely close to + the minimum. By default this early termination is allowed, but it can be turned off via + `allow_early_maxstep_termination` in which case the algorithm continues with the main + loop even in this case. ## Keyword arguments @@ -2714,6 +2737,8 @@ $(_kwargs(:vector_transport_method)) setting to 1 may cause no bracketing to occur when the initial guess satisfies the Wolfe conditions. * `max_function_evaluations::Int = 20`: maximum number of function evaluations per linesearch +* `allow_early_maxstep_termination::Bool = true`: whether to allow early termination when + the maximum stepsize is reached with negative slope and an improvement over the initial point. * `wolfe_condition_mode::Symbol = :adaptive`: one of `:standard`, `:approximate`, or `:adaptive`. Selects between (T1) and (T2) conditions in [HagerZhang:2006:2](@cite). * `ϵ::Real = 1.0e-6`: initial allowed increase in function value in termination condition (T2). From 25407c137a0f0f178bc82dabfb7afc79e9589d5b Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 16 Feb 2026 16:25:22 +0100 Subject: [PATCH 108/135] test new printing methods --- test/solvers/test_quasi_Newton.jl | 2 ++ test/solvers/test_quasi_Newton_box.jl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index e6fe302dff..d94ac88787 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -499,6 +499,8 @@ end # This triggers and cautious update that does not update the Hessian Manopt.update_hessian!(qns.direction_update, mp, qns, p, 1) # But I am not totally sure what to test for afterwards + + @test startswith(repr(qdu), "QuasiNewtonLimitedMemoryDirectionUpdate with memory size") end @testset "Removing zero rho vectors" begin M = Euclidean(2) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 5437cd2fcb..64b1d56911 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -97,6 +97,8 @@ using RecursiveArrayTools ha = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2)) st = QuasiNewtonState(M) + @test startswith(repr(ha), "QuasiNewtonLimitedMemoryBoxDirectionUpdate with internal state:") + f(M, p) = sum(p .^ 2) grad_f(M, p) = 2 * p gmp = ManifoldGradientObjective(f, grad_f) From 0cca30e19ef27b42a0ff3cc0a0b0ca22ba272341 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Mon, 16 Feb 2026 17:05:51 +0100 Subject: [PATCH 109/135] improve coverage --- test/plans/test_stepsize.jl | 68 ++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/test/plans/test_stepsize.jl b/test/plans/test_stepsize.jl index 0f5f4b68ec..48cb67d584 100644 --- a/test/plans/test_stepsize.jl +++ b/test/plans/test_stepsize.jl @@ -515,6 +515,43 @@ end α = hzls_u3(dmp, gs, 1, η) @test α > 0 end + @testset "U3 (b) trigger test" begin + M = Euclidean(1) + # Force U3 (b) in _hz_u3: + # 1) At d=0.5 we need df < 0 and f(d) <= f(0) + ϵₖ with no termination, + # so i_a_bar gets updated to i_d. + # 2) On the next U3 iteration we return from the loop. + function f_u3b(M, q) + return 0.0 + end + + function grad_f_u3b(M, q) + v = q[1] + if isapprox(v, 0.0; atol = 1.0e-12) + return [-1.0] + elseif isapprox(v, 1.0; atol = 1.0e-12) + return [1.0] + elseif isapprox(v, 0.5; atol = 1.0e-12) + return [-1.0] + elseif isapprox(v, 0.75; atol = 1.0e-12) + return [1.0] + end + return [0.0] + end + + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f_u3b, grad_f_u3b)) + p = [0.0] + η = [1.0] + hzls_u3b = Manopt.HagerZhangLinesearchStepsize(M; max_function_evaluations = 4) + Manopt.initialize_stepsize!(hzls_u3b) + Manopt._hz_evaluate_next_step(hzls_u3b, M, dmp, p, η, 0.0) + Manopt._hz_evaluate_next_step(hzls_u3b, M, dmp, p, η, 1.0) + + (i_a, i_b, f_eval, f_wolfe) = Manopt._hz_u3(hzls_u3b, M, dmp, p, η, 1, 2) + @test (i_a, i_b) == (3, 4) + @test f_eval + @test !f_wolfe + end @testset "U3 (c) info trigger test" begin M = Euclidean(1) # Force U3 (c) inside _hz_u3 by making the mid-point have @@ -575,6 +612,29 @@ end @test !f_eval @test !f_wolfe end + @testset "U0 out-of-bracket early return" begin + M = Euclidean(1) + f(M, p) = sum(p .^ 2) + grad_f(M, p) = 2 .* p + dmp = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) + p = [0.0] + η = [1.0] + + hzls_u0 = Manopt.HagerZhangLinesearchStepsize(M; max_function_evaluations = 5) + Manopt.initialize_stepsize!(hzls_u0) + Manopt._hz_evaluate_next_step(hzls_u0, M, dmp, p, η, 0.0) + Manopt._hz_evaluate_next_step(hzls_u0, M, dmp, p, η, 1.0) + + last_eval_before = hzls_u0.last_evaluation_index + + # c is left of bracket [0, 1] -> U0 early return + @test (1, 2, -1, false, false) == Manopt._hz_update(hzls_u0, M, dmp, p, η, 1, 2, -0.1) + @test hzls_u0.last_evaluation_index == last_eval_before + + # c is right of bracket [0, 1] -> U0 early return + @test (1, 2, -1, false, false) == Manopt._hz_update(hzls_u0, M, dmp, p, η, 1, 2, 1.1) + @test hzls_u0.last_evaluation_index == last_eval_before + end @testset "S2 trigger test" begin M = Euclidean(1) @@ -633,14 +693,6 @@ end # 3. Secant gives c=0.2. At c, df=-0.1 and f=0 -> U2. function f_s3(M, q) - v = q[1] - if isapprox(v, 0.0; atol = 1.0e-12) - return 0.0 - elseif isapprox(v, 1.0; atol = 1.0e-12) - return 0.0 - elseif isapprox(v, 0.2; atol = 1.0e-12) - return 0.0 - end return 0.0 end From 19d7df26678a409cc83bd151bef620300fe44f85 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 17 Feb 2026 09:10:56 +0100 Subject: [PATCH 110/135] rename Finder to Subsolver, don't store manifold inside --- src/plans/box_plan.jl | 57 ++++++++++++++------------- test/solvers/test_quasi_Newton_box.jl | 40 +++++++++---------- 2 files changed, 49 insertions(+), 48 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index e026fc9c86..24db43cda0 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -105,8 +105,8 @@ function (d::QuasiNewtonLimitedMemoryBoxDirectionUpdate)( M = get_manifold(mp) p = get_iterate(st) X = get_gradient(st) - gcd = GeneralizedCauchyDirectionFinder(M, p, d) - d.last_gcd_result, d.last_gcd_stepsize = find_generalized_cauchy_direction!(gcd, r, p, r, X) + gcd = GeneralizedCauchyDirectionSubsolver(M, p, d) + d.last_gcd_result, d.last_gcd_stepsize = find_generalized_cauchy_direction!(M, gcd, r, p, r, X) return r end @@ -338,7 +338,7 @@ end abstract type AbstractSegmentHessianUpdater end Abstract type for methods that calculate f' and f'' in the GCD calculation in subsequent -line segments in [`GeneralizedCauchyDirectionFinder`](@ref). +line segments in [`GeneralizedCauchyDirectionSubsolver`](@ref). """ abstract type AbstractSegmentHessianUpdater end @@ -593,7 +593,7 @@ function set_stepsize_bound!(M::ProductManifold, d_out, p, d, t_current::Real) end @doc raw""" - GeneralizedCauchyDirectionFinder{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} + GeneralizedCauchyDirectionSubsolver{TM <: AbstractManifold, TP, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater} Helper container for generalized Cauchy direction search. Stores the manifold `M`, cached original descent direction (`d_original`), the quasi-Newton direction update `ha`, and the @@ -601,12 +601,11 @@ original descent direction (`d_original`), the quasi-Newton direction update `ha Instances are reused across segments during [`find_generalized_cauchy_direction!`](@ref) to avoid allocations. """ -struct GeneralizedCauchyDirectionFinder{ - TM <: AbstractManifold, TX, +struct GeneralizedCauchyDirectionSubsolver{ + TX, T_HA <: AbstractQuasiNewtonDirectionUpdate, TFU <: AbstractSegmentHessianUpdater, TFT <: Tuple{<:Real, Any}, TBI, TO <: Base.Order.Ordering, } - M::TM d_original::TX ha::T_HA hessian_segment_updater::TFU @@ -615,7 +614,7 @@ struct GeneralizedCauchyDirectionFinder{ ordering::TO end -function GeneralizedCauchyDirectionFinder( +function GeneralizedCauchyDirectionSubsolver( M::AbstractManifold, p, ha::AbstractQuasiNewtonDirectionUpdate; hessian_segment_updater::AbstractSegmentHessianUpdater = get_default_hessian_segment_updater(M, p, ha) ) @@ -625,8 +624,8 @@ function GeneralizedCauchyDirectionFinder( F_list = Tuple{TF, TInd}[] sizehint!(F_list, length(bounds_indices) + 1) ordering = Base.By(first) - return GeneralizedCauchyDirectionFinder( - M, zero_vector(M, p), ha, + return GeneralizedCauchyDirectionSubsolver( + zero_vector(M, p), ha, hessian_segment_updater, F_list, bounds_indices, ordering ) end @@ -655,10 +654,13 @@ function collect_isotropic_limits!(M::ProductManifold, F_list::Vector{<:Tuple{TF end """ - find_generalized_cauchy_direction!(gcd::GeneralizedCauchyDirectionFinder, d_out, p, d, X) + find_generalized_cauchy_direction!( + M::AbstractManifold, + gcd::GeneralizedCauchyDirectionSubsolver, d_out, p, d, X + ) -Find generalized Cauchy direction looking from point `p` in direction `d` and save it to `d_out`. -Gradient of the objective at `p` is `X`. +Find generalized Cauchy direction looking from point `p` on manifold `M` in direction `d` +and save it to `d_out`. Gradient of the objective at `p` is `X`. The function returns a pair (status, max_stepsize) where `status` is a symbol describing the result of the search, and `max_stepsize` is the maximum stepsize that can be taken in @@ -672,13 +674,13 @@ The `status` can be one of the following: * `:not_found` if the search cannot be performed in direction `d`. """ function find_generalized_cauchy_direction!( - gcd::GeneralizedCauchyDirectionFinder{ - <:AbstractManifold, <:Any, - <:AbstractQuasiNewtonDirectionUpdate, <:AbstractSegmentHessianUpdater, <:Tuple{TF, Any}, + M::AbstractManifold, + gcd::GeneralizedCauchyDirectionSubsolver{ + <:Any, <:AbstractQuasiNewtonDirectionUpdate, + <:AbstractSegmentHessianUpdater, <:Tuple{TF, Any}, }, d_out, p, d, X ) where {TF <: Real} - M = gcd.M copyto!(M, gcd.d_original, d) copyto!(M, d_out, d) @@ -767,39 +769,38 @@ function find_generalized_cauchy_direction!( end """ - struct MaxStepsizeInDirectionFinder end + struct MaxStepsizeInDirectionSubsolver end Helper container for finding the maximum stepsize in a direction. Stores the manifold `M`, container for the list of bounds `F_list`, and the bound indices. ## Constructor - MaxStepsizeInDirectionFinder(M::AbstractManifold, p) + MaxStepsizeInDirectionSubsolver(M::AbstractManifold, p) -Initialize the `MaxStepsizeInDirectionFinder` for manifold `M` and point `p`. The `F_list` +Initialize the `MaxStepsizeInDirectionSubsolver` for manifold `M` and point `p`. The `F_list` is initialized to be empty and will be populated during the search for the maximum stepsize in a direction. Floating point type of the elements bounds in `F_list` is determined by the number type of `p`. -The `MaxStepsizeInDirectionFinder` can be reused for multiple different points and +The `MaxStepsizeInDirectionSubsolver` can be reused for multiple different points and directions on the same manifold, but it is not thread-safe. """ -struct MaxStepsizeInDirectionFinder{TM <: AbstractManifold, TFT <: Tuple{<:Real, Any}, TBI} - M::TM +struct MaxStepsizeInDirectionSubsolver{TFT <: Tuple{<:Real, Any}, TBI} F_list::Vector{TFT} bounds_indices::TBI end -function MaxStepsizeInDirectionFinder(M::AbstractManifold, p) +function MaxStepsizeInDirectionSubsolver(M::AbstractManifold, p) bounds_indices = get_bounds_index(M) TInd = eltype(bounds_indices) TF = number_eltype(p) F_list = Tuple{TF, TInd}[] sizehint!(F_list, length(bounds_indices) + 1) - return MaxStepsizeInDirectionFinder{typeof(M), Tuple{TF, TInd}, typeof(bounds_indices)}(M, F_list, bounds_indices) + return MaxStepsizeInDirectionSubsolver{Tuple{TF, TInd}, typeof(bounds_indices)}(F_list, bounds_indices) end """ - find_max_stepsize_in_direction(gcd::MaxStepsizeInDirectionFinder, p, d) + find_max_stepsize_in_direction(M::AbstractManifold, gcd::MaxStepsizeInDirectionSubsolver, p, d) Find the maximum stepsize that can be performed from point `p` in direction `d`. @@ -815,11 +816,11 @@ The `status` can be one of the following: * `:not_found` if the search cannot be performed in direction `d`. """ function find_max_stepsize_in_direction( - sdf::MaxStepsizeInDirectionFinder{<:AbstractManifold, <:Tuple{TF, Any}}, + M::AbstractManifold, + sdf::MaxStepsizeInDirectionSubsolver{<:Tuple{TF, Any}}, p, d ) where {TF <: Real} - M = sdf.M F_list = sdf.F_list empty!(F_list) bounds_indices = sdf.bounds_indices diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 64b1d56911..6fb5f829ab 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -160,46 +160,46 @@ using RecursiveArrayTools end end - @testset "GeneralizedCauchyDirectionFinder" begin + @testset "GeneralizedCauchyDirectionSubsolver" begin M = Hyperrectangle([-1.0, -2.0, -Inf], [2.0, Inf, 2.0]) ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS()) p = [0.0, 0.0, 0.0] - gf = Manopt.GeneralizedCauchyDirectionFinder(M, p, ha) + gf = Manopt.GeneralizedCauchyDirectionSubsolver(M, p, ha) X1 = [-5.0, 0.0, 0.0] d = -X1 d_out = similar(d) - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d, X1) === (:found_limited, 1.0) + @test Manopt.find_generalized_cauchy_direction!(M, gf, d_out, p, d, X1) === (:found_limited, 1.0) @test d_out ≈ [2.0, 0.0, 0.0] d_out = similar(d) - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, 0 * d, X1) === (:not_found, NaN) + @test Manopt.find_generalized_cauchy_direction!(M, gf, d_out, p, 0 * d, X1) === (:not_found, NaN) d2 = [0.0, 1.0, 0.0] - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d2, [0.0, -1.0, 0.0]) === (:found_unlimited, Inf) + @test Manopt.find_generalized_cauchy_direction!(M, gf, d_out, p, d2, [0.0, -1.0, 0.0]) === (:found_unlimited, Inf) @test d_out ≈ d2 - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === (:found_limited, 1.0) + @test Manopt.find_generalized_cauchy_direction!(M, gf, d_out, p, [1.0, 1.0, 0.0], [-10.0, -10.0, -10.0]) === (:found_limited, 1.0) @test d_out ≈ [2.0, 10.0, 0.0] p2 = [-1.0, -2.0, 2.0] - gf2 = Manopt.GeneralizedCauchyDirectionFinder(M, p2, ha) + gf2 = Manopt.GeneralizedCauchyDirectionSubsolver(M, p2, ha) - @test Manopt.find_generalized_cauchy_direction!(gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === (:not_found, NaN) + @test Manopt.find_generalized_cauchy_direction!(M, gf2, d_out, p2, [-1.0, -1.0, 1.0], [-10.0, -10.0, -10.0]) === (:not_found, NaN) M2 = Hyperrectangle([-10.0], [10.0]) ha2 = QuasiNewtonMatrixDirectionUpdate(M2, BFGS(), DefaultOrthonormalBasis(), [100.0;;]) p3 = [1.0] - gf3 = Manopt.GeneralizedCauchyDirectionFinder(M2, p3, ha2) + gf3 = Manopt.GeneralizedCauchyDirectionSubsolver(M2, p3, ha2) d_out = similar(p3) - @test Manopt.find_generalized_cauchy_direction!(gf3, d_out, p3, [1.0], [-10.0]) === (:found_limited, 90.0) + @test Manopt.find_generalized_cauchy_direction!(M2, gf3, d_out, p3, [1.0], [-10.0]) === (:found_limited, 90.0) end @testset "Hitting multiple bounds at the same time in GCD" begin @@ -207,13 +207,13 @@ using RecursiveArrayTools ha = QuasiNewtonMatrixDirectionUpdate(M, BFGS(), DefaultOrthonormalBasis(), [1.0 0 0; 0 1 0; 0 0 1]) p = [0.0, 0.0, 0.0] - gf = Manopt.GeneralizedCauchyDirectionFinder(M, p, ha) + gf = Manopt.GeneralizedCauchyDirectionSubsolver(M, p, ha) d = [-2.0, -2.0, -1.0] d_out = similar(d) X = [10.0, 10.0, 10.0] - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p, d, X) === (:found_limited, 1.0) + @test Manopt.find_generalized_cauchy_direction!(M, gf, d_out, p, d, X) === (:found_limited, 1.0) @test d_out ≈ [-1.0, -1.0, -1.0] end @@ -294,10 +294,10 @@ using RecursiveArrayTools @testset "GCD check" begin d = -grad_f(M, p0) ha = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p0, InverseBFGS(), 2)) - gf = Manopt.GeneralizedCauchyDirectionFinder(M, p0, ha) + gf = Manopt.GeneralizedCauchyDirectionSubsolver(M, p0, ha) d_out = similar(d) X = grad_f(M, p0) - @test Manopt.find_generalized_cauchy_direction!(gf, d_out, p0, d, X) === (:found_limited, 1.0) + @test Manopt.find_generalized_cauchy_direction!(M, gf, d_out, p0, d, X) === (:found_limited, 1.0) end p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) @@ -325,8 +325,8 @@ end d = [2.0, 1.0, 1.0] d_before = copy(d) - sdf = Manopt.MaxStepsizeInDirectionFinder(M, p) - @test Manopt.find_max_stepsize_in_direction(sdf, p, d) === (:found_limited, 1.0) + sdf = Manopt.MaxStepsizeInDirectionSubsolver(M, p) + @test Manopt.find_max_stepsize_in_direction(M, sdf, p, d) === (:found_limited, 1.0) @test d == d_before end @@ -336,8 +336,8 @@ end d = [1.0] d_before = copy(d) - sdf = Manopt.MaxStepsizeInDirectionFinder(M, p) - @test Manopt.find_max_stepsize_in_direction(sdf, p, d) === (:found_unlimited, Inf) + sdf = Manopt.MaxStepsizeInDirectionSubsolver(M, p) + @test Manopt.find_max_stepsize_in_direction(M, sdf, p, d) === (:found_unlimited, Inf) @test d == d_before end @@ -347,8 +347,8 @@ end d = [-1.0] d_before = copy(d) - sdf = Manopt.MaxStepsizeInDirectionFinder(M, p) - @test Manopt.find_max_stepsize_in_direction(sdf, p, d) === (:not_found, NaN) + sdf = Manopt.MaxStepsizeInDirectionSubsolver(M, p) + @test Manopt.find_max_stepsize_in_direction(M, sdf, p, d) === (:not_found, NaN) @test d == d_before end end From 7ba3ce607caa00ce6cd5f38d6772489b7b8da577 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 17 Feb 2026 09:20:40 +0100 Subject: [PATCH 111/135] Apply suggestions from code review Co-authored-by: Ronny Bergmann --- docs/src/solvers/generalized_cauchy_direction_subsolver.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index 71e0f24089..283bb4f066 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -1,6 +1,6 @@ -# Generalized Cauchy Direction subsolver +# Generalized Cauchy direction subsolver -The Generalized Cauchy Direction (GCD) subsolver is a component in optimization algorithms that handle problems with bound constraints. It solves the following problem +The generalized Cauchy direction (GCD) subsolver is a component in optimization algorithms that handle problems with bound constraints. It solves the following problem ```math \begin{align*} @@ -11,7 +11,7 @@ The Generalized Cauchy Direction (GCD) subsolver is a component in optimization where $X=(X_{\mathrm{D}}, X_{\mathcal{M}})$ is a given direction, the exponential map handles projection of the tangent vector when reaching the boundary, $D$ is a box domain ([`Hyperrectangle`](@extref Manifolds.Hyperrectangle)), $\mathcal{M}$ is a Riemannian manifold, $X_g$ is the gradient of a scalar function $f$ at point $p=(p_{\mathrm{D}}, p_{\mathcal{M}})$, $A$ is the maximum allowed step size on $\mathcal{M}$ at point $p=(p_{\mathrm{D}}, p_{\mathcal{M}})$ in direction $X_{\mathcal{M}}$ (infinity is supported) and $\mathcal{H}_p$ is a linear operator that approximates the Hessian of $f$ at $p$. -Additionally, the subsolver indicates whether the selected direction $Y$ reaches the boundary of $D$ at some point, in which case the subsequent step size selection in direction $Y$ needs to be limited to the interval $[0, s_{\max}]$, where the number $1 \leq s_{\max} \leq \infty$ is also returned by the subsolver. +Additionally, the subsolver indicates whether the selected direction $Y$ reaches the boundary of $D$ at some point, in which case the subsequent step size selection in direction $Y$ needs to be limited to the interval $[0, s_{\max}]$, where the number $1 ≤ s_{\max} ≤ ∞$ is also returned by the subsolver. Note that the value $s_{\max}=1$ is obtained when the minimum lies at the boundary of $D$, while larger values indicate that we are further away from the boundary along the selected direction $X$. The solver is currently primarily intended for internal use by optimization algorithms that require bound-constrained subproblem solutions. From 0d72c4a6c512d1ee0fc926a8b7f46ac72a9a6d9d Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 17 Feb 2026 09:23:00 +0100 Subject: [PATCH 112/135] address one more issue --- src/plans/stepsize/linesearch.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/plans/stepsize/linesearch.jl b/src/plans/stepsize/linesearch.jl index b48e4d72ed..8266af2c27 100644 --- a/src/plans/stepsize/linesearch.jl +++ b/src/plans/stepsize/linesearch.jl @@ -26,10 +26,11 @@ get_message(::S) where {S <: Stepsize} = "" """ initialize_stepsize!(sm::Stepsize) -Initialize the state of a stepsize functor. This is called at the beginning of a solver run, -and can be used to set up internal state of the stepsize functor that is preserved between -line searches in the same optimization, for example adaptive thresholds for Wolfe criteria -in Hager-Zhang line search. +Initialize the state of a stepsize functor. This function should be called in the +`initialize_solver!` function for solvers that do possess a stepsize and can be used to +set up internal state of the stepsize functor that is preserved between line searches in +the same optimization, for example adaptive thresholds for Wolfe criteria in Hager-Zhang +line search. By default it does nothing. """ From 4156ff590b9fe44ccac3f25dafbb25bf39da944b Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 17 Feb 2026 09:27:51 +0100 Subject: [PATCH 113/135] update docs --- Changelog.md | 2 +- docs/src/solvers/generalized_cauchy_direction_subsolver.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Changelog.md b/Changelog.md index 188bb7295b..953c3bee00 100644 --- a/Changelog.md +++ b/Changelog.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * `nonpositive_curvature_behavior` for `QuasiNewtonLimitedMemoryDirectionUpdate` that determines how transported (y, s) vector pairs are treated after transport; if their inner product gets too low, it may lead to non-positive-definite Hessians which needs to be avoided. This resolves issue #549. (#554) -* `GeneralizedCauchyDirectionFinder` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) +* `GeneralizedCauchyDirectionSubsolver` for handling direction selection in the presence of box (`Hyperrectangle`) constraints in quasi-Newton methods. This allows for L-BFGS-B-style box constraint handling. (#554) * New stopping criteria: `StopWhenRelativeAPosterioriCostChangeLessOrEqual` and `StopWhenProjectedNegativeGradientNormLess`. (#554). * `HagerZhangLinesearch` stepsize, a state-of-the-art line search for smooth objectives with cubic interpolation and adaptive Wolfe condition checking. (#554) * Stopping criteria can now be initialized using `initialize_stepsize!`, similar to solvers. (#554) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index 283bb4f066..b71a8be76c 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -22,7 +22,7 @@ In case there is no Hessian approximation available, a simple stepsize limiting This procedure is available using the following: ```@docs -Manopt.MaxStepsizeInDirectionFinder +Manopt.MaxStepsizeInDirectionSubsolver Manopt.find_max_stepsize_in_direction ``` @@ -35,7 +35,7 @@ These symbols are directly used by solvers to compute the descent direction corr ```@docs Manopt.has_anisotropic_max_stepsize Manopt.find_generalized_cauchy_direction! -Manopt.GeneralizedCauchyDirectionFinder +Manopt.GeneralizedCauchyDirectionSubsolver ``` ### Symbols related to the Hessian approximation From a44c7ca5cc9acfcc5d8f645dbc2606f4e81a992e Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 17 Feb 2026 09:33:55 +0100 Subject: [PATCH 114/135] address two more review comments (Github doesn't seem to be able to apply those suggestions from the browser interface) --- src/plans/box_plan.jl | 2 +- src/plans/stopping_criterion.jl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 24db43cda0..8b837811f4 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -9,7 +9,7 @@ has_anisotropic_max_stepsize(::AbstractManifold) = false has_anisotropic_max_stepsize(M::ProductManifold) = any(has_anisotropic_max_stepsize, M.manifolds) @doc raw""" - mutable struct LimitedMemoryHessianApproximation end + LimitedMemoryHessianApproximation <: AbstractQuasiNewtonDirectionUpdate An approximation of Hessian of a scalar function of the form ``B_0 = θ I``, ``B_{k+1} = B_k - W_k M_k W_k^{\mathrm{T}}``, diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index da1bb13a98..301fd6b5e6 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -1020,13 +1020,13 @@ end Stop the solver when the iterate of the optimization problem from within an [`AbstractManoptProblem`](@ref) contains `NaN` values. -The value is obtained using `get_iterate(s)`. +The value is obtained using [`get_iterate`](@ref)`(s)`. # Constructor StopWhenIterateNaN() -Initialize `at_iteration` to `-1`. +Initialize the stopping criterion. """ mutable struct StopWhenIterateNaN <: StoppingCriterion at_iteration::Int From 03ab065a2a94c8095c68d7543c7cc30547490c8b Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 17 Feb 2026 10:38:13 +0100 Subject: [PATCH 115/135] a bit more comments, a bit more UTF, a bit better name for an internal function --- .../generalized_cauchy_direction_subsolver.md | 2 +- src/plans/box_plan.jl | 14 +++++++------- src/plans/stopping_criterion.jl | 3 ++- test/solvers/test_quasi_Newton_box.jl | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index b71a8be76c..cedee9c719 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -69,5 +69,5 @@ Manopt.set_zero_at_index! ```@docs Manopt.LimitedMemorySegmentHessianUpdater Manopt.hessian_value_from_inner_products -Manopt.set_M_current_scale! +Manopt.update_current_scale! ``` diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 8b837811f4..ff7fc0a1b8 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -13,13 +13,13 @@ has_anisotropic_max_stepsize(M::ProductManifold) = any(has_anisotropic_max_steps An approximation of Hessian of a scalar function of the form ``B_0 = θ I``, ``B_{k+1} = B_k - W_k M_k W_k^{\mathrm{T}}``, -where ``\theta > 0`` is an initial scaling guess. -Matrix ``M_k = \left(\begin{smallmatrix}M_{11} & M_{21}^{\mathrm{T}}\\ M_{21} & M_{22}\end{smallmatrix}\right)`` +where ``θ > 0`` is an initial scaling guess. +Matrix ``M_k = \left(\begin{smallmatrix}M₁₁ & M₂₁^{\mathrm{T}}\\ M₂₁ & M₂₂\end{smallmatrix}\right)`` is stored using its blocks. Blocks ``W_k`` are (implicitly) composed from `memory_y` and `memory_s` stored in `qn_du` of type [`QuasiNewtonLimitedMemoryDirectionUpdate`](@ref). -Initial scale ``\theta`` is stored in the field `initial_scale` but if the memory isn't empty, +Initial scale ``θ`` is stored in the field `initial_scale` but if the memory isn't empty, the current scale is set to squared norm of $s_k$ divided by inner product of ``s_k`` and ``y_k`` where ``k`` is the oldest index for which the denominator is not equal to 0. @@ -210,7 +210,7 @@ function hessian_value(gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate, M::Abstra end @doc raw""" - set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + update_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) Refresh the scaling factor and blockwise Hessian approximation stored in `gh` using the nonzero curvature pairs currently in memory. @@ -218,13 +218,13 @@ nonzero curvature pairs currently in memory. - Identifies the most recent index with nonzero ``ρ_i`` to scale the initial Hessian guess by ``ρ_i‖y_i‖^2 / θ``. - Builds ``L_k`` and ``S_k^\top S_k`` from the stored ``(s_i, y_i)`` pairs and updates the - block matrices ``M_{11}``, ``M_{21}``, and ``M_{22}`` via the blockwise inverse formula. + block matrices ``M₁₁``, ``M₂₁``, and ``M₂₂`` via the blockwise inverse formula. - If all ``ρ_i`` vanish, resets `current_scale` to the inverse of `initial_scale` and clears the block matrices. Returns the mutated `gh`. """ -function set_M_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) +function update_current_scale!(M::AbstractManifold, p, gh::QuasiNewtonLimitedMemoryBoxDirectionUpdate) m = length(gh.qn_du.memory_s) last_safe_index = -1 for i in eachindex(gh.qn_du.ρ) @@ -329,7 +329,7 @@ function update_hessian!( ) (capacity(gh.qn_du.memory_s) == 0) && return gh update_hessian!(gh.qn_du, mp, st, p_old, k) - set_M_current_scale!(get_manifold(mp), get_iterate(st), gh) + update_current_scale!(get_manifold(mp), get_iterate(st), gh) return gh end diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 301fd6b5e6..48a8c54240 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -465,12 +465,13 @@ end A stopping criterion to stop when ````math -\\frac{f_k - f_{k+1}}{\\max(\\lvert f_k \\rvert, \\lvert f_{k+1} \\rvert, 1)} \\leq tol, +\\frac{f_k - f_{k+1}}{\\max(\\lvert f_k \\rvert, \\lvert f_{k+1} \\rvert, 1)} ≤ tol, ```` based on Eq. (1) in [ZhuByrdLuNocedal:1997](@cite) # Fields +* tolerance: the threshold `tol` in the above formula. $(_fields([:at_iteration, :last_change])) * `last_cost``: the last cost value diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 6fb5f829ab..edafd1a938 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -152,7 +152,7 @@ using RecursiveArrayTools ha2 = QuasiNewtonLimitedMemoryBoxDirectionUpdate(QuasiNewtonLimitedMemoryDirectionUpdate(M, p, InverseBFGS(), 2)) idx = Manopt.get_bounds_index(M) @test Manopt.hessian_value(ha2, M, p, Manopt.UnitVector(b), grad) ≈ 4.0 - Manopt.set_M_current_scale!(M, p, ha2) + Manopt.update_current_scale!(M, p, ha2) @test ha2.current_scale == ha2.qn_du.initial_scale @test ha2.M_11 == fill(0.0, 0, 0) @test ha2.M_21 == fill(0.0, 0, 0) From 815cc55cae6343036948c953cf24ee77391499a8 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 17 Feb 2026 11:54:34 +0100 Subject: [PATCH 116/135] Handle decorators in `get_cost` --- src/solvers/solver.jl | 6 ++++++ test/solvers/test_quasi_Newton.jl | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/solvers/solver.jl b/src/solvers/solver.jl index a38faaef3f..ef16a4535a 100644 --- a/src/solvers/solver.jl +++ b/src/solvers/solver.jl @@ -198,5 +198,11 @@ The method may be implemented by particular solvers if they store the cost at th iterate in the state, but by default it is obtained by calling `get_cost(p, get_iterate(s))`. """ function get_cost(p::AbstractManoptProblem, s::AbstractManoptSolverState) + return _get_cost(p, s, dispatch_state_decorator(s)) +end +function _get_cost(p::AbstractManoptProblem, s::AbstractManoptSolverState, ::Val{false}) return get_cost(p, get_iterate(s)) end +function _get_cost(p::AbstractManoptProblem, s::AbstractManoptSolverState, ::Val{true}) + return get_cost(p, s.state) +end diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index d94ac88787..659c5a76f2 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -572,5 +572,9 @@ end solve!(mp, qns) @test get_cost(mp, qns) == f(M, get_iterate(qns)) + @testset "get_cost with DebugSolverState" begin + dqns = DebugSolverState(qns, DebugMessages(:Info, :Always)) + @test get_cost(mp, dqns) == f(M, get_iterate(dqns)) + end end end From a3955acfff5bead7fb664caa961bd0fef73dd187 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 24 Mar 2026 11:13:38 +0100 Subject: [PATCH 117/135] fix a bug noticed while working on RLM --- src/plans/box_plan.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index ff7fc0a1b8..3d9c5a95a1 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -113,7 +113,7 @@ end get_update_vector_transport(u::QuasiNewtonLimitedMemoryBoxDirectionUpdate) = get_update_vector_transport(u.qn_du) function get_at_bound_index(M::ProductManifold, X, b::Tuple{Int, Any}) - return get_at_bound_index(M.manifolds[b[1]], submanifold_component(M, X, Val(1)), b[2]) + return get_at_bound_index(M.manifolds[b[1]], submanifold_component(M, X, b[1]), b[2]) end @doc raw""" From 7bc7eb30011cde3cc09a6b59ade8810077e921fa Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Wed, 8 Apr 2026 09:51:04 +0900 Subject: [PATCH 118/135] Remove and accidentially duplicate method definition. --- src/plans/stopping_criterion.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index c34d94ce47..63b736d148 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -321,7 +321,6 @@ function (c::StopWhenChangeLess)(mp::AbstractManoptProblem, s::AbstractManoptSol c.storage(mp, s, k) return false end -indicates_convergence(c::StopWhenChangeLess) = false function get_reason(c::StopWhenChangeLess) if (c.last_change < c.threshold) && (c.at_iteration >= 0) return "At iteration $(c.at_iteration) the algorithm performed a step with a change ($(c.last_change)) less than $(c.threshold).\n" From 3a2c48baba60cf0dc493a49a4bbc5b4cf992d42a Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Wed, 8 Apr 2026 10:14:27 +0900 Subject: [PATCH 119/135] adapt one new StoppingCriterion to the new show/repr/status_summary style and fix one further bug. --- src/plans/stopping_criterion.jl | 10 +++++++--- test/plans/test_stopping_criteria.jl | 4 ++-- test/solvers/test_quasi_Newton.jl | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 63b736d148..180f4fbdbf 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -903,14 +903,18 @@ function get_reason(c::StopWhenProjectedNegativeGradientNormLess) end return "" end -function status_summary(c::StopWhenProjectedNegativeGradientNormLess) +function status_summary(c::StopWhenProjectedNegativeGradientNormLess; context::Symbol = :default) + (context === :short) && return repr(c) has_stopped = (c.at_iteration >= 0) s = has_stopped ? "reached" : "not reached" - return "|proj (-grad f)| < $(c.threshold): $s" + (context === :inline) && return "|proj (-grad f)| < $(c.threshold): $s" + return "A StoppingCriterion to stop when the negative projected gradient norm is less than a threshold of $(c.threshold):\n$(_MANOPT_INDENT)$s" end indicates_convergence(c::StopWhenProjectedNegativeGradientNormLess) = true function show(io::IO, c::StopWhenProjectedNegativeGradientNormLess) - return print(io, "StopWhenProjectedNegativeGradientNormLess($(c.threshold))\n $(status_summary(c))") + print(io, "StopWhenProjectedNegativeGradientNormLess(", c.threshold, "; norm = ", c.norm) + !ismissing(c.outer_norm) && print(io, ", outer_norm = ", c.outer_norm) + return print(io, ")") end """ diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 193ec9ed0d..00f5fb66bb 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -424,9 +424,9 @@ end @test startswith( to_display_string(sc), - "StopWhenProjectedNegativeGradientNormLess(1.0e-10)\n", + "A StoppingCriterion to stop when the negative projected gradient norm is less than", ) - @test startswith(Manopt.status_summary(sc), "|proj (-grad f)| < 1.0e-10") + @test startswith(Manopt.status_summary(sc; context = :inline), "|proj (-grad f)| < 1.0e-10") Manopt.set_parameter!(sc, Val(:MinGradNorm), 1.0e-5) @test sc.threshold == 1.0e-5 diff --git a/test/solvers/test_quasi_Newton.jl b/test/solvers/test_quasi_Newton.jl index ec13673225..453dd4b60c 100644 --- a/test/solvers/test_quasi_Newton.jl +++ b/test/solvers/test_quasi_Newton.jl @@ -248,7 +248,7 @@ end x = [0.7011245948687502, -0.1726003159556036, 0.38798265967671103, -0.5728026616491424] x_lrbfgs = quasi_Newton( - M, F, grad_f, x; memory_size = -1, + M, F, grad_f, x; basis = get_basis(M, x, DefaultOrthonormalBasis()), memory_size = -1, stopping_criterion = StopWhenGradientNormLess(1.0e-9) | StopAfterIteration(1000), From 705ce1e6547f4b8b73dd889e69e8b9273c248263 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Fri, 10 Apr 2026 13:20:36 +0200 Subject: [PATCH 120/135] add status summary to QuasiNewtonLimitedMemoryBoxDirectionUpdate --- src/plans/box_plan.jl | 6 ++++++ test/solvers/test_quasi_Newton_box.jl | 1 + 2 files changed, 7 insertions(+) diff --git a/src/plans/box_plan.jl b/src/plans/box_plan.jl index 3d9c5a95a1..8a03898135 100644 --- a/src/plans/box_plan.jl +++ b/src/plans/box_plan.jl @@ -50,6 +50,12 @@ mutable struct QuasiNewtonLimitedMemoryBoxDirectionUpdate{ last_gcd_stepsize::F end +function status_summary(d::QuasiNewtonLimitedMemoryBoxDirectionUpdate) + s = "limited memory direction update with support for box constraints; " + s *= "internal direction update status: $(status_summary(d.qn_du))" + return s +end + function get_parameter(d::QuasiNewtonLimitedMemoryBoxDirectionUpdate, ::Val{:max_stepsize}) if d.last_gcd_result === :found_limited return d.last_gcd_stepsize diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index edafd1a938..1430a105bb 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -98,6 +98,7 @@ using RecursiveArrayTools st = QuasiNewtonState(M) @test startswith(repr(ha), "QuasiNewtonLimitedMemoryBoxDirectionUpdate with internal state:") + @test startswith(Manopt.status_summary(ha), "limited memory direction update with support for box constraints; internal direction update status: ") f(M, p) = sum(p .^ 2) grad_f(M, p) = 2 * p From ecebd94f9ba949368c24487ab63670f13f9c56b4 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Wed, 15 Apr 2026 19:47:57 +0200 Subject: [PATCH 121/135] Better support for custom point types in conjugate_gradient_descent (#596) --- Changelog.md | 6 +++++- Project.toml | 2 +- src/solvers/conjugate_gradient_descent.jl | 2 +- test/solvers/test_conjugate_gradient.jl | 12 +++++++++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Changelog.md b/Changelog.md index d2361cad07..3ba8833764 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,7 +6,7 @@ The file was started with Version `0.4`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.5.35] April 3, 2026 +## [0.5.35] April 16, 2026 ### Changed @@ -14,6 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Bump compat for RecursiveArrayTools.jl to include version 4 * deactivate CompatHelper Action and solely use dependabot +### Fixed + +* The default line search in `conjugate_gradient_descent` is now `ArmijoLinesearchStepsize` instead of `ArmijoLinesearch`, which makes it work well with custom point types. + ## [0.5.34] March 3, 2026 ### Fixed diff --git a/Project.toml b/Project.toml index 38a810e6bb..8dedd2eb01 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Manopt" uuid = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5" -version = "0.5.34" +version = "0.5.35" authors = [{family-names = "Bergmann", given-names = "Ronny", alias = "kellertuer", city = "Trondheim", affiliation = "Norwegian University of Science and Technology", country = "NO", email = "manopt@ronnybergmann.net", orcid = "https://orcid.org/0000-0001-8342-7218", website = "https://ronnybergmann.net"}] [workspace] diff --git a/src/solvers/conjugate_gradient_descent.jl b/src/solvers/conjugate_gradient_descent.jl index 697c7fd917..5e11950707 100644 --- a/src/solvers/conjugate_gradient_descent.jl +++ b/src/solvers/conjugate_gradient_descent.jl @@ -4,7 +4,7 @@ function default_stepsize( retraction_method = default_retraction_method(M), ) # take a default with a slightly defensive initial step size. - return ArmijoLinesearchStepsize( + return ArmijoLinesearch( M; retraction_method = retraction_method, initial_stepsize = 1.0 ) end diff --git a/test/solvers/test_conjugate_gradient.jl b/test/solvers/test_conjugate_gradient.jl index 67e3c7aab5..77ffeb6cd7 100644 --- a/test/solvers/test_conjugate_gradient.jl +++ b/test/solvers/test_conjugate_gradient.jl @@ -1,5 +1,6 @@ using Manopt, Manifolds, ManifoldsBase, Test, Random, LinearAlgebra using LinearAlgebra: Diagonal, dot, eigvals, eigvecs +using ManifoldDiff: grad_distance @testset "Conjugate Gradient Descent" begin @testset "Conjugate Gradient coefficient rules" begin @@ -31,7 +32,7 @@ using LinearAlgebra: Diagonal, dot, eigvals, eigvecs initial_gradient = zero_vector(M, x0), ) @test s1.coefficient(dmp, s1, 1) == 0 - @test default_stepsize(M, typeof(s1)) isa Manopt.ArmijoLinesearchStepsize + @test default_stepsize(M, typeof(s1)) isa Manopt.ManifoldDefaultsFactory{Manopt.ArmijoLinesearchStepsize} @test Manopt.get_message(s1) == "" dU = Manopt.ConjugateDescentCoefficient() @@ -394,4 +395,13 @@ using LinearAlgebra: Diagonal, dot, eigvals, eigvecs ) @test q2 ≈ [1, 0, 0] rtol = 1.0e-7 end + + @testset "Custom point types" begin + M = Hyperbolic(2) + data = PoincareBallPoint.([[0.1, 0.2], [0.3, 0.25], [0.35, 0.4]]) + n = length(data) + f(M, p) = sum(1 / (2 * n) * distance.(Ref(M), Ref(p), data) .^ 2) + grad_f(M, p) = sum(1 / n * grad_distance.(Ref(M), data, Ref(p))) + @test conjugate_gradient_descent(M, f, grad_f, data[1]) isa PoincareBallPoint + end end From 0af6c370de36ece7489157659fadb0f1c91cf107 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 08:06:55 +0200 Subject: [PATCH 122/135] Bump crate-ci/typos from 1.45.0 to 1.45.1 (#597) Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.45.0 to 1.45.1. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/v1.45.0...v1.45.1) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-version: 1.45.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/spell_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spell_check.yml b/.github/workflows/spell_check.yml index 91341d7fe4..cada6602ce 100644 --- a/.github/workflows/spell_check.yml +++ b/.github/workflows/spell_check.yml @@ -10,4 +10,4 @@ jobs: - name: Checkout Actions Repository uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@v1.45.0 + uses: crate-ci/typos@v1.45.1 From 0e7305ad59a4dbc80280ed8abedebce5277edf98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 08:35:31 +0200 Subject: [PATCH 123/135] Bump julia-actions/setup-julia from 2 to 3 (#598) * Bump julia-actions/setup-julia from 2 to 3 Bumps [julia-actions/setup-julia](https://github.com/julia-actions/setup-julia) from 2 to 3. - [Release notes](https://github.com/julia-actions/setup-julia/releases) - [Commits](https://github.com/julia-actions/setup-julia/compare/v2...v3) --- updated-dependencies: - dependency-name: julia-actions/setup-julia dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * remove arch keyword since by now they all should run on 64 bit and it was too restructive for mac os --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ronny Bergmann --- .github/workflows/ci.yml | 3 +-- .github/workflows/nightly.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed530cec3f..ad84b169d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,9 @@ jobs: os: [ubuntu-latest, macOS-latest, windows-latest] steps: - uses: actions/checkout@v6 - - uses: julia-actions/setup-julia@v2 + - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.julia-version }} - arch: x64 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d1e08c3c23..3dbcf93111 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -12,10 +12,9 @@ jobs: os: [ubuntu-latest, macOS-latest, windows-latest] steps: - uses: actions/checkout@v6 - - uses: julia-actions/setup-julia@v2 + - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.julia-version }} - arch: x64 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 env: From 571b9097d48d243b51d6bd31cc27dd1383f0ba42 Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Fri, 24 Apr 2026 14:09:19 +0200 Subject: [PATCH 124/135] introduce a stopped_at function for solver states. (#599) * introduce a stopped_at function for solver states. * Add stopped_at to the docs. * Add the AI section to the initial TOC. * a bit of code formatting and one test. * Fix a test. --- CONTRIBUTING.md | 2 + Changelog.md | 10 +++++ Project.toml | 2 +- docs/src/plans/state.md | 1 + src/Manopt.jl | 2 +- src/plans/solver_state.jl | 12 ++++++ src/plans/stopping_criterion.jl | 2 +- test/plans/test_gradient_plan.jl | 1 + test/plans/test_state.jl | 2 +- test/solvers/test_gradient_descent.jl | 56 ++++++--------------------- 10 files changed, 42 insertions(+), 48 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c84c942227..c4dfe141ea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,6 +18,8 @@ The following is a set of guidelines to [`Manopt.jl`](https://juliamanifolds.git - [Code style](#Code-style) - [Concerning the documentation](#Concerning-the-documentation) - [Spell checking](#Spell-checking) + - [On the use of AI](#On-the-use-of-AI) + ## I just have a question The developer can most easily be reached in the Julia Slack channel [#manifolds](https://julialang.slack.com/archives/CP4QF0K5Z). diff --git a/Changelog.md b/Changelog.md index 3ba8833764..cb1c516140 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,16 @@ The file was started with Version `0.4`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.36] April 24, 2026 + +### Added + +* a function `stopped_at(state)` to access the number of iterations it took a solver to stop. (#599) + +### Fixed + +* a small bug where `get_count(sc::StopWhenAny, Val(:Iteration))` wrongly reported it stopped before the first iteration when it actually did not yet stop. (#599) + ## [0.5.35] April 16, 2026 ### Changed diff --git a/Project.toml b/Project.toml index 8dedd2eb01..d711cac8dc 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Manopt" uuid = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5" -version = "0.5.35" +version = "0.5.36" authors = [{family-names = "Bergmann", given-names = "Ronny", alias = "kellertuer", city = "Trondheim", affiliation = "Norwegian University of Science and Technology", country = "NO", email = "manopt@ronnybergmann.net", orcid = "https://orcid.org/0000-0001-8342-7218", website = "https://ronnybergmann.net"}] [workspace] diff --git a/docs/src/plans/state.md b/docs/src/plans/state.md index f35b505048..34912669c0 100644 --- a/docs/src/plans/state.md +++ b/docs/src/plans/state.md @@ -14,6 +14,7 @@ AbstractManoptSolverState get_state Manopt.get_count Manopt.has_converged(::AbstractManoptSolverState) +stopped_at ``` Since every subtype of an [`AbstractManoptSolverState`](@ref) directly relate to a solver, diff --git a/src/Manopt.jl b/src/Manopt.jl index e8c1ffe6b2..a52fd47ea4 100644 --- a/src/Manopt.jl +++ b/src/Manopt.jl @@ -587,7 +587,7 @@ export StopAfter, StopWhenSwarmVelocityLess, StopWhenTrustRegionIsExceeded export get_active_stopping_criteria, - get_stopping_criteria, get_reason, get_stopping_criterion + get_stopping_criteria, get_reason, get_stopping_criterion, stopped_at # # Exports export asymptote_export_S2_signals, asymptote_export_S2_data, asymptote_export_SPD diff --git a/src/plans/solver_state.jl b/src/plans/solver_state.jl index 4562b2f967..ceca627bec 100644 --- a/src/plans/solver_state.jl +++ b/src/plans/solver_state.jl @@ -329,6 +329,18 @@ Make a copy of tangent vector `X` from manifold `M` for storage in [`StoreStateA _storage_copy_vector(M::AbstractManifold, X) = copy(M, X) _storage_copy_vector(::AbstractManifold, X::Number) = StorageRef(X) +@doc """ + stopped_at(state::AbstractManoptSolverState) + +Return the number of iterations the solver represented by the `state` took to stop. +If the solver has not yet stopped, this function returns `-1`. + +By default, this function calls `get_count` function on the state's stopping criterion to access its `:Iteration` count. +""" +function stopped_at(state::AbstractManoptSolverState) + return get_count(get_stopping_criterion(state), Val(:Iterations)) +end + @doc """ StoreStateAction <: AbstractStateAction diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index ca8a5f0b88..5ffdd08463 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -1181,7 +1181,7 @@ function has_converged(c::StopWhenAny) end function get_count(c::StopWhenAny, v::Val{:Iterations}) iters = filter(x -> x > 0, [get_count(ci, v) for ci in c.criteria]) - (length(iters) == 0) && (return 0) + (length(iters) == 0) && (return -1) # None indicated to stop yet, so we also do not return minimum(iters) end function show(io::IO, c::StopWhenAny) diff --git a/test/plans/test_gradient_plan.jl b/test/plans/test_gradient_plan.jl index 7051573137..ba7e5c9e51 100644 --- a/test/plans/test_gradient_plan.jl +++ b/test/plans/test_gradient_plan.jl @@ -16,6 +16,7 @@ using ManifoldsBase, Manopt, Test stopping_criterion = StopAfterIteration(20), stepsize = Manopt.ConstantStepsize(M), ) + @test stopped_at(gst) == -1 set_iterate!(gst, M, q) @test get_iterate(gst) == q set_gradient!(gst, M, p, [1.0, 0.0]) diff --git a/test/plans/test_state.jl b/test/plans/test_state.jl index 57bfe653f5..8b2f2c2ebf 100644 --- a/test/plans/test_state.jl +++ b/test/plans/test_state.jl @@ -72,7 +72,7 @@ struct NoIterateState <: AbstractManoptSolverState end @test_throws ErrorException get_iterate(s2) end - @testset "Iteration and Gradient setters" begin + @testset "Iterate and Gradient setters" begin M = Euclidean(3) s1 = NelderMeadState(M) s2 = GradientDescentState(M) diff --git a/test/solvers/test_gradient_descent.jl b/test/solvers/test_gradient_descent.jl index 033ce3f734..690c424981 100644 --- a/test/solvers/test_gradient_descent.jl +++ b/test/solvers/test_gradient_descent.jl @@ -24,10 +24,7 @@ using ManifoldDiff: grad_distance 500, ) s = gradient_descent( - M, - f, - grad_f, - data[1]; + M, f, grad_f, data[1]; stopping_criterion = StopAfterIteration(200) | StopWhenChangeLess(M, 1.0e-16), stepsize = ArmijoLinesearch(; contraction_factor = 0.99), debug = d, @@ -38,10 +35,7 @@ using ManifoldDiff: grad_distance res_debug = String(take!(my_io)) @test res_debug === " f(x): 1.357071\n" p2 = gradient_descent( - M, - f, - grad_f, - data[1]; + M, f, grad_f, data[1]; stopping_criterion = StopAfterIteration(200) | StopWhenChangeLess(M, 1.0e-16), stepsize = ArmijoLinesearch(; contraction_factor = 0.99), ) @@ -56,10 +50,7 @@ using ManifoldDiff: grad_distance stop_when_stepsize_exceeds = 0.9 * π, ) p3 = gradient_descent( - M, - f, - grad_f, - data[1]; + M, f, grad_f, data[1]; stopping_criterion = StopAfterIteration(1000) | StopWhenChangeLess(M, 1.0e-16), stepsize = step, debug = [], # do not warn about increasing step here @@ -75,10 +66,7 @@ using ManifoldDiff: grad_distance stop_when_stepsize_exceeds = 0.9 * π, ) p4 = gradient_descent( - M, - f, - grad_f, - data[1]; + M, f, grad_f, data[1]; stopping_criterion = StopAfterIteration(1000) | StopWhenChangeLess(M, 1.0e-16), stepsize = step2, debug = [], # do not warn about increasing step here @@ -94,40 +82,28 @@ using ManifoldDiff: grad_distance stop_when_stepsize_exceeds = 0.9 * π, ) p5 = gradient_descent( - M, - f, - grad_f, - data[1]; + M, f, grad_f, data[1]; stopping_criterion = StopAfterIteration(1000) | StopWhenChangeLess(M, 1.0e-16), stepsize = step3, debug = [], # do not warn about increasing step here ) @test isapprox(M, p, p5; atol = 1.0e-13) p6 = gradient_descent( - M, - f, - grad_f, - data[1]; + M, f, grad_f, data[1]; stopping_criterion = StopAfterIteration(1000) | StopWhenChangeLess(M, 1.0e-16), direction = Nesterov(; p = copy(M, data[1])), ) @test isapprox(M, p, p6; atol = 1.0e-13) # Precon in simple scale down by 2 p7 = gradient_descent( - M, - f, - grad_f, - data[1]; + M, f, grad_f, data[1]; stopping_criterion = StopAfterIteration(1000) | StopWhenChangeLess(M, 1.0e-16), direction = PreconditionedDirection((M, p, X) -> 0.5 .* X), ) @test isapprox(M, p, p7; atol = 1.0e-13) # Precon in simple scale down by 2 – inplace p8 = gradient_descent( - M, - f, - grad_f, - data[1]; + M, f, grad_f, data[1]; stopping_criterion = StopAfterIteration(1000) | StopWhenChangeLess(M, 1.0e-16), direction = PreconditionedDirection( (M, Y, p, X) -> (Y .= 0.5 .* X); evaluation = InplaceEvaluation() @@ -170,20 +146,14 @@ using ManifoldDiff: grad_distance # `gradient_descent` allocated n2 newly @test isapprox(M, north, n2a) n3 = gradient_descent( - M, - f, - grad_f, - pts[1]; + M, f, grad_f, pts[1]; direction = MomentumGradient(), stepsize = ConstantLength(), debug = [], # do not warn about increasing step here ) @test isapprox(M, north, n3) n4 = gradient_descent( - M, - f, - grad_f, - pts[1]; + M, f, grad_f, pts[1]; direction = AverageGradient(M; n = 5), stopping_criterion = StopAfterIteration(800), ) @@ -194,14 +164,12 @@ using ManifoldDiff: grad_distance @test startswith(repr(r), "# Solver state for `Manopt.jl`s Gradient Descent") # State and a count objective, putting stats behind print n6 = gradient_descent( - M, - f, - grad_f, - pts[1]; + M, f, grad_f, pts[1]; count = [:Gradient], return_objective = true, return_state = true, ) + @test stopped_at(n6[2]) > 0 @test repr(n6) == "$(n6[2])\n\n$(n6[1])" end @testset "Tutorial mode" begin From c6580a8d90acd1858696b287ebfc7a6d3dddd571 Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Mon, 4 May 2026 09:05:50 +0200 Subject: [PATCH 125/135] Increase GH Action dependabot to only update typos on minor/major and not on patch level (#602) --- .JuliaFormatter.toml | 2 -- .github/dependabot.yml | 6 +++- .github/workflows/spell_check.yml | 2 +- .gitignore | 1 + .vale.ini | 51 ------------------------------- 5 files changed, 7 insertions(+), 55 deletions(-) delete mode 100644 .JuliaFormatter.toml delete mode 100644 .vale.ini diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml deleted file mode 100644 index acd76178c8..0000000000 --- a/.JuliaFormatter.toml +++ /dev/null @@ -1,2 +0,0 @@ -style = "blue" -ignore = ["tutorials", ".git"] \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4dca7af2fb..ec4e3bca10 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,10 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + ignore: + - dependency-name: "crate-ci/typos" + update-types: + - "version-update:semver-patch" - package-ecosystem: "julia" directories: # Location of Julia projects - "/" @@ -21,4 +25,4 @@ updates: # Group all Julia package updates into a single PR: all-julia-packages: patterns: - - "*" \ No newline at end of file + - "*" diff --git a/.github/workflows/spell_check.yml b/.github/workflows/spell_check.yml index cada6602ce..6a045873ad 100644 --- a/.github/workflows/spell_check.yml +++ b/.github/workflows/spell_check.yml @@ -10,4 +10,4 @@ jobs: - name: Checkout Actions Repository uses: actions/checkout@v6 - name: Check spelling - uses: crate-ci/typos@v1.45.1 + uses: crate-ci/typos@v1.46.0 diff --git a/.gitignore b/.gitignore index e9f4ccc67c..9e4bd7eae3 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ docs/src/changelog.md docs/styles/Google tutorials/LocalPreferences.toml temp +docs/node_modules diff --git a/.vale.ini b/.vale.ini deleted file mode 100644 index 0aab5a6f57..0000000000 --- a/.vale.ini +++ /dev/null @@ -1,51 +0,0 @@ -StylesPath = docs/styles -MinAlertLevel = warning -Vocab = Manopt - -Packages = Google - -[formats] -# code blocks with Julia in Markdown do not yet work well -qmd = md -jl = md - -[docs/src/*.md] -BasedOnStyles = Vale, Google - -[{docs/src/contributing.md, Changelog.md, CONTRIBUTING.md}] -BasedOnStyles = Vale, Google -Google.Will = false ; given format and really with intend a _will_ -Google.Headings = false ; some might jeally ahabe [] in their headers -Google.FirstPerson = false ; we pose a few contribution points as first-person questions - -[src/*.md] ; actually .jl -BasedOnStyles = Vale, Google - -[test/*.md] ; actually .jl -BasedOnStyles = Vale, Google - -[docs/src/changelog.md] -; ignore since it is derived -BasedOnStyles = - -[src/plans/debug.md] -Google.Units = false ;w to ignore formats= for now. -Google.Ellipses = false ; since vale gets confused by the DebugFactory Docstring (line 1066) -TokenIgnores = \$(.+)\$,\[.+?\]\(@(ref|id|cite).+?\),`.+`,``.*``,\s{4}.+\n - -[test/plans/test_debug.md] #repeat previous until I find out how to combine them -Google.Units = false #wto ignore formats= for now. -TokenIgnores = \$(.+)\$,\[.+?\]\(@(ref|id|cite).+?\),`.+`,``.*``,\s{4}.+\n - -[tutorials/*.qmd] ; actually .qmd for the first, second autogenerated -BasedOnStyles = Vale, Google -; ignore (1) math (2) ref and cite keys (3) code in docs (4) math in docs (5,6) indented blocks -TokenIgnores = (\$+[^\n$]+\$+) -Google.We = false # For tutorials we want to address the user directly. - -[docs/src/tutorials/*.md] ; Can I somehow just deactivate these? -BasedOnStyles = Vale, Google -; ignore (1) math (2) ref and cite keys (3) code in docs (4) math in docs (5,6) indented blocks -TokenIgnores = (\$+[^\n$]+\$+) -Google.We = false # For tutorials we want to address the user directly. -Google.Spacing = false # one reference uses this From 0e92b741150d55e1483463aeb30fce8c041e3940 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 5 May 2026 15:29:16 +0200 Subject: [PATCH 126/135] Fix some issues with CG and certain coefficient rules when using restart (#604) * refactor CGRule * Fix some issues with CG + HZ * provide a proper test + formatting * fix CG test * Refactor CG rules. * Fix 2 cases where we implicitly relied on storage being updated in case it was not filled and now fill it explicitly. * bump version. --------- Co-authored-by: Ronny Bergmann --- Changelog.md | 8 + Project.toml | 2 +- src/plans/conjugate_gradient_plan.jl | 308 ++++++++++++--------- src/plans/solver_state.jl | 7 +- src/solvers/conjugate_gradient_descent.jl | 5 +- test/plans/test_conjugate_gradient_plan.jl | 15 +- test/solvers/test_conjugate_gradient.jl | 20 ++ 7 files changed, 225 insertions(+), 140 deletions(-) diff --git a/Changelog.md b/Changelog.md index cb1c516140..1b7ba778c6 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,14 @@ The file was started with Version `0.4`. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.37] May 5, 2026 + +### Changed + +* The default restart rule for `conjugate_gradient_descent` is now `RestartOnNonDescent` instead of `NeverRestart`, which makes the algorithm more robust to non-convexity and numerical issues. The old default can still be used by explicitly passing `restart_condition=NeverRestart()`. (#604) +* `HagerZhangCoefficientRule` now has a safeguard against the denominator being too close to zero (the `denom_threshold` field). By default it is set to 1.0e-10. You can set it to a lower positive value (or even zero) to weaken the safeguard, but it is recommended to keep it to avoid numerical issues. (#604) +* introduce for all `Rule`s also a variant without being encapsulated in a memory, where the old values have to be passed as keywords. This is now used by the `ConjugateGradientBealeRestartRule` when evaluating its inner rule. (#604) + ## [0.5.36] April 24, 2026 ### Added diff --git a/Project.toml b/Project.toml index d711cac8dc..b2c5a1b4a1 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Manopt" uuid = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5" -version = "0.5.36" +version = "0.5.37" authors = [{family-names = "Bergmann", given-names = "Ronny", alias = "kellertuer", city = "Trondheim", affiliation = "Norwegian University of Science and Technology", country = "NO", email = "manopt@ronnybergmann.net", orcid = "https://orcid.org/0000-0001-8342-7218", website = "https://ronnybergmann.net"}] [workspace] diff --git a/src/plans/conjugate_gradient_plan.jl b/src/plans/conjugate_gradient_plan.jl index c3bafa3766..2efb22fcd2 100644 --- a/src/plans/conjugate_gradient_plan.jl +++ b/src/plans/conjugate_gradient_plan.jl @@ -52,7 +52,7 @@ The following fields from above hz.denom_threshold + # when abs(denom) is small, we lose numerical stability. + νknormsq = inner(M, cgs.p, ν, ν) + β = inner(M, cgs.p, ν, cgs.X) / denom - 2 * νknormsq * inner(M, cgs.p, δtr, cgs.X) / denom^2 + # Numerical stability from Manopt / Hager-Zhang paper + ξn = norm(M, cgs.p, cgs.X) + η = -1 / (ξn * min(0.01, norm(M, p, X))) + β = max(β, η) + else + β = zero(eltype(denom)) + end + return β +end function (u::DirectionUpdateRuleStorage{<:HagerZhangCoefficientRule})( amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i ) - M = get_manifold(amp) if !has_storage(u.storage, PointStorageKey(:Iterate)) || !has_storage(u.storage, VectorStorageKey(:Gradient)) || !has_storage(u.storage, VectorStorageKey(:δ)) update_storage!(u.storage, amp, cgs) # if not given store current as old return 0.0 end - p_old = get_storage(u.storage, PointStorageKey(:Iterate)) - X_old = get_storage(u.storage, VectorStorageKey(:Gradient)) - δ_old = get_storage(u.storage, VectorStorageKey(:δ)) - - gradienttr = vector_transport_to( - M, p_old, X_old, cgs.p, u.coefficient.vector_transport_method - ) - ν = cgs.X - gradienttr #notation y from [HZ06] - δtr = vector_transport_to(M, p_old, δ_old, cgs.p, u.coefficient.vector_transport_method) - denom = inner(M, cgs.p, δtr, ν) - νknormsq = inner(M, cgs.p, ν, ν) - β = - inner(M, cgs.p, ν, cgs.X) / denom - - 2 * νknormsq * inner(M, cgs.p, δtr, cgs.X) / denom^2 - # Numerical stability from Manopt / Hager-Zhang paper - ξn = norm(M, cgs.p, cgs.X) - η = -1 / (ξn * min(0.01, norm(M, p_old, X_old))) - coef = max(β, η) + p = get_storage(u.storage, PointStorageKey(:Iterate)) + X = get_storage(u.storage, VectorStorageKey(:Gradient)) + δ = get_storage(u.storage, VectorStorageKey(:δ)) + β = u.coefficient(amp, cgs, i; p = p, X = X, δ = δ) update_storage!(u.storage, amp, cgs) - return coef + return β end function show(io::IO, u::HagerZhangCoefficientRule) return print( - io, - "Manopt.HagerZhangCoefficientRule(; vector_transport_method=$(u.vector_transport_method))", + io, "Manopt.HagerZhangCoefficientRule(; vector_transport_method=$(u.vector_transport_method))", ) end @@ -559,39 +591,38 @@ end update_rule_storage_points(::HestenesStiefelCoefficientRule) = Tuple{:Iterate} update_rule_storage_vectors(::HestenesStiefelCoefficientRule) = Tuple{:Gradient, :δ} +# Since the Rule s are “memoryless” their functor accepts old necessary terms as (mandatory) +# keywords, i.e. the state has the current values, the keywords are the old ones +function (hs::HestenesStiefelCoefficientRule)( + amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i; p, X, δ + ) + M = get_manifold(amp) + Xtr = vector_transport_to(M, p, X, cgs.p, hs.vector_transport_method) + δtr = vector_transport_to(M, p, δ, cgs.p, hs.vector_transport_method) + ν = cgs.X - Xtr #notation from [HZ06] + nominator = get_differential(amp, cgs.p, ν; gradient = cgs.X, evaluated = true) + denominator = get_differential(amp, cgs.p, δtr; gradient = cgs.X, evaluated = true) - get_differential(amp, p, δ; gradient = X, evaluated = true) + return max(0, nominator / denominator) +end function (u::DirectionUpdateRuleStorage{<:HestenesStiefelCoefficientRule})( amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i ) - M = get_manifold(amp) if !has_storage(u.storage, PointStorageKey(:Iterate)) || !has_storage(u.storage, VectorStorageKey(:Gradient)) || !has_storage(u.storage, VectorStorageKey(:δ)) update_storage!(u.storage, amp, cgs) # if not given store current as old return 0.0 end - p_old = get_storage(u.storage, PointStorageKey(:Iterate)) - X_old = get_storage(u.storage, VectorStorageKey(:Gradient)) - δ_old = get_storage(u.storage, VectorStorageKey(:δ)) - - gradienttr = vector_transport_to( - M, p_old, X_old, cgs.p, u.coefficient.vector_transport_method - ) - δtr = vector_transport_to(M, p_old, δ_old, cgs.p, u.coefficient.vector_transport_method) - ν = cgs.X - gradienttr #notation from [HZ06] - # old with inners: - # β = inner(M, cgs.p, cgs.X, ν) / inner(M, cgs.p, δtr, ν) - nominator = get_differential(amp, cgs.p, ν; gradient = cgs.X, evaluated = true) - denominator = - get_differential(amp, cgs.p, δtr; gradient = cgs.X, evaluated = true) - - get_differential(amp, p_old, δ_old; gradient = X_old, evaluated = true) - β = nominator / denominator + p = get_storage(u.storage, PointStorageKey(:Iterate)) + X = get_storage(u.storage, VectorStorageKey(:Gradient)) + δ = get_storage(u.storage, VectorStorageKey(:δ)) + β = u.coefficient(amp, cgs, i; p = p, X = X, δ = δ) update_storage!(u.storage, amp, cgs) - return max(0, β) + return β end function show(io::IO, u::HestenesStiefelCoefficientRule) return print( - io, - "Manopt.HestenesStiefelCoefficientRule(; vector_transport_method=$(u.vector_transport_method))", + io, "Manopt.HestenesStiefelCoefficientRule(; vector_transport_method=$(u.vector_transport_method))", ) end @@ -685,27 +716,32 @@ end update_rule_storage_points(::LiuStoreyCoefficientRule) = Tuple{:Iterate} update_rule_storage_vectors(::LiuStoreyCoefficientRule) = Tuple{:Gradient, :δ} +# Since the Rule s are “memoryless” their functor accepts old necessary terms as (mandatory) +# keywords, i.e. the state has the current values, the keywords are the old ones +function (ls::LiuStoreyCoefficientRule)( + amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i; p, X, δ + ) + M = get_manifold(amp) + Xtr = vector_transport_to(M, p, X, cgs.p, ls.vector_transport_method) + ν = cgs.X - Xtr # notation y from [HZ06] + # old: + # β = inner(M, cgs.p, cgs.X, ν) / inner(M, p, -δ, X) + nominator = get_differential(amp, cgs.p, ν; gradient = cgs.X, evaluated = true) + denominator = get_differential(amp, p, δ; gradient = X, evaluated = true) + return -nominator / denominator +end function (u::DirectionUpdateRuleStorage{<:LiuStoreyCoefficientRule})( amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i ) - M = get_manifold(amp) if !has_storage(u.storage, PointStorageKey(:Iterate)) || !has_storage(u.storage, VectorStorageKey(:Gradient)) || !has_storage(u.storage, VectorStorageKey(:δ)) update_storage!(u.storage, amp, cgs) # if not given store current as old end - p_old = get_storage(u.storage, PointStorageKey(:Iterate)) - X_old = get_storage(u.storage, VectorStorageKey(:Gradient)) - δ_old = get_storage(u.storage, VectorStorageKey(:δ)) - gradienttr = vector_transport_to( - M, p_old, X_old, cgs.p, u.coefficient.vector_transport_method - ) - ν = cgs.X - gradienttr # notation y from [HZ06] - # old: - # β = inner(M, cgs.p, cgs.X, ν) / inner(M, p_old, -δ_old, X_old) - nominator = get_differential(amp, cgs.p, ν; gradient = cgs.X, evaluated = true) - denominator = get_differential(amp, p_old, δ_old; gradient = X_old, evaluated = true) - β = -nominator / denominator + p = get_storage(u.storage, PointStorageKey(:Iterate)) + X = get_storage(u.storage, VectorStorageKey(:Gradient)) + δ = get_storage(u.storage, VectorStorageKey(:δ)) + β = u.coefficient(amp, cgs, i; p = p, X = X, δ = δ) update_storage!(u.storage, amp, cgs) return β end @@ -781,34 +817,39 @@ end update_rule_storage_points(::PolakRibiereCoefficientRule) = Tuple{:Iterate} update_rule_storage_vectors(::PolakRibiereCoefficientRule) = Tuple{:Gradient} + +# Since the Rule s are “memoryless” their functor accepts old necessary terms as (mandatory) +# keywords, i.e. the state has the current values, the keywords are the old ones +function (pr::PolakRibiereCoefficientRule)( + amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i; p, X + ) + M = get_manifold(amp) + Xtr = vector_transport_to(M, p, X, cgs.p, pr.vector_transport_method) + ν = cgs.X - Xtr + # old + # β = real(inner(M, cgs.p, cgs.X, ν)) / real(inner(M, p, X, X)) + nominator = get_differential(amp, cgs.p, ν; gradient = cgs.X, evaluated = true) + denominator = get_differential(amp, p, X; gradient = X, evaluated = true) + β = nominator / denominator + # numerical stability from Manopt + return max(zero(β), β) +end function (u::DirectionUpdateRuleStorage{<:PolakRibiereCoefficientRule})( amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i ) - M = get_manifold(amp) if !has_storage(u.storage, PointStorageKey(:Iterate)) || !has_storage(u.storage, VectorStorageKey(:Gradient)) update_storage!(u.storage, amp, cgs) # if not given store current as old end - p_old = get_storage(u.storage, PointStorageKey(:Iterate)) - X_old = get_storage(u.storage, VectorStorageKey(:Gradient)) - - gradienttr = vector_transport_to( - M, p_old, X_old, cgs.p, u.coefficient.vector_transport_method - ) - ν = cgs.X - gradienttr - # old - # β = real(inner(M, cgs.p, cgs.X, ν)) / real(inner(M, p_old, X_old, X_old)) - nominator = get_differential(amp, cgs.p, ν; gradient = cgs.X, evaluated = true) - denominator = get_differential(amp, p_old, X_old; gradient = X_old, evaluated = true) - β = nominator / denominator - # numerical stability from Manopt + p = get_storage(u.storage, PointStorageKey(:Iterate)) + X = get_storage(u.storage, VectorStorageKey(:Gradient)) + β = u.coefficient(amp, cgs, i; p = p, X = X) update_storage!(u.storage, amp, cgs) - return max(zero(β), β) + return β end function show(io::IO, u::PolakRibiereCoefficientRule) return print( - io, - "Manopt.PolakRibiereCoefficientRule(; vector_transport_method=$(u.vector_transport_method))", + io, "Manopt.PolakRibiereCoefficientRule(; vector_transport_method=$(u.vector_transport_method))", ) end @@ -864,11 +905,16 @@ struct SteepestDescentCoefficientRule <: DirectionUpdateRule end update_rule_storage_points(::SteepestDescentCoefficientRule) = Tuple{} update_rule_storage_vectors(::SteepestDescentCoefficientRule) = Tuple{} -function (u::DirectionUpdateRuleStorage{SteepestDescentCoefficientRule})( - ::DefaultManoptProblem, ::ConjugateGradientDescentState, i +function (sd::SteepestDescentCoefficientRule)( + ::DefaultManoptProblem, ::ConjugateGradientDescentState, i; kwargs... ) return 0.0 end +function (u::DirectionUpdateRuleStorage{SteepestDescentCoefficientRule})( + amp::DefaultManoptProblem, cgs::ConjugateGradientDescentState, i + ) + return u.coefficient(amp, cgs, i) +end @doc """ SteepestDescentCoefficient() SteepestDescentCoefficient(M::AbstractManifold) @@ -967,29 +1013,28 @@ function (u::DirectionUpdateRuleStorage{<:ConjugateGradientBealeRestartRule})( amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, k ) M = get_manifold(amp) - if !has_storage(u.storage, PointStorageKey(:Iterate)) || - !has_storage(u.storage, VectorStorageKey(:Gradient)) - update_storage!(u.storage, amp, cgs) # if not given store current as old + if k == 0 + # store current values as old and return 0 + update_storage!(u.storage, amp, cgs) + return 0.0 end - p_old = get_storage(u.storage, PointStorageKey(:Iterate)) - X_old = get_storage(u.storage, VectorStorageKey(:Gradient)) - + # If a rule does not have these, they should return nothing + p = get_storage(u.storage, PointStorageKey(:Iterate)) + X = get_storage(u.storage, VectorStorageKey(:Gradient)) + δ = get_storage(u.storage, VectorStorageKey(:δ)) # call actual rule - β = u.coefficient.direction_update(amp, cgs, k) + β = u.coefficient.direction_update(amp, cgs, k; p = p, X = X, δ = δ) denom = norm(M, cgs.p, cgs.X) - Xoldpk = vector_transport_to( - M, p_old, X_old, cgs.p, u.coefficient.vector_transport_method - ) - num = inner(M, cgs.p, cgs.X, Xoldpk) + Xtr = vector_transport_to(M, p, X, cgs.p, u.coefficient.vector_transport_method) + num = inner(M, cgs.p, cgs.X, Xtr) # update storage only after that in case they share update_storage!(u.storage, amp, cgs) return real(num / denom) > u.coefficient.threshold ? zero(β) : β end function show(io::IO, u::ConjugateGradientBealeRestartRule) return print( - io, - "Manopt.ConjugateGradientBealeRestartRule($(repr(u.direction_update)); threshold=$(u.threshold), vector_transport_method=$(u.vector_transport_method))", + io, "Manopt.ConjugateGradientBealeRestartRule($(repr(u.direction_update)); threshold=$(u.threshold), vector_transport_method=$(u.vector_transport_method))", ) end @@ -1081,12 +1126,17 @@ end update_rule_storage_points(::HybridCoefficientRule) = Tuple{} update_rule_storage_vectors(::HybridCoefficientRule) = Tuple{} +function (hc::HybridCoefficientRule)( + amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i; kwargs... + ) + βs = [c(amp, cgs, i) for c in hc.coefficients] + β_lower_bound = hc.lower_bound(amp, cgs, i) + return max(hc.lower_bound_scale * β_lower_bound, min(βs...)) +end function (u::DirectionUpdateRuleStorage{<:HybridCoefficientRule})( amp::AbstractManoptProblem, cgs::ConjugateGradientDescentState, i ) - βs = [c(amp, cgs, i) for c in u.coefficient.coefficients] - β_lower_bound = u.coefficient.lower_bound(amp, cgs, i) - return max(u.coefficient.lower_bound_scale * β_lower_bound, min(βs...)) + return u.coefficient(amp, cgs, i) end function show(io::IO, u::HybridCoefficientRule) coefficient_str = join([repr(c.coefficient) for c in u.coefficients], ", ") @@ -1182,7 +1232,7 @@ function (corr::RestartOnNonDescent)( end @doc """ -RestartOnNonSufficientDescent <: AbstractRestartCondition + RestartOnNonSufficientDescent <: AbstractRestartCondition ## Fields * `κ`: the sufficient decrease factor diff --git a/src/plans/solver_state.jl b/src/plans/solver_state.jl index ceca627bec..b25c4c7b5c 100644 --- a/src/plans/solver_state.jl +++ b/src/plans/solver_state.jl @@ -500,10 +500,11 @@ end get_storage(a::AbstractStateAction, key::Symbol) Return the internal value of the [`AbstractStateAction`](@ref) `a` at the -`Symbol` `key`. +`Symbol` `key`. Returns `nothing` if the key does not exist """ -get_storage(a::AbstractStateAction, key::Symbol) = a.values[key] - +function get_storage(a::AbstractStateAction, key::Symbol) + return get(a.values, key, nothing) +end """ get_storage(a::AbstractStateAction, ::PointStorageKey{key}) where {key} diff --git a/src/solvers/conjugate_gradient_descent.jl b/src/solvers/conjugate_gradient_descent.jl index 5e11950707..600bd4aef2 100644 --- a/src/solvers/conjugate_gradient_descent.jl +++ b/src/solvers/conjugate_gradient_descent.jl @@ -142,7 +142,7 @@ function conjugate_gradient_descent!( mgo::O, p; coefficient::Union{DirectionUpdateRule, ManifoldDefaultsFactory} = ConjugateDescentCoefficient(), - restart_condition::AbstractRestartCondition = NeverRestart(), + restart_condition::AbstractRestartCondition = RestartOnNonDescent(), retraction_method::AbstractRetractionMethod = default_retraction_method(M, typeof(p)), stepsize::Union{Stepsize, ManifoldDefaultsFactory} = default_stepsize( M, ConjugateGradientDescentState; retraction_method = retraction_method @@ -195,7 +195,8 @@ function step_solver!(amp::AbstractManoptProblem, cgs::ConjugateGradientDescentS cgs.δ .-= cgs.X if (cgs.restart_condition(amp, cgs, k)) # restart solver; set dir to -grad - cgs.δ = -copy(get_manifold(amp), cgs.p, cgs.X) + copyto!(M, cgs.δ, cgs.X) + cgs.δ .*= -1 update_storage!(cgs.coefficient.storage, amp, cgs) cgs.β = 0.0 end diff --git a/test/plans/test_conjugate_gradient_plan.jl b/test/plans/test_conjugate_gradient_plan.jl index 3743c708ac..82dc6098cd 100644 --- a/test/plans/test_conjugate_gradient_plan.jl +++ b/test/plans/test_conjugate_gradient_plan.jl @@ -1,7 +1,8 @@ using Manopt, Manifolds, Test struct DummyCGCoeff <: DirectionUpdateRule end -(u::DummyCGCoeff)(p, s, k) = 0.2 +(::DummyCGCoeff)(pr, st, k; kwagrs...) = 0.2 +(::Manopt.DirectionUpdateRuleStorage{DummyCGCoeff})(pr, st, k) = 0.2 Manopt.update_rule_storage_points(::DummyCGCoeff) = Tuple{} Manopt.update_rule_storage_vectors(::DummyCGCoeff) = Tuple{} @@ -16,22 +17,26 @@ Manopt.update_rule_storage_vectors(::DummyCGCoeff) = Tuple{} p0 = [1.0, 0.0] pr = DefaultManoptProblem(M, ManifoldGradientObjective(f, grad_f)) cgs2 = ConjugateGradientDescentState( - M; - p = p0, + M; p = p0, stopping_criterion = StopAfterIteration(2), stepsize = Manopt.ConstantStepsize(M, 1.0), coefficient = dur2, ) cgs2.X = [0.0, 0.2] + # Fake update history to get a certain old X and old p + cgs2.coefficient(pr, cgs2, 0) + # the inner check is 0.2 which is still less than 0.3 @test cgs2.coefficient(pr, cgs2, 1) != 0 cgs3 = ConjugateGradientDescentState( - M; - p = p0, + M; p = p0, stopping_criterion = StopAfterIteration(2), stepsize = Manopt.ConstantStepsize(M, 1.0), coefficient = dur3, ) cgs3.X = [0.0, 0.2] + # Fake update history to get a certain old X and old p + cgs3.coefficient(pr, cgs3, 0) + # then we are above the threshold 0.1 (namely at 0.2) and we get a descent step @test cgs3.coefficient(pr, cgs3, 1) == 0 end @testset "representation and summary of Coefficients" begin diff --git a/test/solvers/test_conjugate_gradient.jl b/test/solvers/test_conjugate_gradient.jl index 77ffeb6cd7..c7f29ff1c6 100644 --- a/test/solvers/test_conjugate_gradient.jl +++ b/test/solvers/test_conjugate_gradient.jl @@ -404,4 +404,24 @@ using ManifoldDiff: grad_distance grad_f(M, p) = sum(1 / n * grad_distance.(Ref(M), data, Ref(p))) @test conjugate_gradient_descent(M, f, grad_f, data[1]) isa PoincareBallPoint end + + @testset "Issue #603: CG with HZ rule on a numerically challenging problem" begin + M = Sphere(2) + p0 = [1.0, 0.0, 0.0] + + a = [0.0, 1.0, 0.0] + + f(M, p) = 0.5 * norm(p - a)^2 + grad_f(M, p) = project(M, p, p - a) + + cgs = conjugate_gradient_descent( + M, + f, + grad_f, + p0; + coefficient = ConjugateGradientBealeRestart(HagerZhangCoefficient()), + return_state = true, + ) + @test norm(M, cgs.p, grad_f(M, cgs.p)) < 1.0e-8 + end end From 13ff14a6f6d2faabd1e1c689b0c12e1d5cdeadc6 Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Tue, 12 May 2026 09:44:47 +0200 Subject: [PATCH 127/135] Unify REPL print and add a test. --- src/plans/stopping_criterion.jl | 36 ++++++++++++--------------- test/solvers/test_quasi_Newton_box.jl | 8 ++++-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 180f4fbdbf..6deef53cfa 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -480,22 +480,22 @@ A stopping criterion to stop when based on Eq. (1) in [ZhuByrdLuNocedal:1997](@cite) # Fields -* tolerance: the threshold `tol` in the above formula. +* _`threshold`: the threshold `tol` in the above formula. $(_fields([:at_iteration, :last_change])) -* `last_cost``: the last cost value +* `last_cost`: the last cost value # Constructor - StopWhenRelativeAPosterioriCostChangeLessOrEqual(tolerance::F) + StopWhenRelativeAPosterioriCostChangeLessOrEqual(threshold::F) -Initialize the stopping criterion to a threshold `tolerance` for the change of the cost function. +Initialize the stopping criterion to a `threshold` for the change of the cost function. StopWhenRelativeAPosterioriCostChangeLessOrEqual(; factr::Real=1.0e7) -Initialize tolerance to `factr * eps(factr)`, following the convention in [ZhuByrdLuNocedal:1997](@cite). +Initialize threshold to `factr * eps(factr)`, following the convention in [ZhuByrdLuNocedal:1997](@cite). """ mutable struct StopWhenRelativeAPosterioriCostChangeLessOrEqual{F <: Real} <: StoppingCriterion - tolerance::F + threshold::F at_iteration::Int last_cost::F last_change::F @@ -510,12 +510,12 @@ function (c::StopWhenRelativeAPosterioriCostChangeLessOrEqual)( if iteration <= 0 # reset on init c.at_iteration = -1 c.last_cost = Inf - c.last_change = 2 * c.tolerance + c.last_change = 2 * c.threshold end current_cost = get_cost(problem, state) c.last_change = (c.last_cost - current_cost) / max(abs(c.last_cost), abs(current_cost), 1) c.last_cost = current_cost - if iteration > 1 && c.last_change <= c.tolerance + if iteration > 1 && c.last_change <= c.threshold c.at_iteration = iteration return true end @@ -524,20 +524,18 @@ end indicates_convergence(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) = false function get_reason(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) if c.at_iteration >= 0 - return "At iteration $(c.at_iteration) the algorithm performed a step with a relative a posteriori cost change ($(abs(c.last_change))) less than or equal to $(c.tolerance)." + return "At iteration $(c.at_iteration) the algorithm performed a step with a relative a posteriori cost change ($(abs(c.last_change))) less than or equal to $(c.threshold)." end return "" end -function status_summary(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) +function status_summary(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual; context::Symbol = short) + (context == :short) && return repr(c) has_stopped = (c.at_iteration >= 0) s = has_stopped ? "reached" : "not reached" - return "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = $(abs(c.last_change)) ≤ $(c.tolerance):\t$s" + return (_is_inline(context) ? "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = $(abs(c.last_change)) ≤ $(c.threshold):$(_MANOPT_INDENT)" : "A stopping criterion to stop when the relative posteriori cost change is less than $(c.threshold)\n$(_MANOPT_INDENT)") * "$s" end -function Base.show(io::IO, ::MIME"text/plain", c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) - return print( - io, - "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold $(c.tolerance).\n $(status_summary(c))", - ) +function Base.show(io::IO, c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) + return print(io, "StopWhenRelativeAPosterioriCostChangeLessOrEqual($(c.threshold))") end @doc """ @@ -911,10 +909,8 @@ function status_summary(c::StopWhenProjectedNegativeGradientNormLess; context::S return "A StoppingCriterion to stop when the negative projected gradient norm is less than a threshold of $(c.threshold):\n$(_MANOPT_INDENT)$s" end indicates_convergence(c::StopWhenProjectedNegativeGradientNormLess) = true -function show(io::IO, c::StopWhenProjectedNegativeGradientNormLess) - print(io, "StopWhenProjectedNegativeGradientNormLess(", c.threshold, "; norm = ", c.norm) - !ismissing(c.outer_norm) && print(io, ", outer_norm = ", c.outer_norm) - return print(io, ")") +function Base.show(io::IO, c::StopWhenProjectedNegativeGradientNormLess) + return print(io, "StopWhenProjectedNegativeGradientNormLess($(c.threshold); norm = $(c.norm))") end """ diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 1430a105bb..467f7383c1 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -23,7 +23,6 @@ using RecursiveArrayTools @test Manopt.get_stepsize_bound(M, p, d, 2) ≈ Inf end - @testset "update_fp_fpp - basic d = -g" begin M = Hyperrectangle([0.0, 1.0], [3.0, 3.0]) @@ -57,7 +56,6 @@ using RecursiveArrayTools @test hv_eb_d == original_hv_eb_d end - @testset "update_fp_fpp - basic d = [-2.0, -1.0]" begin M = Hyperrectangle([0.0, 1.0], [3.0, 3.0]) @@ -317,6 +315,12 @@ using RecursiveArrayTools p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) @test distance(M, p_opt, ArrayPartition(px, [0 2; 0 0])) < 0.1 end + + @testset "Specialised Stopping criteria" begin + sc = StopWhenProjectedNegativeGradientNormLess(1.0e-9) + @test startswith(repr(sc), "StopWhenProjectedNegativeGradientNormLess(1.0e-9") + @test startswith(Manopt.status_summary(sc), "A StoppingCriterion to stop when the negative projected gradient norm is less than") + end end @testset "MaxStepsizeInDirection" begin From a5038f9ece874c10a79dd201821a4c026775d98f Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Tue, 12 May 2026 10:12:39 +0200 Subject: [PATCH 128/135] unify tests. --- src/plans/stopping_criterion.jl | 2 +- test/plans/test_stopping_criteria.jl | 9 ++++++--- test/solvers/test_quasi_Newton_box.jl | 6 ------ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/plans/stopping_criterion.jl b/src/plans/stopping_criterion.jl index 855c14d150..8426dc2f9c 100644 --- a/src/plans/stopping_criterion.jl +++ b/src/plans/stopping_criterion.jl @@ -528,7 +528,7 @@ function get_reason(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual) end return "" end -function status_summary(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual; context::Symbol = short) +function status_summary(c::StopWhenRelativeAPosterioriCostChangeLessOrEqual; context::Symbol = :default) (context == :short) && return repr(c) has_stopped = (c.at_iteration >= 0) s = has_stopped ? "reached" : "not reached" diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 00f5fb66bb..35ee308928 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -403,15 +403,18 @@ end @test sc(prob, s, 2) @test length(get_reason(sc)) > 0 @test startswith( - to_display_string(sc), - "StopWhenRelativeAPosterioriCostChangeLessOrEqual with threshold", + Manopt.status_summary(sc), + "A stopping criterion to stop when the relative posteriori cost change is less than", ) - @test startswith(Manopt.status_summary(sc), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") + @test startswith(Manopt.status_summary(sc; context = :inline), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") @test !Manopt.indicates_convergence(sc) end @testset "StopWhenProjectedNegativeGradientNormLess" begin sc = StopWhenProjectedNegativeGradientNormLess(1.0e-10) + @test startswith(repr(sc), "StopWhenProjectedNegativeGradientNormLess(") + @test startswith(Manopt.status_summary(sc), "A StoppingCriterion to stop when the negative projected gradient norm is less than") + M = Hyperrectangle([1.0], [2.0]) prob = DefaultManoptProblem( M, ManifoldGradientObjective((M, x) -> x^2, x -> 2x) diff --git a/test/solvers/test_quasi_Newton_box.jl b/test/solvers/test_quasi_Newton_box.jl index 467f7383c1..5c268b8a59 100644 --- a/test/solvers/test_quasi_Newton_box.jl +++ b/test/solvers/test_quasi_Newton_box.jl @@ -315,12 +315,6 @@ using RecursiveArrayTools p_opt = quasi_Newton(M, f, grad_f, p0; stopping_criterion = StopWhenProjectedNegativeGradientNormLess(1.0e-6) | StopAfterIteration(100)) @test distance(M, p_opt, ArrayPartition(px, [0 2; 0 0])) < 0.1 end - - @testset "Specialised Stopping criteria" begin - sc = StopWhenProjectedNegativeGradientNormLess(1.0e-9) - @test startswith(repr(sc), "StopWhenProjectedNegativeGradientNormLess(1.0e-9") - @test startswith(Manopt.status_summary(sc), "A StoppingCriterion to stop when the negative projected gradient norm is less than") - end end @testset "MaxStepsizeInDirection" begin From 3ed029c10e5335cf1eb1b3fe0602d4edafd4db9c Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Tue, 12 May 2026 10:40:58 +0200 Subject: [PATCH 129/135] Improve code cov. --- test/plans/test_stopping_criteria.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/plans/test_stopping_criteria.jl b/test/plans/test_stopping_criteria.jl index 35ee308928..77da09743a 100644 --- a/test/plans/test_stopping_criteria.jl +++ b/test/plans/test_stopping_criteria.jl @@ -407,6 +407,7 @@ end "A stopping criterion to stop when the relative posteriori cost change is less than", ) @test startswith(Manopt.status_summary(sc; context = :inline), "(fₖ- fₖ₊₁)/max(|fₖ|, |fₖ₊₁|, 1) = ") + @test startswith(repr(sc), "StopWhenRelativeAPosterioriCostChangeLessOrEqual(") @test !Manopt.indicates_convergence(sc) end From 15a74ce5bf937ce2cddd9842ab6e5fabd05865a4 Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Tue, 12 May 2026 13:57:20 +0200 Subject: [PATCH 130/135] Update ext/ManoptManifoldsExt/manifold_functions.jl Co-authored-by: Patryk Przybysz --- ext/ManoptManifoldsExt/manifold_functions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/ManoptManifoldsExt/manifold_functions.jl b/ext/ManoptManifoldsExt/manifold_functions.jl index 44b5b2b317..52e2ee0e9b 100644 --- a/ext/ManoptManifoldsExt/manifold_functions.jl +++ b/ext/ManoptManifoldsExt/manifold_functions.jl @@ -236,7 +236,7 @@ end """ Manopt.has_anisotropic_max_stepsize(::Hyperrectangle) -Returns `true`, as `Hyperrectangle` manifold requires generalized Cauchy point computation in solvers. +Returns `true`, as [`Hyperrectangle`](@extref `Manifolds.Hyperrectangle`) manifold requires generalized Cauchy point computation in solvers. """ Manopt.has_anisotropic_max_stepsize(::Hyperrectangle) = true From be9c53ef8773eee74cacd0737b4bf332adbeb224 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 12 May 2026 15:17:38 +0200 Subject: [PATCH 131/135] add citation of the preprint --- docs/src/references.bib | 10 ++++++++++ .../solvers/generalized_cauchy_direction_subsolver.md | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/src/references.bib b/docs/src/references.bib index b3fb0e5573..c93389e940 100644 --- a/docs/src/references.bib +++ b/docs/src/references.bib @@ -87,6 +87,16 @@ @article{BacakBergmannSteidlWeinmann:2016 VOLUME = {38}, } +@misc{BaranBergmannPrzybysz:2026, + title = {A {Riemannian} quasi-{Newton} algorithm for optimization with {Euclidean} bounds}, + doi = {10.48550/arXiv.2605.10573}, + publisher = {arXiv}, + author = {Baran, Mateusz and Bergmann, Ronny and Przybysz, Patryk}, + month = may, + year = {2026}, + note = {arXiv:2605.10573 [math.OC]}, +} + @inproceedings{Beale:1972, ADDRESS = {London}, AUTHOR = {Beale, E. M. L.}, diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index cedee9c719..66e3f87619 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -1,6 +1,6 @@ # Generalized Cauchy direction subsolver -The generalized Cauchy direction (GCD) subsolver is a component in optimization algorithms that handle problems with bound constraints. It solves the following problem +The generalized Cauchy direction (GCD) subsolver is a component in optimization algorithms that handle problems with bound constraints [BaranBergmannPrzybysz:2026](@cite). It solves the following problem ```math \begin{align*} From 9b261bace23b89f264475699ebe9906e0a5f8446 Mon Sep 17 00:00:00 2001 From: Mateusz Baran Date: Tue, 12 May 2026 15:42:38 +0200 Subject: [PATCH 132/135] add bibliography section --- docs/src/solvers/generalized_cauchy_direction_subsolver.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/solvers/generalized_cauchy_direction_subsolver.md b/docs/src/solvers/generalized_cauchy_direction_subsolver.md index 66e3f87619..73aaf780ad 100644 --- a/docs/src/solvers/generalized_cauchy_direction_subsolver.md +++ b/docs/src/solvers/generalized_cauchy_direction_subsolver.md @@ -71,3 +71,7 @@ Manopt.LimitedMemorySegmentHessianUpdater Manopt.hessian_value_from_inner_products Manopt.update_current_scale! ``` + +```@bibliography +Canonical=false +``` From a263b7838a649a043085fd9eb72a388421f8ecdb Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Wed, 13 May 2026 11:43:39 +0200 Subject: [PATCH 133/135] Add Patryk to the about. --- docs/src/about.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/about.md b/docs/src/about.md index 44a4c46403..3228a05d9e 100644 --- a/docs/src/about.md +++ b/docs/src/about.md @@ -15,6 +15,7 @@ Thanks to the following contributors to `Manopt.jl`: * Mathias Ravn Munkvold contributed most of the implementation of the [Adaptive Regularization with Cubics](solvers/adaptive-regularization-with-cubics.md) solver as well as its [Lanczos](@ref arc-Lanczos) subsolver * [Sander Engen Oddsen](https://github.com/oddsen) contributed to the implementation of the [LTMADS](solvers/mesh_adaptive_direct_search.md) solver. * [Jonas Püschel](https://www.uni-augsburg.de/de/fakultaet/mntf/math/prof/numa/team/jonas-pueschel/) contributed [restart rules for the conjugate gradient solver](@ref cg-restart). +* [Patryk Przybysz](https://www.linkedin.com/in/patryk-przybysz-5644aa1a1/) contributed to the [Generalized Cauchy Direection](solvers/generalized_cauchy_direction_subsolver.md) * [Tom-Christian Riemer](https://www.tu-chemnitz.de/mathematik/wire/mitarbeiter.php) implemented the [trust regions](solvers/trust_regions.md) and [quasi Newton](solvers/quasi_Newton.md) solvers as well as the [truncated conjugate gradient descent](solvers/truncated_conjugate_gradient_descent.md) subsolver. * [Markus A. Stokkenes](https://www.linkedin.com/in/markus-a-stokkenes-b41bba17b/) contributed most of the implementation of the [Interior Point Newton Method](solvers/interior_point_Newton.md) as well as its default [Conjugate Residual](solvers/conjugate_residual.md) subsolver * [Laura Weigl](https://num.math.uni-bayreuth.de/en/team/laura-weigl/index.php) implemented the [Vector bundle Newton Method](solvers/vectorbundle_newton.md). From 4bbf42081ba71742da0615fd5fb74b2a563fb0b2 Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Wed, 13 May 2026 11:45:01 +0200 Subject: [PATCH 134/135] Fix a typo. --- docs/src/about.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/about.md b/docs/src/about.md index 3228a05d9e..f15a835251 100644 --- a/docs/src/about.md +++ b/docs/src/about.md @@ -15,7 +15,7 @@ Thanks to the following contributors to `Manopt.jl`: * Mathias Ravn Munkvold contributed most of the implementation of the [Adaptive Regularization with Cubics](solvers/adaptive-regularization-with-cubics.md) solver as well as its [Lanczos](@ref arc-Lanczos) subsolver * [Sander Engen Oddsen](https://github.com/oddsen) contributed to the implementation of the [LTMADS](solvers/mesh_adaptive_direct_search.md) solver. * [Jonas Püschel](https://www.uni-augsburg.de/de/fakultaet/mntf/math/prof/numa/team/jonas-pueschel/) contributed [restart rules for the conjugate gradient solver](@ref cg-restart). -* [Patryk Przybysz](https://www.linkedin.com/in/patryk-przybysz-5644aa1a1/) contributed to the [Generalized Cauchy Direection](solvers/generalized_cauchy_direction_subsolver.md) +* [Patryk Przybysz](https://www.linkedin.com/in/patryk-przybysz-5644aa1a1/) contributed to the [Generalized Cauchy Direection](solvers/generalized_cauchy_direction_subsolver.md). * [Tom-Christian Riemer](https://www.tu-chemnitz.de/mathematik/wire/mitarbeiter.php) implemented the [trust regions](solvers/trust_regions.md) and [quasi Newton](solvers/quasi_Newton.md) solvers as well as the [truncated conjugate gradient descent](solvers/truncated_conjugate_gradient_descent.md) subsolver. * [Markus A. Stokkenes](https://www.linkedin.com/in/markus-a-stokkenes-b41bba17b/) contributed most of the implementation of the [Interior Point Newton Method](solvers/interior_point_Newton.md) as well as its default [Conjugate Residual](solvers/conjugate_residual.md) subsolver * [Laura Weigl](https://num.math.uni-bayreuth.de/en/team/laura-weigl/index.php) implemented the [Vector bundle Newton Method](solvers/vectorbundle_newton.md). From e8b4603630d680214be9b79e40c2b509e8c70a63 Mon Sep 17 00:00:00 2001 From: Ronny Bergmann Date: Wed, 13 May 2026 13:39:03 +0200 Subject: [PATCH 135/135] Fix a typo. --- docs/src/about.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/about.md b/docs/src/about.md index f15a835251..4ff1057d78 100644 --- a/docs/src/about.md +++ b/docs/src/about.md @@ -15,7 +15,7 @@ Thanks to the following contributors to `Manopt.jl`: * Mathias Ravn Munkvold contributed most of the implementation of the [Adaptive Regularization with Cubics](solvers/adaptive-regularization-with-cubics.md) solver as well as its [Lanczos](@ref arc-Lanczos) subsolver * [Sander Engen Oddsen](https://github.com/oddsen) contributed to the implementation of the [LTMADS](solvers/mesh_adaptive_direct_search.md) solver. * [Jonas Püschel](https://www.uni-augsburg.de/de/fakultaet/mntf/math/prof/numa/team/jonas-pueschel/) contributed [restart rules for the conjugate gradient solver](@ref cg-restart). -* [Patryk Przybysz](https://www.linkedin.com/in/patryk-przybysz-5644aa1a1/) contributed to the [Generalized Cauchy Direection](solvers/generalized_cauchy_direction_subsolver.md). +* [Patryk Przybysz](https://www.linkedin.com/in/patryk-przybysz-5644aa1a1/) contributed to the [Generalized Cauchy Direction](solvers/generalized_cauchy_direction_subsolver.md). * [Tom-Christian Riemer](https://www.tu-chemnitz.de/mathematik/wire/mitarbeiter.php) implemented the [trust regions](solvers/trust_regions.md) and [quasi Newton](solvers/quasi_Newton.md) solvers as well as the [truncated conjugate gradient descent](solvers/truncated_conjugate_gradient_descent.md) subsolver. * [Markus A. Stokkenes](https://www.linkedin.com/in/markus-a-stokkenes-b41bba17b/) contributed most of the implementation of the [Interior Point Newton Method](solvers/interior_point_Newton.md) as well as its default [Conjugate Residual](solvers/conjugate_residual.md) subsolver * [Laura Weigl](https://num.math.uni-bayreuth.de/en/team/laura-weigl/index.php) implemented the [Vector bundle Newton Method](solvers/vectorbundle_newton.md).