Skip to content

Commit 3338769

Browse files
AdrienTaylorAdrienTaylor
committed
online learning algos + docs
1 parent e79f973 commit 3338769

17 files changed

Lines changed: 781 additions & 336 deletions

File tree

‎PEPit/examples/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@
1010
'low_dimensional_worst_cases_scenarios',
1111
'tutorials',
1212
'continuous_time_models',
13+
'online_learning',
1314
]

‎PEPit/examples/monotone_inclusions_variational_inequalities/douglas_rachford_splitting_2.py‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ def wc_douglas_rachford_splitting_2(beta, mu, alpha, theta, wrapper="cvxpy", sol
6969
theoretical_tau (float): theoretical value.
7070
7171
Example:
72-
>>> pepit_tau, theoretical_tau = wc_douglas_rachford_splitting(beta=1, mu=.1, alpha=1.3, theta=.9, wrapper="cvxpy", solver=None, verbose=1)
72+
>>> beta, mu, alpha, theta = 1.2, 0.1, 0.3, 1.5
73+
>>> pepit_tau, theoretical_tau = wc_douglas_rachford_splitting_2(beta=beta, mu=mu, alpha=alpha, theta=theta, wrapper="cvxpy", solver=None, verbose=1)
7374
(PEPit) Setting up the problem: size of the Gram matrix: 6x6
7475
(PEPit) Setting up the problem: performance measure is the minimum of 1 element(s)
7576
(PEPit) Setting up the problem: Adding initial conditions and general constraints ...
@@ -84,18 +85,18 @@ def wc_douglas_rachford_splitting_2(beta, mu, alpha, theta, wrapper="cvxpy", sol
8485
(PEPit) Calling SDP solver
8586
(PEPit) Solver status: optimal (wrapper:cvxpy, solver: MOSEK); optimal value: 0.928770707839351
8687
(PEPit) Primal feasibility check:
87-
The solver found a Gram matrix that is positive semi-definite up to an error of 3.297473722026212e-09
88-
All the primal scalar constraints are verified up to an error of 1.64989273354621e-08
89-
(PEPit) Dual feasibility check:
90-
The solver found a residual matrix that is positive semi-definite
91-
All the dual scalar values associated with inequality constraints are nonnegative
92-
(PEPit) The worst-case guarantee proof is perfectly reconstituted up to an error of 3.4855088898444464e-07
93-
(PEPit) Final upper bound (dual): 0.9287707057295752 and lower bound (primal example): 0.928770707839351
94-
(PEPit) Duality gap: absolute: -2.109775798508906e-09 and relative: -2.2715787445719413e-09
95-
*** Example file: worst-case performance of the Douglas Rachford Splitting***
96-
PEPit guarantee: ||w_(t+1)^0 - w_(t+1)^1||^2 <= 0.928771 ||w_(t)^0 - w_(t)^1||^2
97-
Theoretical guarantee: ||w_(t+1)^0 - w_(t+1)^1||^2 <= 0.928771 ||w_(t)^0 - w_(t)^1||^2
98-
88+
The solver found a Gram matrix that is positive semi-definite up to an error of 5.115160308023067e-10
89+
All the primal scalar constraints are verified up to an error of 1.8039681970449806e-09
90+
(PEPit) Dual feasibility check:
91+
The solver found a residual matrix that is positive semi-definite
92+
All the dual scalar values associated with inequality constraints are nonnegative
93+
(PEPit) The worst-case guarantee proof is perfectly reconstituted up to an error of 1.2557649586852904e-08
94+
(PEPit) Final upper bound (dual): 0.9145301167694716 and lower bound (primal example): 0.9145301164750972
95+
(PEPit) Duality gap: absolute: 2.9437441373403317e-10 and relative: 3.218859701074142e-10
96+
*** Example file: worst-case performance of the Douglas Rachford Splitting***
97+
PEPit guarantee: ||w_(t+1)^0 - w_(t+1)^1||^2 <= 0.91453 ||w_(t)^0 - w_(t)^1||^2
98+
Theoretical guarantee: ||w_(t+1)^0 - w_(t+1)^1||^2 <= 0.91453 ||w_(t)^0 - w_(t)^1||^2
99+
99100
"""
100101

101102
# Instantiate PEP

‎PEPit/examples/online_learning/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from .online_gradient_descent import wc_online_gradient_descent
2+
from .online_frank_wolfe import wc_online_frank_wolfe
3+
from .online_follow_leader import wc_online_follow_leader
4+
from .online_follow_regularized_leader import wc_online_follow_regularized_leader
5+
6+
__all__ = ['online_gradient_descent', 'wc_online_gradient_descent',
7+
'online_frank_wolfe', 'wc_online_frank_wolfe',
8+
'online_follow_leader', 'wc_online_follow_leader',
9+
'online_follow_regularized_leader', 'wc_online_follow_regularized_leader',
10+
]

‎PEPit/examples/online_learning/online_follow_leader.py‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
import numpy as np
2+
from PEPit import PEP
3+
from PEPit.functions import ConvexLipschitzFunction
4+
from PEPit.functions import ConvexIndicatorFunction
5+
from PEPit.primitive_steps import proximal_step
6+
7+
def wc_online_follow_leader(M, D, n, wrapper="cvxpy", solver=None, verbose=1):
8+
"""
9+
Consider the online convex minimization problem, whose goal is to sequentially minimize the regret
10+
11+
.. math:: R_n \\triangleq \\min_{x\\in Q} \sum_{i=1}^n f_i(x_i)-f_i(x_\\star),
12+
13+
where the functions :math:`f_i` are :math:`M`-Lipschitz and convex, and where :math:`Q` is a
14+
bounded closed convex set with diameter upper bounded by :math:`D`, and where :math:`x_\\star\\in Q`
15+
is a reference point. Classical references on the topic include [1, 2].
16+
17+
This code computes a worst-case guarantee for **follow the leader** (FTL).
18+
That is, it computes the smallest possible :math:`\\tau(n, M, D)` such that the guarantee
19+
20+
.. math:: R_n \\leqslant \\tau(n, M, D)
21+
22+
is valid for any such sequence of queries of FTL; that is, :math:`x_t` are the query points of OGD.
23+
24+
In short, for given values of :math:`n`, :math:`M`, :math:`D`:
25+
:math:`\\tau(n, M, D)` is computed as the worst-case value of :math:`R_n`.
26+
27+
**Algorithm**:
28+
Follow the leader is described by
29+
30+
.. math:: x_{t+1} \\in \\text{argmin}_{x\\in Q} \\left( \sum_{i=1}^t f_i(x) \\right).
31+
32+
**Theoretical guarantee**: The follow the leader strategy is known to have a linear regret
33+
(see, e.g., [1, Chapter 5]); we do not compare to any guarantee here.
34+
35+
36+
**References**:
37+
38+
`[1] E. Hazan (2016).
39+
Introduction to online convex optimization.
40+
Foundations and Trends in Optimization, 2(3-4), 157-325.
41+
<https://arxiv.org/pdf/1912.13213>`_
42+
43+
`[2] F. Orabona (2025).
44+
A Modern Introduction to Online Learning.
45+
<https://arxiv.org/pdf/1912.13213>`_
46+
47+
Args:
48+
M (float): the Lipschitz parameter.
49+
D (float): the diameter of the set.
50+
n (int): time horizon.
51+
wrapper (str): the name of the wrapper to be used.
52+
solver (str): the name of the solver the wrapper should use.
53+
verbose (int): level of information details to print.
54+
55+
- -1: No verbose at all.
56+
- 0: This example's output.
57+
- 1: This example's output + PEPit information.
58+
- 2: This example's output + PEPit information + solver details.
59+
60+
Returns:
61+
pepit_tau (float): worst-case value
62+
theoretical_tau (float): theoretical value
63+
64+
Example:
65+
>>> M, D, n = 1,.5,2
66+
>>> pepit_tau, theoretical_tau = wc_online_follow_leader(M, D, n, wrapper="cvxpy", solver=None, verbose=1)
67+
(PEPit) Setting up the problem: size of the Gram matrix: 10x10
68+
(PEPit) Setting up the problem: performance measure is the minimum of 1 element(s)
69+
(PEPit) Setting up the problem: Adding initial conditions and general constraints ...
70+
(PEPit) Setting up the problem: initial conditions and general constraints (0 constraint(s) added)
71+
(PEPit) Setting up the problem: interpolation conditions for 3 function(s)
72+
Function 1 : Adding 9 scalar constraint(s) ...
73+
Function 1 : 9 scalar constraint(s) added
74+
Function 2 : Adding 4 scalar constraint(s) ...
75+
Function 2 : 4 scalar constraint(s) added
76+
Function 3 : Adding 15 scalar constraint(s) ...
77+
Function 3 : 15 scalar constraint(s) added
78+
(PEPit) Setting up the problem: additional constraints for 0 function(s)
79+
(PEPit) Compiling SDP
80+
(PEPit) Calling SDP solver
81+
(PEPit) Solver status: optimal (wrapper:cvxpy, solver: MOSEK); optimal value: 0.9330127036100446
82+
(PEPit) Primal feasibility check:
83+
The solver found a Gram matrix that is positive semi-definite
84+
All the primal scalar constraints are verified up to an error of 5.581592632530885e-09
85+
(PEPit) Dual feasibility check:
86+
The solver found a residual matrix that is positive semi-definite
87+
All the dual scalar values associated with inequality constraints are nonnegative up to an error of 2.1864238886638586e-10
88+
(PEPit) The worst-case guarantee proof is perfectly reconstituted up to an error of 2.363987770546799e-08
89+
(PEPit) Final upper bound (dual): 0.9330127041164931 and lower bound (primal example): 0.9330127036100446
90+
(PEPit) Duality gap: absolute: 5.064484387418133e-10 and relative: 5.428097996760877e-10
91+
*** Example file: worst-case regret of online follow the leader ***
92+
PEPit guarantee: R_n <= 0.933013
93+
94+
"""
95+
# Instantiate PEP
96+
problem = PEP()
97+
98+
M_list = [ M for i in range(n)]
99+
gamma = D/M/np.sqrt(n)
100+
101+
# Declare a sequence of M-Lipschitz convex functions fi and an indicator function with Diameter D
102+
fis = [problem.declare_function(ConvexLipschitzFunction, M=M_list[i]) for i in range(n)]
103+
h = problem.declare_function(function_class=ConvexIndicatorFunction, D=D)
104+
105+
F = np.sum(fis)
106+
Ftot = F + h
107+
108+
# Defining a reference point
109+
xRef = problem.set_initial_point()
110+
xRef,_,_ = proximal_step(xRef, h, 1) # project the reference point
111+
gRef, FRef = F.oracle(xRef)
112+
113+
x1 = problem.set_initial_point()
114+
x1,_,_ = proximal_step(x1, h, 1) # project the reference point
115+
116+
# Run n steps of gradient descent with step-size gamma
117+
x = x1
118+
x_saved = list()
119+
g_saved = list()
120+
f_saved = list()
121+
f_occ = h
122+
for i in range(n):
123+
x_saved.append(x-xRef)
124+
g, f = fis[i].oracle(x)
125+
f_saved.append(f-fis[i].value(xRef))
126+
g_saved.append(g)
127+
f_occ = f_occ + fis[i]
128+
if i < n-1:
129+
x = f_occ.stationary_point()
130+
131+
# Set the performance metric to the function values accuracy
132+
problem.set_performance_metric(np.sum(f_saved))
133+
134+
# Solve the PEP
135+
pepit_verbose = max(verbose, 0)
136+
pepit_tau = problem.solve(wrapper=wrapper, solver=solver, verbose=pepit_verbose)
137+
138+
# Compute theoretical guarantee
139+
theoretical_tau = None
140+
141+
# Print conclusion if required
142+
if verbose != -1:
143+
print('*** Example file: worst-case regret of online follow the leader ***')
144+
print('\tPEPit guarantee:\t R_n <= {:.6}'.format(pepit_tau))
145+
146+
# Return the worst-case guarantee of the evaluated method (and the reference theoretical value)
147+
return pepit_tau, theoretical_tau
148+
149+
if __name__ == "__main__":
150+
M, D, n = 1,.5,2
151+
pepit_tau, theoretical_tau = wc_online_follow_leader(M, D, n, wrapper="cvxpy", solver=None, verbose=1)

‎PEPit/examples/online_learning/online_follow_regularized_leader.py‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import numpy as np
2+
from PEPit import PEP
3+
from PEPit.functions import ConvexLipschitzFunction
4+
from PEPit.functions import ConvexIndicatorFunction
5+
from PEPit.primitive_steps import proximal_step
6+
7+
def wc_online_follow_regularized_leader(M, D, n, wrapper="cvxpy", solver=None, verbose=1):
8+
"""
9+
Consider the online convex minimization problem, whose goal is to sequentially minimize the regret
10+
11+
.. math:: R_n \\triangleq \\min_{x\\in Q} \sum_{i=1}^n f_i(x_i)-f_i(x_\\star),
12+
13+
where the functions :math:`f_i` are :math:`M`-Lipschitz and convex, and where :math:`Q` is a
14+
bounded closed convex set with diameter upper bounded by :math:`D`, and where :math:`x_\\star\\in Q`
15+
is a reference point. Classical references on the topic include [1, 2]; such algorithms were studied
16+
using the performance estimation technique in [3].
17+
18+
This code computes a worst-case guarantee for **follow the regularized leader** (FTRL).
19+
That is, it computes the smallest possible :math:`\\tau(n, M, D)` such that the guarantee
20+
21+
.. math:: R_n \\leqslant \\tau(n, M, D)
22+
23+
is valid for any such sequence of queries of FTRL; that is, :math:`x_t` are the query points of OGD.
24+
25+
In short, for given values of :math:`n`, :math:`M`, :math:`D`:
26+
:math:`\\tau(n, M, D)` is computed as the worst-case value of :math:`R_n`.
27+
28+
**Algorithm**:
29+
Follow the regularized leader is described by
30+
31+
.. math:: x_{t+1} \\in \\text{argmin}_{x\\in Q} \\left( \sum_{i=1}^t f_i(x) + \\tfrac{\eta}{2}\\|x-x_1\\|^2 \\right).
32+
33+
**Theoretical guarantee**: The follow the regularized leader strategy is known to enjoy sublinear regret
34+
(see, e.g., [1, Theorem 5.2]); we compare with the bound:
35+
36+
.. math:: R_n \\leqslant MD\\sqrt{n}
37+
38+
with a regularization strength :math:`\\eta=D/M/\\sqrt{n}`.
39+
40+
41+
**References**:
42+
43+
`[1] E. Hazan (2016).
44+
Introduction to online convex optimization.
45+
Foundations and Trends in Optimization, 2(3-4), 157-325.
46+
<https://arxiv.org/pdf/1912.13213>`_
47+
48+
`[2] F. Orabona (2025).
49+
A Modern Introduction to Online Learning.
50+
<https://arxiv.org/pdf/1912.13213>`_
51+
52+
`[3] J. Weibel, P. Gaillard, W.M. Koolen, A. Taylor (2025).
53+
Optimized projection-free algorithms for online learning: construction and worst-case analysis
54+
<https://arxiv.org/pdf/2506.05855>`_
55+
56+
Args:
57+
M (float): the Lipschitz parameter.
58+
D (float): the diameter of the set.
59+
n (int): time horizon.
60+
wrapper (str): the name of the wrapper to be used.
61+
solver (str): the name of the solver the wrapper should use.
62+
verbose (int): level of information details to print.
63+
64+
- -1: No verbose at all.
65+
- 0: This example's output.
66+
- 1: This example's output + PEPit information.
67+
- 2: This example's output + PEPit information + solver details.
68+
69+
Returns:
70+
pepit_tau (float): worst-case value
71+
theoretical_tau (float): theoretical value
72+
73+
Example:
74+
>>> M, D, n = 1,.5,2
75+
>>> pepit_tau, theoretical_tau = wc_online_follow_regularized_leader(M=M, D=D, n=n, wrapper="cvxpy", solver=None, verbose=1)
76+
(PEPit) Setting up the problem: size of the Gram matrix: 10x10
77+
(PEPit) Setting up the problem: performance measure is the minimum of 1 element(s)
78+
(PEPit) Setting up the problem: Adding initial conditions and general constraints ...
79+
(PEPit) Setting up the problem: initial conditions and general constraints (0 constraint(s) added)
80+
(PEPit) Setting up the problem: interpolation conditions for 3 function(s)
81+
Function 1 : Adding 9 scalar constraint(s) ...
82+
Function 1 : 9 scalar constraint(s) added
83+
Function 2 : Adding 4 scalar constraint(s) ...
84+
Function 2 : 4 scalar constraint(s) added
85+
Function 3 : Adding 15 scalar constraint(s) ...
86+
Function 3 : 15 scalar constraint(s) added
87+
(PEPit) Setting up the problem: additional constraints for 0 function(s)
88+
(PEPit) Compiling SDP
89+
(PEPit) Calling SDP solver
90+
(PEPit) Solver status: prosta.prim_and_dual_feas (wrapper:mosek, solver: MOSEK); optimal value: 0.7071067900029648
91+
(PEPit) Primal feasibility check:
92+
The solver found a Gram matrix that is positive semi-definite
93+
All the primal scalar constraints are verified up to an error of 1.7911056637842648e-08
94+
(PEPit) Dual feasibility check:
95+
The solver found a residual matrix that is positive semi-definite
96+
All the dual scalar values associated with inequality constraints are nonnegative up to an error of 4.7132283953961866e-08
97+
(PEPit) The worst-case guarantee proof is perfectly reconstituted up to an error of 5.107611759149334e-08
98+
(PEPit) Final upper bound (dual): 0.7071067911277964 and lower bound (primal example): 0.7071067900029648
99+
(PEPit) Duality gap: absolute: 1.1248315612277793e-09 and relative: 1.5907520294396592e-09
100+
*** Example file: worst-case regret of online follow the regularized leader ***
101+
PEPit guarantee: R_n <= 0.707107
102+
Theoretical guarantee: R_n <= 0.707107
103+
104+
"""
105+
# Instantiate PEP
106+
problem = PEP()
107+
108+
M_list = [ M for i in range(n)]
109+
eta = D/M/np.sqrt(n)
110+
111+
# Declare a sequence of M-Lipschitz convex functions fi and an indicator function with Diameter D
112+
fis = [problem.declare_function(ConvexLipschitzFunction, M=M_list[i]) for i in range(n)]
113+
h = problem.declare_function(function_class=ConvexIndicatorFunction, D=D)
114+
115+
F = np.sum(fis)
116+
# Defining a reference point
117+
xRef = problem.set_initial_point()
118+
xRef,_,_ = proximal_step(xRef, h, 1) # project the reference point
119+
gRef, FRef = F.oracle(xRef)
120+
121+
# Then define the starting point x0 of the algorithm
122+
x1 = problem.set_initial_point()
123+
x1,_,_ = proximal_step(x1, h, 1) # project the initial point
124+
125+
# Run n steps of FTRL (regularization eta)
126+
f_occ = h
127+
x = x1
128+
x_saved = list()
129+
g_saved = list()
130+
f_saved = list()
131+
for i in range(n):
132+
x_saved.append(x-xRef)
133+
g, f = fis[i].oracle(x)
134+
f_saved.append(f)
135+
g_saved.append(g)
136+
f_occ = f_occ + fis[i]
137+
if i < n-1:
138+
x, _, _ = proximal_step(x1, f_occ, eta)
139+
140+
# Set the performance metric to the function values accuracy
141+
problem.set_performance_metric(np.sum(f_saved) - FRef)
142+
143+
# Solve the PEP
144+
pepit_verbose = max(verbose, 0)
145+
pepit_tau = problem.solve(wrapper=wrapper, solver=solver, verbose=pepit_verbose)
146+
147+
# Compute theoretical guarantee
148+
theoretical_tau = M * D * np.sqrt(n)
149+
150+
# Print conclusion if required
151+
if verbose != -1:
152+
print('*** Example file: worst-case regret of online follow the regularized leader ***')
153+
print('\tPEPit guarantee:\t R_n <= {:.6}'.format(pepit_tau))
154+
print('\tTheoretical guarantee:\t R_n <= {:.6}'.format(theoretical_tau))
155+
156+
# Return the worst-case guarantee of the evaluated method (and the reference theoretical value)
157+
return pepit_tau, theoretical_tau
158+
159+
if __name__ == "__main__":
160+
M, D, n = 1, .5, 2
161+
pepit_tau, theoretical_tau = wc_online_follow_regularized_leader(M, D, n, wrapper="cvxpy", solver=None, verbose=1)

0 commit comments

Comments
 (0)