@@ -178,4 +178,158 @@ def _factory(idx: int) -> gym.Env:
178178 )
179179
180180
181- __all__ = ["CurriculumEnv" , "regime_curriculum" ]
181+ # -- adaptive difficulty-targeting curriculum (Prioritized Level Replay) ----------
182+
183+
184+ class AdaptiveScheduler :
185+ """Prioritized-Level-Replay difficulty targeting over a fixed candidate level set.
186+
187+ A fixed rotation replays trivially-solved levels and hopeless ones in equal measure.
188+ PLR instead spends the next episode on a level in the agent's *zone of proximal
189+ development*: one it solves *sometimes* (the 30-70%-solve band), where the learning
190+ signal is richest. This tracks a per-level solve rate from recorded outcomes and
191+ scores each level by the ZPD weight ``p * (1 - p)`` (Bernoulli variance): maximal at
192+ ``p = 0.5``, decaying to zero as a level becomes trivially easy (``p -> 1``) or
193+ hopeless (``p -> 0``). :meth:`select_next` is a **pure deterministic function of the
194+ recorded history** (argmax weight, ties broken by lowest index, no RNG). Unseen levels
195+ take a ``prior`` pseudo-rate (default ``0.5``, the peak) so each is explored once
196+ before the mid band is replayed.
197+ """
198+
199+ def __init__ (self , levels : Sequence [int ], * , prior : float = 0.5 ) -> None :
200+ deduped : list [int ] = []
201+ for x in levels :
202+ xi = int (x )
203+ if xi not in deduped :
204+ deduped .append (xi )
205+ if not deduped :
206+ raise ValueError ("levels must be non-empty" )
207+ self ._levels = deduped
208+ self ._solves : dict [int , int ] = {x : 0 for x in deduped }
209+ self ._attempts : dict [int , int ] = {x : 0 for x in deduped }
210+ self ._prior = float (prior )
211+
212+ @property
213+ def levels (self ) -> list [int ]:
214+ """The scheduled candidate levels (seeds), in tie-break order (copy)."""
215+ return list (self ._levels )
216+
217+ def success_rate (self , level : int ) -> float :
218+ """Observed ``solves / attempts`` for ``level``, or the ``prior`` when unseen."""
219+ level = int (level )
220+ if level not in self ._attempts :
221+ raise KeyError (f"level { level } )
222+ a = self ._attempts [level ]
223+ return self ._prior if a == 0 else self ._solves [level ] / a
224+
225+ def weight (self , level : int ) -> float :
226+ """ZPD replay weight ``p * (1 - p)`` (peaks at ``p = 0.5``, zero at both tails)."""
227+ p = self .success_rate (level )
228+ return p * (1.0 - p )
229+
230+ def record (self , level : int , solved : bool ) -> None :
231+ """Record one episode outcome for ``level`` (``solved`` = success criterion met)."""
232+ level = int (level )
233+ if level not in self ._attempts :
234+ raise KeyError (f"level { level } )
235+ self ._attempts [level ] += 1
236+ self ._solves [level ] += 1 if solved else 0
237+
238+ def select_next (self ) -> int :
239+ """The next level to replay: highest-weight candidate, ties broken by lowest index."""
240+ best = self ._levels [0 ]
241+ best_w = self .weight (best )
242+ for lv in self ._levels [1 :]:
243+ w = self .weight (lv )
244+ if w > best_w :
245+ best , best_w = lv , w
246+ return best
247+
248+
249+ class AdaptiveCurriculumEnv (gym .Wrapper ):
250+ """A curriculum whose next scenario seed is chosen adaptively by the agent's online
251+ success rate (Prioritized Level Replay) rather than a fixed rotation.
252+
253+ On every ``reset()`` the wrapper asks an :class:`AdaptiveScheduler` for the
254+ highest-learning-signal (mid-difficulty) level and points the env at that seed; as the
255+ episode runs it accumulates reward, and on episode end it records a solved/failed
256+ outcome (``solved_fn(total_reward)``, default: a net-positive episode return) back into
257+ the scheduler. The seed choice is deterministic given the observed outcome history, so
258+ the same run replays identically.
259+
260+ Parameters
261+ ----------
262+ levels:
263+ The candidate scenario seeds to target adaptively.
264+ solved_fn:
265+ ``(total_episode_return) -> bool`` success criterion. Defaults to "made money"
266+ (``total > 0``), a deterministic proxy for a trading "solve".
267+ prior:
268+ Unseen-level pseudo success rate (default ``0.5``, the ZPD peak).
269+ env_factory:
270+ Optional ``(seed) -> gym.Env`` builder rebuilt per episode (e.g. to fix a
271+ construction-time ``distribution_mode``). When ``None`` a single
272+ :class:`OpenOutcryEnv` is built from ``env_kwargs`` and re-pointed via
273+ ``reset(seed=...)``.
274+ **env_kwargs:
275+ Forwarded to :class:`OpenOutcryEnv` when ``env_factory`` is ``None``.
276+ """
277+
278+ def __init__ (
279+ self ,
280+ levels : Sequence [int ],
281+ * ,
282+ solved_fn : Optional [Callable [[float ], bool ]] = None ,
283+ prior : float = 0.5 ,
284+ env_factory : Optional [EnvFactory ] = None ,
285+ ** env_kwargs ,
286+ ) -> None :
287+ self ._scheduler = AdaptiveScheduler (levels , prior = prior )
288+ self ._solved_fn = solved_fn or (lambda total : total > 0.0 )
289+ self ._env_factory = env_factory
290+ self ._active_seed : Optional [int ] = None
291+ self ._episode_return = 0.0
292+
293+ first = self ._scheduler .levels [0 ]
294+ env = env_factory (first ) if env_factory is not None else OpenOutcryEnv (** env_kwargs )
295+ super ().__init__ (env )
296+
297+ @property
298+ def scheduler (self ) -> AdaptiveScheduler :
299+ return self ._scheduler
300+
301+ def reset (self , * , seed : Optional [int ] = None , options : Optional [dict ] = None ):
302+ """Reset onto the scheduler's next (mid-difficulty) seed; any external ``seed`` is
303+ ignored so the adaptive sequence stays deterministic in the outcome history."""
304+ active = self ._scheduler .select_next ()
305+ self ._active_seed = active
306+ self ._episode_return = 0.0
307+ if self ._env_factory is not None :
308+ self .env = self ._env_factory (active )
309+ obs , info = self .env .reset ()
310+ else :
311+ obs , info = self .env .reset (seed = active )
312+ info ["curriculum" ] = {
313+ "seed" : active ,
314+ "success_rate" : self ._scheduler .success_rate (active ),
315+ "weight" : self ._scheduler .weight (active ),
316+ }
317+ return obs , info
318+
319+ def step (self , action ):
320+ """Advance one bar; on episode end record the solved/failed outcome for the seed."""
321+ obs , reward , terminated , truncated , info = self .env .step (action )
322+ self ._episode_return += float (reward )
323+ if bool (terminated ) or bool (truncated ):
324+ solved = bool (self ._solved_fn (self ._episode_return ))
325+ self ._scheduler .record (self ._active_seed , solved )
326+ info ["curriculum_solved" ] = solved
327+ return obs , reward , terminated , truncated , info
328+
329+
330+ __all__ = [
331+ "CurriculumEnv" ,
332+ "regime_curriculum" ,
333+ "AdaptiveScheduler" ,
334+ "AdaptiveCurriculumEnv" ,
335+ ]
0 commit comments