nevergrad import fails when profiler is active

[stephanos-stephani]

To reproduce:

from pyinstrument import Profiler

profiler = Profiler()
profiler.start()

import nevergrad as ng

profiler.stop()
profiler.print()

This is under python 3.11, nevergrad 0.13.0, and pyinstrument 4.6.1

Traceback:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[1], line 6
      3 profiler = Profiler()
      4 profiler.start()
----> 6 import nevergrad as ng
      8 profiler.stop()
     10 profiler.print()

File ~/micromamba/envs/dev/lib/python3.11/site-packages/nevergrad/__init__.py:8
      6 from .common import typing as typing
      7 from .parametrization import parameter as p
----> 8 from .optimization import optimizerlib as optimizers  # busy namespace, likely to be simplified
      9 from .optimization import families as families
     10 from .optimization import callbacks as callbacks

File ~/micromamba/envs/dev/lib/python3.11/site-packages/nevergrad/optimization/__init__.py:7
      1 # Copyright (c) Meta Platforms, Inc. and affiliates.
      2 #
      3 # This source code is licensed under the MIT license found in the
      4 # LICENSE file in the root directory of this source tree.
      6 from .base import Optimizer  # abstract class, for type checking
----> 7 from . import optimizerlib
      8 from .optimizerlib import registry as registry

File ~/micromamba/envs/dev/lib/python3.11/site-packages/nevergrad/optimization/optimizerlib.py:26
     24 from nevergrad.parametrization import _layering
     25 from nevergrad.parametrization import _datalayers
---> 26 from . import oneshot
     27 from . import base
     28 from . import mutations

File ~/micromamba/envs/dev/lib/python3.11/site-packages/nevergrad/optimization/oneshot.py:461
    455 ScrHammersleySearch = SamplingSearch(sampler="Hammersley", scrambled=True).set_name(
    456     "ScrHammersleySearch", register=True
    457 )
    458 QOScrHammersleySearch = SamplingSearch(
    459     sampler="Hammersley", scrambled=True, opposition_mode="quasi"
    460 ).set_name("QOScrHammersleySearch", register=True)
--> 461 OScrHammersleySearch = SamplingSearch(
    462     sampler="Hammersley", scrambled=True, opposition_mode="opposite"
    463 ).set_name("OScrHammersleySearch", register=True)
    464 CauchyScrHammersleySearch = SamplingSearch(cauchy=True, sampler="Hammersley", scrambled=True).set_name(
    465     "CauchyScrHammersleySearch", register=True
    466 )
    467 LHSSearch = SamplingSearch(sampler="LHS").set_name("LHSSearch", register=True)

File ~/micromamba/envs/dev/lib/python3.11/site-packages/nevergrad/optimization/oneshot.py:407, in SamplingSearch.__init__(self, sampler, scrambled, middle_point, opposition_mode, cauchy, autorescale, scale, rescaled, recommendation_rule)
    394 def __init__(
    395     self,
    396     *,
   (...)
    405     recommendation_rule: str = "pessimistic",
    406 ) -> None:
--> 407     super().__init__(_SamplingSearch, locals())

File ~/micromamba/envs/dev/lib/python3.11/site-packages/nevergrad/optimization/base.py:776, in ConfiguredOptimizer.__init__(self, OptimizerClass, config, as_config)
    774 self._as_config = as_config
    775 self._config = config  # keep all, to avoid weird behavior at mismatch between optim and configoptim
--> 776 diff = ngtools.different_from_defaults(instance=self, instance_dict=config, check_mismatches=True)
    777 params = ", ".join(f"{x}={y!r}" for x, y in sorted(diff.items()))
    778 self.name = f"{self.__class__.__name__}({params})"

File ~/micromamba/envs/dev/lib/python3.11/site-packages/nevergrad/common/tools.py:185, in different_from_defaults(instance, instance_dict, check_mismatches)
    183     miss = set(instance_dict.keys()) - set(defaults.keys())
    184     if add or miss:  # this is to help during development
--> 185         raise RuntimeError(
    186             f"Mismatch between attributes and arguments of {instance.__class__}:\n"
    187             f"- additional: {add}\n- missing: {miss}"
    188         )
    189 else:
    190     defaults = {x: y for x, y in defaults.items() if x in instance.__dict__}

RuntimeError: Mismatch between attributes and arguments of <class 'nevergrad.optimization.oneshot.SamplingSearch'>:
- additional: set()
- missing: {'__class__', 'self'}

[joerick]

I've taken a good look at this and it appears to be a CPython bug, similar to this issue regarding tracing.

The problem is that nevergrad grabs a function's locals using locals(), and then modifies it in a subfunction. This hits an edge-case of locals(), where the return value isn't a snapshot but a reference to the underlying frame's data, where some bugs are lurking.

The good news is that since PEP 667, this is now fixed in Python 3.13 onwards. Alternatively, nevergrad could work around this issue by copying the contents of locals() before modifying it, like dict(locals()). @stephanos-stephani, would you like to raise an issue in the nevergrad repo to suggest that?