Merge branch 'master' into fix/19427/double-iter

Author: Borda

_notebooks

@@ -2,7 +2,7 @@ setuptools <70.1.1
 wheel <0.46.0
 awscli >=1.30.0, <1.41.0
 twine ==6.0.1
-importlib-metadata <8.0.0
+importlib-metadata <9.0.0
 wget
-pkginfo ==1.12.0
+pkginfo ==1.12.1.2
 packaging <25.1

requirements/ci.txt

@@ -3,7 +3,7 @@ myst-parser >=0.18.1, <3.0.0
 nbsphinx >=0.8.5, <=0.9.2
 nbconvert <7.14  # temporary fix for https://github.com/jupyter/nbconvert/issues/2092
 pandoc >=1.0, <=2.3
-docutils >=0.16, <0.21
+docutils >=0.16, <0.22
 sphinxcontrib-fulltoc >=1.0, <=1.2.0
 sphinxcontrib-mockautodoc
 sphinx-autobuild

requirements/docs.txt

@@ -4,5 +4,5 @@
 torch >=2.1.0, <2.8.0
 fsspec[http] >=2022.5.0, <2025.4.0
 packaging >=20.0, <=25.0
-typing-extensions >=4.4.0, <4.11.0
+typing-extensions >=4.4.0, <4.14.0
 lightning-utilities >=0.10.0, <0.15.0

requirements/fabric/base.txt

@@ -2,8 +2,8 @@ coverage ==7.8.0
 numpy >=1.17.2, <1.27.0
 pytest ==8.3.5
 pytest-cov ==6.1.1
-pytest-timeout ==2.1.0
+pytest-timeout ==2.3.1
 pytest-rerunfailures ==12.0
-pytest-random-order ==1.1.0
+pytest-random-order ==1.1.1
 click ==8.1.7
 tensorboardX >=2.2, <2.7.0  # min version is set by torch.onnx missing attribute

requirements/fabric/test.txt

@@ -7,5 +7,5 @@ PyYAML >=5.4, <6.1.0
 fsspec[http] >=2022.5.0, <2025.4.0
 torchmetrics >=0.7.0, <1.8.0
 packaging >=20.0, <=25.0
-typing-extensions >=4.4.0, <4.11.0
+typing-extensions >=4.4.0, <4.14.0
 lightning-utilities >=0.10.0, <0.15.0

requirements/pytorch/base.txt

@@ -1,7 +1,7 @@
 -r ../docs.txt
 
 nbformat  # used for generate empty notebook
-ipython[notebook] <8.7.0
+ipython[notebook] <8.19.0
 setuptools<58.0  # workaround for `error in ipython setup command: use_2to3 is invalid.`
 
 #-r ../../_notebooks/.actions/requires.txt

requirements/pytorch/docs.txt

@@ -5,7 +5,7 @@
 matplotlib>3.1, <3.9.0
 omegaconf >=2.2.3, <2.4.0
 hydra-core >=1.2.0, <1.4.0
-jsonargparse[signatures] >=4.27.7, <=4.35.0
+jsonargparse[signatures] >=4.28.0, <=4.40.0
 rich >=12.3.0, <13.6.0
 tensorboardX >=2.2, <2.7.0  # min version is set by torch.onnx missing attribute
 bitsandbytes >=0.45.2,<0.45.3; platform_system != "Darwin"

requirements/pytorch/extra.txt

@@ -1,17 +1,17 @@
 coverage ==7.8.0
 pytest ==8.3.5
 pytest-cov ==6.1.1
-pytest-timeout ==2.1.0
+pytest-timeout ==2.3.1
 pytest-rerunfailures ==12.0
-pytest-random-order ==1.1.0
+pytest-random-order ==1.1.1
 
 # needed in tests
 cloudpickle >=1.3, <3.2.0
 scikit-learn >0.22.1, <1.7.0
 numpy >=1.17.2, <1.27.0
-onnx >=1.12.0, <1.17.0
+onnx >=1.12.0, <1.18.0
 onnxruntime >=1.12.0, <1.21.0
-psutil <5.9.6 # for `DeviceStatsMonitor`
+psutil <7.0.1 # for `DeviceStatsMonitor`
 pandas >1.0, <2.3.0  # needed in benchmarks
 fastapi  # for `ServableModuleValidator`  # not setting version as re-defined in App
 uvicorn  # for `ServableModuleValidator`  # not setting version as re-defined in App

requirements/pytorch/test.txt

@@ -26,7 +26,6 @@
 from lightning.pytorch.demos.boring_classes import BoringModel
 from lightning.pytorch.loggers import TensorBoardLogger, WandbLogger
 from lightning.pytorch.utilities.exceptions import MisconfigurationException
-from tests_pytorch.test_cli import _xfail_python_ge_3_11_9
 
 
 def test_wandb_project_name(wandb_mock):
@@ -645,7 +644,6 @@ def test_wandb_logger_download_artifact(wandb_mock, tmp_path):
     wandb_mock.Api().artifact.assert_called_once_with("test_artifact", type="model")
 
 
-@_xfail_python_ge_3_11_9
 @pytest.mark.parametrize(("log_model", "expected"), [("True", True), ("False", False), ("all", "all")])
 def test_wandb_logger_cli_integration(log_model, expected, wandb_mock, monkeypatch, tmp_path):
     """Test that the WandbLogger can be used with the LightningCLI."""

tests/tests_pytorch/loggers/test_wandb.py

@@ -29,7 +29,6 @@
 import yaml
 from lightning_utilities import compare_version
 from lightning_utilities.test.warning import no_warning_call
-from packaging.version import Version
 from tensorboard.backend.event_processing import event_accumulator
 from tensorboard.plugins.hparams.plugin_data_pb2 import HParamsPluginData
 from torch.optim import SGD
@@ -65,14 +64,6 @@ def lazy_instance(*args, **kwargs):
         return None
 
 
-_xfail_python_ge_3_11_9 = pytest.mark.xfail(
-    # https://github.com/omni-us/jsonargparse/issues/484
-    Version(f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}") >= Version("3.11.9"),
-    strict=False,
-    reason="jsonargparse + Python 3.11.9 compatibility issue",
-)
-
-
 @contextmanager
 def mock_subclasses(baseclass, *subclasses):
     """Mocks baseclass so that it only has the given child subclasses."""
@@ -356,7 +347,6 @@ def test_save_to_log_dir_false_error():
         )
 
 
-@_xfail_python_ge_3_11_9
 def test_lightning_cli_logger_save_config(cleandir):
     class LoggerSaveConfigCallback(SaveConfigCallback):
         def __init__(self, *args, **kwargs) -> None:
@@ -753,7 +743,6 @@ def add_arguments_to_parser(self, parser):
     assert cli.trainer.lr_scheduler_configs[0].scheduler.step_size == 50
 
 
-@_xfail_python_ge_3_11_9
 @RunIf(min_torch="2.2")
 @pytest.mark.parametrize("use_generic_base_class", [False, True])
 def test_lightning_cli_optimizers_and_lr_scheduler_with_link_to(use_generic_base_class):
@@ -801,7 +790,6 @@ def __init__(self, optim1: dict, optim2: dict, scheduler: dict):
     assert isinstance(cli.model.scheduler, torch.optim.lr_scheduler.ExponentialLR)
 
 
-@_xfail_python_ge_3_11_9
 @RunIf(min_torch="2.2")
 def test_lightning_cli_optimizers_and_lr_scheduler_with_callable_type():
     class TestModel(BoringModel):
@@ -1118,7 +1106,6 @@ def __init__(self, foo, bar=5):
         self.bar = bar
 
 
-@_xfail_python_ge_3_11_9
 def test_lightning_cli_model_short_arguments():
     with (
         mock.patch("sys.argv", ["any.py", "fit", "--model=BoringModel"]),
@@ -1146,7 +1133,6 @@ def __init__(self, foo, bar=5):
         self.bar = bar
 
 
-@_xfail_python_ge_3_11_9
 def test_lightning_cli_datamodule_short_arguments():
     # with set model
     with (
@@ -1200,7 +1186,6 @@ def test_lightning_cli_datamodule_short_arguments():
         assert cli.parser.groups["data"].group_class is BoringDataModule
 
 
-@_xfail_python_ge_3_11_9
 @pytest.mark.parametrize("use_class_path_callbacks", [False, True])
 def test_callbacks_append(use_class_path_callbacks):
     """This test validates registries are used when simplified command line are being used."""
@@ -1244,7 +1229,6 @@ def test_callbacks_append(use_class_path_callbacks):
     assert all(t in callback_types for t in expected)
 
 
-@_xfail_python_ge_3_11_9
 def test_optimizers_and_lr_schedulers_reload(cleandir):
     base = ["any.py", "--trainer.max_epochs=1"]
     input = base + [
@@ -1276,7 +1260,6 @@ def test_optimizers_and_lr_schedulers_reload(cleandir):
         LightningCLI(BoringModel, run=False)
 
 
-@_xfail_python_ge_3_11_9
 def test_optimizers_and_lr_schedulers_add_arguments_to_parser_implemented_reload(cleandir):
     class TestLightningCLI(LightningCLI):
         def __init__(self, *args):
@@ -1540,7 +1523,6 @@ def test_cli_help_message():
     assert "Implements Adam" in shorthand_help.getvalue()
 
 
-@_xfail_python_ge_3_11_9
 def test_cli_reducelronplateau():
     with mock.patch(
         "sys.argv", ["any.py", "--optimizer=Adam", "--lr_scheduler=ReduceLROnPlateau", "--lr_scheduler.monitor=foo"]
@@ -1551,7 +1533,6 @@ def test_cli_reducelronplateau():
     assert config["lr_scheduler"]["scheduler"].monitor == "foo"
 
 
-@_xfail_python_ge_3_11_9
 def test_cli_configureoptimizers_can_be_overridden():
     class MyCLI(LightningCLI):
         def __init__(self):
@@ -1596,7 +1577,6 @@ def __init__(self, activation: torch.nn.Module = lazy_instance(torch.nn.LeakyReL
         assert cli.model.activation is not model.activation
 
 
-@_xfail_python_ge_3_11_9
 def test_ddpstrategy_instantiation_and_find_unused_parameters(mps_count_0):
     strategy_default = lazy_instance(DDPStrategy, find_unused_parameters=True)
     with mock.patch("sys.argv", ["any.py", "--trainer.strategy.process_group_backend=group"]):
@@ -1612,7 +1592,6 @@ def test_ddpstrategy_instantiation_and_find_unused_parameters(mps_count_0):
     assert strategy_default is not cli.config_init.trainer.strategy
 
 
-@_xfail_python_ge_3_11_9
 def test_cli_logger_shorthand():
     with mock.patch("sys.argv", ["any.py"]):
         cli = LightningCLI(TestModel, run=False, trainer_defaults={"logger": False})
@@ -1643,7 +1622,6 @@ def _test_logger_init_args(logger_name, init, unresolved=None):
         assert data["dict_kwargs"] == unresolved
 
 
-@_xfail_python_ge_3_11_9
 def test_comet_logger_init_args():
     _test_logger_init_args(
         "CometLogger",
@@ -1664,7 +1642,6 @@ def test_comet_logger_init_args():
     strict=False,
     reason="TypeError on Windows when parsing",
 )
-@_xfail_python_ge_3_11_9
 def test_neptune_logger_init_args():
     _test_logger_init_args(
         "NeptuneLogger",
@@ -1673,7 +1650,6 @@ def test_neptune_logger_init_args():
     )
 
 
-@_xfail_python_ge_3_11_9
 def test_tensorboard_logger_init_args():
     _test_logger_init_args(
         "TensorBoardLogger",
@@ -1685,7 +1661,6 @@ def test_tensorboard_logger_init_args():
     )
 
 
-@_xfail_python_ge_3_11_9
 def test_wandb_logger_init_args():
     _test_logger_init_args(
         "WandbLogger",
@@ -1770,7 +1745,6 @@ def __init__(self, a_func: Callable = torch.nn.Softmax):
     assert "a_func: torch.nn.Softmax" in out.getvalue()
 
 
-@_xfail_python_ge_3_11_9
 def test_pytorch_profiler_init_args():
     from lightning.pytorch.profilers import Profiler, PyTorchProfiler