[BugFix] Fix offline-to-online CI failures

Author: vmoens

.github/unittest/linux_libs/scripts_ataridqn/install.sh

@@ -29,15 +29,19 @@ git submodule sync && git submodule update --init --recursive
 printf "Installing PyTorch with cu128"
 if [[ "$TORCH_VERSION" == "nightly" ]]; then
   if [ "${CU_VERSION:-}" == cpu ] ; then
-      pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
+      pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U
+      pip3 install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U --no-deps
   else
-      pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U
+      pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U
+      pip3 install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U --no-deps
   fi
 elif [[ "$TORCH_VERSION" == "stable" ]]; then
     if [ "${CU_VERSION:-}" == cpu ] ; then
-      pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+      pip3 install torch --index-url https://download.pytorch.org/whl/cpu
+      pip3 install torchvision --index-url https://download.pytorch.org/whl/cpu --no-deps
   else
-      pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128
+      pip3 install torch --index-url https://download.pytorch.org/whl/cu128
+      pip3 install torchvision --index-url https://download.pytorch.org/whl/cu128 --no-deps
   fi
 else
   printf "Failed to install pytorch"

.github/unittest/linux_libs/scripts_gen-dgrl/install.sh

@@ -29,15 +29,19 @@ git submodule sync && git submodule update --init --recursive
 printf "Installing PyTorch with cu128"
 if [[ "$TORCH_VERSION" == "nightly" ]]; then
   if [ "${CU_VERSION:-}" == cpu ] ; then
-      pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
+      pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U
+      pip3 install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U --no-deps
   else
-      pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U
+      pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U
+      pip3 install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U --no-deps
   fi
 elif [[ "$TORCH_VERSION" == "stable" ]]; then
     if [ "${CU_VERSION:-}" == cpu ] ; then
-      pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+      pip3 install torch --index-url https://download.pytorch.org/whl/cpu
+      pip3 install torchvision --index-url https://download.pytorch.org/whl/cpu --no-deps
   else
-      pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128
+      pip3 install torch --index-url https://download.pytorch.org/whl/cu128
+      pip3 install torchvision --index-url https://download.pytorch.org/whl/cu128 --no-deps
   fi
 else
   printf "Failed to install pytorch"

.github/unittest/linux_libs/scripts_openx/install.sh

@@ -29,15 +29,19 @@ git submodule sync && git submodule update --init --recursive
 printf "Installing PyTorch with cu128"
 if [[ "$TORCH_VERSION" == "nightly" ]]; then
   if [ "${CU_VERSION:-}" == cpu ] ; then
-      pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
+      pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U
+      pip3 install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U --no-deps
   else
-      pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U
+      pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U
+      pip3 install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U --no-deps
   fi
 elif [[ "$TORCH_VERSION" == "stable" ]]; then
     if [ "${CU_VERSION:-}" == cpu ] ; then
-      pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U
+      pip3 install torch --index-url https://download.pytorch.org/whl/cpu -U
+      pip3 install torchvision --index-url https://download.pytorch.org/whl/cpu -U --no-deps
   else
-      pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128 -U
+      pip3 install torch --index-url https://download.pytorch.org/whl/cu128 -U
+      pip3 install torchvision --index-url https://download.pytorch.org/whl/cu128 -U --no-deps
   fi
 else
   printf "Failed to install pytorch"

.github/unittest/linux_libs/scripts_vd4rl/install.sh

@@ -29,15 +29,19 @@ git submodule sync && git submodule update --init --recursive
 printf "Installing PyTorch with cu128"
 if [[ "$TORCH_VERSION" == "nightly" ]]; then
   if [ "${CU_VERSION:-}" == cpu ] ; then
-      pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U
+      pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu -U
+      pip3 install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu -U --no-deps
   else
-      pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U
+      pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128 -U
+      pip3 install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 -U --no-deps
   fi
 elif [[ "$TORCH_VERSION" == "stable" ]]; then
     if [ "${CU_VERSION:-}" == cpu ] ; then
-      pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu -U
+      pip3 install torch --index-url https://download.pytorch.org/whl/cpu -U
+      pip3 install torchvision --index-url https://download.pytorch.org/whl/cpu -U --no-deps
   else
-      pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cu128 -U
+      pip3 install torch --index-url https://download.pytorch.org/whl/cu128 -U
+      pip3 install torchvision --index-url https://download.pytorch.org/whl/cu128 -U --no-deps
   fi
 else
   printf "Failed to install pytorch"

test/test_offline_to_online.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import argparse
+import importlib.util
 import inspect
 
 import pytest
@@ -16,13 +17,17 @@
 from torchrl.data.datasets.utils import load_dataset, register_dataset
 from torchrl.data.replay_buffers.offline_to_online import prefill_replay_buffer
 from torchrl.envs.libs.gym import _has_gym
+from torchrl.testing.gym_helpers import PENDULUM_VERSIONED
 
 # Running a SAC loss requires a tensordict new enough to support
 # ``to_module(preserve_module_state=...)``; the offline-to-online wiring itself
 # does not.
 _LOSS_RUNNABLE = (
-    "preserve_module_state"
-    in __import__("inspect").signature(TensorDict.to_module).parameters
+    "preserve_module_state" in inspect.signature(TensorDict.to_module).parameters
+)
+_CONFIGS_AVAILABLE = (
+    importlib.util.find_spec("hydra") is not None
+    and importlib.util.find_spec("omegaconf") is not None
 )
 
 
@@ -583,6 +588,10 @@ def test_constructor_exposes_sac_key_and_logging_kwargs(self):
         ):
             assert name in parameters
 
+    @pytest.mark.skipif(
+        not _CONFIGS_AVAILABLE,
+        reason="Config system requires hydra-core and omegaconf",
+    )
     def test_config_registered(self):
         from torchrl.trainers.algorithms.configs import OfflineToOnlineTrainerConfig
 
@@ -637,7 +646,7 @@ def test_train_grows_online_and_anneals(self, tmp_path):
         from torchrl.trainers.algorithms.offline_to_online import OfflineToOnlineTrainer
 
         torch.manual_seed(0)
-        env = GymEnv("Pendulum-v1")
+        env = GymEnv(PENDULUM_VERSIONED())
         obs_dim = env.observation_spec["observation"].shape[-1]
         action_dim = env.action_spec.shape[-1]
 

torchrl/trainers/algorithms/configs/trainers.py

@@ -221,8 +221,7 @@ def _make_sac_trainer(*args, **kwargs) -> SACTrainer:
 
 @dataclass
 class OfflineToOnlineTrainerConfig(SACTrainerConfig):
-    """Hydra configuration for
-    :class:`~torchrl.trainers.algorithms.OfflineToOnlineTrainer`.
+    """Hydra configuration for :class:`~torchrl.trainers.algorithms.OfflineToOnlineTrainer`.
 
     Every kwarg accepted by ``OfflineToOnlineTrainer.__init__`` is exposed as a
     field here, with SAC network-construction helper fields inherited from
@@ -267,9 +266,7 @@ def _make_offline_to_online_trainer(*args, **kwargs) -> OfflineToOnlineTrainer:
     target_net_updater = kwargs.pop("target_net_updater")
     async_collection = kwargs.pop("async_collection", False)
     if async_collection:
-        raise ValueError(
-            "OfflineToOnlineTrainer does not support async_collection."
-        )
+        raise ValueError("OfflineToOnlineTrainer does not support async_collection.")
     log_timings = kwargs.pop("log_timings", False)
     auto_log_optim_steps = kwargs.pop("auto_log_optim_steps", True)
     batch_size = kwargs.pop("batch_size", None)

torchrl/trainers/algorithms/offline_to_online.py

@@ -148,8 +148,7 @@ def register(self, trainer, name: str = "offline_to_online_anneal") -> None:
 class OfflineToOnlineTrainer(SACTrainer):
     """A SAC trainer for the offline-pretrain -> online-finetune transition.
 
-    See also
-    :class:`~torchrl.trainers.algorithms.configs.OfflineToOnlineTrainerConfig`
+    See also :class:`~torchrl.trainers.algorithms.configs.OfflineToOnlineTrainerConfig`
     for the Hydra configuration counterpart.
 
     Builds on :class:`~torchrl.trainers.algorithms.SACTrainer`, swapping the