feat: introduce requirements as layers

src/isolate/backends/common.py

@@ -9,6 +9,7 @@
 import threading
 import time
 from contextlib import contextmanager, suppress
+from dataclasses import dataclass, field
 from functools import lru_cache
 from pathlib import Path
 from types import ModuleType
@@ -220,6 +221,41 @@ def _normalize(text: str | bytes) -> bytes:
     return hashlib.sha256(inner_text).hexdigest()
 
 
+@dataclass
+class Requirements:
+    layers: list[list[str]] = field(default_factory=list)
+
+    @classmethod
+    def from_raw(cls, raw: list[str] | list[list[str]]) -> Requirements:
+        if not raw:
+            return cls()
+
+        if isinstance(raw, list) and all(isinstance(item, str) for item in raw):
+            return cls([raw])  # type: ignore[list-item]
+
+        if isinstance(raw, list) and all(isinstance(item, list) for item in raw):
+            layers: list[list[str]] = []
+            for layer in raw:
+                if not all(isinstance(item, str) for item in layer):
+                    raise TypeError("Requirements layers must contain strings.")
+                layers.append(layer)  # type: ignore[arg-type]
+            return cls(layers)
+
+        raise TypeError(
+            "Requirements must be a list of strings or list of lists of strings."
+        )
+
+    def keys(self) -> list[str]:
+        if not self.layers:
+            return []
+        if len(self.layers) == 1:
+            return list(self.layers[0])
+        return [
+            f"layer{index}:\n" + "\n".join(layer)
+            for index, layer in enumerate(self.layers)
+        ]
+
+
 def active_python() -> str:
     """Return the active Python version that can be used for caching
     and re-creating this environment. Currently only covers major and

src/isolate/backends/container.py

@@ -6,7 +6,7 @@
 from typing import Any, ClassVar
 
 from isolate.backends import BaseEnvironment
-from isolate.backends.common import sha256_digest_of
+from isolate.backends.common import Requirements, sha256_digest_of
 from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings
 from isolate.connections import PythonIPC
 
@@ -17,8 +17,7 @@ class ContainerizedPythonEnvironment(BaseEnvironment[Path]):
 
     image: dict[str, Any] = field(default_factory=dict)
     python_version: str | None = None
-    requirements: list[str] = field(default_factory=list)
-    install_requirements: list[str] = field(default_factory=list)
+    requirements: Requirements = field(default_factory=Requirements)
     tags: list[str] = field(default_factory=list)
     resolver: str | None = None
 
@@ -28,7 +27,11 @@ def from_config(
         config: dict[str, Any],
         settings: IsolateSettings = DEFAULT_SETTINGS,
     ) -> BaseEnvironment:
-        environment = cls(**config)
+        prepared = dict(config)
+        prepared["requirements"] = Requirements.from_raw(
+            config.get("requirements") or []
+        )
+        environment = cls(**prepared)
         environment.apply_settings(settings)
         if environment.resolver not in ("uv", None):
             raise ValueError(
@@ -47,8 +50,7 @@ def key(self) -> str:
         dockerfile_str = self.image.get("dockerfile_str", "")
         return sha256_digest_of(
             dockerfile_str,
-            *self.install_requirements,
-            *self.requirements,
+            *self.requirements.keys(),
             *sorted(self.tags),
             *extras,
         )

src/isolate/backends/virtualenv.py

@@ -10,6 +10,7 @@
 
 from isolate.backends import BaseEnvironment, EnvironmentCreationError
 from isolate.backends.common import (
+    Requirements,
     active_python,
     get_executable,
     get_executable_path,
@@ -29,8 +30,7 @@
 class VirtualPythonEnvironment(BaseEnvironment[Path]):
     BACKEND_NAME: ClassVar[str] = "virtualenv"
 
-    requirements: list[str] = field(default_factory=list)
-    install_requirements: list[str] = field(default_factory=list)
+    requirements: Requirements = field(default_factory=Requirements)
     constraints_file: os.PathLike | None = None
     python_version: str | None = None
     extra_index_urls: list[str] = field(default_factory=list)
@@ -43,7 +43,11 @@ def from_config(
         config: dict[str, Any],
         settings: IsolateSettings = DEFAULT_SETTINGS,
     ) -> BaseEnvironment:
-        environment = cls(**config)
+        prepared = dict(config)
+        prepared["requirements"] = Requirements.from_raw(
+            config.get("requirements") or []
+        )
+        environment = cls(**prepared)
         environment.apply_settings(settings)
         if environment.resolver not in ("uv", None):
             raise ValueError(
@@ -66,8 +70,7 @@ def key(self) -> str:
         active_python_version = self.python_version or active_python()
         return sha256_digest_of(
             active_python_version,
-            *self.install_requirements,
-            *self.requirements,
+            *self.requirements.keys(),
             *constraints,
             *self.extra_index_urls,
             *sorted(self.tags),
@@ -185,8 +188,8 @@ def create(self, *, force: bool = False) -> Path:
                     f"Failed to create the environment at '{venv_path}': {exc}"
                 )
 
-            self._install_packages(venv_path, self.install_requirements)
-            self._install_packages(venv_path, self.requirements)
+            for layer in self.requirements.layers:
+                self._install_packages(venv_path, layer)
             completion_marker.touch()
 
         self.log(f"New environment cached at '{venv_path}'")

src/isolate/server/server.py

@@ -25,7 +25,7 @@
     EnvironmentCreationError,
     IsolateSettings,
 )
-from isolate.backends.common import active_python
+from isolate.backends.common import Requirements, active_python
 from isolate.backends.local import LocalPythonEnvironment
 from isolate.backends.virtualenv import VirtualPythonEnvironment
 from isolate.connections.grpc import AgentError, LocalPythonGRPC
@@ -266,7 +266,7 @@ def _run_task(self, task: RunTask) -> Iterator[definitions.PartialRunResult]:
                 primary_environment, "python_version", active_python()
             )
             agent_environ = VirtualPythonEnvironment(
-                requirements=AGENT_REQUIREMENTS,
+                requirements=Requirements.from_raw(AGENT_REQUIREMENTS),
                 python_version=python_version,
             )
             agent_environ.apply_settings(run_settings)

tests/test_backends.py

@@ -9,7 +9,7 @@
 import isolate
 import pytest
 from isolate.backends import BaseEnvironment, EnvironmentCreationError
-from isolate.backends.common import get_executable, sha256_digest_of
+from isolate.backends.common import Requirements, get_executable, sha256_digest_of
 from isolate.backends.conda import CondaEnvironment
 from isolate.backends.local import LocalPythonEnvironment
 from isolate.backends.pyenv import PyenvEnvironment, _get_pyenv_executable
@@ -392,18 +392,34 @@ def test_tags_in_key(self, tmp_path, monkeypatch):
             tagged_environment.key == tagged_environment_2.key
         ), "Tag order should not matter"
 
-    def test_install_requirements_in_key(self, tmp_path):
-        base = self.get_environment(tmp_path, {"requirements": ["pyjokes==0.6.0"]})
-        with_install = self.get_environment(
+    def test_layered_requirements_in_key(self, tmp_path):
+        base = self.get_environment(
+            tmp_path, {"requirements": ["pyjokes==0.6.0", "pip==23.0.1"]}
+        )
+        layered = self.get_environment(
             tmp_path,
             {
-                "requirements": ["pyjokes==0.6.0"],
-                "install_requirements": ["pip==23.0.1"],
+                "requirements": [["pyjokes==0.6.0"], ["pip==23.0.1"]],
             },
         )
-        assert base.key != with_install.key
+        assert base.key != layered.key
 
-    def test_install_requirements_order(self, tmp_path, monkeypatch):
+    def test_layered_requirements_order_affects_key(self, tmp_path):
+        first = self.get_environment(
+            tmp_path,
+            {
+                "requirements": [["pyjokes==0.6.0"], ["pip==23.0.1"]],
+            },
+        )
+        second = self.get_environment(
+            tmp_path,
+            {
+                "requirements": [["pip==23.0.1"], ["pyjokes==0.6.0"]],
+            },
+        )
+        assert first.key != second.key
+
+    def test_requirements_layers_install_order(self, tmp_path, monkeypatch):
         installed = []
 
         def fake_install_packages(self, path, requirements):
@@ -422,8 +438,7 @@ def cli_run(args):
         )
 
         environment = VirtualPythonEnvironment(
-            requirements=["pyjokes==0.6.0"],
-            install_requirements=["pip==23.0.1"],
+            requirements=Requirements.from_raw([["pip==23.0.1"], ["pyjokes==0.6.0"]]),
         )
         environment.apply_settings(IsolateSettings(Path(tmp_path)))
         environment.create()
@@ -444,6 +459,24 @@ def test_try_using_uv(self, tmp_path):
         assert pyjokes_version == "0.5.0"
 
 
+class TestRequirements:
+    def test_from_raw_single_layer(self):
+        requirements = Requirements.from_raw(["a", "b"])
+        assert requirements.layers == [["a", "b"]]
+
+    def test_from_raw_multi_layer(self):
+        requirements = Requirements.from_raw([["a"], ["b", "c"]])
+        assert requirements.layers == [["a"], ["b", "c"]]
+
+    def test_keys_single_layer_backward_compat(self):
+        requirements = Requirements.from_raw(["a", "b"])
+        assert requirements.keys() == ["a", "b"]
+
+    def test_keys_multi_layer_prefixes(self):
+        requirements = Requirements.from_raw([["a"], ["b", "c"]])
+        assert requirements.keys() == ["layer0:\na", "layer1:\nb\nc"]
+
+
 # Since mamba is an external dependency, we'll skip tests using it
 # if it is not installed.
 try:
@@ -918,7 +951,7 @@ def test_pyenv_environment(python_version, tmp_path):
 @pytest.mark.skipif(not IS_PYENV_AVAILABLE, reason="Pyenv is not available")
 def test_virtual_env_custom_python_version_with_pyenv(tmp_path, monkeypatch):
     pyjokes_env = VirtualPythonEnvironment(
-        requirements=["pyjokes==0.6.0"],
+        requirements=Requirements.from_raw(["pyjokes==0.6.0"]),
         python_version="3.9",
     )
 

tests/test_connections.py

@@ -7,6 +7,7 @@
 
 import pytest
 from isolate.backends import BaseEnvironment, EnvironmentConnection
+from isolate.backends.common import Requirements
 from isolate.backends.local import LocalPythonEnvironment
 from isolate.backends.settings import IsolateSettings
 from isolate.backends.virtualenv import VirtualPythonEnvironment
@@ -42,7 +43,7 @@ def make_venv(
         self, tmp_path: Any, requirements: List[str]
     ) -> VirtualPythonEnvironment:
         """Create a new virtual env with the specified requirements."""
-        env = VirtualPythonEnvironment(requirements)
+        env = VirtualPythonEnvironment(Requirements.from_raw(requirements))
         env.apply_settings(IsolateSettings(Path(tmp_path)))
         return env
 
@@ -206,6 +207,8 @@ def make_venv(
     ) -> VirtualPythonEnvironment:
         # Since gRPC agent requires isolate to be installed, we
         # have to add it to the requirements.
-        env = VirtualPythonEnvironment(requirements + [f"{REPO_DIR}[grpc]"])
+        env = VirtualPythonEnvironment(
+            Requirements.from_raw(requirements + [f"{REPO_DIR}[grpc]"])
+        )
         env.apply_settings(IsolateSettings(Path(tmp_path)))
         return env