diff --git a/src/isolate/backends/common.py b/src/isolate/backends/common.py index c5ae50c..857138f 100644 --- a/src/isolate/backends/common.py +++ b/src/isolate/backends/common.py @@ -9,6 +9,7 @@ import threading import time from contextlib import contextmanager, suppress +from dataclasses import dataclass, field from functools import lru_cache from pathlib import Path from types import ModuleType @@ -220,6 +221,41 @@ def _normalize(text: str | bytes) -> bytes: return hashlib.sha256(inner_text).hexdigest() +@dataclass +class Requirements: + layers: list[list[str]] = field(default_factory=list) + + @classmethod + def from_raw(cls, raw: list[str] | list[list[str]]) -> Requirements: + if not raw: + return cls() + + if isinstance(raw, list) and all(isinstance(item, str) for item in raw): + return cls([raw]) # type: ignore[list-item] + + if isinstance(raw, list) and all(isinstance(item, list) for item in raw): + layers: list[list[str]] = [] + for layer in raw: + if not all(isinstance(item, str) for item in layer): + raise TypeError("Requirements layers must contain strings.") + layers.append(layer) # type: ignore[arg-type] + return cls(layers) + + raise TypeError( + "Requirements must be a list of strings or list of lists of strings." + ) + + def keys(self) -> list[str]: + if not self.layers: + return [] + if len(self.layers) == 1: + return list(self.layers[0]) + return [ + f"layer{index}:\n" + "\n".join(layer) + for index, layer in enumerate(self.layers) + ] + + def active_python() -> str: """Return the active Python version that can be used for caching and re-creating this environment. Currently only covers major and diff --git a/src/isolate/backends/container.py b/src/isolate/backends/container.py index 4e211dd..008661d 100644 --- a/src/isolate/backends/container.py +++ b/src/isolate/backends/container.py @@ -6,7 +6,7 @@ from typing import Any, ClassVar from isolate.backends import BaseEnvironment -from isolate.backends.common import sha256_digest_of +from isolate.backends.common import Requirements, sha256_digest_of from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings from isolate.connections import PythonIPC @@ -17,8 +17,7 @@ class ContainerizedPythonEnvironment(BaseEnvironment[Path]): image: dict[str, Any] = field(default_factory=dict) python_version: str | None = None - requirements: list[str] = field(default_factory=list) - install_requirements: list[str] = field(default_factory=list) + requirements: Requirements = field(default_factory=Requirements) tags: list[str] = field(default_factory=list) resolver: str | None = None @@ -28,7 +27,11 @@ def from_config( config: dict[str, Any], settings: IsolateSettings = DEFAULT_SETTINGS, ) -> BaseEnvironment: - environment = cls(**config) + prepared = dict(config) + prepared["requirements"] = Requirements.from_raw( + config.get("requirements") or [] + ) + environment = cls(**prepared) environment.apply_settings(settings) if environment.resolver not in ("uv", None): raise ValueError( @@ -47,8 +50,7 @@ def key(self) -> str: dockerfile_str = self.image.get("dockerfile_str", "") return sha256_digest_of( dockerfile_str, - *self.install_requirements, - *self.requirements, + *self.requirements.keys(), *sorted(self.tags), *extras, ) diff --git a/src/isolate/backends/virtualenv.py b/src/isolate/backends/virtualenv.py index 1751d26..184131e 100644 --- a/src/isolate/backends/virtualenv.py +++ b/src/isolate/backends/virtualenv.py @@ -10,6 +10,7 @@ from isolate.backends import BaseEnvironment, EnvironmentCreationError from isolate.backends.common import ( + Requirements, active_python, get_executable, get_executable_path, @@ -29,8 +30,7 @@ class VirtualPythonEnvironment(BaseEnvironment[Path]): BACKEND_NAME: ClassVar[str] = "virtualenv" - requirements: list[str] = field(default_factory=list) - install_requirements: list[str] = field(default_factory=list) + requirements: Requirements = field(default_factory=Requirements) constraints_file: os.PathLike | None = None python_version: str | None = None extra_index_urls: list[str] = field(default_factory=list) @@ -43,7 +43,11 @@ def from_config( config: dict[str, Any], settings: IsolateSettings = DEFAULT_SETTINGS, ) -> BaseEnvironment: - environment = cls(**config) + prepared = dict(config) + prepared["requirements"] = Requirements.from_raw( + config.get("requirements") or [] + ) + environment = cls(**prepared) environment.apply_settings(settings) if environment.resolver not in ("uv", None): raise ValueError( @@ -66,8 +70,7 @@ def key(self) -> str: active_python_version = self.python_version or active_python() return sha256_digest_of( active_python_version, - *self.install_requirements, - *self.requirements, + *self.requirements.keys(), *constraints, *self.extra_index_urls, *sorted(self.tags), @@ -185,8 +188,8 @@ def create(self, *, force: bool = False) -> Path: f"Failed to create the environment at '{venv_path}': {exc}" ) - self._install_packages(venv_path, self.install_requirements) - self._install_packages(venv_path, self.requirements) + for layer in self.requirements.layers: + self._install_packages(venv_path, layer) completion_marker.touch() self.log(f"New environment cached at '{venv_path}'") diff --git a/src/isolate/server/server.py b/src/isolate/server/server.py index 77af155..f659e91 100644 --- a/src/isolate/server/server.py +++ b/src/isolate/server/server.py @@ -25,7 +25,7 @@ EnvironmentCreationError, IsolateSettings, ) -from isolate.backends.common import active_python +from isolate.backends.common import Requirements, active_python from isolate.backends.local import LocalPythonEnvironment from isolate.backends.virtualenv import VirtualPythonEnvironment from isolate.connections.grpc import AgentError, LocalPythonGRPC @@ -266,7 +266,7 @@ def _run_task(self, task: RunTask) -> Iterator[definitions.PartialRunResult]: primary_environment, "python_version", active_python() ) agent_environ = VirtualPythonEnvironment( - requirements=AGENT_REQUIREMENTS, + requirements=Requirements.from_raw(AGENT_REQUIREMENTS), python_version=python_version, ) agent_environ.apply_settings(run_settings) diff --git a/tests/test_backends.py b/tests/test_backends.py index 8236d84..2fd5e56 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -9,7 +9,7 @@ import isolate import pytest from isolate.backends import BaseEnvironment, EnvironmentCreationError -from isolate.backends.common import get_executable, sha256_digest_of +from isolate.backends.common import Requirements, get_executable, sha256_digest_of from isolate.backends.conda import CondaEnvironment from isolate.backends.local import LocalPythonEnvironment from isolate.backends.pyenv import PyenvEnvironment, _get_pyenv_executable @@ -392,18 +392,34 @@ def test_tags_in_key(self, tmp_path, monkeypatch): tagged_environment.key == tagged_environment_2.key ), "Tag order should not matter" - def test_install_requirements_in_key(self, tmp_path): - base = self.get_environment(tmp_path, {"requirements": ["pyjokes==0.6.0"]}) - with_install = self.get_environment( + def test_layered_requirements_in_key(self, tmp_path): + base = self.get_environment( + tmp_path, {"requirements": ["pyjokes==0.6.0", "pip==23.0.1"]} + ) + layered = self.get_environment( tmp_path, { - "requirements": ["pyjokes==0.6.0"], - "install_requirements": ["pip==23.0.1"], + "requirements": [["pyjokes==0.6.0"], ["pip==23.0.1"]], }, ) - assert base.key != with_install.key + assert base.key != layered.key - def test_install_requirements_order(self, tmp_path, monkeypatch): + def test_layered_requirements_order_affects_key(self, tmp_path): + first = self.get_environment( + tmp_path, + { + "requirements": [["pyjokes==0.6.0"], ["pip==23.0.1"]], + }, + ) + second = self.get_environment( + tmp_path, + { + "requirements": [["pip==23.0.1"], ["pyjokes==0.6.0"]], + }, + ) + assert first.key != second.key + + def test_requirements_layers_install_order(self, tmp_path, monkeypatch): installed = [] def fake_install_packages(self, path, requirements): @@ -422,8 +438,7 @@ def cli_run(args): ) environment = VirtualPythonEnvironment( - requirements=["pyjokes==0.6.0"], - install_requirements=["pip==23.0.1"], + requirements=Requirements.from_raw([["pip==23.0.1"], ["pyjokes==0.6.0"]]), ) environment.apply_settings(IsolateSettings(Path(tmp_path))) environment.create() @@ -444,6 +459,24 @@ def test_try_using_uv(self, tmp_path): assert pyjokes_version == "0.5.0" +class TestRequirements: + def test_from_raw_single_layer(self): + requirements = Requirements.from_raw(["a", "b"]) + assert requirements.layers == [["a", "b"]] + + def test_from_raw_multi_layer(self): + requirements = Requirements.from_raw([["a"], ["b", "c"]]) + assert requirements.layers == [["a"], ["b", "c"]] + + def test_keys_single_layer_backward_compat(self): + requirements = Requirements.from_raw(["a", "b"]) + assert requirements.keys() == ["a", "b"] + + def test_keys_multi_layer_prefixes(self): + requirements = Requirements.from_raw([["a"], ["b", "c"]]) + assert requirements.keys() == ["layer0:\na", "layer1:\nb\nc"] + + # Since mamba is an external dependency, we'll skip tests using it # if it is not installed. try: @@ -918,7 +951,7 @@ def test_pyenv_environment(python_version, tmp_path): @pytest.mark.skipif(not IS_PYENV_AVAILABLE, reason="Pyenv is not available") def test_virtual_env_custom_python_version_with_pyenv(tmp_path, monkeypatch): pyjokes_env = VirtualPythonEnvironment( - requirements=["pyjokes==0.6.0"], + requirements=Requirements.from_raw(["pyjokes==0.6.0"]), python_version="3.9", ) diff --git a/tests/test_connections.py b/tests/test_connections.py index e81aa65..f1ebb4a 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -7,6 +7,7 @@ import pytest from isolate.backends import BaseEnvironment, EnvironmentConnection +from isolate.backends.common import Requirements from isolate.backends.local import LocalPythonEnvironment from isolate.backends.settings import IsolateSettings from isolate.backends.virtualenv import VirtualPythonEnvironment @@ -42,7 +43,7 @@ def make_venv( self, tmp_path: Any, requirements: List[str] ) -> VirtualPythonEnvironment: """Create a new virtual env with the specified requirements.""" - env = VirtualPythonEnvironment(requirements) + env = VirtualPythonEnvironment(Requirements.from_raw(requirements)) env.apply_settings(IsolateSettings(Path(tmp_path))) return env @@ -206,6 +207,8 @@ def make_venv( ) -> VirtualPythonEnvironment: # Since gRPC agent requires isolate to be installed, we # have to add it to the requirements. - env = VirtualPythonEnvironment(requirements + [f"{REPO_DIR}[grpc]"]) + env = VirtualPythonEnvironment( + Requirements.from_raw(requirements + [f"{REPO_DIR}[grpc]"]) + ) env.apply_settings(IsolateSettings(Path(tmp_path))) return env