From 2be142238f79847c54e56138019a8468ce16c512 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 28 Jan 2026 01:25:59 +0200 Subject: [PATCH 1/4] feat: introduce requirements as layers --- src/isolate/backends/common.py | 38 ++++++++++++++++++++- src/isolate/backends/container.py | 14 ++++---- src/isolate/backends/virtualenv.py | 17 +++++---- src/isolate/server/server.py | 4 +-- tests/test_backends.py | 55 ++++++++++++++++++++++++------ tests/test_connections.py | 7 ++-- 6 files changed, 106 insertions(+), 29 deletions(-) diff --git a/src/isolate/backends/common.py b/src/isolate/backends/common.py index c5ae50cb..612ddd83 100644 --- a/src/isolate/backends/common.py +++ b/src/isolate/backends/common.py @@ -9,10 +9,11 @@ import threading import time from contextlib import contextmanager, suppress +from dataclasses import dataclass, field from functools import lru_cache from pathlib import Path from types import ModuleType -from typing import Callable, Iterator +from typing import Callable, Iterator, List, cast # For ensuring that the lock is created and not forgotten # (e.g. the process which acquires it crashes, so it is never @@ -220,6 +221,41 @@ def _normalize(text: str | bytes) -> bytes: return hashlib.sha256(inner_text).hexdigest() +@dataclass +class Requirements: + layers: list[list[str]] = field(default_factory=list) + + @classmethod + def from_raw(cls, raw: list[str] | list[list[str]]) -> Requirements: + if not raw: + return cls() + + if isinstance(raw, list) and all(isinstance(item, str) for item in raw): + return cls([list(cast(List[str], raw))]) + + if isinstance(raw, list) and all(isinstance(item, list) for item in raw): + layers: list[list[str]] = [] + for layer in raw: + if not all(isinstance(item, str) for item in layer): + raise TypeError("Requirements layers must contain strings.") + layers.append(list(layer)) + return cls(layers) + + raise TypeError( + "Requirements must be a list of strings or list of lists of strings." + ) + + def keys(self) -> list[str]: + if not self.layers: + return [] + if len(self.layers) == 1: + return list(self.layers[0]) + return [ + f"layer{index}:\n" + "\n".join(layer) + for index, layer in enumerate(self.layers) + ] + + def active_python() -> str: """Return the active Python version that can be used for caching and re-creating this environment. Currently only covers major and diff --git a/src/isolate/backends/container.py b/src/isolate/backends/container.py index 4e211dd8..008661db 100644 --- a/src/isolate/backends/container.py +++ b/src/isolate/backends/container.py @@ -6,7 +6,7 @@ from typing import Any, ClassVar from isolate.backends import BaseEnvironment -from isolate.backends.common import sha256_digest_of +from isolate.backends.common import Requirements, sha256_digest_of from isolate.backends.settings import DEFAULT_SETTINGS, IsolateSettings from isolate.connections import PythonIPC @@ -17,8 +17,7 @@ class ContainerizedPythonEnvironment(BaseEnvironment[Path]): image: dict[str, Any] = field(default_factory=dict) python_version: str | None = None - requirements: list[str] = field(default_factory=list) - install_requirements: list[str] = field(default_factory=list) + requirements: Requirements = field(default_factory=Requirements) tags: list[str] = field(default_factory=list) resolver: str | None = None @@ -28,7 +27,11 @@ def from_config( config: dict[str, Any], settings: IsolateSettings = DEFAULT_SETTINGS, ) -> BaseEnvironment: - environment = cls(**config) + prepared = dict(config) + prepared["requirements"] = Requirements.from_raw( + config.get("requirements") or [] + ) + environment = cls(**prepared) environment.apply_settings(settings) if environment.resolver not in ("uv", None): raise ValueError( @@ -47,8 +50,7 @@ def key(self) -> str: dockerfile_str = self.image.get("dockerfile_str", "") return sha256_digest_of( dockerfile_str, - *self.install_requirements, - *self.requirements, + *self.requirements.keys(), *sorted(self.tags), *extras, ) diff --git a/src/isolate/backends/virtualenv.py b/src/isolate/backends/virtualenv.py index 1751d267..184131e1 100644 --- a/src/isolate/backends/virtualenv.py +++ b/src/isolate/backends/virtualenv.py @@ -10,6 +10,7 @@ from isolate.backends import BaseEnvironment, EnvironmentCreationError from isolate.backends.common import ( + Requirements, active_python, get_executable, get_executable_path, @@ -29,8 +30,7 @@ class VirtualPythonEnvironment(BaseEnvironment[Path]): BACKEND_NAME: ClassVar[str] = "virtualenv" - requirements: list[str] = field(default_factory=list) - install_requirements: list[str] = field(default_factory=list) + requirements: Requirements = field(default_factory=Requirements) constraints_file: os.PathLike | None = None python_version: str | None = None extra_index_urls: list[str] = field(default_factory=list) @@ -43,7 +43,11 @@ def from_config( config: dict[str, Any], settings: IsolateSettings = DEFAULT_SETTINGS, ) -> BaseEnvironment: - environment = cls(**config) + prepared = dict(config) + prepared["requirements"] = Requirements.from_raw( + config.get("requirements") or [] + ) + environment = cls(**prepared) environment.apply_settings(settings) if environment.resolver not in ("uv", None): raise ValueError( @@ -66,8 +70,7 @@ def key(self) -> str: active_python_version = self.python_version or active_python() return sha256_digest_of( active_python_version, - *self.install_requirements, - *self.requirements, + *self.requirements.keys(), *constraints, *self.extra_index_urls, *sorted(self.tags), @@ -185,8 +188,8 @@ def create(self, *, force: bool = False) -> Path: f"Failed to create the environment at '{venv_path}': {exc}" ) - self._install_packages(venv_path, self.install_requirements) - self._install_packages(venv_path, self.requirements) + for layer in self.requirements.layers: + self._install_packages(venv_path, layer) completion_marker.touch() self.log(f"New environment cached at '{venv_path}'") diff --git a/src/isolate/server/server.py b/src/isolate/server/server.py index 77af155e..f659e913 100644 --- a/src/isolate/server/server.py +++ b/src/isolate/server/server.py @@ -25,7 +25,7 @@ EnvironmentCreationError, IsolateSettings, ) -from isolate.backends.common import active_python +from isolate.backends.common import Requirements, active_python from isolate.backends.local import LocalPythonEnvironment from isolate.backends.virtualenv import VirtualPythonEnvironment from isolate.connections.grpc import AgentError, LocalPythonGRPC @@ -266,7 +266,7 @@ def _run_task(self, task: RunTask) -> Iterator[definitions.PartialRunResult]: primary_environment, "python_version", active_python() ) agent_environ = VirtualPythonEnvironment( - requirements=AGENT_REQUIREMENTS, + requirements=Requirements.from_raw(AGENT_REQUIREMENTS), python_version=python_version, ) agent_environ.apply_settings(run_settings) diff --git a/tests/test_backends.py b/tests/test_backends.py index 8236d843..2fd5e562 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -9,7 +9,7 @@ import isolate import pytest from isolate.backends import BaseEnvironment, EnvironmentCreationError -from isolate.backends.common import get_executable, sha256_digest_of +from isolate.backends.common import Requirements, get_executable, sha256_digest_of from isolate.backends.conda import CondaEnvironment from isolate.backends.local import LocalPythonEnvironment from isolate.backends.pyenv import PyenvEnvironment, _get_pyenv_executable @@ -392,18 +392,34 @@ def test_tags_in_key(self, tmp_path, monkeypatch): tagged_environment.key == tagged_environment_2.key ), "Tag order should not matter" - def test_install_requirements_in_key(self, tmp_path): - base = self.get_environment(tmp_path, {"requirements": ["pyjokes==0.6.0"]}) - with_install = self.get_environment( + def test_layered_requirements_in_key(self, tmp_path): + base = self.get_environment( + tmp_path, {"requirements": ["pyjokes==0.6.0", "pip==23.0.1"]} + ) + layered = self.get_environment( tmp_path, { - "requirements": ["pyjokes==0.6.0"], - "install_requirements": ["pip==23.0.1"], + "requirements": [["pyjokes==0.6.0"], ["pip==23.0.1"]], }, ) - assert base.key != with_install.key + assert base.key != layered.key - def test_install_requirements_order(self, tmp_path, monkeypatch): + def test_layered_requirements_order_affects_key(self, tmp_path): + first = self.get_environment( + tmp_path, + { + "requirements": [["pyjokes==0.6.0"], ["pip==23.0.1"]], + }, + ) + second = self.get_environment( + tmp_path, + { + "requirements": [["pip==23.0.1"], ["pyjokes==0.6.0"]], + }, + ) + assert first.key != second.key + + def test_requirements_layers_install_order(self, tmp_path, monkeypatch): installed = [] def fake_install_packages(self, path, requirements): @@ -422,8 +438,7 @@ def cli_run(args): ) environment = VirtualPythonEnvironment( - requirements=["pyjokes==0.6.0"], - install_requirements=["pip==23.0.1"], + requirements=Requirements.from_raw([["pip==23.0.1"], ["pyjokes==0.6.0"]]), ) environment.apply_settings(IsolateSettings(Path(tmp_path))) environment.create() @@ -444,6 +459,24 @@ def test_try_using_uv(self, tmp_path): assert pyjokes_version == "0.5.0" +class TestRequirements: + def test_from_raw_single_layer(self): + requirements = Requirements.from_raw(["a", "b"]) + assert requirements.layers == [["a", "b"]] + + def test_from_raw_multi_layer(self): + requirements = Requirements.from_raw([["a"], ["b", "c"]]) + assert requirements.layers == [["a"], ["b", "c"]] + + def test_keys_single_layer_backward_compat(self): + requirements = Requirements.from_raw(["a", "b"]) + assert requirements.keys() == ["a", "b"] + + def test_keys_multi_layer_prefixes(self): + requirements = Requirements.from_raw([["a"], ["b", "c"]]) + assert requirements.keys() == ["layer0:\na", "layer1:\nb\nc"] + + # Since mamba is an external dependency, we'll skip tests using it # if it is not installed. try: @@ -918,7 +951,7 @@ def test_pyenv_environment(python_version, tmp_path): @pytest.mark.skipif(not IS_PYENV_AVAILABLE, reason="Pyenv is not available") def test_virtual_env_custom_python_version_with_pyenv(tmp_path, monkeypatch): pyjokes_env = VirtualPythonEnvironment( - requirements=["pyjokes==0.6.0"], + requirements=Requirements.from_raw(["pyjokes==0.6.0"]), python_version="3.9", ) diff --git a/tests/test_connections.py b/tests/test_connections.py index e81aa654..f1ebb4a3 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -7,6 +7,7 @@ import pytest from isolate.backends import BaseEnvironment, EnvironmentConnection +from isolate.backends.common import Requirements from isolate.backends.local import LocalPythonEnvironment from isolate.backends.settings import IsolateSettings from isolate.backends.virtualenv import VirtualPythonEnvironment @@ -42,7 +43,7 @@ def make_venv( self, tmp_path: Any, requirements: List[str] ) -> VirtualPythonEnvironment: """Create a new virtual env with the specified requirements.""" - env = VirtualPythonEnvironment(requirements) + env = VirtualPythonEnvironment(Requirements.from_raw(requirements)) env.apply_settings(IsolateSettings(Path(tmp_path))) return env @@ -206,6 +207,8 @@ def make_venv( ) -> VirtualPythonEnvironment: # Since gRPC agent requires isolate to be installed, we # have to add it to the requirements. - env = VirtualPythonEnvironment(requirements + [f"{REPO_DIR}[grpc]"]) + env = VirtualPythonEnvironment( + Requirements.from_raw(requirements + [f"{REPO_DIR}[grpc]"]) + ) env.apply_settings(IsolateSettings(Path(tmp_path))) return env From 25d49bd2142b1b2409a1b793eedb77ad7b50d0c0 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 28 Jan 2026 22:05:47 +0200 Subject: [PATCH 2/4] Update src/isolate/backends/common.py Co-authored-by: Matteo Ferrando --- src/isolate/backends/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isolate/backends/common.py b/src/isolate/backends/common.py index 612ddd83..882b4350 100644 --- a/src/isolate/backends/common.py +++ b/src/isolate/backends/common.py @@ -231,7 +231,7 @@ def from_raw(cls, raw: list[str] | list[list[str]]) -> Requirements: return cls() if isinstance(raw, list) and all(isinstance(item, str) for item in raw): - return cls([list(cast(List[str], raw))]) + return cls([raw]) if isinstance(raw, list) and all(isinstance(item, list) for item in raw): layers: list[list[str]] = [] From 4205363842e1ec0b809d971475e054a26c296c24 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 28 Jan 2026 22:05:53 +0200 Subject: [PATCH 3/4] Update src/isolate/backends/common.py Co-authored-by: Matteo Ferrando --- src/isolate/backends/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isolate/backends/common.py b/src/isolate/backends/common.py index 882b4350..26aafb8c 100644 --- a/src/isolate/backends/common.py +++ b/src/isolate/backends/common.py @@ -238,7 +238,7 @@ def from_raw(cls, raw: list[str] | list[list[str]]) -> Requirements: for layer in raw: if not all(isinstance(item, str) for item in layer): raise TypeError("Requirements layers must contain strings.") - layers.append(list(layer)) + layers.append(layer) return cls(layers) raise TypeError( From 02fc640c56bd0b8b0f0d8f85d37ef93353065c78 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 28 Jan 2026 22:16:09 +0200 Subject: [PATCH 4/4] lint --- src/isolate/backends/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/isolate/backends/common.py b/src/isolate/backends/common.py index 26aafb8c..857138ff 100644 --- a/src/isolate/backends/common.py +++ b/src/isolate/backends/common.py @@ -13,7 +13,7 @@ from functools import lru_cache from pathlib import Path from types import ModuleType -from typing import Callable, Iterator, List, cast +from typing import Callable, Iterator # For ensuring that the lock is created and not forgotten # (e.g. the process which acquires it crashes, so it is never @@ -231,14 +231,14 @@ def from_raw(cls, raw: list[str] | list[list[str]]) -> Requirements: return cls() if isinstance(raw, list) and all(isinstance(item, str) for item in raw): - return cls([raw]) + return cls([raw]) # type: ignore[list-item] if isinstance(raw, list) and all(isinstance(item, list) for item in raw): layers: list[list[str]] = [] for layer in raw: if not all(isinstance(item, str) for item in layer): raise TypeError("Requirements layers must contain strings.") - layers.append(layer) + layers.append(layer) # type: ignore[arg-type] return cls(layers) raise TypeError(