[docker] rayimg base deps lock files (ray-project#57073)

generating lock files for base-deps image

Including setuptools in base-deps requirements due to
https://buildkite.com/ray-project/release/builds/67063#019a7066-7935-4eea-ac7b-3bf62705b81d/142-3586

Copying lock file to image and uv pip installing the deps

Hello world release test run:
https://buildkite.com/ray-project/release/builds/67552

---------

Signed-off-by: elliot-barn <elliot.barnwell@anyscale.com>
Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com>

Author: elliot-barn

.buildkite/_images.rayci.yml

@@ -14,6 +14,7 @@ steps:
       - "3.12"
     env:
       PYTHON_VERSION: "{{matrix}}"
+      REQUIREMENTS_FILE: "ray_base_deps_py{{matrix}}.lock"
       ARCH_SUFFIX: ""
 
   - name: raycpubaseextra
@@ -56,6 +57,7 @@ steps:
     env:
       PYTHON_VERSION: "{{matrix.python}}"
       CUDA_VERSION: "{{matrix.cuda}}"
+      REQUIREMENTS_FILE: "ray_base_deps_py{{matrix.python}}.lock"
       ARCH_SUFFIX: ""
 
   - name: raycudabaseextra

.buildkite/linux_aarch64.rayci.yml

@@ -39,6 +39,7 @@ steps:
     env:
       PYTHON_VERSION: "{{matrix}}"
       ARCH_SUFFIX: "-aarch64"
+      REQUIREMENTS_FILE: "ray_base_deps_py{{matrix}}.lock"
 
   - name: raycpubaseextra-aarch64
     label: "wanda: ray.py{{matrix}}.cpu.base-extra (aarch64)"
@@ -81,6 +82,7 @@ steps:
       PYTHON_VERSION: "{{matrix.python}}"
       CUDA_VERSION: "{{matrix.cuda}}"
       ARCH_SUFFIX: "-aarch64"
+      REQUIREMENTS_FILE: "ray_base_deps_py{{matrix.python}}.lock"
 
   - name: raycudabaseextra-aarch64
     label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra (aarch64)"

build-docker.sh

@@ -64,11 +64,16 @@ RAY_DEPS_BUILD_DIR="$(mktemp -d)"
 cp docker/base-deps/Dockerfile "${RAY_DEPS_BUILD_DIR}/."
 mkdir -p "${RAY_DEPS_BUILD_DIR}/python"
 cp python/requirements_compiled.txt "${RAY_DEPS_BUILD_DIR}/python/requirements_compiled.txt"
+PYTHON_DEPSET_FILE_NAME="ray_base_deps_py${PYTHON_VERSION}.lock"
+REQUIREMENTS_FILE_BASE_DEPS="python/deplocks/base_deps/${PYTHON_DEPSET_FILE_NAME}"
+
+cp "${REQUIREMENTS_FILE_BASE_DEPS}" "${RAY_DEPS_BUILD_DIR}/."
 
 BUILD_CMD=(
     docker build "${BUILD_ARGS[@]}"
-    --build-arg BASE_IMAG="$BASE_IMAGE"
+    --build-arg BASE_IMAGE="$BASE_IMAGE"
     --build-arg PYTHON_VERSION="${PYTHON_VERSION}"
+    --build-arg PYTHON_DEPSET="${PYTHON_DEPSET_FILE_NAME}"
     -t "rayproject/base-deps:dev$GPU" "${RAY_DEPS_BUILD_DIR}"
 )
 

ci/raydepsets/configs/rayimg.depsets.yaml

@@ -88,8 +88,8 @@ depsets:
       - ray_img_depset_${PYTHON_SHORT}
     output: release/ray_release/byod/ray_base_extra_testdeps_gpu_py${PYTHON_VERSION}.lock
     append_flags:
-      - --unsafe-package ray
       - --extra-index-url https://download.pytorch.org/whl/cu128
+      - --unsafe-package ray
       - --python-version=${PYTHON_VERSION}
       - --python-platform=linux
     build_arg_sets:
@@ -135,6 +135,25 @@ depsets:
       - py39
       - py310
 
+  # First layer of the ray release images. Contains all base layer python dependencies
+  # and is a subset of the base extra test deps defined above (release image depset)
+  - name: ray_base_deps_${PYTHON_SHORT}
+    operation: subset
+    source_depset: ray_base_extra_testdeps_${PYTHON_SHORT}
+    requirements:
+      - docker/base-deps/requirements.in
+    output: python/deplocks/base_deps/ray_base_deps_py${PYTHON_VERSION}.lock
+    append_flags:
+      - --unsafe-package ray
+      - --python-version=${PYTHON_VERSION}
+      - --python-platform=linux
+    build_arg_sets:
+      - py39
+      - py310
+      - py311
+      - py312
+    include_setuptools: true
+
     # Second layer of the ray release images. Contains anyscale cli deps (and others)
     # and is a subset of the base extra test deps defined above (release image depset)
   - name: ray_base_extra_${PYTHON_SHORT}

ci/raydepsets/tests/test_workspace.py

@@ -172,7 +172,6 @@ def test_invalid_build_arg_set_in_config():
         workspace = Workspace(dir=tmpdir)
         with unittest.TestCase().assertRaises(KeyError) as e:
             workspace.load_config(config_path=Path(tmpdir) / "test.depsets.yaml")
-        print(str(e.exception))
         assert (
             "Build arg set invalid_build_arg_set not found in config test.depsets.yaml"
             in str(e.exception)

docker/base-deps/Dockerfile

@@ -25,6 +25,8 @@ ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/usr/local/nvidia/lib64"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG PYTHON_VERSION=3.9
 
+ARG PYTHON_DEPSET
+
 ARG RAY_UID=1000
 ARG RAY_GID=100
 
@@ -68,6 +70,7 @@ ENV HOME=/home/ray
 WORKDIR /home/ray
 
 COPY python/requirements_compiled.txt /home/ray/requirements_compiled.txt
+COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
 
 SHELL ["/bin/bash", "-c"]
 
@@ -100,27 +103,6 @@ $HOME/anaconda3/bin/conda install -y libgcc-ng python=$PYTHON_VERSION
 $HOME/anaconda3/bin/conda install -y -c conda-forge libffi=3.4.2
 $HOME/anaconda3/bin/conda clean -y --all
 
-PIP_PKGS=(
-    # Required a recent version of setuptools to be compatible with python 3.12+.
-    setuptools==80.9.0
-
-    flatbuffers
-    cython
-    numpy  # Necessary for Dataset to work properly.
-    psutil
-
-    # For the ease to submit jobs on various cloud providers.
-    "smart_open[s3,gcs,azure,http]"
-
-    six
-    boto3
-    pyopenssl
-    cryptography
-    google-api-python-client
-    google-oauth
-    "adlfs[abfs]"
-)
-
 # Install uv
 wget -qO- https://astral.sh/uv/install.sh | sudo -E env UV_UNMANAGED_INSTALL="/usr/local/bin" sh
 
@@ -130,9 +112,8 @@ export PATH=$HOME/anaconda3/bin:$PATH
 # Some packages are on PyPI as well as other indices, but the latter
 # (unhelpfully) take precedence. We use `--index-strategy unsafe-best-match`
 # to ensure that the best match is chosen from the available indices.
-uv pip install --system --no-cache-dir --index-strategy unsafe-best-match \
-    -c $HOME/requirements_compiled.txt \
-    "${PIP_PKGS[@]}"
+uv pip install --system --no-cache-dir --no-deps --index-strategy unsafe-best-match \
+    -r $HOME/python_depset.lock
 
 # We install cmake temporarily to get psutil
 sudo apt-get autoremove -y cmake zlib1g-dev

docker/base-deps/cpu.wanda.yaml

@@ -3,8 +3,10 @@ froms: ["ubuntu:22.04"]
 dockerfile: docker/base-deps/Dockerfile
 srcs:
   - python/requirements_compiled.txt
+  - python/deplocks/base_deps/$REQUIREMENTS_FILE
 build_args:
   - PYTHON_VERSION
   - BASE_IMAGE=ubuntu:22.04
+  - PYTHON_DEPSET=python/deplocks/base_deps/$REQUIREMENTS_FILE
 tags:
   - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX

docker/base-deps/cuda.wanda.yaml

@@ -3,8 +3,10 @@ froms: ["nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04"]
 dockerfile: docker/base-deps/Dockerfile
 srcs:
   - python/requirements_compiled.txt
+  - python/deplocks/base_deps/$REQUIREMENTS_FILE
 build_args:
   - PYTHON_VERSION
   - BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04
+  - PYTHON_DEPSET=python/deplocks/base_deps/$REQUIREMENTS_FILE
 tags:
   - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX

docker/base-deps/requirements.in

@@ -1,3 +1,4 @@
+setuptools==80.9.0
 flatbuffers
 cython
 numpy  # Necessary for Dataset to work properly.