Merge branch 'master' of https://github.com/ray-project/ray

Author: akyang-anyscale

.buildkite/_images.rayci.yml

@@ -226,6 +226,7 @@ steps:
           - "12.5.1"
           - "12.6.3"
           - "12.8.1"
+          - "12.9.1"
     env:
       PYTHON_VERSION: "{{matrix.python}}"
       CUDA_VERSION: "{{matrix.cuda}}"

.buildkite/base.rayci.yml

@@ -7,10 +7,10 @@ steps:
     label: "wanda: oss-ci-base_test-py{{matrix}}"
     wanda: ci/docker/base.test.wanda.yaml
     matrix:
-      - "3.9"
       - "3.10"
       - "3.11"
       - "3.12"
+      - "3.13"
     env:
       PYTHON: "{{matrix}}"
 
@@ -23,7 +23,6 @@ steps:
     label: "wanda: oss-ci-base_build-py{{matrix}}"
     wanda: ci/docker/base.build.wanda.yaml
     matrix:
-      - "3.9"
       - "3.10"
       - "3.11"
       - "3.12"
@@ -39,6 +38,8 @@ steps:
   - name: oss-ci-base_build-aarch64
     wanda: ci/docker/base.build.aarch64.wanda.yaml
     depends_on: oss-ci-base_test-aarch64
+    env:
+      PYTHON: "3.10"
     instance_type: builder-arm64
 
   - name: oss-ci-base_ml

.buildkite/core.rayci.yml

@@ -35,6 +35,7 @@ steps:
   - name: minbuild-core
     label: "wanda: minbuild-core-py{{matrix}}"
     wanda: ci/docker/min.build.wanda.yaml
+    depends_on: oss-ci-base_test-multipy
     tags: cibase
     matrix:
       - "3.10"
@@ -425,8 +426,8 @@ steps:
       # cpp worker tests has one that tests cross-language with Java.
       - RAY_INSTALL_JAVA=1 ci/ci.sh build
       - ci/ci.sh test_cpp
-    depends_on: oss-ci-base_build
-    job_env: oss-ci-base_build
+    depends_on: oss-ci-base_build-multipy
+    job_env: oss-ci-base_build-py3.10
 
   - label: ":ray: core: java worker tests"
     tags:
@@ -447,7 +448,8 @@ steps:
       - docker
     instance_type: large
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- docker --platform cpu --canonical-tag ha_integration
+      - bazel run //ci/ray_ci:build_in_docker -- docker --python-version 3.10
+        --platform cpu --canonical-tag ha_integration
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core --only-tags ha_integration
         --python-version 3.10 --build-name corebuild-py3.10
     depends_on:
@@ -463,8 +465,8 @@ steps:
       - oss
     instance_type: medium
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- docker --platform cpu
-        --canonical-tag test_container
+      - bazel run //ci/ray_ci:build_in_docker -- docker --python-version 3.10
+        --platform cpu --canonical-tag test_container
       - docker build --progress=plain --build-arg BASE_IMAGE="rayproject/ray:test_container"
         -t rayproject/ray:runtime_env_container -f ci/docker/runtime_env_container/Dockerfile .
       # Disable test DB, these tests will never succeed if run in the flaky step.

.buildkite/data.rayci.yml

@@ -51,6 +51,14 @@ steps:
       PYTHON: "3.10"
     tags: cibase
 
+  - name: datatfdsbuild-multipy
+    label: "wanda: datatfdsbuild-py3.12"
+    wanda: ci/docker/datatfds.build.wanda.yaml
+    env:
+      PYTHON: "3.12"
+      REQUIREMENTS_FILE: "datatfds_py3.12.lock"
+    tags: cibase
+
   # tests
   - label: ":database: data: arrow v9 tests"
     tags:
@@ -62,9 +70,21 @@ steps:
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name data9build-py3.10 --python-version 3.10
-        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
+        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials,tensorflow_datasets
     depends_on: data9build-multipy
 
+  - label: ":database: data: tensorflow-datasets tests"
+    key: data_tensorflow_datasets_tests
+    tags:
+      - python
+      - data
+    instance_type: medium
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
+        --build-name datatfdsbuild-py3.12 --python-version 3.12
+        --only-tags tensorflow_datasets
+    depends_on: datatfdsbuild-multipy
+
   - label: ":database: data: arrow v9 tests (data_non_parallel)"
     tags:
       - data
@@ -90,7 +110,7 @@ steps:
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name datalbuild-py3.10 --python-version 3.10
-        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
+        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials,tensorflow_datasets
     depends_on: datalbuild-multipy
 
   - label: ":database: data: arrow v21 tests (data_non_parallel)"
@@ -118,7 +138,7 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
         --workers 2 --worker-id {{matrix.worker_id}} --parallelism-per-worker 3
         --build-name datalbuild-py{{matrix.python}} --python-version {{matrix.python}}
-        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
+        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials,tensorflow_datasets
     depends_on: datalbuild-multipy
     matrix:
       setup:
@@ -153,7 +173,7 @@ steps:
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name datanbuild-py3.10 --python-version 3.10
-        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
+        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials,tensorflow_datasets
     depends_on: datanbuild-multipy
     soft_fail: true
 

.buildkite/kuberay.rayci.yml

@@ -3,7 +3,7 @@ steps:
   - name: k8sbuild
     wanda: ci/docker/k8s.build.wanda.yaml
     depends_on:
-      - oss-ci-base_build
+      - oss-ci-base_build-multipy
     tags: cibase
 
   - label: ":kubernetes: operator"
@@ -19,6 +19,8 @@ steps:
       - manylinux
       - forge
       - raycpubase
+      - ray-core-build
+      - ray-dashboard-build
 
   - label: ":kubernetes: chaos {{matrix.workload}} under {{matrix.fault}}"
     key: kuberay_tests
@@ -45,3 +47,5 @@ steps:
       - manylinux
       - forge
       - raycpubase
+      - ray-core-build
+      - ray-dashboard-build

.buildkite/linux_aarch64.rayci.yml

@@ -211,6 +211,7 @@ steps:
         --parallelism-per-worker 3
         --only-tags post_wheel_build
         --test-env=RAY_CI_POST_WHEEL_TESTS=True
+        --python-version 3.10
     depends_on:
       - manylinux-aarch64
       - oss-ci-base_build-aarch64
@@ -227,6 +228,7 @@ steps:
         --parallelism-per-worker 3
         --only-tags post_wheel_build
         --test-env=RAY_CI_POST_WHEEL_TESTS=True
+        --python-version 3.10
     depends_on:
       - manylinux-aarch64
       - oss-ci-base_build-aarch64

.buildkite/ml.rayci.yml

@@ -8,9 +8,8 @@ steps:
   - name: minbuild-ml
     label: "wanda: minbuild-ml-py{{matrix}}"
     wanda: ci/docker/min.build.wanda.yaml
-    depends_on: oss-ci-base_build
+    depends_on: oss-ci-base_test-multipy
     matrix:
-      - "3.9"
       - "3.10"
     env:
       PYTHON_VERSION: "{{matrix}}"
@@ -42,16 +41,7 @@ steps:
 
   - name: mllightning2gpubuild
     wanda: ci/docker/mllightning2gpu.build.wanda.yaml
-    depends_on: oss-ci-base_gpu
-
-  - name: mlgpubuild
-    wanda: ci/docker/ml.build.wanda.yaml
-    depends_on: oss-ci-base_gpu
-    env:
-      IMAGE_FROM: cr.ray.io/rayproject/oss-ci-base_gpu
-      IMAGE_TO: mlgpubuild
-      RAYCI_IS_GPU_BUILD: "true"
-    tags: cibase
+    depends_on: oss-ci-base_gpu-multipy
 
   - name: mlgpubuild-multipy
     label: "wanda: mlgpubuild-py{{matrix}}"
@@ -188,8 +178,8 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tune/... ml
         --only-tags soft_imports
-        --build-name oss-ci-base_build
-    depends_on: [ "oss-ci-base_build", "forge" ]
+        --build-name oss-ci-base_build-py3.10 --python-version 3.10
+    depends_on: [ "oss-ci-base_build-multipy", "forge" ]
 
   - label: ":train: ml: air tests"
     tags: ml
@@ -284,7 +274,7 @@ steps:
     instance_type: gpu-large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/train/... ml
-        --build-name mllightning2gpubuild
+        --build-name mllightning2gpubuild --python-version 3.10
         --only-tags ptl_v2
     depends_on: [ "mllightning2gpubuild", "forge" ]