Merge branch 'master' into fix-timezone-ui

Author: alanwguo

.buildkite/build.rayci.yml

@@ -22,7 +22,7 @@ steps:
     tags:
       - linux_wheels
       - oss
-    instance_type: medium
+    instance_type: large
     commands:
       - bazel run //ci/ray_ci:build_in_docker -- wheel --build-type debug --upload
     depends_on:

.buildkite/core.rayci.yml

@@ -56,12 +56,24 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/_common/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
         --install-mask all-ray-libraries
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
+        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu,spark_on_ray,ray_client,compiled_graphs
         --install-mask all-ray-libraries
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --only-tags use_all_core --skip-ray-installation
 
+  - label: ":ray: core: cgraph python tests"
+    tags:
+      - compiled_graphs
+    instance_type: large
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/dag/... core
+        --install-mask all-ray-libraries
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --only-tags compiled_graphs
+        --except-tags multi_gpu
+        --install-mask all-ray-libraries
+
   - label: ":ray: core: python {{matrix.python}} tests ({{matrix.worker_id}})"
     if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
     tags:
@@ -73,7 +85,7 @@ steps:
         --install-mask all-ray-libraries
         --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
         --python-version {{matrix.python}}
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
+        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu,spark_on_ray,ray_client
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
         --install-mask all-ray-libraries
         --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
@@ -85,6 +97,16 @@ steps:
         python: ["3.12"]
         worker_id: ["0", "1", "2", "3"]
 
+  - label: ":ray: core: ray client tests"
+    tags:
+      - ray_client
+    instance_type: large
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/...   core
+        --install-mask all-ray-libraries
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
+        --only-tags ray_client
+
   - label: ":ray: core: redis tests"
     tags:
       - python
@@ -97,7 +119,7 @@ steps:
         --install-mask all-ray-libraries
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --test-env=TEST_EXTERNAL_REDIS=1
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
+        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu,spark_on_ray,ray_client
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --only-tags use_all_core --skip-ray-installation
@@ -176,7 +198,7 @@ steps:
         --except-tags gpu
         --skip-ray-installation
 
-  - label: ":ray: core: data tests"
+  - label: ":ray: core: dask & modin tests"
     tags: python
     instance_type: medium
     commands:
@@ -306,12 +328,20 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --except-tags=cgroup --build-type clang
         --cache-test-results --parallelism-per-worker 2
 
+  # block on premerge and microcheck
+  - block: "run cpp sanitizer tests"
+    if: build.env("BUILDKITE_PIPELINE_ID") == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("BUILDKITE_PIPELINE_ID") == "018f4f1e-1b73-4906-9802-92422e3badaa"
+    key: block-core-cpp-sanitizer-tests
+    depends_on: []
+
   - label: ":ray: core: cpp asan tests"
     tags: core_cpp
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --except-tags=cgroup
         --build-type asan-clang --cache-test-results --parallelism-per-worker 2
+    depends_on:
+      - block-core-cpp-sanitizer-tests
 
   - label: ":ray: core: cpp ubsan tests"
     tags: core_cpp
@@ -320,6 +350,8 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core
         --build-type ubsan --except-tags no_ubsan,cgroup
         --cache-test-results --parallelism-per-worker 2
+    depends_on:
+      - block-core-cpp-sanitizer-tests
 
   - label: ":ray: core: cpp tsan tests"
     tags: core_cpp
@@ -328,6 +360,8 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core
         --build-type tsan-clang --except-tags no_tsan,cgroup
         --cache-test-results --parallelism-per-worker 2
+    depends_on:
+      - block-core-cpp-sanitizer-tests
 
   - label: ":ray: core: flaky cpp tests"
     key: core_flaky_cpp_tests
@@ -417,6 +451,19 @@ steps:
       - raycpubase
       - corebuild
 
+  - label: ":core: core: spark-on-ray tests"
+    tags:
+      - spark_on_ray
+    instance_type: medium
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core
+        --build-type debug
+        --test-env=RAY_ON_SPARK_BACKGROUND_JOB_STARTUP_WAIT=1
+        --test-env=RAY_ON_SPARK_RAY_WORKER_NODE_STARTUP_INTERVAL=5
+        --parallelism-per-worker 3
+        --only-tags spark_on_ray
+        --except-tags kubernetes
+
   # block on premerge and microcheck
   - block: "run multi gpu tests"
     if: build.env("BUILDKITE_PIPELINE_ID") == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("BUILDKITE_PIPELINE_ID") == "018f4f1e-1b73-4906-9802-92422e3badaa"
@@ -426,7 +473,7 @@ steps:
   - label: ":ray: core: multi gpu tests"
     key: core-multi-gpu-tests
     tags:
-      - accelerated_dag
+      - compiled_graphs
       - gpu
     instance_type: gpu-large
     # we're running some cgraph doc tests here as well since they need gpus

.buildkite/data.rayci.yml

@@ -143,6 +143,7 @@ steps:
     tags:
       - python
       - data
+      - skip-on-premerge
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data

.buildkite/others.rayci.yml

@@ -78,7 +78,6 @@ steps:
       - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --test-prefix windows:__ --consider-master-branch --production
     matrix:
       - "core"
-      - "serverless"
       - "serve"
       - "data"
       - "ml"

.buildkite/serverless.rayci.yml

@@ -62,22 +62,6 @@ steps:
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 2
     depends_on: windowsbuild
 
-  - label: ":serverless: serverless: :windows: tests"
-    tags: python
-    job_env: WINDOWS
-    instance_type: windows
-    commands:
-      - bash ci/ray_ci/windows/install_tools.sh
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... serverless
-        --build-name windowsbuild
-        --operating-system windows
-        --except-tags no_windows
-        --test-env=CI="1"
-        --test-env=RAY_CI_POST_WHEEL_TESTS="1"
-        --test-env=USERPROFILE
-        --parallelism-per-worker 3
-    depends_on: windowsbuild
-
   - label: ":ray-serve: serve: :windows: tests"
     tags: serve
     job_env: WINDOWS
@@ -146,26 +130,6 @@ steps:
     depends_on: windowsbuild
     soft_fail: true
 
-  - label: "flaky :windows: serverless tests"
-    key: windows_serverless_flaky_tests
-    tags:
-      - flaky
-      - skip-on-premerge
-    job_env: WINDOWS
-    instance_type: windows
-    commands:
-      - bash ci/ray_ci/windows/install_tools.sh
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... serverless
-        --except-tags no_windows
-        --run-flaky-tests
-        --build-name windowsbuild
-        --operating-system windows
-        --test-env=CI="1"
-        --test-env=RAY_CI_POST_WHEEL_TESTS="1"
-        --test-env=USERPROFILE
-    depends_on: windowsbuild
-    soft_fail: true
-
   - label: "flaky :windows: serve tests"
     key: windows_serve_flaky_tests
     tags:

.buildkite/windows.rayci.yml

@@ -43,7 +43,29 @@ repos:
         args: [ --fix, --exit-non-zero-on-fix ]
       - id: ruff
         args: [ --select, "I", --fix, --exit-non-zero-on-fix ]
-        files: '^python/ray/serve/|^python/ray/train'
+        files: '^python/ray/serve/|^python/ray/train|^python/ray/data'
+
+  - repo: https://github.com/jsh9/pydoclint
+    rev: "0.6.6"
+    hooks:
+      - id: pydoclint
+        args: [
+          --style=google,
+          --baseline=ci/lint/pydoclint-baseline.txt,
+          --exclude=thirdparty,
+          # --generate-baseline=True, # Not generally needed, but documenting since this is how we generate the initial baseline
+          --auto-regenerate-baseline=True,
+          # Current settings (not because we think they're right, but because we
+          # don't want a baseline the size of the codebase)
+          --arg-type-hints-in-docstring=False,
+          --skip-checking-raises=True,
+          --check-return-types=False,
+          --allow-init-docstring=True,
+          --check-class-attributes=False,
+          # --check-style-mismatch=True, # Bring this back once things are a bit cleaner
+        ]
+        types: [python]
+        files: '^python/ray/'
 
   - repo: https://github.com/cpplint/cpplint
     rev: 2.0.0

.pre-commit-config.yaml

@@ -1,22 +1,29 @@
 # Use 'API' judiciously: https://developers.google.com/style/word-list#api.
-API[s]
-CPU[s]
-GPU[s]
-NumPy
-URI[s]
 [Aa]pplication
 [Cc]odec
+[Cc]omposable
+[Dd]eduplication
 [Dd]ict(s)?
+[Ee]xoshuffle
+[Gg]rafana
 [Ii]nteroperates
 [Ii]nterpretability
+[Mm]ultimodal
 [Pp]arallelization
 [Pp]erformant
+[Pp]rofiler
 [Ss]ubclassing
-Kueue
-GKE
-namespace
+Alibaba
+Alpaca
+Anyscale
+API[s]
 ARM
+async
+autoscales
 breakpoint
+BTS
+CLI
+CPU[s]
 deduplicate[s]
 [Dd]eduplication
 [Aa]sync
@@ -30,22 +37,37 @@ UUID
 bool
 deserializes
 PACK
+configs
 disable
 deduplicate(s)?
-[Dd]eduplication
-Alibaba
+DeepSpeed
+deserialization
+deserialize
+dev
+dev to prod
+DPO
+FSDP
+GGUF
+GKE
+GPTQ
+GPU[s]
+KTO
+Kueue
+LMs
 LSH
-BTS
-[Mm]ultimodal
+Megatron
+namespace
+NER
+Nsight
+NumPy
+pretraining
 Pythonic
-[Gg]rafana
-CLI
-[Ee]xoshuffle
+QPS
 RISECamp
-Anyscale
-[Cc]omposable
-Nsight
-[Pp]rofiler
-deserialize
+rollouts
+SFT
+ShareGPT
 teardown
-deserialization
+uncaptured
+URI[s]
+[Vv]ision LMs