Merge branch 'master' of https://github.com/ray-project/ray

Author: akyang-anyscale

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ exclude: |
     python/ray/_private/runtime_env/_clonevirtualenv.py|
     python/ray/data/examples/data/|
     release/release_logs/|
-    rllib/tests/data|
+    rllib/offline/tests/data|
     thirdparty/patches/|
     src/ray/thirdparty/|
     doc/external/|
@@ -30,10 +30,7 @@ repos:
         exclude: |
           (?x)^(
             # Intentionally bad json schema
-            python/ray/tests/unit/test_runtime_env_validation_bad_schema.json|
-            # json5 comments prevent parsing
-            python/asv.conf.json|
-            rllib/asv.conf.json
+            python/ray/tests/unit/test_runtime_env_validation_bad_schema.json
           )
       - id: check-toml
 
@@ -45,38 +42,10 @@ repos:
       - id: ruff
         args: [ --select, "I", --fix, --exit-non-zero-on-fix ]
 
-  # pydoclint-local is for local commits only due to pre-commit-hook only passing
-  # updated files to the hook and overwriting the baseline text file
   - repo: https://github.com/jsh9/pydoclint
-    rev: "0.8.1"
+    rev: "0.8.3"
     hooks:
       - id: pydoclint
-        name: pydoclint-local
-        stages: [pre-commit, pre-push]
-        args: [
-          --style=google,
-          --baseline=ci/lint/pydoclint-baseline.txt,
-          --exclude=thirdparty|^python/ray/serve/tests/test_config_files/syntax_error\.py$|^python/ray/_private/parameter\.py$,
-          --auto-regenerate-baseline=False,
-          # Current settings (not because we think they're right, but because we
-          # don't want a baseline the size of the codebase)
-          --arg-type-hints-in-docstring=False,
-          --skip-checking-raises=True,
-          --check-return-types=False,
-          --allow-init-docstring=True,
-          --check-class-attributes=False,
-          # --check-style-mismatch=True, # Bring this back once things are a bit cleaner
-        ]
-        types: [python]
-        files: '^python/ray/'
-
-  # pydoclint-ci is for CI, overwrites the baseline text file, and is run with the manual stage flag
-  - repo: https://github.com/jsh9/pydoclint
-    rev: "0.8.1"
-    hooks:
-      - id: pydoclint
-        name: pydoclint-ci
-        stages: [manual]
         args: [
           --style=google,
           --baseline=ci/lint/pydoclint-baseline.txt,

.vale/styles/config/vocabularies/Data/accept.txt

@@ -1,13 +1,15 @@
+assess_quality
 autoscaler
 [Aa]vro
 [Bb]ackpressure
 Dask
 [Dd]ata('s)?
 [Dd]atasource(s)?
 [Dd]iscretizer(s)?
+docstrings
 dtype
 FLAC
-[Gg]roupby
+[Gg]roup[bB]y
 [Hh]asher(s)?
 [Hh]udi
 [Ii]ndexable
@@ -19,14 +21,18 @@ MCAP
 Modin
 [Mm]ultiget(s)?
 ndarray(s)?
+NLP
 [Oo]utqueue(s)?
+PDFs
 [Pp]ipelined
 Predibase('s)?
 [Pp]refetch
 [Pp]refetching
 [Pp]reprocess
 [Pp]reprocessor(s)?
+process_file
 [Pp]ushdown
+queryable
 RGB
 runai
 [Ss]calers
@@ -36,5 +42,3 @@ UDF(s)?
 VLM(s)?
 XGBoost
 YOLO
-[Ss]harding
-[Ss]harded

ci/ci.sh

@@ -13,43 +13,6 @@ suppress_output() {
   "${WORKSPACE_DIR}"/ci/suppress_output "$@"
 }
 
-# Calls the provided command with set -x temporarily suppressed
-suppress_xtrace() {
-  {
-    local restore_shell_state=""
-    if [ -o xtrace ]; then set +x; restore_shell_state="set -x"; fi
-  } 2> /dev/null
-  local status=0
-  "$@" || status=$?
-  ${restore_shell_state}
-  { return "${status}"; } 2> /dev/null
-}
-
-# Idempotent environment loading
-reload_env() {
-  # Try to only modify CI-specific environment variables here (TRAVIS_... or GITHUB_...),
-  # e.g. for CI cross-compatibility.
-  # Normal environment variables should be set up at software installation time, not here.
-
-  if [ -n "${GITHUB_PULL_REQUEST-}" ]; then
-    case "${GITHUB_PULL_REQUEST}" in
-      [1-9]*) TRAVIS_PULL_REQUEST="${GITHUB_PULL_REQUEST}";;
-      *) TRAVIS_PULL_REQUEST=false;;
-    esac
-    export TRAVIS_PULL_REQUEST
-  fi
-
-  if [ "${GITHUB_ACTIONS-}" = true ] && [ -z "${TRAVIS_BRANCH-}" ]; then
-    # Define TRAVIS_BRANCH to make Travis scripts run on GitHub Actions.
-    TRAVIS_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF}}"  # For pull requests, the base branch name
-    TRAVIS_BRANCH="${TRAVIS_BRANCH#refs/heads/}"  # Remove refs/... prefix
-    # TODO(mehrdadn): Make TRAVIS_BRANCH be a named ref (e.g. 'master') like it's supposed to be.
-    # For now we use a hash because GitHub Actions doesn't clone refs the same way as Travis does.
-    TRAVIS_BRANCH="${GITHUB_HEAD_SHA:-${TRAVIS_BRANCH}}"
-    export TRAVIS_BRANCH
-  fi
-}
-
 compile_pip_dependencies() {
   # Compile boundaries
   TARGET="${1-requirements_compiled.txt}"
@@ -167,9 +130,9 @@ _install_npm_project() {
 }
 
 build_dashboard_front_end() {
-  if [ "${OSTYPE}" = msys ]; then
+  if [[ "${OSTYPE}" == msys ]]; then
     { echo "WARNING: Skipping dashboard due to NPM incompatibilities with Windows"; } 2> /dev/null
-  elif [ "${NO_DASHBOARD-}" = "1" ]; then
+  elif [[ "${NO_DASHBOARD-}" == "1" ]]; then
     echo "Skipping dashboard build"
   else
     (
@@ -196,7 +159,7 @@ build_sphinx_docs() {
 
   (
     cd "${WORKSPACE_DIR}"/doc
-    if [ "${OSTYPE}" = msys ]; then
+    if [[ "${OSTYPE}" == msys ]]; then
       echo "WARNING: Documentation not built on Windows due to currently-unresolved issues"
     else
       make html
@@ -208,7 +171,7 @@ build_sphinx_docs() {
 check_sphinx_links() {
   (
     cd "${WORKSPACE_DIR}"/doc
-    if [ "${OSTYPE}" = msys ]; then
+    if [[ "${OSTYPE}" == msys ]]; then
       echo "WARNING: Documentation not built on Windows due to currently-unresolved issues"
     else
       make linkcheck
@@ -345,15 +308,7 @@ build() {
   install_ray
 }
 
-_main() {
-  if [ "${GITHUB_ACTIONS-}" = true ]; then
-    exec 2>&1  # Merge stdout and stderr to prevent out-of-order buffering issues
-    reload_env
-  fi
-  "$@"
-}
-
-_main "$@"
+"$@"
 
 # Pop caller's shell options (quietly)
 { set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null

ci/lint/lint.sh

@@ -49,11 +49,7 @@ pre_commit() {
 pre_commit_pydoclint() {
   # Run pre-commit pydoclint on all files
   pip install -c python/requirements_compiled.txt pre-commit clang-format
-  pre-commit run pydoclint --hook-stage manual --all-files --show-diff-on-failure
-  git diff --quiet -- ci/lint/pydoclint-baseline.txt || {
-  echo "Baseline needs update. Run the CI-style hook: \"pre-commit run pydoclint --hook-stage manual --all-files --show-diff-on-failure\" locally and commit the baseline."
-  exit 1
-  }
+  pre-commit run pydoclint --all-files --show-diff-on-failure
 }
 
 code_format() {

ci/lint/pydoclint-baseline.txt

@@ -1317,7 +1317,6 @@ python/ray/data/read_api.py
     DOC101: Function `read_bigquery`: Docstring contains fewer arguments than in function signature.
     DOC103: Function `read_bigquery`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [query: Optional[str]].
     DOC103: Function `read_parquet`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**arrow_parquet_args: ]. Arguments in the docstring but not in the function signature: [arrow_parquet_args: ].
-    DOC103: Function `read_parquet_bulk`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**arrow_parquet_args: ]. Arguments in the docstring but not in the function signature: [arrow_parquet_args: ].
     DOC103: Function `read_json`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**arrow_json_args: ]. Arguments in the docstring but not in the function signature: [arrow_json_args: ].
     DOC103: Function `read_csv`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**arrow_csv_args: ]. Arguments in the docstring but not in the function signature: [arrow_csv_args: ].
     DOC101: Function `read_text`: Docstring contains fewer arguments than in function signature.

doc/BUILD.bazel

@@ -700,9 +700,15 @@ doctest(
     env = {"RAY_TRAIN_V2_ENABLED": "1"},
 )
 
+# --------------------------------------------------------------------
+# Discover the Anyscale Jobs compute configs .yaml for release tests in CI
+# --------------------------------------------------------------------
+
 filegroup(
     name = "example_configs",
-    srcs = glob(["source/ray-overview/examples/**/*.yaml"]),
+    srcs = glob([
+        "source/ray-overview/examples/**/*.yaml"
+    ]),
     visibility = ["//release:__pkg__"],
 )
 

doc/source/cluster/kubernetes/examples/mobilenet-rayservice.md

@@ -19,7 +19,7 @@ Note that the YAML file in this example uses `serveConfigV2`. You need KubeRay v
 
 ```sh
 # Create a RayService
-kubectl apply -f https://raw.githubusercontent.com/ray-project/kuberay/v1.5.0/ray-operator/config/samples/ray-service.mobilenet.yaml
+kubectl apply -f https://raw.githubusercontent.com/ray-project/kuberay/v1.5.1/ray-operator/config/samples/ray-service.mobilenet.yaml
 ```
 
 * The [mobilenet.py](https://github.com/ray-project/serve_config_examples/blob/master/mobilenet/mobilenet.py) file needs `tensorflow` as a dependency. Hence, the YAML file uses `rayproject/ray-ml` image instead of `rayproject/ray` image.

doc/source/cluster/kubernetes/examples/rayjob-batch-inference-example.md

@@ -37,12 +37,12 @@ The KubeRay operator Pod must be on the CPU node if you have set up the taint fo
 
 ## Step 2: Submit the RayJob
 
-Create the RayJob custom resource with [ray-job.batch-inference.yaml](https://github.com/ray-project/kuberay/blob/v1.5.0/ray-operator/config/samples/ray-job.batch-inference.yaml).
+Create the RayJob custom resource with [ray-job.batch-inference.yaml](https://github.com/ray-project/kuberay/blob/v1.5.1/ray-operator/config/samples/ray-job.batch-inference.yaml).
 
 Download the file with `curl`:
 
 ```bash
-curl -LO https://raw.githubusercontent.com/ray-project/kuberay/v1.5.0/ray-operator/config/samples/ray-job.batch-inference.yaml
+curl -LO https://raw.githubusercontent.com/ray-project/kuberay/v1.5.1/ray-operator/config/samples/ray-job.batch-inference.yaml
 ```
 
 Note that the `RayJob` spec contains a spec for the `RayCluster`. This tutorial uses a single-node cluster with 4 GPUs. For production use cases, use a multi-node cluster where the head node doesn't have GPUs, so that Ray can automatically schedule GPU workloads on worker nodes which won't interfere with critical Ray processes on the head node.

doc/source/cluster/kubernetes/getting-started/kuberay-operator-installation.md

@@ -17,15 +17,15 @@ kind create cluster --image=kindest/node:v1.26.0
 ```sh
 helm repo add kuberay https://ray-project.github.io/kuberay-helm/
 helm repo update
-# Install both CRDs and KubeRay operator v1.5.0.
-helm install kuberay-operator kuberay/kuberay-operator --version 1.5.0
+# Install both CRDs and KubeRay operator v1.5.1.
+helm install kuberay-operator kuberay/kuberay-operator --version 1.5.1
 ```
 
 ### Method 2: Kustomize
 
 ```sh
 # Install CRD and KubeRay operator.
-kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v1.5.0"
+kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v1.5.1"
 ```
 
 ## Step 3: Validate Installation

doc/source/cluster/kubernetes/getting-started/raycluster-quick-start.md

@@ -28,7 +28,7 @@ Once the KubeRay operator is running, you're ready to deploy a RayCluster. Creat
 
 ```sh
 # Deploy a sample RayCluster CR from the KubeRay Helm chart repo:
-helm install raycluster kuberay/ray-cluster --version 1.5.0
+helm install raycluster kuberay/ray-cluster --version 1.5.1
 ```