Merge branch 'master' of https://github.com/ray-project/ray

Author: akyang-anyscale

.vale/styles/Google/WordList.yml

@@ -11,7 +11,7 @@ swap:
   "(?:dev|developer|APIs) console": API console
   "(?:e-mail|Email|E-mail)": email
   "(?:file ?path|path ?name)": path
-  "(?:kill|terminate|abort)": stop|exit|cancel|end
+  "(?:terminate|abort)": stop|exit|cancel|end
   "(?:OAuth ?2|Oauth)": OAuth 2.0
   "(?:ok|Okay)": OK|okay
   "(?:WiFi|wifi)": Wi-Fi

.vale/styles/config/vocabularies/Core/accept.txt

@@ -2,7 +2,7 @@
 [Ss]tderr
 [Ss]tdout
 job_id
-Autoscaler
+[Aa]utoscaler
 ANSI
 PID
 tqdm

.vale/styles/config/vocabularies/General/accept.txt

@@ -88,7 +88,7 @@ NVIDIA
 NVLink
 OOM
 open-source
-PACK
+[Pp][Aa][Cc][Kk]
 pipelining
 Podman
 preemptible

ci/build/build-manylinux-forge.sh

@@ -74,8 +74,8 @@ chmod +x /tmp/bazelisk
 sudo mv /tmp/bazelisk /usr/local/bin/bazelisk
 sudo ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel
 
-# Use python3.9 as default python3
-sudo ln -sf /usr/local/bin/python3.9 /usr/local/bin/python3
+# Use python3.10 as default python3
+sudo ln -sf /usr/local/bin/python3.10 /usr/local/bin/python3
 
 {
   echo "build --config=ci"

ci/build/build-ray-docker.sh

@@ -14,15 +14,14 @@ CPU_TMP="$(mktemp -d)"
 
 cp -r .whl "${CPU_TMP}/.whl"
 cp docker/ray/Dockerfile "${CPU_TMP}/Dockerfile"
-cp python/requirements_compiled.txt "${CPU_TMP}/."
+cp python/"${CONSTRAINTS_FILE}" "${CPU_TMP}/requirements_compiled.txt"
 
 # Build the image.
 cd "${CPU_TMP}"
 tar --mtime="UTC 2020-01-01" -c -f - . \
     | docker build --progress=plain \
         --build-arg FULL_BASE_IMAGE="$SOURCE_IMAGE" \
         --build-arg WHEEL_PATH=".whl/${WHEEL_NAME}" \
-        --build-arg CONSTRAINTS_FILE="$CONSTRAINTS_FILE" \
         --label "io.ray.ray-version=$RAY_VERSION" \
         --label "io.ray.ray-commit=$RAY_COMMIT" \
         -t "$DEST_IMAGE" -f Dockerfile -

ci/docker/base.gpu.Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.3-labs
-ARG BASE_IMAGE=nvidia/cuda:12.1.1-cudnn8-devel-ubuntu20.04
+ARG BASE_IMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu20.04
 FROM $BASE_IMAGE
 
 ARG BUILDKITE_BAZEL_CACHE_URL
@@ -8,7 +8,7 @@ ARG PYTHON=3.10
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/Los_Angeles
 
-ENV RAY_BUILD_ENV=ubuntu20.04_cuda12.1_py$PYTHON
+ENV RAY_BUILD_ENV=ubuntu20.04_cuda12.8.1_py$PYTHON
 ENV BUILDKITE=true
 ENV CI=true
 ENV PYTHON=$PYTHON

ci/docker/base.gpu.wanda.yaml

@@ -1,5 +1,5 @@
 name: "oss-ci-base_gpu-py$PYTHON"
-froms: ["nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04"]
+froms: ["nvidia/cuda:12.8.1-cudnn-devel-ubuntu20.04"]
 dockerfile: ci/docker/base.gpu.Dockerfile
 srcs:
   - .bazelrc

ci/env/install-miniforge.sh

@@ -95,6 +95,12 @@ install_miniforge() {
     )
   fi
 
+  (
+    set +x
+    echo "Cleaning conda caches..."
+    "${WORKSPACE_DIR}"/ci/suppress_output conda clean --all -q -y
+  )
+
   command -V python
   test -x "${CONDA_PYTHON_EXE}"  # make sure conda is activated
 }

ci/lint/pydoclint-baseline.txt

@@ -192,8 +192,6 @@ python/ray/_private/services.py
     DOC101: Function `start_raylet`: Docstring contains fewer arguments than in function signature.
     DOC107: Function `start_raylet`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints
     DOC103: Function `start_raylet`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [cluster_id: str, socket_to_use: Optional[int]].
-    DOC101: Function `determine_plasma_store_config`: Docstring contains fewer arguments than in function signature.
-    DOC103: Function `determine_plasma_store_config`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [temp_dir: str].
     DOC101: Function `start_monitor`: Docstring contains fewer arguments than in function signature.
     DOC103: Function `start_monitor`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [autoscaler_v2: bool, fate_share: Optional[bool]].
     DOC101: Function `start_ray_client_server`: Docstring contains fewer arguments than in function signature.
@@ -867,8 +865,6 @@ python/ray/dashboard/modules/event/event_utils.py
     DOC201: Function `monitor_events` does not have a return section in docstring
 --------------------
 python/ray/dashboard/modules/job/cli.py
-    DOC101: Function `submit`: Docstring contains fewer arguments than in function signature.
-    DOC103: Function `submit`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [address: Optional[str], entrypoint: Tuple[str], entrypoint_memory: Optional[int], entrypoint_num_cpus: Optional[Union[int, float]], entrypoint_num_gpus: Optional[Union[int, float]], entrypoint_resources: Optional[str], headers: Optional[str], job_id: Optional[str], metadata_json: Optional[str], no_wait: bool, runtime_env: Optional[str], runtime_env_json: Optional[str], submission_id: Optional[str], verify: Union[bool, str], working_dir: Optional[str]].
     DOC101: Function `status`: Docstring contains fewer arguments than in function signature.
     DOC103: Function `status`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [address: Optional[str], headers: Optional[str], job_id: str, verify: Union[bool, str]].
     DOC101: Function `stop`: Docstring contains fewer arguments than in function signature.
@@ -896,8 +892,6 @@ python/ray/dashboard/modules/job/job_supervisor.py
     DOC103: Method `JobSupervisor._exec_entrypoint`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [env: dict].
 --------------------
 python/ray/dashboard/modules/job/sdk.py
-    DOC104: Method `JobSubmissionClient.submit_job`: Arguments are the same in the docstring and the function signature, but are in a different order.
-    DOC105: Method `JobSubmissionClient.submit_job`: Argument names match, but type hints in these args do not match: entrypoint, job_id, runtime_env, metadata, submission_id, entrypoint_num_cpus, entrypoint_num_gpus, entrypoint_memory, entrypoint_resources
     DOC402: Method `JobSubmissionClient.tail_job_logs` has "yield" statements, but the docstring does not have a "Yields" section
     DOC404: Method `JobSubmissionClient.tail_job_logs` yield type(s) in docstring not consistent with the return annotation. Return annotation exists, but docstring "yields" section does not exist or has 0 type(s).
 --------------------
@@ -1150,10 +1144,6 @@ python/ray/data/_internal/logical/operators/join_operator.py
     DOC103: Method `Join.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [aggregator_ray_remote_args: Optional[Dict[str, Any]], join_type: str, left_columns_suffix: Optional[str], left_input_op: LogicalOperator, left_key_columns: Tuple[str], num_partitions: int, partition_size_hint: Optional[int], right_columns_suffix: Optional[str], right_input_op: LogicalOperator, right_key_columns: Tuple[str]].
 --------------------
 python/ray/data/_internal/logical/operators/map_operator.py
-    DOC001: Method `__init__` Potential formatting errors in docstring. Error message: No specification for "Args": ""
-    DOC001: Function/method `__init__`: Potential formatting errors in docstring. Error message: No specification for "Args": "" (Note: DOC001 could trigger other unrelated violations under this function/method too. Please fix the docstring formatting first.)
-    DOC101: Method `AbstractUDFMap.__init__`: Docstring contains fewer arguments than in function signature.
-    DOC103: Method `AbstractUDFMap.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [compute: Optional[ComputeStrategy], fn: UserDefinedFunction, fn_args: Optional[Iterable[Any]], fn_constructor_args: Optional[Iterable[Any]], fn_constructor_kwargs: Optional[Dict[str, Any]], fn_kwargs: Optional[Dict[str, Any]], input_op: LogicalOperator, min_rows_per_bundled_input: Optional[int], name: str, ray_remote_args: Optional[Dict[str, Any]], ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]]].
     DOC101: Method `StreamingRepartition.__init__`: Docstring contains fewer arguments than in function signature.
     DOC103: Method `StreamingRepartition.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [input_op: LogicalOperator].
 --------------------
@@ -1163,12 +1153,6 @@ python/ray/data/_internal/logical/operators/n_ary_operator.py
     DOC101: Method `NAry.__init__`: Docstring contains fewer arguments than in function signature.
     DOC103: Method `NAry.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [*input_ops: LogicalOperator, num_outputs: Optional[int]].
 --------------------
-python/ray/data/_internal/logical/operators/one_to_one_operator.py
-    DOC001: Method `__init__` Potential formatting errors in docstring. Error message: No specification for "Args": ""
-    DOC001: Function/method `__init__`: Potential formatting errors in docstring. Error message: No specification for "Args": "" (Note: DOC001 could trigger other unrelated violations under this function/method too. Please fix the docstring formatting first.)
-    DOC101: Method `AbstractOneToOne.__init__`: Docstring contains fewer arguments than in function signature.
-    DOC103: Method `AbstractOneToOne.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [input_op: Optional[LogicalOperator], name: str, num_outputs: Optional[int]].
---------------------
 python/ray/data/_internal/metadata_exporter.py
     DOC101: Method `Topology.create_topology_metadata`: Docstring contains fewer arguments than in function signature.
     DOC103: Method `Topology.create_topology_metadata`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [op_to_id: Dict['PhysicalOperator', str]].

ci/raydepsets/cli.py

@@ -96,6 +96,15 @@ def __init__(
         check: Optional[bool] = False,
         build_all_configs: Optional[bool] = False,
     ):
+        """Initialize the dependency set manager.
+
+        Args:
+            config_path: Path to the depsets config file.
+            workspace_dir: Path to the workspace directory.
+            uv_cache_dir: Directory to cache uv dependencies.
+            check: Whether to check if lock files are up to date.
+            build_all_configs: Whether to build all configs or just the specified one.
+        """
         self.workspace = Workspace(workspace_dir)
         self.config = self.workspace.load_configs(config_path)
         self.config_name = os.path.basename(config_path)
@@ -109,6 +118,7 @@ def __init__(
             self.copy_to_temp_dir()
 
     def get_output_paths(self) -> List[Path]:
+        """Get all output paths for depset nodes in topological order."""
         output_paths = []
         for node in topological_sort(self.build_graph):
             if self.build_graph.nodes[node]["node_type"] == "depset":
@@ -126,6 +136,7 @@ def copy_to_temp_dir(self):
             )
 
     def get_diffs(self) -> List[str]:
+        """Compare current lock files with previously saved copies and return unified diffs."""
         diffs = []
         for output_path in self.output_paths:
             new_lock_file_fp, old_lock_file_fp = self.get_source_and_dest(output_path)
@@ -142,6 +153,7 @@ def get_diffs(self) -> List[str]:
         return diffs
 
     def diff_lock_files(self):
+        """Check if lock files are up to date and raise an error if not."""
         diffs = self.get_diffs()
         if len(diffs) > 0:
             raise RuntimeError(
@@ -151,9 +163,11 @@ def diff_lock_files(self):
         click.echo("Lock files are up to date.")
 
     def get_source_and_dest(self, output_path: str) -> tuple[Path, Path]:
+        """Get the source workspace path and temporary destination path for a lock file."""
         return (self.get_path(output_path), (Path(self.temp_dir) / output_path))
 
     def _build(self, build_all_configs: Optional[bool] = False):
+        """Build the dependency graph from config depsets."""
         for depset in self.config.depsets:
             if depset.operation == "compile":
                 self.build_graph.add_node(
@@ -201,11 +215,13 @@ def _build(self, build_all_configs: Optional[bool] = False):
             self.subgraph_config_nodes()
 
     def subgraph_dependency_nodes(self, depset_name: str):
+        """Reduce the build graph to only include the specified depset and its ancestors."""
         dependency_nodes = networkx_ancestors(self.build_graph, depset_name)
         nodes = dependency_nodes | {depset_name}
         self.build_graph = self.build_graph.subgraph(nodes).copy()
 
     def subgraph_config_nodes(self):
+        """Reduce the build graph to nodes matching the current config and their ancestors."""
         # Get all nodes that have the target config name
         config_nodes = [
             node
@@ -224,6 +240,7 @@ def subgraph_config_nodes(self):
         self.build_graph = self.build_graph.subgraph(nodes).copy()
 
     def execute(self, single_depset_name: Optional[str] = None):
+        """Execute all depsets in topological order, optionally limited to a single depset."""
         if single_depset_name:
             # check if the depset exists
             _get_depset(self.config.depsets, single_depset_name)
@@ -240,6 +257,7 @@ def execute(self, single_depset_name: Optional[str] = None):
     def exec_uv_cmd(
         self, cmd: str, args: List[str], stdin: Optional[bytes] = None
     ) -> str:
+        """Execute a uv pip command with the given arguments."""
         cmd = [self._uv_binary, "pip", cmd, *args]
         click.echo(f"Executing command: {' '.join(cmd)}")
         status = subprocess.run(
@@ -252,6 +270,7 @@ def exec_uv_cmd(
         return status.stdout.decode("utf-8")
 
     def execute_pre_hook(self, pre_hook: str):
+        """Execute a pre-hook shell command."""
         status = subprocess.run(
             shlex.split(pre_hook),
             cwd=self.workspace.dir,
@@ -265,6 +284,7 @@ def execute_pre_hook(self, pre_hook: str):
         click.echo(f"Executed pre_hook {pre_hook} successfully")
 
     def execute_depset(self, depset: Depset):
+        """Execute a single depset based on its operation type (compile, subset, or expand)."""
         if depset.operation == "compile":
             self.compile(
                 constraints=depset.constraints,
@@ -389,15 +409,18 @@ def expand(
         )
 
     def read_lock_file(self, file_path: Path) -> List[str]:
+        """Read and return the contents of a lock file as a list of lines."""
         if not file_path.exists():
             raise RuntimeError(f"Lock file {file_path} does not exist")
         with open(file_path, "r") as f:
             return f.readlines()
 
     def get_path(self, path: str) -> Path:
+        """Convert a relative path to an absolute path within the workspace."""
         return Path(self.workspace.dir) / path
 
     def check_subset_exists(self, source_depset: Depset, requirements: List[str]):
+        """Verify that all requirements exist in the source depset."""
         for req in requirements:
             if req not in self.get_expanded_depset_requirements(source_depset.name, []):
                 raise RuntimeError(
@@ -424,15 +447,18 @@ def get_expanded_depset_requirements(
         return list(set(requirements_list))
 
     def cleanup(self):
+        """Remove the temporary directory used for lock file comparisons."""
         if self.temp_dir:
             shutil.rmtree(self.temp_dir)
 
 
 def _get_bytes(packages: List[str]) -> bytes:
+    """Convert a list of package names to newline-separated UTF-8 bytes."""
     return ("\n".join(packages) + "\n").encode("utf-8")
 
 
 def _get_depset(depsets: List[Depset], name: str) -> Depset:
+    """Find and return a depset by name from a list of depsets."""
     for depset in depsets:
         if depset.name == name:
             return depset
@@ -452,6 +478,7 @@ def _flatten_flags(flags: List[str]) -> List[str]:
 
 
 def _override_uv_flags(flags: List[str], args: List[str]) -> List[str]:
+    """Override existing uv flags in args with new values from flags."""
     flag_names = {f.split()[0] for f in flags if f.startswith("--")}
     new_args = []
     skip_next = False
@@ -468,6 +495,7 @@ def _override_uv_flags(flags: List[str], args: List[str]) -> List[str]:
 
 
 def _uv_binary():
+    """Get the path to the uv binary for the current platform."""
     r = runfiles.Create()
     system = platform.system()
     processor = platform.processor()