[pull] master from ray-project:master

.bazelrc

@@ -1,5 +1,13 @@
 # Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd
 build --enable_platform_specific_config
+
+# Provides users an option to turn on strict action env.
+# TODO(aslonnie): make this default; fix the python tests..
+build:strict --incompatible_strict_action_env
+
+# To distinguish different incompatible environments.
+build --action_env=RAY_BUILD_ENV
+
 ###############################################################################
 # On       Windows, provide: BAZEL_SH, and BAZEL_LLVM (if using clang-cl)
 # On all platforms, provide: PYTHON3_BIN_PATH=python
@@ -8,14 +16,19 @@ build:windows --action_env=PATH
 # For --compilation_mode=dbg, consider enabling checks in the standard library as well (below).
 build --compilation_mode=opt
 # Using C++ 17 on all platforms.
+build:linux --host_cxxopt="-std=c++17"
+build:macos --host_cxxopt="-std=c++17"
+build:clang-cl --host_cxxopt="-std=c++17"
+build:msvc-cl --host_cxxopt="/std:c++17"
+build:windows --host_cxxopt="/std:c++17"
 build:linux --cxxopt="-std=c++17"
 build:macos --cxxopt="-std=c++17"
 build:clang-cl --cxxopt="-std=c++17"
 build:msvc-cl --cxxopt="/std:c++17"
 build:windows --cxxopt="/std:c++17"
 # This workaround is needed to prevent Bazel from compiling the same file twice (once PIC and once not).
-build:linux --force_pic
-build:macos --force_pic
+build:linux --copt="-fPIC"
+build:macos --copt="-fPIC"
 build:clang-cl --compiler=clang-cl
 build:msvc-cl --compiler=msvc-cl
 # `LC_ALL` and `LANG` is needed for cpp worker tests, because they will call "ray start".
@@ -44,20 +57,23 @@ build:clang-cl --per_file_copt="-\\.(asm|S)$@-Werror"
 build:msvc-cl     --per_file_copt="-\\.(asm|S)$@-WX"
 # Ignore warnings for protobuf generated files and external projects.
 build --per_file_copt="\\.pb\\.cc$@-w"
-build:linux --per_file_copt="-\\.(asm|S)$,external/.*@-w,-Wno-error=implicit-function-declaration"
-build:macos --per_file_copt="-\\.(asm|S)$,external/.*@-w,-Wno-error=implicit-function-declaration"
-# Ignore minor warnings for host tools, which we generally can't control
+build --per_file_copt="src/ray/thirdparty/.*$@-w"
+build:linux --per_file_copt="-\\.(asm|S)$,external/.*@-w,-Wno-error=implicit-function-declaration,-Wno-error=unused-function"
+build:macos --per_file_copt="-\\.(asm|S)$,external/.*@-w,-Wno-error=implicit-function-declaration,-Wno-error=unused-function,-Wno-missing-template-arg-list-after-template-kw"
+# Ignore warnings for host tools, which we generally can't control.
+# Ideally we only want to ignore warnings for external project
+# but the current bazel version doesn't support host_per_file_copt yet.
+build:macos --host_copt="-w"
 build:clang-cl --host_copt="-Wno-inconsistent-missing-override"
 build:clang-cl --host_copt="-Wno-microsoft-unqualified-friend"
 # This workaround is needed due to https://github.com/bazelbuild/bazel/issues/4341
 build --per_file_copt="-\\.(asm|S)$,external/com_github_grpc_grpc/.*@-DGRPC_BAZEL_BUILD"
-# Don't generate warnings about kernel features we don't need https://github.com/ray-project/ray/issues/6832
-build:linux --per_file_copt="-\\.(asm|S)$,external/com_github_grpc_grpc/.*@-DGPR_MANYLINUX1"
 # Ignore wchar_t -> char conversion warning on MSVC
 build:msvc-cl --per_file_copt="external/boost/libs/regex/src/wc_regex_traits\\.cpp@-wd4244"
 build --http_timeout_scaling=5.0
 build --verbose_failures
-build:iwyu --experimental_action_listener=//:iwyu_cpp
+build:iwyu --aspects @com_github_storypku_bazel_iwyu//bazel/iwyu:iwyu.bzl%iwyu_aspect
+build:iwyu --output_groups=report
 
 # Print relative paths when possible
 build:windows --attempt_to_print_relative_paths
@@ -71,16 +87,17 @@ build:windows --color=yes
 build:clang-cl --per_file_copt="-\\.(asm|S)$@-fansi-escape-codes"
 build:clang-cl --per_file_copt="-\\.(asm|S)$@-fcolor-diagnostics"
 
-build:manylinux2010 --copt="-Wno-unused-result"
-build:manylinux2010 --linkopt="-lrt"
 
 # Debug build flags. Uncomment in '-c dbg' builds to enable checks in the C++ standard library:
-#build:linux --cxxopt="-D_GLIBCXX_DEBUG=1"
-#build:linux --cxxopt="-D_GLIBCXX_DEBUG_PEDANTIC=1"
-#build:linux --cxxopt="-D_LIBCPP_DEBUG=1"
-#build:macos --cxxopt="-D_GLIBCXX_DEBUG=1"
-#build:macos --cxxopt="-D_GLIBCXX_DEBUG_PEDANTIC=1"
-#build:windows --cxxopt="-D_ITERATOR_DEBUG_LEVEL=2"
+build:linux-debug --config linux
+build:linux-debug --cxxopt="-D_GLIBCXX_DEBUG=1"
+build:linux-debug --cxxopt="-D_GLIBCXX_DEBUG_PEDANTIC=1"
+build:linux-debug --cxxopt="-D_LIBCPP_DEBUG=1"
+build:macos-debug --config macos
+build:macos-debug --cxxopt="-D_GLIBCXX_DEBUG=1"
+build:macos-debug --cxxopt="-D_GLIBCXX_DEBUG_PEDANTIC=1"
+build:windows-debug --config windows
+build:windows-debug --cxxopt="-D_ITERATOR_DEBUG_LEVEL=2"
 
 # LLVM (clang & libc++) build flags common across Linux installations and systems.
 # On Ubuntu, the remaining configurations can be generated by running ci/env/install-llvm-binaries.sh
@@ -101,6 +118,7 @@ build:tsan --copt -fno-omit-frame-pointer
 build:tsan --copt -Wno-uninitialized
 build:tsan --linkopt -fsanitize=thread
 build:tsan --cxxopt="-D_RAY_TSAN_BUILD"
+build:tsan --no//:jemalloc_flag
 # This config is only for running TSAN with LLVM toolchain on Linux.
 build:tsan-clang --config=tsan
 build:tsan-clang --config=llvm
@@ -113,6 +131,7 @@ build:asan --copt -fsanitize=address
 build:asan --copt -DADDRESS_SANITIZER
 build:asan --copt -fno-omit-frame-pointer
 build:asan --linkopt -fsanitize=address
+build:asan --no//:jemalloc_flag
 test:asan --jobs=1
 test:asan --test_env=ASAN_OPTIONS="detect_leaks=0"
 # This config is only for running ASAN with LLVM toolchain on Linux.
@@ -139,33 +158,32 @@ test:asan-buildkite --test_env=LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libasan.so.
 # CI configuration:
 aquery:ci --color=no
 aquery:ci --noshow_progress
+
+test:ci-base --test_output=errors
+test:ci-base --test_verbose_timeout_warnings
+test:ci-base --flaky_test_attempts=3
+
 build:ci --color=yes
 build:ci --curses=no
 build:ci --keep_going
 build:ci --progress_report_interval=100
 build:ci --show_progress_rate_limit=15
-build:ci --show_task_finish
 build:ci --ui_actions_shown=1024
 build:ci --show_timestamps
-build:ci-travis --disk_cache=~/ray-bazel-cache
-build:ci-github --experimental_repository_cache_hardlinks # GitHub Actions has low disk space, so prefer hardlinks there.
-build:ci-github --disk_cache=~/ray-bazel-cache
-test:ci --flaky_test_attempts=3
 # Disable test result caching because py_test under Bazel can import from outside of sandbox, but Bazel only looks at
 # declared dependencies to determine if a result should be cached. More details at:
 # https://github.com/bazelbuild/bazel/issues/7091, https://github.com/bazelbuild/rules_python/issues/382
+test:ci --config=ci-base
 test:ci --nocache_test_results
 test:ci --spawn_strategy=local
-test:ci --test_output=errors
-test:ci --test_verbose_timeout_warnings
+test:ci --experimental_ui_max_stdouterr_bytes=-1
+
+test:ci-debug --config=ci-base
 test:ci-debug -c dbg
 test:ci-debug --copt="-g"
-test:ci-debug --flaky_test_attempts=3
 # Disable test result caching for the same reason above.
 test:ci-debug --nocache_test_results
 test:ci-debug --spawn_strategy=local
-test:ci-debug --test_output=errors
-test:ci-debug --test_verbose_timeout_warnings
 
 aquery:get-toolchain --include_commandline=false
 aquery:get-toolchain --noimplicit_deps
@@ -181,16 +199,20 @@ test:segfault --run_under="bash -c 'unset GREP_OPTIONS && if ! grep -q -o Micros
 build:debug -c dbg
 build:debug --copt="-g"
 build:debug --copt -fno-omit-frame-pointer
+build:debug --copt -fstack-protector-strong
+build:debug --copt -fcf-protection=full
 build:debug --strip="never"
 
 # Undefined Behavior Sanitizer
 build:ubsan --strip=never
-build:ubsan --copt -fsanitize=undefined
 build:ubsan --copt -fno-sanitize=vptr
 build:ubsan --copt -fno-sanitize-recover=all
 build:ubsan --copt -g
 build:ubsan --linkopt -fsanitize=undefined
 build:ubsan --linkopt -fno-sanitize-recover=all
+build:ubsan --per_file_copt="-external/com_github_grpc_grpc/.*@-fsanitize=undefined"
+
+build:cgroup --sandbox_writable_path=/sys/fs/cgroup --config=llvm
 
 # Import local specific llvm config options, which can be generated by
 # ci/env/install-llvm-dependencies.sh

.bazelversion

@@ -0,0 +1 @@
+6.5.0

.buildkite/.sunset_civ1_linux

@@ -0,0 +1 @@
+2023-12-20

.buildkite/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
+
+py_binary(
+    name = "copy_files",
+    srcs = ["copy_files.py"],
+    visibility = ["//visibility:private"],
+    deps = [
+        ci_require("boto3"),
+        ci_require("aws_requests_auth"),
+    ],
+)

.buildkite/README.md

@@ -2,7 +2,7 @@
 
 This directory contains buildkite pipelines used to start CI tests.
 
-Each step contains a buildkite step that is parsed and executed according to the 
+Each step contains a buildkite step that is parsed and executed according to the
 [Buildkite pipeline specification](https://buildkite.com/docs/pipelines).
 
 ## Conditions

.buildkite/_forge.rayci.yml

@@ -0,0 +1,100 @@
+group: forge
+steps:
+  - name: forge
+    wanda: ci/docker/forge.wanda.yaml
+
+  - name: manylinux
+    wanda: ci/docker/manylinux.wanda.yaml
+
+  - name: raycudabase
+    label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: ci/docker/ray.cuda.base.wanda.yaml
+    matrix:
+      setup:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        cuda:
+          - "11.7.1-cudnn8"
+          - "11.8.0-cudnn8"
+          - "12.1.1-cudnn8"
+          - "12.3.2-cudnn9"
+          - "12.4.1-cudnn"
+          - "12.5.1-cudnn"
+          - "12.8.1-cudnn"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+
+
+  - name: raycpubase
+    label: "wanda: ray.py{{matrix}}.cpu.base"
+    tags:
+      - python_dependencies
+      - python
+      - docker
+      - tune
+      - serve
+    wanda: ci/docker/ray.cpu.base.wanda.yaml
+    matrix:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+
+  - name: ray-llmbase
+    label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: ci/docker/ray-llm.base.wanda.yaml
+    depends_on: raycudabase
+    matrix:
+      setup:
+        python:
+          - "3.11"
+        cuda:
+          - "12.4.1-cudnn"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+
+  - name: ray-mlcudabase
+    label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: ci/docker/ray-ml.cuda.base.wanda.yaml
+    depends_on: raycudabase
+    matrix:
+      setup:
+        python:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+        cuda:
+          - "12.1.1-cudnn8"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+
+  - name: ray-mlcpubase
+    label: "wanda: ray-ml.py{{matrix}}.cpu.base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: ci/docker/ray-ml.cpu.base.wanda.yaml
+    depends_on: raycpubase
+    matrix:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+    env:
+      PYTHON_VERSION: "{{matrix}}"

.buildkite/base.rayci.yml

@@ -0,0 +1,78 @@
+group: base
+steps:
+  - name: oss-ci-base_test
+    wanda: ci/docker/base.test.py39.wanda.yaml
+
+  - name: oss-ci-base_test-multipy
+    label: "wanda: oss-ci-base_test-py{{matrix}}"
+    wanda: ci/docker/base.test.wanda.yaml
+    matrix:
+      - "3.9"
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON: "{{matrix}}"
+
+  - name: oss-ci-base_build
+    wanda: ci/docker/base.build.py39.wanda.yaml
+    depends_on: oss-ci-base_test
+
+  - name: oss-ci-base_build-multipy
+    label: "wanda: oss-ci-base_build-py{{matrix}}"
+    wanda: ci/docker/base.build.wanda.yaml
+    matrix:
+      - "3.9"
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON: "{{matrix}}"
+    depends_on: oss-ci-base_test-multipy
+
+  - name: oss-ci-base_test-aarch64
+    wanda: ci/docker/base.test.aarch64.wanda.yaml
+    instance_type: builder-arm64
+
+  - name: oss-ci-base_build-aarch64
+    wanda: ci/docker/base.build.aarch64.wanda.yaml
+    depends_on: oss-ci-base_test-aarch64
+    instance_type: builder-arm64
+
+  - name: oss-ci-base_ml
+    wanda: ci/docker/base.ml.py39.wanda.yaml
+    depends_on: oss-ci-base_test
+
+  - name: oss-ci-base_ml-multipy
+    label: "wanda: oss-ci-base_ml-py{{matrix}}"
+    wanda: ci/docker/base.ml.wanda.yaml
+    matrix:
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON: "{{matrix}}"
+    depends_on: oss-ci-base_test-multipy
+
+  - name: oss-ci-base_gpu
+    wanda: ci/docker/base.gpu.py39.wanda.yaml
+
+  - name: oss-ci-base_gpu-multipy
+    label: "wanda: oss-ci-base_gpu-py{{matrix}}"
+    wanda: ci/docker/base.gpu.wanda.yaml
+    matrix:
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON: "{{matrix}}"
+
+  - name: docbuild
+    label: "wanda: docbuild-py{{matrix}}"
+    wanda: ci/docker/doc.build.wanda.yaml
+    depends_on: oss-ci-base_build-multipy
+    matrix:
+      - "3.9"
+      - "3.12"
+    env:
+      PYTHON: "{{matrix}}"
+
+  - name: docgpubuild
+    wanda: ci/docker/docgpu.build.wanda.yaml
+    depends_on: oss-ci-base_gpu

.buildkite/bisect/_forge.rayci.yml

@@ -0,0 +1,4 @@
+group: forge
+steps:
+  - name: forge
+    wanda: ci/docker/forge.wanda.yaml