From 774e08b7c5a2bbddf9b0fab88472c0ee739a6ed8 Mon Sep 17 00:00:00 2001 From: ashione Date: Sun, 21 Apr 2024 22:19:52 +0800 Subject: [PATCH 1/4] java upgrade to 2.11.0 --- streaming/java/dependencies.bzl | 4 ++-- streaming/java/pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/streaming/java/dependencies.bzl b/streaming/java/dependencies.bzl index df5d43f..c27776e 100644 --- a/streaming/java/dependencies.bzl +++ b/streaming/java/dependencies.bzl @@ -28,13 +28,13 @@ def gen_streaming_java_deps(): maven.artifact( group = "io.ray", artifact = "ray-api", - version = "2.9.3", + version = "2.11.0", neverlink = True ), maven.artifact( group = "io.ray", artifact = "ray-runtime", - version = "2.9.3", + version = "2.11.0", neverlink = True ), maven.artifact( diff --git a/streaming/java/pom.xml b/streaming/java/pom.xml index dfbf1a4..64c0d83 100644 --- a/streaming/java/pom.xml +++ b/streaming/java/pom.xml @@ -58,7 +58,7 @@ 1.8 11 UTF-8 - 2.9.3 + 2.11.0 0.0.1 1.10.19 From fd7ea88c228a602c8da8fc366d7b74fad1531943 Mon Sep 17 00:00:00 2001 From: ashione Date: Sun, 5 May 2024 17:18:39 +0800 Subject: [PATCH 2/4] lint --- scripts/install-ray.sh | 2 +- streaming/java/BUILD.bazel | 64 +++++++++++++++++---------------- streaming/java/dependencies.bzl | 12 ++++--- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/scripts/install-ray.sh b/scripts/install-ray.sh index 26d85b3..b512e70 100644 --- a/scripts/install-ray.sh +++ b/scripts/install-ray.sh @@ -2,7 +2,7 @@ minor_version=`python -c "import sys;print(sys.version_info[1])"` -python -m pip install ray +python -m pip install -U ray # if [ "$minor_version" == "8" ] ; then # python -m pip install https://ray-mobius-us.oss-us-west-1.aliyuncs.com/ci/linux/ubuntu/b7d148815e427ccd47046d1c69ee3ab55bf7db3c/ray-2.0.0.dev0-cp38-cp38-linux_x86_64.whl # elif [ "$minor_version" == "7" ] ; then diff --git a/streaming/java/BUILD.bazel b/streaming/java/BUILD.bazel index dd030d7..648db98 100644 --- a/streaming/java/BUILD.bazel +++ b/streaming/java/BUILD.bazel @@ -45,7 +45,7 @@ define_java_module( ":io_ray_ray_streaming-common", "@maven//:org_testng_testng", "@ray_streaming_maven//:com_google_guava_guava", - ], + ], visibility = ["//visibility:public"], deps = [ "@maven//:org_apache_logging_log4j_log4j_api", @@ -53,20 +53,20 @@ define_java_module( "@maven//:org_apache_logging_log4j_log4j_slf4j_impl", "@maven//:org_slf4j_slf4j_api", "@maven//:org_testng_testng", - "@ray_streaming_maven//:io_ray_ray_api", - "@ray_streaming_maven//:io_ray_ray_runtime", - "@ray_streaming_maven//:com_google_guava_guava", "@ray_streaming_maven//:com_esotericsoftware_kryo", "@ray_streaming_maven//:com_esotericsoftware_minlog_minlog", "@ray_streaming_maven//:com_esotericsoftware_reflectasm_reflectasm", - "@ray_streaming_maven//:de_ruedigermoeller_fst", + "@ray_streaming_maven//:com_google_guava_guava", "@ray_streaming_maven//:com_typesafe_config", + "@ray_streaming_maven//:commons_io_commons_io", "@ray_streaming_maven//:de_javakaffee_kryo_serializers", - "@ray_streaming_maven//:org_aeonbits_owner_owner", - "@ray_streaming_maven//:org_apache_commons_commons_lang3", + "@ray_streaming_maven//:de_ruedigermoeller_fst", + "@ray_streaming_maven//:io_ray_ray_api", + "@ray_streaming_maven//:io_ray_ray_runtime", "@ray_streaming_maven//:joda_time_joda_time", - "@ray_streaming_maven//:commons_io_commons_io", "@ray_streaming_maven//:net_java_dev_jna_jna", + "@ray_streaming_maven//:org_aeonbits_owner_owner", + "@ray_streaming_maven//:org_apache_commons_commons_lang3", "@ray_streaming_maven//:org_objenesis_objenesis", ], ) @@ -89,13 +89,13 @@ define_java_module( "@maven//:org_apache_logging_log4j_log4j_core", "@maven//:org_apache_logging_log4j_log4j_slf4j_impl", "@maven//:org_slf4j_slf4j_api", - "@ray_streaming_maven//:com_google_guava_guava", "@ray_streaming_maven//:com_esotericsoftware_kryo", "@ray_streaming_maven//:com_esotericsoftware_minlog_minlog", "@ray_streaming_maven//:com_esotericsoftware_reflectasm_reflectasm", + "@ray_streaming_maven//:com_google_guava_guava", "@ray_streaming_maven//:de_javakaffee_kryo_serializers", - "@ray_streaming_maven//:org_apache_commons_commons_lang3", "@ray_streaming_maven//:org_aeonbits_owner_owner", + "@ray_streaming_maven//:org_apache_commons_commons_lang3", "@ray_streaming_maven//:org_objenesis_objenesis", ], ) @@ -104,8 +104,8 @@ define_java_module( name = "streaming-api", define_test_lib = True, test_deps = [ - ":io_ray_ray_streaming-common", ":io_ray_ray_streaming-api", + ":io_ray_ray_streaming-common", "@maven//:org_slf4j_slf4j_api", "@maven//:org_testng_testng", "@ray_streaming_maven//:com_google_guava_guava", @@ -119,14 +119,14 @@ define_java_module( "@maven//:org_apache_logging_log4j_log4j_core", "@maven//:org_apache_logging_log4j_log4j_slf4j_impl", "@maven//:org_slf4j_slf4j_api", - "@ray_streaming_maven//:io_ray_ray_api", - "@ray_streaming_maven//:io_ray_ray_runtime", - "@ray_streaming_maven//:com_google_guava_guava", "@ray_streaming_maven//:com_google_code_findbugs_jsr305", "@ray_streaming_maven//:com_google_code_gson_gson", - "@ray_streaming_maven//:org_ow2_asm_asm", - "@ray_streaming_maven//:org_apache_commons_commons_lang3", + "@ray_streaming_maven//:com_google_guava_guava", + "@ray_streaming_maven//:io_ray_ray_api", + "@ray_streaming_maven//:io_ray_ray_runtime", "@ray_streaming_maven//:org_apache_arrow_arrow_vector", + "@ray_streaming_maven//:org_apache_commons_commons_lang3", + "@ray_streaming_maven//:org_ow2_asm_asm", ], ) @@ -150,25 +150,25 @@ define_java_module( "streaming-runtime/src/main/java/io/ray/streaming/runtime/generated/*.java", ], test_deps = [ - ":io_ray_ray_streaming-common", ":io_ray_ray_streaming-api", + ":io_ray_ray_streaming-common", ":io_ray_ray_streaming-runtime", "@maven//:org_slf4j_slf4j_api", "@maven//:org_testng_testng", - "@ray_streaming_maven//:io_ray_ray_api", - "@ray_streaming_maven//:io_ray_ray_runtime", "@ray_streaming_maven//:com_google_guava_guava", "@ray_streaming_maven//:de_ruedigermoeller_fst", + "@ray_streaming_maven//:io_ray_ray_api", + "@ray_streaming_maven//:io_ray_ray_runtime", + "@ray_streaming_maven//:org_aeonbits_owner_owner", "@ray_streaming_maven//:org_apache_commons_commons_lang3", "@ray_streaming_maven//:org_mockito_mockito_all", - "@ray_streaming_maven//:org_aeonbits_owner_owner", "@ray_streaming_maven//:org_powermock_powermock_api_mockito", "@ray_streaming_maven//:org_powermock_powermock_module_testng", ], visibility = ["//visibility:public"], deps = [ - ":io_ray_ray_streaming-common", ":io_ray_ray_streaming-api", + ":io_ray_ray_streaming-common", ":io_ray_ray_streaming-state", "@maven//:com_google_protobuf_protobuf_java", "@maven//:org_apache_logging_log4j_log4j_api", @@ -176,17 +176,17 @@ define_java_module( "@maven//:org_apache_logging_log4j_log4j_slf4j_impl", "@maven//:org_msgpack_msgpack_core", "@maven//:org_slf4j_slf4j_api", - "@ray_streaming_maven//:io_ray_ray_api", - "@ray_streaming_maven//:io_ray_ray_runtime", - "@ray_streaming_maven//:com_google_guava_guava", - "@ray_streaming_maven//:org_apache_commons_commons_lang3", - "@ray_streaming_maven//:commons_io_commons_io", - "@ray_streaming_maven//:net_java_dev_jna_jna", "@ray_streaming_maven//:com_github_davidmoten_flatbuffers_java", "@ray_streaming_maven//:com_google_code_findbugs_jsr305", "@ray_streaming_maven//:com_google_code_gson_gson", + "@ray_streaming_maven//:com_google_guava_guava", "@ray_streaming_maven//:commons_collections_commons_collections", + "@ray_streaming_maven//:commons_io_commons_io", + "@ray_streaming_maven//:io_ray_ray_api", + "@ray_streaming_maven//:io_ray_ray_runtime", + "@ray_streaming_maven//:net_java_dev_jna_jna", "@ray_streaming_maven//:org_aeonbits_owner_owner", + "@ray_streaming_maven//:org_apache_commons_commons_lang3", ], ) @@ -196,10 +196,12 @@ java_binary( data = ["testng.xml"], main_class = "org.testng.TestNG", runtime_deps = [ - ":io_ray_ray_streaming-common_test", - ":io_ray_ray_streaming-state_test", ":io_ray_ray_streaming-api_test", + ":io_ray_ray_streaming-common_test", ":io_ray_ray_streaming-runtime_test", + ":io_ray_ray_streaming-state_test", + "@ray_streaming_maven//:io_ray_ray_api", + "@ray_streaming_maven//:io_ray_ray_runtime", ], ) @@ -302,10 +304,10 @@ java_binary( # fat jar file. It's not really an executable jar. So we set its `main_class` to empty. main_class = "", runtime_deps = [ - "//java:io_ray_ray_streaming-common", "//java:io_ray_ray_streaming-api", - "//java:io_ray_ray_streaming-state", + "//java:io_ray_ray_streaming-common", "//java:io_ray_ray_streaming-runtime", + "//java:io_ray_ray_streaming-state", ], ) diff --git a/streaming/java/dependencies.bzl b/streaming/java/dependencies.bzl index c27776e..db20a45 100644 --- a/streaming/java/dependencies.bzl +++ b/streaming/java/dependencies.bzl @@ -29,13 +29,13 @@ def gen_streaming_java_deps(): group = "io.ray", artifact = "ray-api", version = "2.11.0", - neverlink = True + neverlink = True, ), maven.artifact( group = "io.ray", artifact = "ray-runtime", version = "2.11.0", - neverlink = True + neverlink = True, ), maven.artifact( group = "org.apache.arrow", @@ -44,15 +44,19 @@ def gen_streaming_java_deps(): exclusions = [ "io.netty:netty-buffer", "io.netty:netty-common", - ] + ], ), maven.artifact( group = "de.ruedigermoeller", artifact = "fst", version = "2.56", exclusions = [ + maven.exclusion( + group = "com.fasterxml.jackson.core", + artifact = "jackson-core", + ), "com.fasterxml.jackson.core:jackson-core", - ] + ], ), ], repositories = [ From 0523b9e2935f6c027ac026dbf088d8f042085dff Mon Sep 17 00:00:00 2001 From: ashione Date: Fri, 21 Jun 2024 21:50:43 +0800 Subject: [PATCH 3/4] upgrade python version --- .github/workflows/macos_building.yml | 2 +- .github/workflows/ubuntu_building.yml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos_building.yml b/.github/workflows/macos_building.yml index a6c3e7f..78fbd29 100644 --- a/.github/workflows/macos_building.yml +++ b/.github/workflows/macos_building.yml @@ -15,7 +15,7 @@ env: jobs: streaming-mac-pipeline: timeout-minutes: 90 - runs-on: macos-11 + runs-on: macos-14 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/ubuntu_building.yml b/.github/workflows/ubuntu_building.yml index b268d24..eb10b87 100644 --- a/.github/workflows/ubuntu_building.yml +++ b/.github/workflows/ubuntu_building.yml @@ -29,6 +29,7 @@ jobs: apt-get install -yq software-properties-common add-apt-repository -y ppa:deadsnakes/ppa apt-get install -yq wget gcc g++ openjdk-8-jdk maven python3.8 zlib1g-dev zip git pip + apt install python3.10 python3.10-venv python3.10-dev - name: Install bazel run: | @@ -40,8 +41,8 @@ jobs: - name: Install ray wheel run: | - python3 -m pip install virtualenv - python3 -m virtualenv -p python3 py3 + python3.10 -m pip install virtualenv + python3.10 -m virtualenv -p python3.10 py3 . py3/bin/activate pip install pytest sh -c "bash scripts/install-ray.sh" From b05c49793cc2c402483ed1ca586c98597aa629da Mon Sep 17 00:00:00 2001 From: ashione Date: Fri, 21 Jun 2024 22:50:32 +0800 Subject: [PATCH 4/4] upgrade python version to 0.1.0 --- .github/workflows/ubuntu_building.yml | 2 +- streaming/python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu_building.yml b/.github/workflows/ubuntu_building.yml index eb10b87..0f1a5a0 100644 --- a/.github/workflows/ubuntu_building.yml +++ b/.github/workflows/ubuntu_building.yml @@ -29,7 +29,7 @@ jobs: apt-get install -yq software-properties-common add-apt-repository -y ppa:deadsnakes/ppa apt-get install -yq wget gcc g++ openjdk-8-jdk maven python3.8 zlib1g-dev zip git pip - apt install python3.10 python3.10-venv python3.10-dev + apt-get install python3.10 python3.10-venv python3.10-dev - name: Install bazel run: | diff --git a/streaming/python/setup.py b/streaming/python/setup.py index d0abc86..04bcac3 100644 --- a/streaming/python/setup.py +++ b/streaming/python/setup.py @@ -26,7 +26,7 @@ # Package raystreaming setup( name="raystreaming", - version="0.0.1", + version="0.1.0", author="Ray Streaming Team", description="streaming module", packages=find_packages(),