Prepare 0.4.1 Release (#193)

* upgrade ray dependency to 1.8.0 (#192)

* upgrade ray dependency

* fix

* require 1.8.0

* Prepare for 0.4.1

Co-authored-by: Zhi Lin <zl1nn@outlook.com>

Author: carsonwang

core/pom.xml

@@ -6,7 +6,7 @@
 
   <groupId>com.intel</groupId>
   <artifactId>raydp</artifactId>
-  <version>0.4.0</version>
+  <version>0.4.1</version>
 
   <name>raydp</name>
 
@@ -21,7 +21,7 @@
     <fasterxml.jackson.version>2.12.2</fasterxml.jackson.version>
     <jersey.version>2.31</jersey.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <ray.version>1.7.0</ray.version>
+    <ray.version>1.8.0</ray.version>
   </properties>
 
   <repositories>

core/src/main/scala/org/apache/spark/sql/raydp/ObjectStoreWriter.scala

@@ -64,7 +64,7 @@ class ObjectStoreWriter(@transient val df: DataFrame) extends Serializable {
     val objectRefImpl = RayDPUtils.convert(objectRef)
     val objectId = objectRefImpl.getId
     val runtime = Ray.internal.asInstanceOf[RayRuntimeInternal]
-    val addressInfo = runtime.getObjectStore.promoteAndGetOwnershipInfo(objectId)
+    val addressInfo = runtime.getObjectStore.getOwnershipInfo(objectId)
     RecordBatch(addressInfo, objectId.getBytes, numRecords)
   }
 

python/raydp/__init__.py

@@ -17,6 +17,6 @@
 
 from raydp.context import init_spark, stop_spark
 
-__version__ = "0.4.0"
+__version__ = "0.4.1"
 
 __all__ = ["init_spark", "stop_spark"]

python/raydp/spark/dataset.py

@@ -27,8 +27,7 @@
 from pyspark.sql.types import StructType
 from pyspark.sql.pandas.types import from_arrow_type
 import ray
-from ray.data import from_arrow
-from ray.data import Dataset
+from ray.data import Dataset, from_arrow_refs
 from ray.types import ObjectRef
 import ray.util.data as ml_dataset
 import ray.util.iter as parallel_it
@@ -466,7 +465,7 @@ def spark_dataframe_to_ray_dataset(df: sql.DataFrame,
         elif parallelism > num_part:
             df = df.repartition(parallelism)
     blocks, _ = _save_spark_df_to_object_store(df, False)
-    return from_arrow(blocks)
+    return from_arrow_refs(blocks)
 
 @ray.remote
 class RayDPConversionHelper():

python/setup.py

@@ -29,7 +29,7 @@
 if package_name == 'raydp_nightly':
     VERSION = datetime.today().strftime("%Y.%m.%d.dev0")
 else:
-    VERSION = "0.4.0"
+    VERSION = "0.4.1"
 
 ROOT_DIR = os.path.dirname(__file__)
 
@@ -95,7 +95,7 @@ def run(self):
         "pandas >= 1.1.4",
         "psutil",
         "pyarrow >= 0.10",
-        "ray >= 1.7.0",
+        "ray >= 1.8.0",
         "pyspark >= 3.2.0",
         "netifaces"
     ]