[Python]fix hybird stream test (#66)

* fix hybird stream test

* hybirdtest

* make workaround for hybirdstream test

* fix pytest current all of running tests

* check python test exit code

* add java 11 compatible compiler arguments for sun.ni package

* lint

Author: ashione

README.rst

@@ -165,6 +165,7 @@ Build
 ----------------
 
 Build from source code :
+
 - Build a docker using docker/Dockerfile-env
 - Execute `scripts/install.sh`
 

streaming/buildtest.sh

@@ -111,9 +111,12 @@ function test_streaming_python()
     fi
     #python3 -m pytest $script_dir/python/raystreaming/tests/simple --capture=no
     bazel build java:streaming_java_pkg
-    python3 -m pytest "$script_dir"/python/raystreaming/tests/ > "$TMP_LOG_OUTPUT"/python-test/python-test.log 2>&1
-    zip_and_upload_log "$TMP_LOG_OUTPUT"/python-test/ "${script_dir}/${ZIP_FILE}" "/${GITHUB_SHA}/${TIME}/${ZIP_FILE}"
-    exit $?
+    python3 -m pytest "$script_dir"/python/raystreaming/tests/ # > "$TMP_LOG_OUTPUT"/python-test/python-test.log 2>&1
+    exit_code=$?
+    echo "Running python test exit code : ${exit_code}"
+    echo "[Disabled] Uploding output to remote file."
+    #zip_and_upload_log "$TMP_LOG_OUTPUT"/python-test/ "${script_dir}/${ZIP_FILE}" "/${GITHUB_SHA}/${TIME}/${ZIP_FILE}"
+    exit $exit_code
 
     popd || exit
 }

streaming/java/pom.xml

@@ -56,6 +56,7 @@
 
   <properties>
     <java.version>1.8</java.version>
+    <java.new.version>11</java.new.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <ray.version>2.9.3</ray.version>
     <streaming.version>0.0.1</streaming.version>
@@ -128,6 +129,22 @@
           </configuration>
         </plugin>
 
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.6.1</version>
+          <configuration>
+            <source>${java.new.version}</source>
+            <target>${java.new.version}</target>
+            <encoding>${project.build.sourceEncoding}</encoding>
+            <compilerArgs>
+              <arg>--add-modules=jdk.unsupported</arg>
+              <arg>--add-exports=java.base/sun.nio.ch=ALL-UNNAMED</arg>
+            </compilerArgs>
+          </configuration>
+        </plugin>
+
+
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-source-plugin</artifactId>

streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java

@@ -56,6 +56,7 @@ public static StreamingContext buildContext() {
   public void execute(String jobName) {
     JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(this.streamSinks, jobName);
     JobGraph originalJobGraph = jobGraphBuilder.build();
+    originalJobGraph.printJobGraph();
     this.jobGraph = new JobGraphOptimizer(originalJobGraph).optimize();
     jobGraph.printJobGraph();
     LOG.info("JobGraph digraph\n{}", jobGraph.generateDigraph());

streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/impl/PythonPartitionFunction.java

@@ -23,7 +23,7 @@ public class PythonPartitionFunction implements Partition<Object> {
   public static final PythonPartitionFunction KeyPartition =
       new PythonPartitionFunction("raystreaming.partition", "KeyPartition");
   public static final PythonPartitionFunction RoundRobinPartition =
-      new PythonPartitionFunction("raystreaming.partition", "RoundRobinPartitionFunction");
+      new PythonPartitionFunction("raystreaming.partition", "RoundRobinPartition");
   public static final String FORWARD_PARTITION_CLASS = "ForwardPartition";
   public static final PythonPartitionFunction ForwardPartition =
       new PythonPartitionFunction("raystreaming.partition", FORWARD_PARTITION_CLASS);

streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/impl/RoundRobinPartitionFunction.java

@@ -25,8 +25,9 @@ public int[] partition(T value, int currentIndex, int numPartition) {
 
   @Override
   public int[] partition(T record, int numPartition) {
-    // TODO
-    return new int[0];
+    seq = (seq + 1) % numPartition;
+    partitions[0] = seq;
+    return partitions;
   }
 
   @Override

streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/AbstractStreamOperator.java

@@ -54,6 +54,7 @@ public void setFunction(F function) {
 
   @Override
   public void open(List<Collector> collectorList, RuntimeContext runtimeContext) {
+    LOG.info("Abstract {}, {} open : {}.", this.getId(), this.getName(), collectorList.size());
     this.collectorList = collectorList;
     this.runtimeContext = runtimeContext;
     if (runtimeContext != null && runtimeContext.getOpConfig() != null) {

streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/chain/ChainedOperator.java

@@ -59,7 +59,7 @@ public ChainedOperator(
   @Override
   public void open(List<Collector> collectorList, RuntimeContext runtimeContext) {
     // Dont' call super.open() as we `open` every operator separately.
-    LOG.info("chainedOperator open.");
+    LOG.info("ChainedOperator open.");
     for (int i = 0; i < operators.size(); i++) {
       StreamOperator operator = operators.get(i);
       List<Collector> succeedingCollectors = new ArrayList<>();
@@ -77,6 +77,14 @@ public void open(List<Collector> collectorList, RuntimeContext runtimeContext) {
                                   (collector.getId() == operator.getId()
                                       && collector.getDownStreamOpId() == subOperator.getId()))
                           .collect(Collectors.toList()));
+                  // FIXME(lingxuan.zlx): Workaround for edge mismatch, see more detail from
+                  // https://github.com/ray-project/mobius/issues/67.
+                  if (succeedingCollectors.isEmpty()) {
+                    succeedingCollectors.addAll(
+                        collectorList.stream()
+                            .filter(x -> (x.getDownStreamOpId() == subOperator.getId()))
+                            .collect(Collectors.toList()));
+                  }
                 }
               });
       operator.open(succeedingCollectors, createRuntimeContext(runtimeContext, i));

…reaming/runtime/util/BinaryFileUtil.java …reaming/common/utils/BinaryFileUtil.javastreaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/BinaryFileUtil.java renamed to streaming/java/streaming-common/src/main/java/io/ray/streaming/common/utils/BinaryFileUtil.java streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/BinaryFileUtil.java renamed to streaming/java/streaming-common/src/main/java/io/ray/streaming/common/utils/BinaryFileUtil.java

@@ -1,4 +1,4 @@
-package io.ray.streaming.runtime.util;
+package io.ray.streaming.common.utils;
 
 import com.google.common.base.Preconditions;
 import java.io.File;

streaming/java/streaming-common/src/main/java/io/ray/streaming/common/utils/JniUtils.java

@@ -2,7 +2,6 @@
 
 import com.google.common.collect.Sets;
 import com.sun.jna.NativeLibrary;
-import io.ray.runtime.util.BinaryFileUtil;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;