Apache Arrow upgrade (#17)

* upgrade apache arrow to 9.0.0

* testing with multiple jdk versions

* changelog updates

Author: NRHelmi

.github/workflows/maven-build.yaml

@@ -9,16 +9,22 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        java-version: [ 11, 17, 18 ]
+
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 11
+      - name: Set up JDK
         uses: actions/setup-java@v2
         with:
-          java-version: '11'
+          java-version: ${{ matrix.java-version }}
           distribution: 'adopt'
       - name: Build with Maven
         env:
           CLIENT_ID: ${{ secrets.CLIENT_ID }}
           CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
           CLIENT_CREDENTIALS_URL: ${{ secrets.CLIENT_CREDENTIALS_URL }}
-        run: mvn clean install
+        run: |
+          export JAVA_TOOL_OPTIONS="--add-opens=java.base/java.nio=ALL-UNNAMED"
+          mvn clean install

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v0.4.1-alpha
+* Adding support to new java versions.
+* Upgrading `apache arrow` version to `9.0.0`.
+* Fix protobuf metadata integration test.
+* Run integration tests using jdks 11, 17 & 18.
+* Fix memory leak issue
+* If seeing errors like module java.base does not "opens java.nio" to unnamed module
+please make sure to run java with `--add-opens=java.base/java.nio=ALL-UNNAMED` argument
+or set JAVA_TOOL_OPTIONS environment variable
+```shell
+export JAVA_TOOL_OPTIONS="--add-opens=java.base/java.nio=ALL-UNNAMED"
+```
 ## v0.4.0-alpha
 * Renamed:
   - `execute` to `executeV1`.

README.md

@@ -57,6 +57,10 @@ Note that `mvn install` is required to build and run the examples.
 
     mvn clean
 
+**If facing errors like ` module java.base does not "opens java.nio" to unnamed module`**
+
+    export JAVA_TOOL_OPTIONS="--add-opens=java.base/java.nio=ALL-UNNAMED"
+
 ### Create a configuration file
 
 In order to run the examples you will need to create an SDK config file.
@@ -86,7 +90,7 @@ In order to use the `rai-sdk-java`, you need add this dependency to your project
     <dependency>
         <groupId>com.relationalai</groupId>
         <artifactId>rai-sdk</artifactId>
-        <version>0.4.0-alpha</version>
+        <version>0.4.1-alpha</version>
     </dependency>
 
 You need also to point maven to the SDK GitHub packages repository in the project's POM:

pom.xml

@@ -21,7 +21,7 @@
     <description>The RelationalAI Software Development Kit (SDK) for Java</description>
     <groupId>com.relationalai</groupId>
     <artifactId>rai-sdk-pom</artifactId>
-    <version>0.4.0-alpha</version>
+    <version>0.4.1-alpha</version>
     <packaging>pom</packaging>
     <url></url>
 
@@ -37,7 +37,7 @@
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
         <jsoniter.version>0.9.19</jsoniter.version>
-        <apache.arrow.version>7.0.0</apache.arrow.version>
+        <apache.arrow.version>9.0.0</apache.arrow.version>
         <protobuf.version>3.21.1</protobuf.version>
         <protobuf-java-format.version>1.4</protobuf-java-format.version>
     </properties>

rai-sdk-examples/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <groupId>com.relationalai</groupId>
         <artifactId>rai-sdk-pom</artifactId>
-        <version>0.4.0-alpha</version>
+        <version>0.4.1-alpha</version>
     </parent>
 
     <name>RelationalAI SDK for Java Examples</name>

rai-sdk-protos/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <groupId>com.relationalai</groupId>
         <artifactId>rai-sdk-pom</artifactId>
-        <version>0.4.0-alpha</version>
+        <version>0.4.1-alpha</version>
     </parent>
 
     <name>RelationalAI SDK Protobuf for Java</name>

rai-sdk/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <groupId>com.relationalai</groupId>
         <artifactId>rai-sdk-pom</artifactId>
-        <version>0.4.0-alpha</version>
+        <version>0.4.1-alpha</version>
     </parent>
 
     <name>RelationalAI SDK for Java Package</name>

rai-sdk/src/main/java/com/relationalai/ArrowUtils.java

@@ -0,0 +1,20 @@
+package com.relationalai;
+
+import org.apache.arrow.memory.RootAllocator;
+
+public class ArrowUtils implements AutoCloseable {
+    private static RootAllocator allocator;
+
+    public static RootAllocator getOrCreateRootAllocator() {
+        if (allocator == null) {
+            allocator = new RootAllocator(Long.MAX_VALUE);
+        }
+
+        return allocator;
+    }
+
+    @Override
+    public void close() throws Exception {
+        allocator.close();
+    }
+}

rai-sdk/src/main/java/com/relationalai/Client.java

@@ -286,7 +286,7 @@ private List<ArrowRelation> readArrowFiles(List<TransactionAsyncFile> files) thr
             if ("application/vnd.apache.arrow.stream".equals(file.contentType.toLowerCase())) {
                 ByteArrayInputStream in = new ByteArrayInputStream(file.data);
                 List<FieldVector> fieldVectors = null;
-                RootAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+                RootAllocator allocator = ArrowUtils.getOrCreateRootAllocator();
 
                 try(ArrowStreamReader arrowStreamReader = new ArrowStreamReader(in, allocator)){
                     VectorSchemaRoot root = arrowStreamReader.getVectorSchemaRoot();
@@ -313,7 +313,6 @@ private List<ArrowRelation> readArrowFiles(List<TransactionAsyncFile> files) thr
                             }
                         }
                     }
-                    allocator.close();
                 }
             }
         }

rai-sdk/src/test/resources/metadata.pb

@@ -2,8 +2,8 @@
 >
 3
 "
-

-�
output
+

+�
output
 

 

 


rai-sdk/src/test/resources/metadata.pb.txt

@@ -5,11 +5,11 @@ relations {
       constant_type {
         rel_type {
           tag: PRIMITIVE_TYPE
-          primitive_type: SYMBOL
+          primitive_type: STRING
         }
         value {
           arguments {
-            tag: SYMBOL
+            tag: STRING
             string_val: "output"
           }
         }