Bring Apache Arrow classes into arrow-spi library so those could be shared between plugins

Signed-off-by: Andriy Redko <[email protected]>

Author: reta

gradle/libs.versions.toml

@@ -83,7 +83,7 @@ opentelemetry         = "1.46.0"
 opentelemetrysemconv  = "1.29.0-alpha"
 
 # arrow dependencies
-arrow                 = "18.1.0"
+arrow                 = "18.2.0"
 flatbuffers           = "2.0.0"
 
 [libraries]

libs/arrow-spi/build.gradle

@@ -13,8 +13,20 @@ testingConventions.enabled = false
 
 dependencies {
   api project(':libs:opensearch-core')
+  api "org.apache.arrow:arrow-memory-core:${versions.arrow}"
 }
 
 tasks.named('forbiddenApisMain').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
+
+tasks.named('thirdPartyAudit').configure {
+  ignoreMissingClasses(
+    'org.slf4j.Logger',
+    'org.slf4j.LoggerFactory'
+  )
+  ignoreViolations(
+    'org.apache.arrow.memory.util.MemoryUtil',
+    'org.apache.arrow.memory.util.MemoryUtil$1'
+  )
+}

libs/arrow-spi/licenses/arrow-memory-core-18.2.0.jar.sha1

@@ -0,0 +1 @@
+af2e83d46089243245decb0c4a6f8bc4231a35f0

plugins/arrow-flight-rpc/licenses/arrow-memory-core-LICENSE.txt renamed to libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt

@@ -34,7 +34,7 @@ public interface StreamManager extends AutoCloseable {
      * @return A StreamTicket that can be used to access the stream
      * @throws IllegalArgumentException if producer is null or parentTaskId is invalid
      */
-    <VectorRoot, Allocator> StreamTicket registerStream(StreamProducer<VectorRoot, Allocator> producer, TaskId parentTaskId);
+    <VectorRoot> StreamTicket registerStream(StreamProducer<VectorRoot> producer, TaskId parentTaskId);
 
     /**
      * Creates a stream reader for consuming Arrow data using a valid ticket.

plugins/arrow-flight-rpc/licenses/arrow-memory-core-NOTICE.txt renamed to libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt

@@ -8,6 +8,7 @@
 
 package org.opensearch.arrow.spi;
 
+import org.apache.arrow.memory.BufferAllocator;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.core.tasks.TaskId;
 
@@ -75,7 +76,7 @@
  * @see StreamReader
  */
 @ExperimentalApi
-public interface StreamProducer<VectorRoot, Allocator> extends Closeable {
+public interface StreamProducer<VectorRoot> extends Closeable {
 
     /**
      * Creates a VectorSchemaRoot that defines the schema for this stream. This schema will be used
@@ -84,7 +85,7 @@ public interface StreamProducer<VectorRoot, Allocator> extends Closeable {
      * @param allocator The allocator to use for creating vectors
      * @return A new VectorSchemaRoot instance
      */
-    VectorRoot createRoot(Allocator allocator);
+    VectorRoot createRoot(BufferAllocator allocator);
 
     /**
      * Creates a job that will produce the stream data in batches. The job will populate
@@ -93,7 +94,7 @@ public interface StreamProducer<VectorRoot, Allocator> extends Closeable {
      * @param allocator The allocator to use for any additional memory allocations
      * @return A new BatchedJob instance
      */
-    BatchedJob<VectorRoot> createJob(Allocator allocator);
+    BatchedJob<VectorRoot> createJob(BufferAllocator allocator);
 
     /**
      * Provides an estimate of the total number of rows that will be produced.

libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java

@@ -23,7 +23,6 @@ dependencies {
   implementation "org.apache.arrow:arrow-vector:${versions.arrow}"
   implementation "org.apache.arrow:arrow-format:${versions.arrow}"
   implementation "org.apache.arrow:flight-core:${versions.arrow}"
-  implementation "org.apache.arrow:arrow-memory-core:${versions.arrow}"
 
   runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}"
   runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}"
@@ -227,7 +226,8 @@ tasks.named('thirdPartyAudit').configure {
     'reactor.blockhound.BlockHound$Builder',
     'reactor.blockhound.integration.BlockHoundIntegration',
 
-    'com.google.protobuf.util.Timestamps'
+    'com.google.protobuf.util.Timestamps',
+    'io.grpc.stub.BlockingClientCall'
   )
   ignoreViolations(
     // Guava internal classes
@@ -293,9 +293,6 @@ tasks.named('thirdPartyAudit').configure {
     'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField',
     'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
     'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess',
-    'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
-    'org.apache.arrow.memory.util.MemoryUtil',
-    'org.apache.arrow.memory.util.MemoryUtil$1'
-
+    'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess'
   )
 }

libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java

@@ -0,0 +1 @@
+34c4c28ddc01038498b89125c3a9f2329c2227ae