repo-sync-2024-12-18T16:45:42+0800 (#13)

### Added

- [Feature] SCQL is supported to query ODPS data sources

### Fixed

- Fixed the ODPS resource read and write issues

### Removed

- Removed support for localfs, OSS, and MySQL data sources, which are provided by Kuscia

Author: YanZhuangz

build/Dockerfiles/dataproxy.Dockerfile

@@ -20,9 +20,10 @@ WORKDIR /app
 # fix: RunP proot + java bug
 RUN ln -s ${JAVA_HOME}/lib/libjli.so /lib64
 
-COPY target/*.jar dataproxy.jar
-COPY config/application.yaml application.yaml
-COPY scripts/start_dp.sh start_dp.sh
-ENV JAVA_OPTS="" SPRING_PROFILES_ACTIVE="default"
+COPY dataproxy-server/target/dataproxy-server-0.0.1-SNAPSHOT.jar dataproxy.jar
+COPY libs/*.jar libs/
+
+ENV JAVA_OPTS=""
+ENV LOG_LEVEL=INFO
 EXPOSE 8023
-ENTRYPOINT ${JAVA_HOME}/bin/java ${JAVA_OPTS} -Dsun.net.http.allowRestrictedHeaders=true --add-opens=java.base/java.nio=ALL-UNNAMED -jar -Dspring.profiles.active=${SPRING_PROFILES_ACTIVE} ./dataproxy.jar
+ENTRYPOINT ${JAVA_HOME}/bin/java ${JAVA_OPTS} -Dsun.net.http.allowRestrictedHeaders=true --add-opens=java.base/java.nio=ALL-UNNAMED -jar ./dataproxy.jar

config/application.yaml

@@ -7,6 +7,7 @@
         <groupId>org.secretflow</groupId>
         <artifactId>dataproxy</artifactId>
         <version>0.0.1-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
     </parent>
 
     <artifactId>dataproxy-api</artifactId>
@@ -21,16 +22,19 @@
             <artifactId>protobuf-java-util</artifactId>
         </dependency>
         <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>flight-grpc</artifactId>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-protobuf</artifactId>
+            <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
-            <artifactId>grpc-netty-shaded</artifactId>
+            <artifactId>grpc-stub</artifactId>
+            <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>javax.annotation</groupId>
             <artifactId>javax.annotation-api</artifactId>
+            <scope>compile</scope>
         </dependency>
     </dependencies>
 

dataproxy-api/pom.xml

@@ -7,6 +7,7 @@
         <groupId>org.secretflow</groupId>
         <artifactId>dataproxy</artifactId>
         <version>0.0.1-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
     </parent>
 
     <artifactId>dataproxy-common</artifactId>
@@ -20,23 +21,24 @@
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
+            <scope>compile</scope>
         </dependency>
-        <dependency>
+        <!--<dependency>
             <groupId>com.fasterxml.jackson.datatype</groupId>
             <artifactId>jackson-datatype-jsr310</artifactId>
-        </dependency>
-        <dependency>
+        </dependency>-->
+        <!--<dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-collections4</artifactId>
-        </dependency>
-        <dependency>
+        </dependency>-->
+        <!--<dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
-        </dependency>
+        </dependency>-->
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-all</artifactId>
@@ -45,10 +47,10 @@
             <groupId>io.netty</groupId>
             <artifactId>netty-tcnative-boringssl-static</artifactId>
         </dependency>
-        <dependency>
+        <!--<dependency>
             <groupId>com.squareup.okio</groupId>
             <artifactId>okio</artifactId>
-        </dependency>
+        </dependency>-->
 
         <dependency>
             <groupId>com.google.protobuf</groupId>
@@ -80,12 +82,17 @@
             <groupId>org.apache.arrow</groupId>
             <artifactId>flight-core</artifactId>
         </dependency>
-        <dependency>
+        <!--<dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>flight-grpc</artifactId>
-        </dependency>
+        </dependency>-->
 
         <!-- apache arrow end-->
+
+        <!--<dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+        </dependency>-->
     </dependencies>
 
 </project>

dataproxy-common/pom.xml

@@ -21,8 +21,6 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import org.secretflow.dataproxy.common.model.datasource.conn.ConnConfig;
-import org.secretflow.dataproxy.common.utils.IdUtils;
-import org.secretflow.dataproxy.common.utils.JsonUtils;
 
 /**
  * Datasource connection config
@@ -46,7 +44,4 @@ public class DatasourceConnConfig {
      */
     private ConnConfig connConfig;
 
-    public String generateUniqueId() {
-        return IdUtils.combineIds(JsonUtils.toJSONString(this));
-    }
 }