Use file based schema and named args for SqlContext main

pom.xml

@@ -106,7 +106,7 @@
         <scala.mayor.version>2.12</scala.mayor.version>
         <spark.version>3.4.4</spark.version>
         <flink.version>1.20.0</flink.version>
-        <calcite.version>1.39.0</calcite.version>       
+        <calcite.version>1.39.0</calcite.version>
 
         <java.version>17</java.version>
         <source.level>17</source.level>
@@ -785,6 +785,12 @@
                 <artifactId>guava</artifactId>
                 <version>${guava.version}</version>
             </dependency>
+            <!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
+            <dependency>
+                <groupId>commons-cli</groupId>
+                <artifactId>commons-cli</artifactId>
+                <version>1.3.1</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

wayang-api/pom.xml

@@ -40,6 +40,7 @@
         <module>wayang-api-python</module>
         <module>wayang-api-sql</module>
         <module>wayang-api-json</module>
+        <module>wayang-api-utils</module>
     </modules>
 
 </project>

wayang-api/wayang-api-scala-java/pom.xml

@@ -105,6 +105,11 @@
           <artifactId>wayang-spark</artifactId>
           <version>1.0.1-SNAPSHOT</version>
         </dependency>
+        <dependency>
+          <groupId>org.apache.wayang</groupId>
+          <artifactId>wayang-tensorflow</artifactId>
+          <version>1.0.1-SNAPSHOT</version>
+        </dependency>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-core_2.12</artifactId>

wayang-api/wayang-api-sql/pom.xml

@@ -25,7 +25,7 @@
         <version>1.0.1-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-    
+
     <artifactId>wayang-api-sql</artifactId>
     <dependencies>
         <dependency>
@@ -49,6 +49,12 @@
             <version>1.0.1-SNAPSHOT</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.wayang</groupId>
+            <artifactId>wayang-api-utils</artifactId>
+            <version>1.0.1-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.wayang</groupId>
             <artifactId>wayang-postgres</artifactId>

wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/context/SqlContext.java

@@ -19,7 +19,6 @@
 
 import org.apache.commons.lang3.StringUtils;
 
-
 import org.apache.calcite.jdbc.CalciteSchema;
 import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
 import org.apache.calcite.rel.RelNode;
@@ -38,22 +37,30 @@
 import org.apache.wayang.basic.data.Record;
 import org.apache.wayang.core.api.Configuration;
 import org.apache.wayang.core.plugin.Plugin;
+import org.apache.wayang.api.utils.Parameters;
 import org.apache.wayang.core.api.WayangContext;
 import org.apache.wayang.core.util.ReflectionUtils;
 import org.apache.wayang.core.plan.wayangplan.WayangPlan;
 import org.apache.wayang.java.Java;
 import org.apache.wayang.postgres.Postgres;
 import org.apache.wayang.spark.Spark;
+import org.apache.commons.cli.*;
+
+import com.google.common.io.Resources;
 
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
 
+import scala.collection.JavaConversions;
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.charset.Charset;
 import java.nio.file.Paths;
+import java.net.URL;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Properties;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -103,43 +110,46 @@ public static void main(final String[] args) throws Exception {
             throw new IllegalArgumentException(
                     "Usage: ./bin/wayang-submit org.apache.wayang.api.sql.SqlContext <SQL statement path> <JDBC driver> <JDBC URL> <JDBC user> <JDBC password> <Result output path> [platforms...]");
 
-        final String queryPath = args[0];
-        final String jdbcDriver = args[1];
-        final String jdbcUrl = args[2];
-        final String jdbcUser = args[3];
-        final String jdbcPassword = args[4];
-        final String outputPath = args[5];
+        //Specify the named arguments
+        Options options = new Options();
+        options.addOption("p", "platforms", true, "[platforms...]");
+        options.addOption("s", "schema", true, "Schema path");
+        options.addOption("q", "query", true, "SQL statement path");
+        options.addOption("o", "outputPath", true, "Output path");
+        options.addOption("d", "data", true, "Data path for file-based schema");
+        options.addOption("c", "config", true, "File path for config file");
+        options.addOption("jdbcDriver", true, "JDBC driver");
+        options.addOption("jdbcUrl", true, "JDBC URL");
+        options.addOption("jdbcPassword", true, "JDBC URL");
+
+        CommandLineParser parser = new DefaultParser();
+        CommandLine cmd = parser.parse(options, args);
+
+        final String queryPath = cmd.getOptionValue("q");
+        final String jdbcDriver = cmd.getOptionValue("jdbcDriver");
+        final String jdbcUrl = cmd.getOptionValue("jdbcUrl");
+        final String jdbcUser = cmd.getOptionValue("jdbcUser");
+        final String jdbcPassword = cmd.getOptionValue("jdbcPassword");
+        final String outputPath = cmd.getOptionValue("o");
+        final String dataPath = cmd.getOptionValue("d");
+        final String schemaPath = cmd.getOptionValue("s");
 
         final String query = StringUtils.chop(
                 Files.readString(Paths.get(queryPath))
                         .stripTrailing());
 
         final String driverPlatform = jdbcDriver.split("\\.")[0];
 
-        final String calciteModel = String.format(
-                "{\r\n" +
-                        "\"calcite\": {\r\n" +
-                        "    \"version\": \"1.0\",\n" +
-                        "    \"defaultSchema\": \"wayang\",\n" +
-                        "    \"schemas\": [\n" +
-                        "        {\n" +
-                        "            \"name\": \"postgres\",\n" +
-                        "            \"type\": \"custom\",\n" +
-                        "            \"factory\": \"org.apache.wayang.api.sql.calcite.jdbc.JdbcSchema$Factory\",\n" +
-                        "            \"operand\": {\n" +
-                        "                \"jdbcDriver\": \"%s\",\n" +
-                        "                \"jdbcUrl\": \"%s\",\n" +
-                        "                \"jdbcUser\": \"%s\",\n" +
-                        "                \"jdbcPassword\": \"%s\"\n" +
-                        "            }\n" +
-                        "        }\n" +
-                        "    ]\n" +
-                        "}\r\n" +
-                        "}",
-                jdbcDriver, jdbcUrl, jdbcUser, jdbcPassword);
-
         final Configuration configuration = new Configuration();
-        configuration.load(ReflectionUtils.loadResource("wayang-defaults.properties"));
+
+        if (cmd.hasOption("c")) {
+            configuration.load(cmd.getOptionValue("c"));
+        }
+
+        final String calciteModel = Resources.toString(
+                new URL(schemaPath),
+                Charset.defaultCharset()
+        );
 
         configuration.setProperty("wayang.calcite.model", calciteModel);
         configuration.setProperty(String.format("wayang.%s.jdbc.url", driverPlatform), jdbcUrl);
@@ -153,23 +163,8 @@ public static void main(final String[] args) throws Exception {
         final SqlContext context = new SqlContext(parseModel,
                 List.of(Java.channelConversionPlugin(), Postgres.conversionPlugin()));
 
-        for (int i = 6; i < args.length; i++) {
-            final String platform = args[i];
-
-            switch (platform.toLowerCase()) {
-                case "spark":
-                    context.withPlugin(Spark.basicPlugin());
-                    break;
-                case "java":
-                    context.withPlugin(Java.basicPlugin());
-                    break;
-                case "postgres":
-                    context.withPlugin(Postgres.plugin());
-                    break;
-                default:
-                    throw new IllegalArgumentException("platform not supported " + platform);
-            }
-        }
+        List<Plugin> plugins = JavaConversions.seqAsJavaList(Parameters.loadPlugins(cmd.getOptionValue("p")));
+        plugins.stream().forEach(context::register);
 
         final Collection<Record> result = context.executeSql(query);
 

wayang-api/wayang-api-utils/pom.xml

@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>wayang-api</artifactId>
+        <groupId>org.apache.wayang</groupId>
+        <version>1.0.1-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>wayang-api-utils</artifactId>
+    <version>1.0.1-SNAPSHOT</version>
+
+    <name>Wayang API Utils</name>
+    <description>Wayang implementation of an API of Scala-Java to be enable to work in functional style</description>
+
+    <properties>
+        <java-module-name>org.apache.wayang.api</java-module-name>
+        <tensorflow.version>1.0.0</tensorflow.version>
+    </properties>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.apache.wayang</groupId>
+                <artifactId>wayang-commons</artifactId>
+                <version>1.0.1-SNAPSHOT</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.wayang</groupId>
+            <artifactId>wayang-core</artifactId>
+            <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.wayang</groupId>
+            <artifactId>wayang-basic</artifactId>
+            <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.wayang</groupId>
+            <artifactId>wayang-utils-profile-db</artifactId>
+            <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.wayang</groupId>
+            <artifactId>wayang-api-python</artifactId>
+            <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.12.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.wayang</groupId>
+            <artifactId>wayang-java</artifactId>
+            <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.wayang</groupId>
+            <artifactId>wayang-sqlite3</artifactId>
+            <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.wayang</groupId>
+            <artifactId>wayang-postgres</artifactId>
+            <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-hdfs</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.wayang</groupId>
+          <artifactId>wayang-spark</artifactId>
+          <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.wayang</groupId>
+          <artifactId>wayang-tensorflow</artifactId>
+          <version>1.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-core_2.12</artifactId>
+          <version>${spark.version}</version>
+          <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-graphx_2.12</artifactId>
+          <version>${spark.version}</version>
+          <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-mllib_2.12</artifactId>
+          <version>${spark.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.scala-lang</groupId>
+          <artifactId>scala-library</artifactId>
+          <version>${scala.version}</version>
+          <scope>provided</scope>
+        </dependency>
+        <dependency>
+          <groupId>io.netty</groupId>
+          <artifactId>netty-all</artifactId>
+          <version>4.1.89.Final</version>
+        </dependency>
+        <dependency>
+          <groupId>com.google.protobuf</groupId>
+          <artifactId>protobuf-java</artifactId>
+          <version>3.25.5</version>
+        </dependency>
+         <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.15.4</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-scala_2.12</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.tensorflow</groupId>
+            <artifactId>tensorflow-core-platform</artifactId>
+            <version>${tensorflow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.tensorflow</groupId>
+            <artifactId>tensorflow-framework</artifactId>
+            <version>${tensorflow.version}</version>
+        </dependency>
+    </dependencies>
+</project>