Skip to content

Commit 5446160

Browse files
msprucmspruc
committed
initial commit
1 parent 68d49c9 commit 5446160

1,385 files changed

Lines changed: 35242 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎wayang-plugins/wayang-ml/.project‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<projectDescription>
3+
<name>wayang-ml</name>
4+
<comment></comment>
5+
<projects>
6+
</projects>
7+
<buildSpec>
8+
<buildCommand>
9+
<name>org.eclipse.jdt.core.javabuilder</name>
10+
<arguments>
11+
</arguments>
12+
</buildCommand>
13+
<buildCommand>
14+
<name>org.eclipse.m2e.core.maven2Builder</name>
15+
<arguments>
16+
</arguments>
17+
</buildCommand>
18+
</buildSpec>
19+
<natures>
20+
<nature>org.eclipse.jdt.core.javanature</nature>
21+
<nature>org.eclipse.m2e.core.maven2Nature</nature>
22+
</natures>
23+
<filteredResources>
24+
<filter>
25+
<id>1715926807014</id>
26+
<name></name>
27+
<type>30</type>
28+
<matcher>
29+
<id>org.eclipse.core.resources.regexFilterMatcher</id>
30+
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
31+
</matcher>
32+
</filter>
33+
</filteredResources>
34+
</projectDescription>

‎wayang-plugins/wayang-ml/pom.xml‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing,
14+
~ software distributed under the License is distributed on an
15+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
~ KIND, either express or implied. See the License for the
17+
~ specific language governing permissions and limitations
18+
~ under the License.
19+
~
20+
-->
21+
22+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
25+
<parent>
26+
<groupId>org.apache.wayang</groupId>
27+
<artifactId>wayang-plugins</artifactId>
28+
<version>0.7.1</version>
29+
</parent>
30+
31+
<artifactId>wayang-ml</artifactId>
32+
<version>0.7.1</version>
33+
34+
<properties>
35+
<java-module-name>org.apache.wayang.extensions.ml</java-module-name>
36+
</properties>
37+
38+
<dependencies>
39+
<dependency>
40+
<groupId>org.apache.wayang</groupId>
41+
<artifactId>wayang-api-sql</artifactId>
42+
<version>0.7.1</version>
43+
</dependency>
44+
<dependency>
45+
<groupId>com.microsoft.onnxruntime</groupId>
46+
<artifactId>onnxruntime</artifactId>
47+
<version>1.21.1</version>
48+
</dependency>
49+
<!--dependency>
50+
<groupId>com.microsoft.onnxruntime</groupId>
51+
<artifactId>onnxruntime_gpu</artifactId>
52+
<version>1.18.0</version>
53+
</dependency-->
54+
<dependency>
55+
<groupId>org.apache.wayang</groupId>
56+
<artifactId>wayang-core</artifactId>
57+
<version>0.7.1</version>
58+
</dependency>
59+
<dependency>
60+
<groupId>org.apache.wayang</groupId>
61+
<artifactId>wayang-basic</artifactId>
62+
<version>0.7.1</version>
63+
</dependency>
64+
<dependency>
65+
<groupId>org.apache.wayang</groupId>
66+
<artifactId>wayang-java</artifactId>
67+
<version>0.7.1</version>
68+
</dependency>
69+
<dependency>
70+
<groupId>org.apache.wayang</groupId>
71+
<artifactId>wayang-spark</artifactId>
72+
<version>0.7.1</version>
73+
</dependency>
74+
<dependency>
75+
<groupId>org.apache.wayang</groupId>
76+
<artifactId>wayang-flink</artifactId>
77+
<version>0.7.1</version>
78+
</dependency>
79+
<dependency>
80+
<groupId>org.apache.flink</groupId>
81+
<artifactId>flink-java</artifactId>
82+
<version>${flink.version}</version>
83+
</dependency>
84+
<dependency>
85+
<groupId>org.apache.wayang</groupId>
86+
<artifactId>wayang-giraph</artifactId>
87+
<version>0.7.1</version>
88+
</dependency>
89+
<dependency>
90+
<groupId>org.apache.wayang</groupId>
91+
<artifactId>wayang-generic-jdbc</artifactId>
92+
<version>0.7.1</version>
93+
</dependency>
94+
<dependency>
95+
<groupId>org.reflections</groupId>
96+
<artifactId>reflections</artifactId>
97+
<version>0.10.2</version>
98+
</dependency>
99+
<dependency>
100+
<groupId>org.apache.wayang</groupId>
101+
<artifactId>wayang-benchmark</artifactId>
102+
<version>0.7.1</version>
103+
</dependency>
104+
<dependency>
105+
<groupId>org.apache.wayang</groupId>
106+
<artifactId>wayang-api-python</artifactId>
107+
<version>0.7.1</version>
108+
</dependency>
109+
<dependency>
110+
<groupId>org.apache.commons</groupId>
111+
<artifactId>commons-dbcp2</artifactId>
112+
<version>2.7.0</version>
113+
</dependency>
114+
<dependency>
115+
<groupId>org.apache.spark</groupId>
116+
<artifactId>spark-core_2.12</artifactId>
117+
<version>${spark.version}</version>
118+
</dependency>
119+
<dependency>
120+
<groupId>org.apache.spark</groupId>
121+
<artifactId>spark-graphx_2.12</artifactId>
122+
<version>${spark.version}</version>
123+
</dependency>
124+
<dependency>
125+
<groupId>org.apache.spark</groupId>
126+
<artifactId>spark-mllib_2.12</artifactId>
127+
<version>${spark.version}</version>
128+
</dependency>
129+
<dependency>
130+
<groupId>com.google.protobuf</groupId>
131+
<artifactId>protobuf-java</artifactId>
132+
<version>3.16.3</version>
133+
</dependency>
134+
<dependency>
135+
<groupId>org.apache.calcite</groupId>
136+
<artifactId>calcite-core</artifactId>
137+
<version>${calcite.version}</version>
138+
</dependency>
139+
<dependency>
140+
<groupId>org.apache.calcite</groupId>
141+
<artifactId>calcite-linq4j</artifactId>
142+
<version>${calcite.version}</version>
143+
</dependency>
144+
<dependency>
145+
<groupId>org.apache.calcite</groupId>
146+
<artifactId>calcite-file</artifactId>
147+
<version>${calcite.version}</version>
148+
</dependency>
149+
</dependencies>
150+
<build>
151+
<resources>
152+
<resource>
153+
<directory>src/main/resources</directory>
154+
</resource>
155+
</resources>
156+
</build>
157+
</project>

‎wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.wayang.ml;
20+
21+
import org.apache.wayang.core.api.WayangContext;
22+
import org.apache.wayang.core.api.exception.WayangException;
23+
import org.apache.logging.log4j.Level;
24+
import org.apache.wayang.core.api.Configuration;
25+
import org.apache.wayang.core.api.Job;
26+
import org.apache.wayang.core.plan.wayangplan.WayangPlan;
27+
import org.apache.wayang.core.plan.executionplan.ExecutionPlan;
28+
import org.apache.wayang.core.optimizer.DefaultOptimizationContext;
29+
import org.apache.wayang.core.optimizer.OptimizationContext;
30+
import org.apache.wayang.core.util.ReflectionUtils;
31+
import org.apache.wayang.ml.costs.PairwiseCost;
32+
import org.apache.wayang.ml.encoding.OneHotMappings;
33+
import org.apache.wayang.ml.encoding.OrtTensorEncoder;
34+
import org.apache.wayang.ml.encoding.TreeEncoder;
35+
import org.apache.wayang.ml.encoding.TreeNode;
36+
import org.apache.wayang.ml.util.EnumerationStrategy;
37+
import org.apache.wayang.ml.util.Logging;
38+
import org.apache.wayang.core.util.Tuple;
39+
import org.apache.logging.log4j.Level;
40+
41+
import java.io.IOException;
42+
import java.io.BufferedWriter;
43+
import java.io.FileWriter;
44+
import java.time.Instant;
45+
import java.time.Duration;
46+
47+
import java.util.ArrayList;
48+
import java.util.Optional;
49+
import java.util.Collection;
50+
51+
/**
52+
* This is the entry point for users to work with Wayang ML.
53+
*/
54+
public class MLContext extends WayangContext {
55+
56+
private OrtMLModel model;
57+
58+
private EnumerationStrategy enumerationStrategy = EnumerationStrategy.NONE;
59+
60+
public MLContext() {
61+
super();
62+
}
63+
64+
public MLContext(Configuration configuration) {
65+
super(configuration);
66+
}
67+
68+
/**
69+
* Execute a plan.
70+
*
71+
* @param wayangPlan the plan to execute
72+
* @param udfJars JARs that declare the code for the UDFs
73+
* @see ReflectionUtils#getDeclaringJar(Class)
74+
*/
75+
@Override
76+
public void execute(WayangPlan wayangPlan, String... udfJars) {
77+
this.setLogLevel(Level.ERROR);
78+
Job wayangJob = this.createJob("", wayangPlan, udfJars);
79+
OneHotMappings.setOptimizationContext(wayangJob.getOptimizationContext());
80+
81+
Configuration config = this.getConfiguration();
82+
Configuration jobConfig = wayangJob.getConfiguration();
83+
84+
wayangJob.execute();
85+
86+
if (config.getBooleanProperty("wayang.ml.experience.enabled")) {
87+
String original;
88+
89+
Optional<String> originalOption = config.getOptionalStringProperty("wayang.ml.experience.original");
90+
if (originalOption.isPresent()) {
91+
original = originalOption.get();
92+
} else {
93+
original = TreeEncoder.encode(wayangPlan).toString();
94+
}
95+
96+
String withChoices;
97+
98+
Optional<String> choicesOption = config.getOptionalStringProperty("wayang.ml.experience.with-platforms");
99+
if (choicesOption.isPresent()) {
100+
withChoices = choicesOption.get();
101+
} else {
102+
withChoices = jobConfig.getStringProperty("wayang.ml.experience.with-platforms");
103+
}
104+
105+
long execTime = jobConfig.getLongProperty("wayang.ml.experience.exec-time");
106+
107+
this.logExperience(original, withChoices, execTime);
108+
}
109+
}
110+
111+
public void executeVAE(WayangPlan wayangPlan, String ...udfJars) {
112+
this.setLogLevel(Level.ERROR);
113+
try {
114+
Job job = this.createJob("", wayangPlan, udfJars);
115+
Configuration jobConfig = job.getConfiguration();
116+
//job.prepareWayangPlan();
117+
job.estimateKeyFigures();
118+
OneHotMappings.setOptimizationContext(job.getOptimizationContext());
119+
OneHotMappings.encodeIds = true;
120+
121+
// Log Encoding time
122+
Instant start = Instant.now();
123+
TreeNode wayangNode = TreeEncoder.encode(wayangPlan);
124+
125+
Instant end = Instant.now();
126+
long execTime = Duration.between(start, end).toMillis();
127+
Logging.writeToFile(
128+
String.format("Encoding: %d", execTime),
129+
this.getConfiguration().getStringProperty("wayang.ml.optimizations.file")
130+
);
131+
OrtMLModel model = OrtMLModel.getInstance(job.getConfiguration());
132+
// Log inference time
133+
start = Instant.now();
134+
Tuple<WayangPlan, TreeNode> resultTuple = model.runVAE(wayangPlan, wayangNode);
135+
end = Instant.now();
136+
execTime = Duration.between(start, end).toMillis();
137+
138+
WayangPlan platformPlan = resultTuple.field0;
139+
140+
this.getConfiguration().setProperty(
141+
"wayang.ml.experience.original",
142+
wayangNode.toStringEncoding()
143+
);
144+
145+
this.getConfiguration().setProperty(
146+
"wayang.ml.experience.with-platforms",
147+
resultTuple.field1.toString()
148+
);
149+
150+
this.execute(platformPlan, udfJars);
151+
} catch (Exception e) {
152+
e.printStackTrace();
153+
throw new WayangException("Executing WayangPlan with VAE model failed");
154+
}
155+
}
156+
157+
public ExecutionPlan buildWithVAE(WayangPlan wayangPlan, String ...udfJars) {
158+
try {
159+
Job job = this.createJob("", wayangPlan, udfJars);
160+
job.estimateKeyFigures();
161+
OneHotMappings.setOptimizationContext(job.getOptimizationContext());
162+
OneHotMappings.encodeIds = true;
163+
164+
TreeNode wayangNode = TreeEncoder.encode(wayangPlan);
165+
OrtMLModel model = OrtMLModel.getInstance(job.getConfiguration());
166+
Tuple<WayangPlan, TreeNode> resultTuple = model.runVAE(wayangPlan, wayangNode);
167+
WayangPlan platformPlan = resultTuple.field0;
168+
169+
return this.buildInitialExecutionPlan("", platformPlan, udfJars);
170+
} catch (Exception e) {
171+
e.printStackTrace();
172+
throw new WayangException("Executing WayangPlan with VAE model failed");
173+
}
174+
}
175+
176+
public void setModel(OrtMLModel model) {
177+
this.model = model;
178+
}
179+
180+
private void logExperience(String original, String withChoices, long execTime) {
181+
if (!this.getConfiguration().getBooleanProperty("wayang.ml.experience.enabled")) {
182+
return;
183+
}
184+
185+
String content = String.format("%s:%s:%d", original, withChoices, execTime);
186+
Logging.writeToFile(
187+
content,
188+
this.getConfiguration().getStringProperty("wayang.ml.experience.file")
189+
);
190+
}
191+
}

0 commit comments

Comments
 (0)