Skip to content

Commit 886938e

Browse files
novatechflownovatechflow
authored
Merge pull request #468 from apache/kamir-patch-2
Kamir patch 2
2 parents a2f5ed7 + b95e18c commit 886938e

24 files changed

Lines changed: 1097 additions & 24 deletions

File tree

‎bin/wayang-submit‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818

1919
CLASS=$1
2020

21-
2221
if [ -z "${CLASS}" ]; then
2322
echo "Target Class for execution was not provided"
2423
exit 1
@@ -120,5 +119,7 @@ do
120119
ARGS="$ARGS \"${arg}\""
121120
done
122121

122+
WAYANG_CLASSPATH="${WAYANG_CLASSPATH}:${WAYANG_APP_HOME}"
123+
123124
eval "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${ARGS}"
124125

‎env_template_osx.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
export BOOTSTRAP_SERVER= ...
20+
export CLUSTER_API_KEY= ...
21+
export CLUSTER_API_SECRET= ...
22+
export SR_ENDPOINT= ...
23+
export SR_API_KEY= ...
24+
export SR_API_SECRET= ...
25+
export SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO=" ... : .... "
26+
export SCHEMA_REGISTRY_URL="https://.... "
27+
28+
export SPARK_HOME= ...
29+
export HADOOP_HOME= ...
30+
export PATH=$PATH:$HADOOP_HOME/bin
31+
export WAYANG_VERSION= ...
32+
export WAYANG_HOME= ...
33+
export WAYANG_APP_HOME= ...
34+
35+
echo "Hadoop home : $HADOOP_HOME"
36+
echo "Spark home : $SPARK_HOME"
37+
echo "Wayang home : $WAYANG_HOME"
38+
echo "Wayang app : $WAYANG_APP_HOME"
39+
echo "Wayang version : $WAYANG_VERSION"
40+
41+

‎pom.xml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1339,14 +1339,15 @@
13391339
<modules>
13401340
<module>wayang-commons</module>
13411341
<module>wayang-platforms</module>
1342-
<module>wayang-tests-integration</module>
13431342
<module>wayang-api</module>
13441343
<module>wayang-profiler</module>
13451344
<module>wayang-plugins</module>
13461345
<module>wayang-resources</module>
1347-
<module>wayang-benchmark</module>
13481346
<module>wayang-assembly</module>
13491347
<module>wayang-ml4all</module>
1350-
<!-- <module>wayang-docs</module> -->
1348+
<module>wayang-applications</module>
1349+
<module>wayang-benchmark</module>
1350+
<module>wayang-tests-integration</module>
1351+
<!--module>wayang-docs</module-->
13511352
</modules>
13521353
</project>

‎wayang-api/wayang-api-scala-java/src/main/scala/org/apache/wayang/api/DataQuanta.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -966,7 +966,8 @@ class DataQuanta[Out: ClassTag](val operator: ElementaryOperator, outputIndex: I
966966
topicName,
967967
new TransformationDescriptor(formatterUdf, basicDataUnitType[Out], basicDataUnitType[String], udfLoad)
968968
)
969-
sink.setName(s"Write to KafkaTopic $topicName")
969+
sink.setName(s"*#-> Write to KafkaTopic $topicName")
970+
println(s"*#-> Write to KafkaTopic $topicName")
970971
this.connectTo(sink, 0)
971972

972973
// Do the execution.
@@ -991,6 +992,7 @@ class DataQuanta[Out: ClassTag](val operator: ElementaryOperator, outputIndex: I
991992
new TransformationDescriptor(formatterUdf, basicDataUnitType[Out], basicDataUnitType[String], udfLoad)
992993
)
993994
sink.setName(s"Write to $url")
995+
994996
this.connectTo(sink, 0)
995997

996998
// Do the execution.

‎wayang-applications/README.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Wayang Applications
2+
3+
This module provides some example applications for using Apache Wayang in industrie specific scenarios.
4+
5+
## Example 1
6+
Our traditional word-count example for Kafka topics is provided in the script:
7+
8+
```bash
9+
run_wordcount_kafka.sh
10+
```
11+
12+
This script needs the configuration files:
13+
14+
```bash
15+
source .env.sh
16+
source env.demo1.sh
17+
```
18+
19+
Furthermore, the cluster properties are stored in the _default.properties_ file in the module with Kafka-Source and Kafka-Sink components.
20+
21+
**TODO:** We will improve this, by making the path to the Kafka client properties configurable soon.
22+
23+
24+
## Prerequisites
25+
26+
The following scripts use Apache Kafka topics as source and sink:
27+
28+
- run_wordcount_kafka.sh
29+
30+
In order to make the demo working, you need a proper cluster setup.
31+
Over time, we aim on a robust and reusable DEMO environment.
32+
For the beginning, we use a Confluent cloud cluster, and its specific CLI tool to setup and teardown the topics.
33+
34+
Later on, an improved solution will follow.
35+
36+
For now you need the following tools installed, in addition to the Wayang, Spark, Hadoop libraries:
37+
38+
- Confluent CLI tool.
39+
- jq
40+
41+
In OSX, both can be installed using homebrew. The installation and setup process for other environment are different.
42+
```bash
43+
brew install confluentinc/tap/cli
44+
brew install jq
45+
```
46+
47+
## Configuration
48+
49+
### Application environment
50+
The file named _.env.sh_ is ignored by git, hence it is the place
51+
for your personal configuration including credentials and cluster coordinates.
52+
An example is given in _env_template.sh_.
53+
54+
### DEMO Setup
55+
The file _env.demo1.sh_ contains additional properties, need in a particlar demo application.
56+
57+
In this file we will never see cluster or user specific details, only properties which are specific to the
58+
particular application are listed here.
59+
60+

‎wayang-applications/bin/bootstrap_ccloud_kafka_topics.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
source .env.sh
19+
source env.demo1.sh
20+
21+
confluent kafka topic delete $topic_l1_a --cluster $DEMO1_CLUSTER1
22+
confluent kafka topic delete $topic_l1_b --cluster $DEMO1_CLUSTER1
23+
confluent kafka topic delete $topic_l1_c --cluster $DEMO1_CLUSTER1
24+
confluent kafka topic delete $topic_l2_a --cluster $DEMO1_CLUSTER1
25+
confluent kafka topic delete $topic_l2_b --cluster $DEMO1_CLUSTER1
26+
27+
confluent kafka topic create $topic_l1_a --cluster $DEMO1_CLUSTER1
28+
confluent kafka topic create $topic_l1_b --cluster $DEMO1_CLUSTER1
29+
confluent kafka topic create $topic_l1_c --cluster $DEMO1_CLUSTER1
30+
confluent kafka topic create $topic_l2_a --cluster $DEMO1_CLUSTER1
31+
confluent kafka topic create $topic_l2_b --cluster $DEMO1_CLUSTER1
32+
33+
confluent kafka topic list --cluster $DEMO1_CLUSTER1
34+
35+
curl --silent -X GET -u $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO $SCHEMA_REGISTRY_URL/subjects | jq .

‎wayang-applications/bin/cleanup_ccloud_kafka_topics.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
source .env.sh
19+
source env.demo1.sh
20+
21+
confluent kafka topic delete $topic_l1_a --cluster $DEMO1_CLUSTER1
22+
confluent kafka topic delete $topic_l1_b --cluster $DEMO1_CLUSTER1
23+
confluent kafka topic delete $topic_l1_c --cluster $DEMO1_CLUSTER1
24+
confluent kafka topic delete $topic_l2_a --cluster $DEMO1_CLUSTER1
25+
confluent kafka topic delete $topic_l2_b --cluster $DEMO1_CLUSTER1
26+
27+
confluent kafka topic list --cluster $DEMO1_CLUSTER1

‎wayang-applications/bin/env.demo1.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
# topics for demo1
20+
export topic_l1_a=region_emea_counts
21+
export topic_l1_b=region_apac_counts
22+
export topic_l1_c=region_uswest_counts
23+
export topic_l2_a=global_contribution
24+
export topic_l2_b=global_averages

‎wayang-applications/bin/env_template.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
export JAVA_HOME=...
20+
export SPARK_HOME=...
21+
export HADOOP_HOME=...
22+
export PATH=$PATH:$HADOOP_HOME/bin
23+
export WAYANG_HOME=...
24+
export WAYANG_APP_HOME=...
25+
26+
# properties of brokers and schema registry of Ccloud cluster for demo 1
27+
export BOOTSTRAP_SERVER=
28+
export CLUSTER_API_KEY=
29+
export CLUSTER_API_SECRET=
30+
export SR_ENDPOINT=
31+
export SR_API_KEY=
32+
export SR_API_SECRET=
33+
34+
export SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO="...:..."
35+
export SCHEMA_REGISTRY_URL="..."
36+
37+
# cluster-id of Ccloud cluster...
38+
export DEMO1_CLUSTER1=...

‎wayang-applications/bin/run_wordcount.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
######
20+
# to adjust variables to your own environment, please configure them in env.sh
21+
#
22+
source .env.sh
23+
24+
cd ../..
25+
26+
mvn clean compile package install -pl :wayang-assembly -Pdistribution -DskipTests
27+
28+
cd wayang-applications
29+
30+
mvn compile package install -DskipTests
31+
32+
cd ..
33+
34+
bin/wayang-submit org.apache.wayang.applications.WordCount java file://$(pwd)/wayang-applications/data/case-study/DATA_REPO_001/README.md

0 commit comments

Comments
 (0)