Skip to content

Commit 649bf1c

Browse files
author
lupengfan1
committed
chore(pegasus-spark): Merge pegasus-spark from pegasus-kv/pegasus-spark
#945 Merge pegasus-spark from pegasus-kv/pegasus-spark
2 parents 933ac39 + 48f4c6c commit 649bf1c

File tree

67 files changed

+5340
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+5340
-0
lines changed

‎.gitignore‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,3 +350,39 @@ thirdparty/output/
350350

351351
#collector
352352
collector/collector
353+
354+
# ================= #
355+
# pegasus-spark #
356+
# ================= #
357+
pegasus-spark/*.class
358+
pegasus-spark/
359+
# Log file
360+
pegasus-spark/*.log
361+
pegasus-spark/*.ctxt
362+
# Mobile Tools for Java (J2ME)
363+
pegasus-spark/.mtj.tmp/
364+
pegasus-spark/*.jar
365+
pegasus-spark/*.war
366+
pegasus-spark/*.nar
367+
pegasus-spark/*.ear
368+
pegasus-spark/*.zip
369+
pegasus-spark/*.tar.gz
370+
pegasus-spark/*.rar
371+
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
372+
pegasus-spark/hs_err_pid*
373+
pegasus-spark/core
374+
pegasus-spark/.idea
375+
pegasus-spark/Rocksdb.iml
376+
pegasus-spark/target
377+
pegasus-spark/*.iml
378+
pegasus-spark/dependency-reduced-pom.xml
379+
pegasus-spark/core-site.xml
380+
pegasus-spark/.classpath
381+
pegasus-spark/.factorypath
382+
pegasus-spark/.project
383+
pegasus-spark/.settings/
384+
pegasus-spark/scalafmt
385+
pegasus-spark/rolling_log
386+
pegasus-spark/.vscode
387+
pegasus-spark/.metals
388+
pegasus-spark/.bloop

‎.travis.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
dist: bionic
2+
3+
cache:
4+
directories:
5+
- $HOME/.m2
6+
7+
script:
8+
- ./scripts/travis.sh

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ Pegasus has support for several languages:
7676
- [Python](https://github.com/apache/incubator-pegasus/blob/master/python-client)
7777
- [Node.js](https://github.com/apache/incubator-pegasus/blob/master/nodejs-client)
7878
- [Scala](https://github.com/apache/incubator-pegasus/blob/master/scala-client)
79+
- [pegasus-spark](https://github.com/apache/incubator-pegasus/blob/master/pegasus-spark)
7980

8081
## Contact us
8182

‎pom.xml‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
3+
<project xmlns="http://maven.apache.org/POM/4.0.0"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
6+
<modelVersion>4.0.0</modelVersion>
7+
8+
<groupId>com.xiaomi.infra</groupId>
9+
<artifactId>pegasus-spark</artifactId>
10+
<version>1.1.0-mdh-3.1.3-SNAPSHOT</version>
11+
12+
<name>pegasus-spark</name>
13+
14+
<distributionManagement>
15+
<repository>
16+
<id>central</id>
17+
<name>maven-release-virtual</name>
18+
<url>https://repo.maven.apache.org/maven2/</url>
19+
</repository>
20+
<!--&lt;!&ndash; snapshot repository may not be supported&ndash;&gt;-->
21+
<!-- <snapshotRepository>-->
22+
<!-- <id>snapshots</id>-->
23+
<!-- <name>maven-snapshot-virtual</name>-->
24+
<!-- <url>https://oss.sonatype.org/content/repositories/snapshots/</url>-->
25+
<!-- </snapshotRepository>-->
26+
</distributionManagement>
27+
28+
<properties>
29+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
30+
<maven.compiler.source>1.8</maven.compiler.source>
31+
<maven.compiler.target>1.8</maven.compiler.target>
32+
<pegasus.shade.name>com.xiaomi.infra.pegasus.spark.thirdparty</pegasus.shade.name>
33+
</properties>
34+
35+
<dependencies>
36+
<dependency>
37+
<groupId>junit</groupId>
38+
<artifactId>junit</artifactId>
39+
<version>4.13.1</version>
40+
<scope>test</scope>
41+
</dependency>
42+
<dependency>
43+
<groupId>com.github.rholder</groupId>
44+
<artifactId>guava-retrying</artifactId>
45+
<version>2.0.0</version>
46+
</dependency>
47+
<dependency>
48+
<groupId>commons-cli</groupId>
49+
<artifactId>commons-cli</artifactId>
50+
<version>1.4</version>
51+
</dependency>
52+
<dependency>
53+
<groupId>org.apache.commons</groupId>
54+
<artifactId>commons-lang3</artifactId>
55+
<version>3.8.1</version>
56+
</dependency>
57+
<dependency>
58+
<groupId>commons-configuration</groupId>
59+
<artifactId>commons-configuration</artifactId>
60+
<version>1.8</version>
61+
</dependency>
62+
<dependency>
63+
<groupId>commons-beanutils</groupId>
64+
<artifactId>commons-beanutils</artifactId>
65+
<version>1.8.0</version>
66+
</dependency>
67+
<dependency>
68+
<groupId>commons-jxpath</groupId>
69+
<artifactId>commons-jxpath</artifactId>
70+
<version>1.3</version>
71+
</dependency>
72+
<dependency>
73+
<groupId>com.typesafe</groupId>
74+
<artifactId>config</artifactId>
75+
<version>1.4.0</version>
76+
</dependency>
77+
<dependency>
78+
<groupId>com.google.code.gson</groupId>
79+
<artifactId>gson</artifactId>
80+
<version>2.8.6</version>
81+
</dependency>
82+
<dependency>
83+
<groupId>com.revinate</groupId>
84+
<artifactId>guava-rate-limiter</artifactId>
85+
<version>19.0</version>
86+
</dependency>
87+
<dependency>
88+
<groupId>org.rocksdb</groupId>
89+
<artifactId>rocksdbjni</artifactId>
90+
<version>6.6.4-SNAPSHOT</version>
91+
</dependency>
92+
93+
<!--Pegasus: The version add `Serializable` for some class, it will be replaced by release version-->
94+
<dependency>
95+
<groupId>com.xiaomi.infra</groupId>
96+
<artifactId>pegasus-client</artifactId>
97+
<version>1.11.10-for-spark-SNAPSHOT</version>
98+
<exclusions>
99+
<exclusion>
100+
<groupId>io.netty</groupId>
101+
<artifactId>netty-all</artifactId>
102+
</exclusion>
103+
</exclusions>
104+
</dependency>
105+
<!--FDS: Only read fds backup need the dependency-->
106+
<dependency>
107+
<groupId>com.xiaomi.infra.galaxy</groupId>
108+
<artifactId>galaxy-hadoop</artifactId>
109+
<version>2.9.8-SNAPSHOT</version>
110+
</dependency>
111+
<!--HDFS-->
112+
<dependency>
113+
<groupId>org.apache.hadoop</groupId>
114+
<artifactId>hadoop-hdfs</artifactId>
115+
<version>3.1.0-mdh3.1.1.9</version>
116+
<scope>provided</scope>
117+
</dependency>
118+
<dependency>
119+
<groupId>org.apache.hadoop</groupId>
120+
<artifactId>hadoop-hdfs-client</artifactId>
121+
<version>3.1.0-mdh3.1.1.9</version>
122+
<scope>provided</scope>
123+
</dependency>
124+
<dependency>
125+
<groupId>org.apache.hadoop</groupId>
126+
<artifactId>hadoop-common</artifactId>
127+
<version>3.1.0-mdh3.1.1.9</version>
128+
<scope>provided</scope>
129+
</dependency>
130+
<!--Scala-->
131+
<dependency>
132+
<groupId>org.scala-lang</groupId>
133+
<artifactId>scala-library</artifactId>
134+
<version>2.12.10</version>
135+
<scope>provided</scope>
136+
</dependency>
137+
<!--Spark-->
138+
<dependency>
139+
<groupId>org.apache.spark</groupId>
140+
<artifactId>spark-core_2.12</artifactId>
141+
<version>3.1.2</version>
142+
<scope>provided</scope>
143+
</dependency>
144+
<dependency>
145+
<groupId>org.apache.spark</groupId>
146+
<artifactId>spark-sql_2.12</artifactId>
147+
<version>3.1.2</version>
148+
<scope>provided</scope>
149+
</dependency>
150+
<dependency>
151+
<groupId>com.typesafe.play</groupId>
152+
<artifactId>play-json_2.12</artifactId>
153+
<version>2.7.0</version>
154+
</dependency>
155+
</dependencies>
156+
157+
<build>
158+
<plugins>
159+
<plugin>
160+
<groupId>org.scala-tools</groupId>
161+
<artifactId>maven-scala-plugin</artifactId>
162+
<version>2.15.2</version>
163+
<executions>
164+
<execution>
165+
<goals>
166+
<goal>compile</goal>
167+
<goal>testCompile</goal>
168+
</goals>
169+
</execution>
170+
</executions>
171+
</plugin>
172+
<plugin>
173+
<groupId>org.apache.maven.plugins</groupId>
174+
<artifactId>maven-shade-plugin</artifactId>
175+
<version>2.4.3</version>
176+
<executions>
177+
<execution>
178+
<phase>package</phase>
179+
<goals>
180+
<goal>shade</goal>
181+
</goals>
182+
<configuration>
183+
<transformers>
184+
<transformer
185+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
186+
<mainClass>com.xiaomi.infra.pegasus.spark.toolkits.TaskRunner</mainClass>
187+
</transformer>
188+
</transformers>
189+
<filters>
190+
<filter>
191+
<artifact>*:*</artifact>
192+
<excludes>
193+
<exclude>META-INF/*.SF</exclude>
194+
<exclude>META-INF/*.DSA</exclude>
195+
<exclude>META-INF/*.RSA</exclude>
196+
</excludes>
197+
</filter>
198+
</filters>
199+
</configuration>
200+
</execution>
201+
</executions>
202+
</plugin>
203+
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
204+
<plugin>
205+
<artifactId>maven-clean-plugin</artifactId>
206+
<version>3.1.0</version>
207+
</plugin>
208+
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
209+
<plugin>
210+
<artifactId>maven-resources-plugin</artifactId>
211+
<version>3.0.2</version>
212+
</plugin>
213+
<plugin>
214+
<artifactId>maven-compiler-plugin</artifactId>
215+
<version>3.8.0</version>
216+
</plugin>
217+
<plugin>
218+
<artifactId>maven-surefire-plugin</artifactId>
219+
<version>2.22.1</version>
220+
</plugin>
221+
<plugin>
222+
<artifactId>maven-install-plugin</artifactId>
223+
<version>2.5.2</version>
224+
</plugin>
225+
<plugin>
226+
<artifactId>maven-deploy-plugin</artifactId>
227+
<version>2.8.2</version>
228+
</plugin>
229+
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
230+
<plugin>
231+
<artifactId>maven-site-plugin</artifactId>
232+
<version>3.7.1</version>
233+
</plugin>
234+
<plugin>
235+
<artifactId>maven-project-info-reports-plugin</artifactId>
236+
<version>3.0.0</version>
237+
</plugin>
238+
<plugin>
239+
<groupId>com.atlassian.maven.plugins</groupId>
240+
<artifactId>clover-maven-plugin</artifactId>
241+
<version>4.1.2</version>
242+
</plugin>
243+
<plugin>
244+
<groupId>org.codehaus.mojo</groupId>
245+
<artifactId>exec-maven-plugin</artifactId>
246+
<version>1.2.1</version>
247+
<executions>
248+
<execution>
249+
<goals>
250+
<goal>java</goal>
251+
</goals>
252+
</execution>
253+
</executions>
254+
</plugin>
255+
<plugin>
256+
<groupId>org.apache.maven.plugins</groupId>
257+
<artifactId>maven-source-plugin</artifactId>
258+
<version>3.0.1</version>
259+
<executions>
260+
<execution>
261+
<id>attach-sources</id>
262+
<goals>
263+
<goal>jar</goal>
264+
</goals>
265+
</execution>
266+
</executions>
267+
</plugin>
268+
</plugins>
269+
</build>
270+
</project>

‎scripts/format-all.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
3+
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
4+
PROJECT_DIR=$(dirname "${SCRIPT_DIR}")
5+
cd "${PROJECT_DIR}" || exit 1
6+
7+
SRC_FILES=(src/main/java/com/xiaomi/infra/pegasus/spark/common/*.java
8+
src/main/java/com/xiaomi/infra/pegasus/spark/common/utils/*.java
9+
src/main/java/com/xiaomi/infra/pegasus/spark/common/utils/gateway/*.java
10+
src/main/java/com/xiaomi/infra/pegasus/spark/common/utils/metaproxy/*.java
11+
src/main/java/com/xiaomi/infra/pegasus/spark/analyser/*.java
12+
src/main/java/com/xiaomi/infra/pegasus/spark/bulkloader/*.java
13+
)
14+
15+
if [[ ! -f "${PROJECT_DIR}"/google-java-format-1.7-all-deps.jar ]]; then
16+
wget https://github.com/google/google-java-format/releases/download/google-java-format-1.7/google-java-format-1.7-all-deps.jar
17+
fi
18+
if ! java -jar "${PROJECT_DIR}"/google-java-format-1.7-all-deps.jar --replace "${SRC_FILES[@]}"; then
19+
echo "ERROR: failed to format java codes"
20+
exit 1
21+
fi
22+
23+
if [[ ! -f "${PROJECT_DIR}"/scalafmt ]]; then
24+
if ! wget https://github.com/scalameta/scalafmt/releases/download/v2.6.1/scalafmt-linux.zip; then
25+
echo "ERROR: failed to download scalafmt"
26+
exit 1
27+
fi
28+
unzip scalafmt-linux.zip
29+
rm -rf scalafmt-linux.zip
30+
chmod +x scalafmt
31+
fi
32+
33+
if ! "${PROJECT_DIR}"/scalafmt; then
34+
echo "ERROR: failed to format scala codes"
35+
exit 1
36+
fi

‎scripts/travis.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
6+
PROJECT_DIR=$(dirname "${SCRIPT_DIR}")
7+
cd "${PROJECT_DIR}" || exit 1
8+
9+
# lint all scripts, abort if there's any warning.
10+
function shellcheck_must_pass()
11+
{
12+
if [[ $(shellcheck "$1") ]]; then
13+
echo "shellcheck $1 failed"
14+
shellcheck "$1"
15+
exit 1
16+
fi
17+
}
18+
shellcheck_must_pass ./scripts/format-all.sh
19+
shellcheck_must_pass ./scripts/travis.sh
20+
21+
# ensure source files are well formatted
22+
./scripts/format-all.sh
23+
if [[ $(git status -s) ]]; then
24+
git status -s
25+
echo "please format the above files before commit"
26+
exit 1
27+
fi

0 commit comments

Comments
 (0)