Skip to content

Commit ccf241b

Browse files
JeynmannZJeynmannZ
committed
support external shuffle
1 parent 1b9230d commit ccf241b

File tree

1 file changed

+98
-1
lines changed

1 file changed

+98
-1
lines changed

‎pom.xml‎

Lines changed: 98 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ See file LICENSE for terms.
4545
<excludes>
4646
<exclude>**/spark_2_1/**</exclude>
4747
<exclude>**/spark_3_0/**</exclude>
48+
<exclude>org/apache/spark/network/**</exclude>
49+
<exclude>org/apache/spark/shuffle/ucx/external/server/**</exclude>
4850
</excludes>
4951
</configuration>
5052
</plugin>
@@ -71,6 +73,8 @@ See file LICENSE for terms.
7173
<excludes>
7274
<exclude>**/spark_2_1/**</exclude>
7375
<exclude>**/spark_2_4/**</exclude>
76+
<exclude>org/apache/spark/network/**</exclude>
77+
<exclude>org/apache/spark/shuffle/ucx/external/server/**</exclude>
7478
</excludes>
7579
</configuration>
7680
</plugin>
@@ -83,6 +87,93 @@ See file LICENSE for terms.
8387
<sonar.exclusions>**/spark_2_1/**, **/spark_2_4/**</sonar.exclusions>
8488
</properties>
8589
</profile>
90+
<profile>
91+
<id>sparkess-2.4</id>
92+
<build>
93+
<plugins>
94+
<plugin>
95+
<groupId>net.alchim31.maven</groupId>
96+
<artifactId>scala-maven-plugin</artifactId>
97+
<configuration>
98+
<includes>
99+
<include>org/apache/spark/network/**</include>
100+
<include>org/apache/spark/shuffle/ucx/external/**</include>
101+
<include>org/apache/spark/shuffle/ucx/memory/**</include>
102+
<include>org/apache/spark/shuffle/ucx/ShuffleTransport.scala</include>
103+
<include>org/apache/spark/shuffle/utils/**</include>
104+
</includes>
105+
<excludes>
106+
<exclude>**/spark_3_0/**</exclude>
107+
<exclude>org/apache/spark/shuffle/ucx/external/client/**</exclude>
108+
</excludes>
109+
</configuration>
110+
</plugin>
111+
</plugins>
112+
</build>
113+
<properties>
114+
<spark.version>2.4.0</spark.version>
115+
<sonar.exclusions>**/spark_3_0/**, **/spark_2_1/**</sonar.exclusions>
116+
<scala.version>2.11.12</scala.version>
117+
<scala.compat.version>2.11</scala.compat.version>
118+
<fasterxml.jackson.version>2.6.7</fasterxml.jackson.version>
119+
</properties>
120+
<dependencies>
121+
<dependency>
122+
<groupId>org.scala-lang</groupId>
123+
<artifactId>scala-library</artifactId>
124+
<version>${scala.version}</version>
125+
</dependency>
126+
<dependency>
127+
<groupId>com.fasterxml.jackson.core</groupId>
128+
<artifactId>jackson-databind</artifactId>
129+
<version>${fasterxml.jackson.version}</version>
130+
</dependency>
131+
</dependencies>
132+
</profile>
133+
<profile>
134+
<id>sparkess-3.0</id>
135+
<build>
136+
<plugins>
137+
<plugin>
138+
<groupId>net.alchim31.maven</groupId>
139+
<artifactId>scala-maven-plugin</artifactId>
140+
<configuration>
141+
<includes>
142+
<include>org/apache/spark/network/**</include>
143+
<include>org/apache/spark/shuffle/ucx/external/**</include>
144+
<include>org/apache/spark/shuffle/ucx/memory/**</include>
145+
<include>org/apache/spark/shuffle/ucx/ShuffleTransport.scala</include>
146+
<include>org/apache/spark/shuffle/utils/**</include>
147+
</includes>
148+
<excludes>
149+
<exclude>**/spark_2_4/**</exclude>
150+
<exclude>org/apache/spark/shuffle/ucx/external/client/**</exclude>
151+
</excludes>
152+
</configuration>
153+
</plugin>
154+
</plugins>
155+
</build>
156+
<properties>
157+
<spark.version>3.0.1</spark.version>
158+
<scala.version>2.12.10</scala.version>
159+
<scala.compat.version>2.12</scala.compat.version>
160+
<sonar.exclusions>**/spark_2_1/**, **/spark_2_4/**</sonar.exclusions>
161+
<fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
162+
</properties>
163+
<dependencies>
164+
<dependency>
165+
<groupId>org.scala-lang</groupId>
166+
<artifactId>scala-library</artifactId>
167+
<version>${scala.version}</version>
168+
</dependency>
169+
<dependency>
170+
<groupId>com.fasterxml.jackson.core</groupId>
171+
<artifactId>jackson-databind</artifactId>
172+
<version>${fasterxml.jackson.version}</version>
173+
</dependency>
174+
</dependencies>
175+
</profile>
176+
86177
</profiles>
87178

88179

@@ -93,10 +184,16 @@ See file LICENSE for terms.
93184
<version>${spark.version}</version>
94185
<scope>provided</scope>
95186
</dependency>
187+
<dependency>
188+
<groupId>org.apache.spark</groupId>
189+
<artifactId>spark-network-yarn_${scala.compat.version}</artifactId>
190+
<version>${spark.version}</version>
191+
<scope>provided</scope>
192+
</dependency>
96193
<dependency>
97194
<groupId>org.openucx</groupId>
98195
<artifactId>jucx</artifactId>
99-
<version>1.13.1</version>
196+
<version>1.16.0</version>
100197
</dependency>
101198
</dependencies>
102199

0 commit comments

Comments
 (0)