Skip to content

Commit 63dabac

Browse files
GaryShen2008GaryShen2008
authored
Merge pull request #3941 from NVIDIA/branch-21.10
Merge source from branch-21.10 to main [skip ci]
2 parents 76e128f + ba28548 commit 63dabac

File tree

8 files changed

+63
-4
lines changed

8 files changed

+63
-4
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Change log
2-
Generated on 2021-10-26
2+
Generated on 2021-10-28
33

44
## Release 21.10
55

@@ -55,6 +55,7 @@ Generated on 2021-10-26
5555
### Bugs Fixed
5656
|||
5757
|:---|:---|
58+
|[#3929](https://github.com/NVIDIA/spark-rapids/issues/3929)|[BUG] published rapids-4-spark dist artifact references aggregator|
5859
|[#3837](https://github.com/NVIDIA/spark-rapids/issues/3837)|[BUG] Spark-rapids v21.10.0 release candidate jars failed on the OSS validation check.|
5960
|[#3769](https://github.com/NVIDIA/spark-rapids/issues/3769)|[BUG] dedupe fails with find: './parallel-world/spark301/ ...' No such file or directory|
6061
|[#3783](https://github.com/NVIDIA/spark-rapids/issues/3783)|[BUG] spark-rapids v21.10.0 release build failed on script "dist/scripts/binary-dedupe.sh"|
@@ -145,6 +146,8 @@ Generated on 2021-10-26
145146
### PRs
146147
|||
147148
|:---|:---|
149+
|[#3930](https://github.com/NVIDIA/spark-rapids/pull/3930)|Dist artifact with provided aggregator dependency|
150+
|[#3918](https://github.com/NVIDIA/spark-rapids/pull/3918)|Update changelog [skip ci]|
148151
|[#3906](https://github.com/NVIDIA/spark-rapids/pull/3906)|Doc updated for v2110[skip ci]|
149152
|[#3840](https://github.com/NVIDIA/spark-rapids/pull/3840)|Update changelog [skip ci]|
150153
|[#3838](https://github.com/NVIDIA/spark-rapids/pull/3838)|Update deploy script [skip ci]|

‎api_validation/pom.xml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,14 @@
127127
<classifier>${cuda.version}</classifier>
128128
<scope>provided</scope>
129129
</dependency>
130+
<!-- use aggregator jar because accessing internal classes -->
131+
<dependency>
132+
<groupId>com.nvidia</groupId>
133+
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
134+
<version>${project.version}</version>
135+
<classifier>${spark.version.classifier}</classifier>
136+
<scope>provided</scope>
137+
</dependency>
130138
<dependency>
131139
<groupId>com.nvidia</groupId>
132140
<artifactId>rapids-4-spark_${scala.binary.version}</artifactId>
@@ -149,7 +157,7 @@
149157
<configuration>
150158
<mainClass>com.nvidia.spark.rapids.api.ApiValidation</mainClass>
151159
</configuration>
152-
</plugin>
160+
</plugin>
153161
</plugins>
154162
</build>
155163
</project>

‎dist/pom.xml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
<groupId>com.nvidia</groupId>
3838
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
3939
<version>${project.version}</version>
40+
<scope>provided</scope>
4041
</dependency>
4142
<dependency>
4243
<!-- required for conf generation script -->
@@ -358,6 +359,7 @@
358359
<groupId>com.nvidia</groupId>
359360
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
360361
<version>${project.version}</version>
362+
<scope>provided</scope>
361363
</dependency>
362364
<dependency>
363365
<!-- required for conf generation script -->
@@ -648,6 +650,7 @@
648650
<groupId>com.nvidia</groupId>
649651
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
650652
<version>${project.version}</version>
653+
<scope>provided</scope>
651654
</dependency>
652655
<dependency>
653656
<!-- required for conf generation script -->
@@ -860,6 +863,7 @@
860863
<groupId>com.nvidia</groupId>
861864
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
862865
<version>${project.version}</version>
866+
<scope>provided</scope>
863867
</dependency>
864868
<dependency>
865869
<!-- required for conf generation script -->
@@ -1180,6 +1184,7 @@
11801184
<groupId>com.nvidia</groupId>
11811185
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
11821186
<version>${project.version}</version>
1187+
<scope>provided</scope>
11831188
</dependency>
11841189
<dependency>
11851190
<!-- required for conf generation script -->
@@ -1548,6 +1553,7 @@
15481553
<groupId>com.nvidia</groupId>
15491554
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
15501555
<version>${project.version}</version>
1556+
<scope>provided</scope>
15511557
</dependency>
15521558
<dependency>
15531559
<groupId>org.apache.spark</groupId>
@@ -1655,6 +1661,7 @@
16551661
<groupId>com.nvidia</groupId>
16561662
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
16571663
<version>${project.version}</version>
1664+
<scope>provided</scope>
16581665
</dependency>
16591666
<dependency>
16601667
<!-- required for conf generation script -->
@@ -1848,6 +1855,20 @@
18481855
</excludes>
18491856
</configuration>
18501857
</plugin>
1858+
<plugin>
1859+
<groupId>org.apache.maven.plugins</groupId>
1860+
<artifactId>maven-install-plugin</artifactId>
1861+
<version>3.0.0-M1</version>
1862+
<executions>
1863+
<execution>
1864+
<id>default-install</id>
1865+
<phase>install</phase>
1866+
<configuration>
1867+
<pomFile>${project.build.directory}/dependency-reduced-pom.xml</pomFile>
1868+
</configuration>
1869+
</execution>
1870+
</executions>
1871+
</plugin>
18511872
</plugins>
18521873
</build>
18531874

‎dist/scripts/binary-dedupe.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,4 +111,10 @@ done
111111
# Remove unshimmed classes from parallel worlds
112112
echo Removing duplicates of unshimmed classes
113113
xargs --arg-file="$UNSHIMMED_LIST_TXT" -P 6 -n 100 -I% \
114-
find . -path './parallel-world/spark*/%' | xargs rm || exit 255
114+
find . -path './parallel-world/spark*/%' | xargs rm || exit 255
115+
116+
117+
118+
echo "Generating dependency-reduced-pom.xml"
119+
# which is just delete the dependencies list altogether
120+
sed -e '/<dependencies>/,/<\/dependencies>/d' ../pom.xml > dependency-reduced-pom.xml

‎integration_tests/pom.xml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@
6060
<groupId>com.nvidia</groupId>
6161
<artifactId>rapids-4-spark_${scala.binary.version}</artifactId>
6262
<version>${project.version}</version>
63+
<scope>provided</scope>
64+
</dependency>
65+
<!-- use aggregator jar because accessing internal classes -->
66+
<dependency>
67+
<groupId>com.nvidia</groupId>
68+
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
69+
<version>${project.version}</version>
70+
<classifier>${spark.version.classifier}</classifier>
6371
<scope>test</scope>
6472
</dependency>
6573
<dependency>

‎jenkins/deploy.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ $DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
8282
# Distribution jar is a shaded artifact so use the reduced dependency pom.
8383
$DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
8484
$SRC_DOC_JARS \
85-
-Dfile=$FPATH.jar -DgroupId=com.nvidia -DartifactId=$ART_ID -Dversion=$ART_VER -DpomFile=./dist/pom.xml
85+
-Dfile=$FPATH.jar -DgroupId=com.nvidia -DartifactId=$ART_ID -Dversion=$ART_VER -DpomFile=./dist/target/dependency-reduced-pom.xml
8686

8787
###### Deploy integration tests jar(s) ######
8888
TESTS_ART_ID=`mvn help:evaluate -q -pl $TESTS_PL -Dexpression=project.artifactId -DforceStdout`

‎jenkins/spark-tests.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@ $MVN_GET_CMD -DremoteRepositories=$PROJECT_REPO \
3434
-DgroupId=com.nvidia -DartifactId=rapids-4-spark_$SCALA_BINARY_VER -Dversion=$PROJECT_VER
3535
$MVN_GET_CMD -DremoteRepositories=$PROJECT_TEST_REPO \
3636
-DgroupId=com.nvidia -DartifactId=rapids-4-spark-udf-examples_$SCALA_BINARY_VER -Dversion=$PROJECT_TEST_VER
37+
38+
# TODO remove -Dtransitive=false workaround once pom is fixed
3739
$MVN_GET_CMD -DremoteRepositories=$PROJECT_TEST_REPO \
40+
-Dtransitive=false \
3841
-DgroupId=com.nvidia -DartifactId=rapids-4-spark-integration-tests_$SCALA_BINARY_VER -Dversion=$PROJECT_TEST_VER -Dclassifier=$SHUFFLE_SPARK_SHIM
3942
if [ "$CUDA_CLASSIFIER"x == x ];then
4043
CUDF_JAR="$ARTF_ROOT/cudf-$CUDF_VER.jar"
@@ -45,7 +48,9 @@ export RAPIDS_PLUGIN_JAR="$ARTF_ROOT/rapids-4-spark_${SCALA_BINARY_VER}-$PROJECT
4548
RAPIDS_UDF_JAR="$ARTF_ROOT/rapids-4-spark-udf-examples_${SCALA_BINARY_VER}-$PROJECT_TEST_VER.jar"
4649
RAPIDS_TEST_JAR="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-$SHUFFLE_SPARK_SHIM.jar"
4750

51+
# TODO remove -Dtransitive=false workaround once pom is fixed
4852
$MVN_GET_CMD -DremoteRepositories=$PROJECT_TEST_REPO \
53+
-Dtransitive=false \
4954
-DgroupId=com.nvidia -DartifactId=rapids-4-spark-integration-tests_$SCALA_BINARY_VER -Dversion=$PROJECT_TEST_VER -Dclassifier=pytest -Dpackaging=tar.gz
5055

5156
RAPIDS_INT_TESTS_HOME="$ARTF_ROOT/integration_tests/"

‎tests/pom.xml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,14 @@
5858
<groupId>com.nvidia</groupId>
5959
<artifactId>rapids-4-spark_${scala.binary.version}</artifactId>
6060
<version>${project.version}</version>
61+
<scope>provided</scope>
62+
</dependency>
63+
<!-- use aggregator jar because accessing internal classes -->
64+
<dependency>
65+
<groupId>com.nvidia</groupId>
66+
<artifactId>rapids-4-spark-aggregator_${scala.binary.version}</artifactId>
67+
<version>${project.version}</version>
68+
<classifier>${spark.version.classifier}</classifier>
6169
<scope>test</scope>
6270
</dependency>
6371
<dependency>

0 commit comments

Comments
 (0)