Merge pull request #7302 from NvTimLiu/release-tmp

Merge branch 'branch-22.12' into main [skip ci]

Author: NvTimLiu

.github/workflows/add-to-project.yml

@@ -12,19 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Add new issues to project
+name: Add new issues and pull requests to project
 
 on:
   issues:
     types:
       - opened
+  pull_request_target:
+    types:
+      - opened
 
 jobs:
   add-to-project:
     if: github.repository == 'NVIDIA/spark-rapids'
-    name: Add new issues to project
+    name: Add new issues and pull requests to project
     runs-on: ubuntu-latest
     steps:
+      # TODO: update project version when new release supports node 16 instead of 12
       - uses: actions/add-to-project@v0.3.0
         with:
           project-url: https://github.com/orgs/NVIDIA/projects/4

.github/workflows/auto-merge.yml

@@ -18,7 +18,7 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-    - branch-22.10
+    - branch-22.12
     types: [closed]
 
 jobs:
@@ -27,15 +27,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          ref: branch-22.10 # force to fetch from latest upstream instead of PR ref
+          ref: branch-22.12 # force to fetch from latest upstream instead of PR ref
 
       - name: auto-merge job
         uses: ./.github/workflows/auto-merge
         env:
           OWNER: NVIDIA
           REPO_NAME: spark-rapids
-          HEAD: branch-22.10
-          BASE: branch-22.12
+          HEAD: branch-22.12
+          BASE: branch-23.02
           AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR

.github/workflows/blossom-ci.yml

@@ -65,8 +65,6 @@ jobs:
       NVnavkumar,\
       amahussein,\
       mattahrens,\
-      sinkinben,\
-      thirtiseven,\
       YanxuanLiu,\
       cindyyuanjiang,\
       ', format('{0},', github.actor)) && github.event.comment.body == 'build'
@@ -84,17 +82,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
           ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
           lfs: 'true'
 
       # repo specific steps
       - name: Setup java
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: adopt
+          java-version: 8
 
       # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file
       - name: Setup blackduck properties

.github/workflows/markdown-links-check/markdown-links-check-config.json

@@ -8,6 +8,9 @@
     },
     {
       "pattern": "/docs/archives"
+    },
+    {
+      "pattern": "http://localhost*"
     }
   ],
   "timeout": "15s",

.github/workflows/mvn-verify-check.yml

@@ -24,48 +24,61 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  get-noSnapshot-versions-from-dist:
+  get-shim-versions-from-dist:
     runs-on: ubuntu-latest
     outputs:
-      sparkHeadVersion: ${{ steps.noSnapshotVersionsStep.outputs.headVersion }}
-      sparkTailVersions: ${{ steps.noSnapshotVersionsStep.outputs.tailVersions }}
+      sparkHeadVersion: ${{ steps.allShimVersionsStep.outputs.headVersion }}
+      sparkTailVersions: ${{ steps.allShimVersionsStep.outputs.tailVersions }}
+      sparkJDK11Versions: ${{ steps.allShimVersionsStep.outputs.jdk11Versions }}
     steps:
-      - uses: actions/checkout@v2 # refs/pull/:prNumber/merge
+      - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
 
       - name: Setup Java and Maven Env
         uses: actions/setup-java@v3
         with:
           distribution: adopt
           java-version: 8
 
-      - name: all noSnapshot versions
-        id: noSnapshotVersionsStep
+      - name: all shim versions
+        id: allShimVersionsStep
         run: |
           set -x
           . jenkins/version-def.sh
-          svArrBody=$(printf ",{\"spark-version\":\"%s\"}" "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS_TAIL[@]}")
-          svArrBody=${svArrBody:1}
-          svJsonStr=$(printf {\"include\":[%s]} $svArrBody)
-          echo ::set-output name=headVersion::$SPARK_BASE_SHIM_VERSION
-          echo ::set-output name=tailVersions::$svJsonStr
-
-  package-aggregator:
-    needs: get-noSnapshot-versions-from-dist
+          svArrBodyNoSnapshot=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":false}" "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS_TAIL[@]}")
+          svArrBodyNoSnapshot=${svArrBodyNoSnapshot:1}
+          svArrBodySnapshot=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":true}" "${SPARK_SHIM_VERSIONS_SNAPSHOTS_ONLY[@]}")
+
+          # add snapshot versions which are not in snapshot property in pom file
+          svArrBodySnapshot+=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":true}" 340)
+          svArrBodySnapshot=${svArrBodySnapshot:1}
+          svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot,$svArrBodySnapshot)
+          echo "headVersion=$SPARK_BASE_SHIM_VERSION" >> $GITHUB_OUTPUT
+          echo "tailVersions=$svJsonStr" >> $GITHUB_OUTPUT
+          jdkVersionArrBody=$(printf ",{\"spark-version\":\"%s\"}" "${SPARK_SHIM_VERSIONS_JDK11[@]}")
+          jdkVersionArrBody=${jdkVersionArrBody:1}
+          jdkVersionJsonStr=$(printf {\"include\":[%s]} $jdkVersionArrBody)
+          echo "jdk11Versions=$jdkVersionJsonStr" >> $GITHUB_OUTPUT
+
+  package-tests:
+    needs: get-shim-versions-from-dist
+    continue-on-error: ${{ matrix.isSnapshot }}
     strategy:
-      matrix: ${{ fromJSON(needs.get-noSnapshot-versions-from-dist.outputs.sparkTailVersions) }}
+      matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkTailVersions) }}
+      fail-fast: false
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2 # refs/pull/:prNumber/merge
+      - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
 
       - name: Setup Java and Maven Env
         uses: actions/setup-java@v3
         with:
           distribution: adopt
           java-version: 8
 
-      - name: package aggregator check
+      - name: package tests check
         run: >
-          mvn -Dmaven.wagon.http.retryHandler.count=3 -B package -pl aggregator -am
+          mvn -Dmaven.wagon.http.retryHandler.count=3 -B package
+          -pl integration_tests,tests -am
           -P 'individual,pre-merge'
           -Dbuildver=${{ matrix.spark-version }}
           -DskipTests
@@ -74,11 +87,12 @@ jobs:
           -Dmaven.scalastyle.skip=true
           -Drat.skip=true
 
+
   verify-all-modules-with-headSparkVersion:
-    needs: get-noSnapshot-versions-from-dist
+    needs: get-shim-versions-from-dist
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2 # refs/pull/:prNumber/merge
+      - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
 
       - name: Setup Java and Maven Env
         uses: actions/setup-java@v3
@@ -91,7 +105,36 @@ jobs:
         run: >
           mvn -Dmaven.wagon.http.retryHandler.count=3 -B verify
           -P 'individual,pre-merge'
-          -Dbuildver=${{ needs.get-noSnapshot-versions-from-dist.outputs.sparkHeadVersion }}
+          -Dbuildver=${{ needs.get-shim-versions-from-dist.outputs.sparkHeadVersion }}
+          -DskipTests
+          -Dskip
+          -Dmaven.javadoc.skip
+
+  verify-modules-with-jdk11:
+    needs: get-shim-versions-from-dist
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDK11Versions) }}
+    steps:
+      - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
+
+      - name: Setup Java and Maven Env
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 11
+      
+      - name: Build JDK11
+        run: >
+          mvn -Dmaven.wagon.http.retryHandler.count=3 -B verify
+          -P 'individual,pre-merge'
+          -Dbuildver=${{ matrix.spark-version }}
+          -Dmaven.compiler.release=11
+          -Dmaven.compiler.source=11
+          -Dmaven.compiler.target=11
+          -Dscala.plugin.version=4.6.1
+          -DallowConventionalDistJar=true
           -DskipTests
           -Dskip
           -Dmaven.javadoc.skip
+          

.github/workflows/signoff-check.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ jobs:
   signoff-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: sigoff-check job
         uses: ./.github/workflows/signoff-check