Publish release #8
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| versions: | |
| required: true | |
| type: string | |
| description: 'Example: {"include": [{"params": {"spark-version": "4.0.0","scala-version": "2.13.16"}}]}' | |
| default: | | |
| { | |
| "include": [ | |
| {"params": {"spark-version": "3.2.4", "scala-version": "2.12.15", "java-compat-version": "8"}}, | |
| {"params": {"spark-version": "3.3.4", "scala-version": "2.12.15", "java-compat-version": "8"}}, | |
| {"params": {"spark-version": "3.4.4", "scala-version": "2.12.17", "java-compat-version": "8"}}, | |
| {"params": {"spark-version": "3.5.6", "scala-version": "2.12.18", "java-compat-version": "8"}}, | |
| {"params": {"spark-version": "3.2.4", "scala-version": "2.13.5", "java-compat-version": "8"}}, | |
| {"params": {"spark-version": "3.3.4", "scala-version": "2.13.8", "java-compat-version": "8"}}, | |
| {"params": {"spark-version": "3.4.4", "scala-version": "2.13.8", "java-compat-version": "8"}}, | |
| {"params": {"spark-version": "3.5.6", "scala-version": "2.13.8", "java-compat-version": "8"}}, | |
| {"params": {"spark-version": "4.0.0", "scala-version": "2.13.16", "java-compat-version": "17"}} | |
| ] | |
| } | |
| env: | |
| # PySpark 3 versions only work with Python 3.9 | |
| PYTHON_VERSION: "3.9" | |
| jobs: | |
| get-version: | |
| name: Get version | |
| runs-on: ubuntu-latest | |
| outputs: | |
| release-tag: ${{ steps.versions.outputs.release-tag }} | |
| is-snapshot: ${{ steps.versions.outputs.is-snapshot }} | |
| steps: | |
| - name: Checkout release tag | |
| uses: actions/checkout@v4 | |
| - name: Get versions | |
| id: versions | |
| run: | | |
| # get release version | |
| version=$(grep --max-count=1 "<version>.*</version>" pom.xml | sed -E -e "s/\s*<[^>]+>//g" -e "s/-SNAPSHOT//" -e "s/-[0-9.]+//g") | |
| is_snapshot=$(if grep -q "<version>.*-SNAPSHOT</version>" pom.xml; then echo "true"; else echo "false"; fi) | |
| # share versions | |
| echo "release-tag=v${version}" >> "$GITHUB_OUTPUT" | |
| echo "is-snapshot=$is_snapshot" >> "$GITHUB_OUTPUT" | |
| - name: Check tag setup | |
| run: | | |
| # Check tag setup | |
| if [[ "$GITHUB_REF" != "refs/tags/v"* ]] | |
| then | |
| echo "This workflow must be run on a tag, not $GITHUB_REF" | |
| exit 1 | |
| fi | |
| if [ "${{ steps.versions.outputs.is-snapshot }}" == "true" ] | |
| then | |
| echo "This is a tagged SNAPSHOT version. This is not allowed for release!" | |
| exit 1 | |
| fi | |
| if [ "${{ github.ref_name }}" != "${{ steps.versions.outputs.release-tag }}" ] | |
| then | |
| echo "The version in the pom.xml is ${{ steps.versions.outputs.release-tag }}" | |
| echo "This tag is ${{ github.ref_name }}, which is different!" | |
| exit 1 | |
| fi | |
| - name: Show matrix | |
| run: | | |
| echo '${{ github.event.inputs.versions }}' | jq . | |
| maven-release: | |
| name: Publish maven release (Spark ${{ matrix.params.spark-version }}, Scala ${{ matrix.params.scala-version }}) | |
| runs-on: ubuntu-latest | |
| needs: get-version | |
| if: ( ! github.event.repository.fork ) | |
| # secrets are provided by environment | |
| environment: | |
| name: release | |
| # a different URL for each point in the matrix, but the same URLs accross commits | |
| url: 'https://github.com/G-Research/spark-extension?version=${{ needs.get-version.outputs.release-tag }}&spark=${{ matrix.params.spark-version }}&scala=${{ matrix.params.scala-version }}&package=maven' | |
| permissions: {} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(github.event.inputs.versions) }} | |
| steps: | |
| - name: Checkout release tag | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK and publish to Maven Central | |
| uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0 | |
| with: | |
| java-version: ${{ matrix.params.java-compat-version }} | |
| distribution: 'corretto' | |
| server-id: central | |
| server-username: MAVEN_USERNAME | |
| server-password: MAVEN_PASSWORD | |
| gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - name: Inspect GPG | |
| run: gpg -k | |
| - name: Restore Maven packages cache | |
| id: cache-maven | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-mvn-build-${{ matrix.params.spark-version }}-${{ matrix.params.scala-version }}-${{ hashFiles('pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-mvn-build-${{ matrix.params.spark-version }}-${{ matrix.params.scala-version }}-${{ hashFiles('pom.xml') }} | |
| ${{ runner.os }}-mvn-build-${{ matrix.params.spark-version }}-${{ matrix.params.scala-version }}- | |
| - name: Publish maven artifacts | |
| id: publish-maven | |
| run: | | |
| ./set-version.sh ${{ matrix.params.spark-version }} ${{ matrix.params.scala-version }} | |
| mvn clean deploy -Dsign -Dspotless.check.skip -DskipTests -Dmaven.test.skip=true | |
| env: | |
| MAVEN_USERNAME: ${{ secrets.MAVEN_USERNAME }} | |
| MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE}} | |
| pypi-release: | |
| name: Publish PyPi release (Spark ${{ matrix.params.spark-version }}, Scala ${{ matrix.params.scala-version }}) | |
| runs-on: ubuntu-latest | |
| needs: get-version | |
| if: ( ! github.event.repository.fork ) | |
| # secrets are provided by environment | |
| environment: | |
| name: release | |
| # a different URL for each point in the matrix, but the same URLs accross commits | |
| url: 'https://github.com/G-Research/spark-extension?version=${{ needs.get-version.outputs.release-tag }}&spark=${{ matrix.params.spark-version }}&scala=${{ matrix.params.scala-version }}&package=pypi' | |
| permissions: | |
| id-token: write # required for PiPy publish | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(github.event.inputs.versions) }} | |
| steps: | |
| - name: Checkout release tag | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK | |
| uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0 | |
| with: | |
| java-version: ${{ matrix.params.java-compat-version }} | |
| distribution: 'corretto' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Restore Maven packages cache | |
| id: cache-maven | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-mvn-build-${{ matrix.params.spark-version }}-${{ matrix.params.scala-version }}-${{ hashFiles('pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-mvn-build-${{ matrix.params.spark-version }}-${{ matrix.params.scala-version }}-${{ hashFiles('pom.xml') }} | |
| ${{ runner.os }}-mvn-build-${{ matrix.params.spark-version }}-${{ matrix.params.scala-version }}- | |
| - name: Build maven artifacts | |
| id: maven | |
| if: startsWith(matrix.params.spark-version, '3.') && startsWith(matrix.params.scala-version, '2.12.') || startsWith(matrix.params.spark-version, '4.') && startsWith(matrix.params.scala-version, '2.13.') | |
| run: | | |
| ./set-version.sh ${{ matrix.params.spark-version }} ${{ matrix.params.scala-version }} | |
| mvn clean package -Dspotless.check.skip -DskipTests -Dmaven.test.skip=true | |
| - name: Prepare PyPi package | |
| id: prepare-pypi-package | |
| if: steps.maven.outcome == 'success' | |
| run: | | |
| ./build-whl.sh | |
| - name: Publish package distributions to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| if: steps.prepare-pypi-package.outcome == 'success' | |
| with: | |
| packages-dir: python/dist | |
| skip-existing: true | |
| verbose: true |