flink_cdc_base.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Flink CDC Base Workflow

on:
  workflow_call:
    inputs:
      java-versions:
        description: "Jdk versions to test against."
        required: false
        type: string
        default: "['11']"
      flink-versions:
        description: "Flink versions to test against."
        required: false
        type: string
        default: "['generic']"
      modules:
        description: "Flink CDC modules to test against."
        required: true
        type: string
      parallelism:
        description: "Flink parallelism."
        required: false
        type: number
        default: 4
      custom-maven-parameter:
        description: "Custom maven parameter."
        required: false
        type: string

jobs:
  test:
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    strategy:
      max-parallel: 20
      fail-fast: false
      matrix:
        java-version: ${{ fromJSON(inputs.java-versions) }}
        flink-version: ${{ fromJSON(inputs.flink-versions) }}
        module: ${{ fromJSON(inputs.modules) }}
    steps:
      - run: echo "Running CI pipeline for JDK version ${{ matrix.java-version }}"
      - name: Clean up disk space
        run: |
          set -euo pipefail

          echo "Disk space before cleanup"
          df -h 

          echo "Cleaning up disk space"
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force

          echo "Disk space after cleanup"
          df -h

      - name: Check out repository code
        uses: actions/checkout@v6
        with:
          submodules: true

      - name: Set JDK
        uses: actions/setup-java@v5
        with:
          java-version: ${{ matrix.java-version }}
          distribution: 'temurin'
          cache: 'maven'

      - name: Set Maven 3.8.6
        uses: stCarolas/setup-maven@v5
        with:
          maven-version: 3.8.6

      - name: Compile and test
        timeout-minutes: 90
        run: |
          . .github/workflows/utils.sh
          jvm_timezone=$(random_timezone)
          echo "JVM timezone is set to $jvm_timezone"
          set -o pipefail

          modules=$(./.github/workflows/modules.py test "${{ matrix.module }}")
          compile_modules=$(./.github/workflows/modules.py compile "${{ matrix.module }}")
          
          build_maven_parameter="-DspecifiedMongoVersion=8.0.14"

          if [ ! -z "${{ matrix.flink-version }}" ]; then
            build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }-DspecifiedFlinkVersion=${{ matrix.flink-version }}"
          fi
          
          build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }${{ inputs.custom-maven-parameter }}"
          
          mvn --no-snapshot-updates -B -DskipTests ${{ inputs.custom-maven-parameter }} -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules -DspecifiedParallelism=${{ inputs.parallelism }} -Duser.timezone=$jvm_timezone verify

      - name: Print JVM thread dumps when cancelled
        if: ${{ failure() }}
        run: |
          echo "$OSTYPE"
          if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then
            echo "Setting up JVM thread dumps"
            curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach
            if command -v sha256sum &> /dev/null; then
              sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e  /tmp/jattach") || exit 1
            fi
            chmod +x /tmp/jattach
            for java_pid in $(sudo pgrep java); do
              echo "----------------------- pid $java_pid -----------------------"
              echo "command line: $(sudo cat /proc/$java_pid/cmdline | xargs -0 echo)"
              sudo /tmp/jattach $java_pid jcmd VM.command_line || true
              sudo /tmp/jattach $java_pid jcmd "Thread.print -l"
              sudo /tmp/jattach $java_pid jcmd GC.heap_info || true
            done
          else
            for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do
              echo "----------------------- pid $java_pid -----------------------"
              jcmd $java_pid VM.command_line || true
              jcmd $java_pid Thread.print -l
              jcmd $java_pid GC.heap_info || true
            done
          fi
          exit 0