Merge branch 'master' into FLINK-37203

Author: ChaomingZhangCN

.github/labeler.yml

@@ -93,3 +93,7 @@ oceanbase-pipeline-connector:
   - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase/**/*
 maxcompute-pipeline-connector:
   - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute/**/*
+iceberg-pipeline-connector:
+  - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/**/*
+postgres-pipeline-connector:
+  - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres/**/*

.github/workflows/build_docs.yml

@@ -37,7 +37,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Check dead links
-        uses: gaurav-nelson/github-action-markdown-link-check@1.0.15
+        uses: tcort/github-action-markdown-link-check@e7c7a18363c842693fadde5d41a3bd3573a7a225
         with:
           config-file: '.dlc.json'
       - name: Build doc
@@ -54,8 +54,8 @@ jobs:
       matrix:
         branch:
           - master
-          - release-3.3
-          - release-3.2
+          - release-3.5
+          - release-3.4
 
     steps:
       - uses: actions/checkout@v3
@@ -69,8 +69,8 @@ jobs:
           echo "flink_branch=${currentBranch}" >> ${GITHUB_ENV}
           
           if [ "${currentBranch}" = "master" ]; then
-          echo "flink_alias=release-3.4" >> ${GITHUB_ENV}
-          elif [ "${currentBranch}" = "release-3.3" ]; then
+          echo "flink_alias=release-3.6" >> ${GITHUB_ENV}
+          elif [ "${currentBranch}" = "release-3.5" ]; then
           echo "flink_alias=stable" >> ${GITHUB_ENV}
           fi
 
@@ -84,7 +84,7 @@ jobs:
           docker run --rm --volume "$PWD:/root/flink-cdc" chesnay/flink-ci:java_8_11_17_21_maven_386 bash -c "cd /root/flink-cdc && chmod +x ./.github/workflows/docs.sh && ./.github/workflows/docs.sh"
 
       - name: Upload documentation
-        uses: burnett01/rsync-deployments@5.2
+        uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
         with:
           switches: --archive --compress --delete
           path: docs/target/
@@ -96,7 +96,7 @@ jobs:
 
       - name: Upload documentation alias
         if: env.flink_alias != ''
-        uses: burnett01/rsync-deployments@5.2
+        uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
         with:
           switches: --archive --compress --delete
           path: docs/target/

.github/workflows/flink_cdc_base.yml

@@ -18,18 +18,18 @@ name: Flink CDC Base Workflow
 on:
   workflow_call:
     inputs:
-      java-version:
+      java-versions:
         description: "Jdk versions to test against."
         required: false
         type: string
         default: "['8']"
-      flink-version:
+      flink-versions:
         description: "Flink versions to test against."
         required: false
         type: string
         default: "['generic']"
-      module:
-        description: "Flink CDC module to test against."
+      modules:
+        description: "Flink CDC modules to test against."
         required: true
         type: string
       parallelism:
@@ -48,27 +48,38 @@ env:
   flink-cdc-common,\
   flink-cdc-composer,\
   flink-cdc-runtime,\
-  flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base"
+  flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
 
   MODULES_PIPELINE_CONNECTORS: "\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
 
-  MODULES_MYSQL: "\
+  MODULES_MYSQL_SOURCE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
 
-  MODULES_POSTGRES: "\
+  MODULES_MYSQL_PIPELINE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql"
+
+  MODULES_POSTGRES_SOURCE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
 
+  MODULES_POSTGRES_PIPELINE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres"
+
   MODULES_ORACLE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"
@@ -85,10 +96,13 @@ env:
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
 
-  MODULES_OCEANBASE: "\
+  MODULES_OCEANBASE_SOURCE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
 
+  MODULES_OCEANBASE_PIPELINE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase"
+
   MODULES_DB2: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
@@ -97,20 +111,46 @@ env:
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
 
+  MODULES_DORIS: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris"
+
+  MODULES_STARROCKS: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks"
+
+  MODULES_ICEBERG: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg"
+
+  MODULES_KAFKA: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka"
+
+  MODULES_PAIMON: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"
+
+  MODULES_ELASTICSEARCH: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
+
+  MODULES_MAXCOMPUTE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
+
+  MODULES_FLUSS: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss"
+
   MODULES_PIPELINE_E2E: "\
   flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
 
   MODULES_SOURCE_E2E: "\
   flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
 
 jobs:
-  compile_and_test:
+  test:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     strategy:
+      fail-fast: false
       matrix:
-        java-version: ${{ fromJSON(inputs.java-version) }}
-        flink-version: ${{ fromJSON(inputs.flink-version) }}
+        java-version: ${{ fromJSON(inputs.java-versions) }}
+        flink-version: ${{ fromJSON(inputs.flink-versions) }}
+        module: ${{ fromJSON(inputs.modules) }}
     steps:
       - run: echo "Running CI pipeline for JDK version ${{ matrix.java-version }}"
       - name: Clean up disk space
@@ -150,28 +190,34 @@ jobs:
       - name: Compile and test
         timeout-minutes: 90
         run: |
+          . .github/workflows/utils.sh
+          jvm_timezone=$(random_timezone)
+          echo "JVM timezone is set to $jvm_timezone"
           set -o pipefail
 
-          case ${{ inputs.module }} in
+          case ${{ matrix.module }} in
               ("core")
                modules=${{ env.MODULES_CORE }}
               ;;
-              ("pipeline_connectors")
-               modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
+              ("mysql-source")
+               modules=${{ env.MODULES_MYSQL_SOURCE }}
               ;;
-              ("mysql")
-               modules=${{ env.MODULES_MYSQL }}
+              ("mysql-pipeline")
+               modules=${{ env.MODULES_MYSQL_PIPELINE }}
               ;;
-              ("postgres")
-               modules=${{ env.MODULES_POSTGRES }}
+              ("postgres-source")
+               modules=${{ env.MODULES_POSTGRES_SOURCE }}
+              ;;
+              ("postgres-pipeline")
+               modules=${{ env.MODULES_POSTGRES_PIPELINE }}
               ;;
               ("oracle")
                modules=${{ env.MODULES_ORACLE }}
               ;;
-              ("mongodb6")
+              ("mongodb7")
                modules=${{ env.MODULES_MONGODB }}
               ;;
-              ("mongodb7")
+              ("mongodb8")
                modules=${{ env.MODULES_MONGODB }}
               ;;
               ("sqlserver")
@@ -180,35 +226,62 @@ jobs:
               ("tidb")
                modules=${{ env.MODULES_TIDB }}
               ;;
-              ("oceanbase")
-               modules=${{ env.MODULES_OCEANBASE }}
+              ("oceanbase-source")
+               modules=${{ env.MODULES_OCEANBASE_SOURCE }}
+              ;;
+              ("oceanbase-pipeline")
+               modules=${{ env.MODULES_OCEANBASE_PIPELINE }}
               ;;
               ("db2")
                modules=${{ env.MODULES_DB2 }}
               ;;
               ("vitess")
                modules=${{ env.MODULES_VITESS }}
               ;;
+              ("doris")
+               modules=${{ env.MODULES_DORIS }}
+              ;;
+              ("elasticsearch")
+               modules=${{ env.MODULES_ELASTICSEARCH }}
+              ;;
+              ("iceberg")
+               modules=${{ env.MODULES_ICEBERG }}
+              ;;
+              ("kafka")
+               modules=${{ env.MODULES_KAFKA }}
+              ;;
+              ("maxcompute")
+               modules=${{ env.MODULES_MAXCOMPUTE }}
+              ;;
+              ("paimon")
+               modules=${{ env.MODULES_PAIMON }}
+              ;;
+              ("starrocks")
+               modules=${{ env.MODULES_STARROCKS }}
+              ;;
+              ("fluss")
+               modules=${{ env.MODULES_FLUSS }}
+              ;;
               ("pipeline_e2e")
-               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
+               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
                modules=${{ env.MODULES_PIPELINE_E2E }}
               ;;
               ("source_e2e")
-               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
+               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
                modules=${{ env.MODULES_SOURCE_E2E }}
               ;;
           esac
           
-          if [ ${{ inputs.module }} != "pipeline_e2e" ] && [ ${{ inputs.module }} != "source_e2e" ]; then
+          if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
             compile_modules=$modules
           fi
           
           build_maven_parameter=""
 
-          if [ ${{ inputs.module }} == "mongodb6" ]; then
-            build_maven_parameter="-DspecifiedMongoVersion=6.0.16"
-          elif [ ${{ inputs.module }} == "mongodb7" ]; then
-            build_maven_parameter="-DspecifiedMongoVersion=7.0.12"
+          if [ ${{ matrix.module }} == "mongodb7" ]; then
+            build_maven_parameter="-DspecifiedMongoVersion=7.0.24"
+          elif [ ${{ matrix.module }} == "mongodb8" ]; then
+            build_maven_parameter="-DspecifiedMongoVersion=8.0.14"
           fi
           
           if [ ! -z "${{ matrix.flink-version }}" ]; then
@@ -217,7 +290,7 @@ jobs:
           
           build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }${{ inputs.custom-maven-parameter }}"
           
-          mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules -DspecifiedParallelism=${{ inputs.parallelism }} verify
+          mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules -DspecifiedParallelism=${{ inputs.parallelism }} -Duser.timezone=$jvm_timezone verify
 
       - name: Print JVM thread dumps when cancelled
         if: ${{ failure() }}

.github/workflows/flink_cdc_ci.yml

@@ -57,36 +57,40 @@ jobs:
         run: mvn --no-snapshot-updates -B package -DskipTests
       - name: Run license check
         run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
-  ut:
-    strategy:
-      matrix:
-        module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
-    name: Unit Tests
+  common:
+    name: Common Unit Tests
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-versions: "[8]"
+      modules: "['core']"
+  pipeline-ut:
+    name: Pipeline Unit Tests
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[8]"
-      module: ${{ matrix.module }}
+      java-versions: "[8]"
+      modules: "['mysql-pipeline', 'postgres-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss']"
+  source-ut:
+    name: Source Unit Tests
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-versions: "[8]"
+      modules: "['mysql-source', 'postgres-source', 'oracle', 'mongodb7', 'mongodb8', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
   pipeline_e2e:
     strategy:
+      fail-fast: false
       matrix:
         parallelism: [ 1, 4 ]
     name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism)
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[8]"
-      flink-version: "['1.19.1', '1.20.0']"
-      module: pipeline_e2e
+      java-versions: "[8]"
+      flink-versions: "['1.19.3', '1.20.3']"
+      modules: "['pipeline_e2e']"
       parallelism: ${{ matrix.parallelism }}
   source_e2e:
     name: Source E2E Tests
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[8]"
-      flink-version: "['1.19.1', '1.20.0']"
-      module: source_e2e
-  migration_test:
-    name: Migration Tests
-    uses: ./.github/workflows/flink_cdc_migration_test_base.yml
-    with:
-      java-version: "[8]"
-      flink-version: "['1.19.1', '1.20.0']"
+      java-versions: "[8]"
+      flink-versions: "['1.19.3', '1.20.3']"
+      modules: "['source_e2e']"