Skip to content

Commit e7e67af

Browse files
yuxiqianyuxiqian
committed
[ci] Combine test modules together to reduce test matrix size
...as we may need more Flink versions to test against later.
1 parent 533072b commit e7e67af

4 files changed

Lines changed: 189 additions & 197 deletions

File tree

‎.github/workflows/flink_cdc_base.yml‎

Lines changed: 3 additions & 193 deletions
Original file line numberDiff line numberDiff line change
@@ -42,109 +42,6 @@ on:
4242
required: false
4343
type: string
4444

45-
env:
46-
MODULES_CORE: "\
47-
flink-cdc-cli,\
48-
flink-cdc-common,\
49-
flink-cdc-composer,\
50-
flink-cdc-runtime,\
51-
flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base,\
52-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
53-
54-
MODULES_PIPELINE_CONNECTORS: "\
55-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
56-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch,\
57-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg,\
58-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
59-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
60-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
61-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase,\
62-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
63-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres,\
64-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
65-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
66-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss,\
67-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-hudi,\
68-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
69-
70-
MODULES_MYSQL_SOURCE: "\
71-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
72-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
73-
74-
MODULES_MYSQL_PIPELINE: "\
75-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql"
76-
77-
MODULES_POSTGRES_SOURCE: "\
78-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
79-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
80-
81-
MODULES_POSTGRES_PIPELINE: "\
82-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres"
83-
84-
MODULES_ORACLE: "\
85-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
86-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"
87-
88-
MODULES_MONGODB: "\
89-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\
90-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc"
91-
92-
MODULES_SQLSERVER: "\
93-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\
94-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc"
95-
96-
MODULES_TIDB: "\
97-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
98-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
99-
100-
MODULES_OCEANBASE_SOURCE: "\
101-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
102-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
103-
104-
MODULES_OCEANBASE_PIPELINE: "\
105-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase"
106-
107-
MODULES_DB2: "\
108-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
109-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
110-
111-
MODULES_VITESS: "\
112-
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
113-
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
114-
115-
MODULES_DORIS: "\
116-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris"
117-
118-
MODULES_STARROCKS: "\
119-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks"
120-
121-
MODULES_ICEBERG: "\
122-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg"
123-
124-
MODULES_KAFKA: "\
125-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka"
126-
127-
MODULES_PAIMON: "\
128-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"
129-
130-
MODULES_ELASTICSEARCH: "\
131-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
132-
133-
MODULES_MAXCOMPUTE: "\
134-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
135-
136-
MODULES_FLUSS: "\
137-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss"
138-
139-
MODULES_HUDI: "\
140-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-hudi"
141-
142-
MODULES_PIPELINE_E2E: "\
143-
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
144-
145-
MODULES_SOURCE_E2E: "\
146-
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
147-
14845
jobs:
14946
test:
15047
runs-on: ubuntu-latest
@@ -199,98 +96,11 @@ jobs:
19996
echo "JVM timezone is set to $jvm_timezone"
20097
set -o pipefail
20198
202-
case ${{ matrix.module }} in
203-
("core")
204-
modules=${{ env.MODULES_CORE }}
205-
;;
206-
("mysql-source")
207-
modules=${{ env.MODULES_MYSQL_SOURCE }}
208-
;;
209-
("mysql-pipeline")
210-
modules=${{ env.MODULES_MYSQL_PIPELINE }}
211-
;;
212-
("postgres-source")
213-
modules=${{ env.MODULES_POSTGRES_SOURCE }}
214-
;;
215-
("postgres-pipeline")
216-
modules=${{ env.MODULES_POSTGRES_PIPELINE }}
217-
;;
218-
("oracle")
219-
modules=${{ env.MODULES_ORACLE }}
220-
;;
221-
("mongodb7")
222-
modules=${{ env.MODULES_MONGODB }}
223-
;;
224-
("mongodb8")
225-
modules=${{ env.MODULES_MONGODB }}
226-
;;
227-
("sqlserver")
228-
modules=${{ env.MODULES_SQLSERVER }}
229-
;;
230-
("tidb")
231-
modules=${{ env.MODULES_TIDB }}
232-
;;
233-
("oceanbase-source")
234-
modules=${{ env.MODULES_OCEANBASE_SOURCE }}
235-
;;
236-
("oceanbase-pipeline")
237-
modules=${{ env.MODULES_OCEANBASE_PIPELINE }}
238-
;;
239-
("db2")
240-
modules=${{ env.MODULES_DB2 }}
241-
;;
242-
("vitess")
243-
modules=${{ env.MODULES_VITESS }}
244-
;;
245-
("doris")
246-
modules=${{ env.MODULES_DORIS }}
247-
;;
248-
("elasticsearch")
249-
modules=${{ env.MODULES_ELASTICSEARCH }}
250-
;;
251-
("iceberg")
252-
modules=${{ env.MODULES_ICEBERG }}
253-
;;
254-
("kafka")
255-
modules=${{ env.MODULES_KAFKA }}
256-
;;
257-
("maxcompute")
258-
modules=${{ env.MODULES_MAXCOMPUTE }}
259-
;;
260-
("paimon")
261-
modules=${{ env.MODULES_PAIMON }}
262-
;;
263-
("starrocks")
264-
modules=${{ env.MODULES_STARROCKS }}
265-
;;
266-
("fluss")
267-
modules=${{ env.MODULES_FLUSS }}
268-
;;
269-
("hudi")
270-
modules=${{ env.MODULES_HUDI }}
271-
;;
272-
("pipeline_e2e")
273-
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
274-
modules=${{ env.MODULES_PIPELINE_E2E }}
275-
;;
276-
("source_e2e")
277-
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
278-
modules=${{ env.MODULES_SOURCE_E2E }}
279-
;;
280-
esac
99+
modules=$(./.github/workflows/modules.py test "${{ matrix.module }}")
100+
compile_modules=$(./.github/workflows/modules.py compile "${{ matrix.module }}")
281101
282-
if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
283-
compile_modules=$modules
284-
fi
285-
286-
build_maven_parameter=""
102+
build_maven_parameter="-DspecifiedMongoVersion=8.0.14"
287103
288-
if [ ${{ matrix.module }} == "mongodb7" ]; then
289-
build_maven_parameter="-DspecifiedMongoVersion=7.0.24"
290-
elif [ ${{ matrix.module }} == "mongodb8" ]; then
291-
build_maven_parameter="-DspecifiedMongoVersion=8.0.14"
292-
fi
293-
294104
if [ ! -z "${{ matrix.flink-version }}" ]; then
295105
build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }-DspecifiedFlinkVersion=${{ matrix.flink-version }}"
296106
fi

‎.github/workflows/flink_cdc_ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,13 @@ jobs:
6868
uses: ./.github/workflows/flink_cdc_base.yml
6969
with:
7070
java-versions: "[11]"
71-
modules: "['mysql-pipeline', 'postgres-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss', 'hudi']"
71+
modules: "['doris, elasticsearch, fluss, hudi, iceberg, kafka, maxcompute, mysql-pipeline, oceanbase-pipeline, paimon, postgres-pipeline, starrocks']"
7272
source-ut:
7373
name: Source Unit Tests
7474
uses: ./.github/workflows/flink_cdc_base.yml
7575
with:
7676
java-versions: "[11]"
77-
modules: "['mysql-source', 'postgres-source', 'oracle', 'mongodb7', 'mongodb8', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
77+
modules: "['mysql-source', 'postgres-source, oceanbase-source, tidb, vitess', 'oracle, sqlserver', 'db2, mongodb']"
7878
pipeline_e2e:
7979
strategy:
8080
fail-fast: false

‎.github/workflows/flink_cdc_ci_nightly.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,14 @@ jobs:
6060
uses: ./.github/workflows/flink_cdc_base.yml
6161
with:
6262
java-versions: "[17]"
63-
modules: "['mysql-pipeline', 'postgres-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss', 'hudi']"
63+
modules: "['doris, elasticsearch, fluss, hudi, iceberg, kafka, maxcompute, mysql-pipeline, oceanbase-pipeline, paimon, postgres-pipeline, starrocks']"
6464
source-ut:
6565
if: github.repository == 'apache/flink-cdc'
6666
name: Source Unit Tests
6767
uses: ./.github/workflows/flink_cdc_base.yml
6868
with:
6969
java-versions: "[17]"
70-
modules: "['mysql-source', 'postgres-source', 'oracle', 'mongodb7', 'mongodb8', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
70+
modules: "['mysql-source', 'postgres-source, oceanbase-source, tidb, vitess', 'oracle, sqlserver', 'db2, mongodb']"
7171
pipeline_e2e:
7272
if: github.repository == 'apache/flink-cdc'
7373
strategy:

0 commit comments

Comments
 (0)