Skip to content

Commit 51094e9

Browse files
wuchongwuchong
committed
[flink] Add separate fluss-flink-${flink.version} modules for Flink 1.18~1.20
1 parent 93caa42 commit 51094e9

File tree

44 files changed

+1192
-253
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1192
-253
lines changed

‎.github/workflows/ci-flink.yaml‎

Lines changed: 0 additions & 55 deletions
This file was deleted.

‎.github/workflows/ci.yaml‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515
################################################################################
16-
name: Tests on JDK 8
16+
name: CI
1717
on:
1818
push:
1919
branches:
@@ -31,13 +31,22 @@ concurrency:
3131
jobs:
3232
build:
3333
runs-on: self-hosted
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
module: [ core, flink ]
3438
steps:
3539
- name: Checkout code
3640
uses: actions/checkout@v2
41+
- name: Build
42+
run: |
43+
mvn -T 1C -B clean install -DskipTests
3744
- name: Test
3845
timeout-minutes: 60
3946
run: |
40-
mvn -B verify -Ptest-coverage -Dlog.dir=${{ runner.temp }}/fluss-logs -Dlog4j.configurationFile=${{ github.workspace }}/tools/ci/log4j.properties
47+
TEST_MODULES=$(./.github/workflows/stage.sh ${{ matrix.module }})
48+
echo "Start testing modules: $TEST_MODULES"
49+
mvn -B verify $TEST_MODULES -Ptest-coverage -Ptest-${{ matrix.module }} -Dlog.dir=${{ runner.temp }}/fluss-logs -Dlog4j.configurationFile=${{ github.workspace }}/tools/ci/log4j.properties
4150
env:
4251
MAVEN_OPTS: -Xmx4096m
4352
ARTIFACTS_OSS_ENDPOINT: ${{ secrets.ARTIFACTS_OSS_ENDPOINT }}
@@ -51,5 +60,5 @@ jobs:
5160
uses: actions/upload-artifact@v4
5261
if: ${{ failure() }}
5362
with:
54-
name: logs-test-${{ github.run_number}}#${{ github.run_attempt }}
63+
name: logs-test-${{ matrix.module }}-${{ github.run_number}}#${{ github.run_attempt }}
5564
path: ${{ runner.temp }}/fluss-logs/*

‎.github/workflows/stage.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/usr/bin/env bash
2+
################################################################################
3+
# Copyright (c) 2025 Alibaba Group Holding Ltd.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
################################################################################
17+
18+
STAGE_CORE="core"
19+
STAGE_FLINK="flink"
20+
21+
MODULES_FLINK="\
22+
fluss-flink,\
23+
fluss-flink/fluss-flink-common,\
24+
fluss-flink/fluss-flink-1.20,\
25+
fluss-flink/fluss-flink-1.19,\
26+
fluss-flink/fluss-flink-1.18,\
27+
fluss-lakehouse,\
28+
fluss-lakehouse/fluss-lakehouse-cli,\
29+
fluss-lakehouse/fluss-lakehouse-paimon,\
30+
fluss-lake,\
31+
"
32+
33+
function get_test_modules_for_stage() {
34+
local stage=$1
35+
36+
local modules_flink=$MODULES_FLINK
37+
local modules_core=\!${MODULES_FLINK//,/,\!}
38+
39+
case ${stage} in
40+
(${STAGE_CORE})
41+
echo "-pl $modules_core"
42+
;;
43+
(${STAGE_FLINK})
44+
echo "-pl fluss-test-coverage,$modules_flink"
45+
;;
46+
esac
47+
}
48+
49+
get_test_modules_for_stage $1

‎fluss-flink/README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<!--
2+
Copyright (c) 2025 Alibaba Group Holding Ltd.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
-->
16+
17+
# Multi-Version Support for Flink Engine
18+
19+
Fluss supports multiple versions of Apache Flink by providing dedicated modules for each version.
20+
The `fluss-flink-common` module always targets the latest version of Flink, while the `fluss-flink-${flink.version}` modules depend on both `fluss-flink-common` and the corresponding Flink version.
21+
22+
Occasionally, Flink's interfaces may change across versions. For example, the class `org.apache.flink.api.connector.sink2.WriterInitContext` was
23+
introduced in Flink v1.19, while older APIs were deprecated or removed. In such cases, we address compatibility issues within the specific `fluss-flink-${flink.version}` module.
24+
This may involve introducing placeholder classes, such as `org.apache.flink.api.connector.sink2.WriterInitContext`, to ensure successful compilation.

‎fluss-flink/fluss-flink-1.18/pom.xml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright (c) 2025 Alibaba Group Holding Ltd.
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License");
6+
~ you may not use this file except in compliance with the License.
7+
~ You may obtain a copy of the License at
8+
~
9+
~ http://www.apache.org/licenses/LICENSE-2.0
10+
~
11+
~ Unless required by applicable law or agreed to in writing, software
12+
~ distributed under the License is distributed on an "AS IS" BASIS,
13+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
~ See the License for the specific language governing permissions and
15+
~ limitations under the License.
16+
-->
17+
18+
<project xmlns="http://maven.apache.org/POM/4.0.0"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
<parent>
23+
<groupId>com.alibaba.fluss</groupId>
24+
<artifactId>fluss-flink</artifactId>
25+
<version>0.7-SNAPSHOT</version>
26+
</parent>
27+
28+
<artifactId>fluss-flink-1.18</artifactId>
29+
30+
<name>Fluss : Engine Flink : 1.18</name>
31+
32+
<properties>
33+
<flink.major.version>1.18</flink.major.version>
34+
<flink.minor.version>1.18.1</flink.minor.version>
35+
</properties>
36+
37+
<dependencies>
38+
<!-- Fluss dependency -->
39+
<dependency>
40+
<groupId>com.alibaba.fluss</groupId>
41+
<artifactId>fluss-client</artifactId>
42+
<version>${project.version}</version>
43+
</dependency>
44+
45+
<dependency>
46+
<groupId>com.alibaba.fluss</groupId>
47+
<artifactId>fluss-flink-common</artifactId>
48+
<version>${project.version}</version>
49+
<exclusions>
50+
<exclusion>
51+
<groupId>*</groupId>
52+
<artifactId>*</artifactId>
53+
</exclusion>
54+
</exclusions>
55+
</dependency>
56+
57+
<!-- Flink dependency -->
58+
<dependency>
59+
<groupId>org.apache.flink</groupId>
60+
<artifactId>flink-core</artifactId>
61+
<version>${flink.minor.version}</version>
62+
<scope>provided</scope>
63+
</dependency>
64+
65+
<dependency>
66+
<groupId>org.apache.flink</groupId>
67+
<artifactId>flink-table-common</artifactId>
68+
<version>${flink.minor.version}</version>
69+
<scope>provided</scope>
70+
</dependency>
71+
72+
<!-- test dependency -->
73+
<dependency>
74+
<groupId>com.alibaba.fluss</groupId>
75+
<artifactId>fluss-flink-common</artifactId>
76+
<version>${project.version}</version>
77+
<scope>test</scope>
78+
<type>test-jar</type>
79+
</dependency>
80+
81+
<dependency>
82+
<groupId>com.alibaba.fluss</groupId>
83+
<artifactId>fluss-server</artifactId>
84+
<version>${project.version}</version>
85+
<scope>test</scope>
86+
</dependency>
87+
88+
<dependency>
89+
<groupId>org.apache.flink</groupId>
90+
<artifactId>flink-table-test-utils</artifactId>
91+
<version>${flink.minor.version}</version>
92+
<scope>test</scope>
93+
</dependency>
94+
95+
<dependency>
96+
<groupId>org.apache.flink</groupId>
97+
<artifactId>flink-connector-base</artifactId>
98+
<version>${flink.minor.version}</version>
99+
<scope>test</scope>
100+
</dependency>
101+
102+
103+
<dependency>
104+
<groupId>com.alibaba.fluss</groupId>
105+
<artifactId>fluss-server</artifactId>
106+
<version>${project.version}</version>
107+
<scope>test</scope>
108+
<type>test-jar</type>
109+
</dependency>
110+
111+
<!-- for curator TestingServer -->
112+
<dependency>
113+
<groupId>org.apache.curator</groupId>
114+
<artifactId>curator-test</artifactId>
115+
<version>${curator.version}</version>
116+
<scope>test</scope>
117+
</dependency>
118+
119+
<dependency>
120+
<groupId>org.apache.flink</groupId>
121+
<artifactId>flink-table-common</artifactId>
122+
<version>${flink.minor.version}</version>
123+
<scope>test</scope>
124+
<type>test-jar</type>
125+
</dependency>
126+
127+
<dependency>
128+
<groupId>org.apache.flink</groupId>
129+
<artifactId>flink-connector-test-utils</artifactId>
130+
<version>${flink.minor.version}</version>
131+
<scope>test</scope>
132+
</dependency>
133+
134+
135+
<dependency>
136+
<groupId>com.alibaba.fluss</groupId>
137+
<artifactId>fluss-test-utils</artifactId>
138+
</dependency>
139+
140+
<dependency>
141+
<groupId>com.alibaba.fluss</groupId>
142+
<artifactId>fluss-common</artifactId>
143+
<version>${project.version}</version>
144+
<scope>test</scope>
145+
<type>test-jar</type>
146+
</dependency>
147+
</dependencies>
148+
149+
<build>
150+
<plugins>
151+
<plugin>
152+
<groupId>org.apache.maven.plugins</groupId>
153+
<artifactId>maven-shade-plugin</artifactId>
154+
<executions>
155+
<execution>
156+
<id>shade-fluss</id>
157+
<phase>package</phase>
158+
<goals>
159+
<goal>shade</goal>
160+
</goals>
161+
<configuration>
162+
<artifactSet>
163+
<includes combine.children="append">
164+
<include>com.alibaba.fluss:fluss-client</include>
165+
</includes>
166+
</artifactSet>
167+
</configuration>
168+
</execution>
169+
</executions>
170+
</plugin>
171+
</plugins>
172+
</build>
173+
174+
</project>

fluss-flink/fluss-flink-common/src/test/java/org/apache/flink/api/common/JobInfo.java renamed to fluss-flink/fluss-flink-1.18/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.apache.flink.api.common;
17+
package org.apache.flink.api.connector.sink2;
1818

1919
/** Placeholder class to resolve compatibility issues. */
20-
public interface JobInfo {}
20+
public interface WriterInitContext extends Sink.InitContext {}

‎fluss-flink/fluss-flink-1.18/src/test/java/com/alibaba/fluss/flink/catalog/Flink118CatalogITCase.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright (c) 2025 Alibaba Group Holding Ltd.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.alibaba.fluss.flink.catalog;
18+
19+
/** IT case for catalog in Flink 1.18. */
20+
public class Flink118CatalogITCase extends FlinkCatalogITCase {}

0 commit comments

Comments
 (0)