Skip to content

Commit bd9e1c4

Browse files
luoyuxialuoyuxia
authored
[lake][paimon] Create datalake enabled table should also create in lake (#640)
1 parent 0f0f304 commit bd9e1c4

File tree

25 files changed

+1428
-15
lines changed

25 files changed

+1428
-15
lines changed

‎.github/workflows/stage.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ fluss-lakehouse,\
2828
fluss-lakehouse/fluss-lakehouse-cli,\
2929
fluss-lakehouse/fluss-lakehouse-paimon,\
3030
fluss-lake,\
31+
fluss-lake/fluss-lake-paimon
3132
"
3233

3334
function get_test_modules_for_stage() {

‎fluss-common/src/main/java/com/alibaba/fluss/lakehouse/lakestorage/LakeCatalog.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright (c) 2025 Alibaba Group Holding Ltd.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.alibaba.fluss.lakehouse.lakestorage;
18+
19+
import com.alibaba.fluss.annotation.PublicEvolving;
20+
import com.alibaba.fluss.exception.TableAlreadyExistException;
21+
import com.alibaba.fluss.metadata.TableDescriptor;
22+
import com.alibaba.fluss.metadata.TablePath;
23+
24+
/**
25+
* A catalog interface to modify metadata in external datalake.
26+
*
27+
* @since 0.7
28+
*/
29+
@PublicEvolving
30+
public interface LakeCatalog extends AutoCloseable {
31+
32+
/**
33+
* Create a new table in lake.
34+
*
35+
* @param tablePath path of the table to be created
36+
* @param tableDescriptor The descriptor of the table to be created
37+
* @throws TableAlreadyExistException if the table already exists
38+
*/
39+
void createTable(TablePath tablePath, TableDescriptor tableDescriptor)
40+
throws TableAlreadyExistException;
41+
42+
@Override
43+
default void close() throws Exception {
44+
// default do nothing
45+
}
46+
}

‎fluss-common/src/main/java/com/alibaba/fluss/lakehouse/lakestorage/LakeStorage.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,7 @@ public interface LakeStorage {
3434
* @return the lake tiering factory
3535
*/
3636
LakeTieringFactory createLakeTieringFactory();
37+
38+
/** Create lake catalog. */
39+
LakeCatalog createLakeCatalog();
3740
}

‎fluss-common/src/main/java/com/alibaba/fluss/lakehouse/lakestorage/LakeStoragePluginSetUp.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Copyright (c) 2025 Alibaba Group Holding Ltd.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.alibaba.fluss.lakehouse.lakestorage;
18+
19+
import com.alibaba.fluss.config.ConfigOptions;
20+
import com.alibaba.fluss.config.Configuration;
21+
import com.alibaba.fluss.metadata.DataLakeFormat;
22+
import com.alibaba.fluss.plugin.PluginManager;
23+
import com.alibaba.fluss.shaded.guava32.com.google.common.collect.Iterators;
24+
25+
import javax.annotation.Nullable;
26+
27+
import java.util.Iterator;
28+
import java.util.Objects;
29+
import java.util.ServiceLoader;
30+
31+
/**
32+
* Encapsulates everything needed for the instantiation and configuration of a {@link
33+
* LakeStoragePlugin}.
34+
*/
35+
public class LakeStoragePluginSetUp {
36+
37+
@Nullable
38+
public static LakeStoragePlugin fromConfiguration(
39+
final Configuration configuration, @Nullable final PluginManager pluginManager) {
40+
DataLakeFormat dataLakeFormat = configuration.get(ConfigOptions.DATALAKE_FORMAT);
41+
if (dataLakeFormat == null) {
42+
return null;
43+
}
44+
String dataLakeIdentifier = dataLakeFormat.toString();
45+
// now, load lake storage plugin
46+
Iterator<LakeStoragePlugin> lakeStoragePluginIterator =
47+
getAllLakeStoragePlugins(pluginManager);
48+
49+
while (lakeStoragePluginIterator.hasNext()) {
50+
LakeStoragePlugin lakeStoragePlugin = lakeStoragePluginIterator.next();
51+
if (Objects.equals(lakeStoragePlugin.identifier(), dataLakeIdentifier)) {
52+
return lakeStoragePlugin;
53+
}
54+
}
55+
56+
// if come here, means we haven't found LakeStoragePlugin match the configured
57+
// datalake, throw exception
58+
throw new UnsupportedOperationException(
59+
"No LakeStoragePlugin can be found for datalake format: " + dataLakeIdentifier);
60+
}
61+
62+
private static Iterator<LakeStoragePlugin> getAllLakeStoragePlugins(
63+
@Nullable PluginManager pluginManager) {
64+
final Iterator<LakeStoragePlugin> pluginIteratorSPI =
65+
ServiceLoader.load(LakeStoragePlugin.class).iterator();
66+
if (pluginManager == null) {
67+
return pluginIteratorSPI;
68+
} else {
69+
return Iterators.concat(pluginManager.load(LakeStoragePlugin.class), pluginIteratorSPI);
70+
}
71+
}
72+
}

‎fluss-dist/pom.xml‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,26 @@
8989
<scope>provided</scope>
9090
</dependency>
9191

92+
<dependency>
93+
<groupId>com.alibaba.fluss</groupId>
94+
<artifactId>fluss-lake-paimon</artifactId>
95+
<version>${project.version}</version>
96+
<scope>provided</scope>
97+
</dependency>
98+
99+
<dependency>
100+
<groupId>org.apache.flink</groupId>
101+
<artifactId>flink-shaded-hadoop-2-uber</artifactId>
102+
<version>2.8.3-10.0</version>
103+
<scope>compile</scope>
104+
<exclusions>
105+
<exclusion>
106+
<groupId>org.slf4j</groupId>
107+
<artifactId>slf4j-log4j12</artifactId>
108+
</exclusion>
109+
</exclusions>
110+
</dependency>
111+
92112

93113
<!-- Concrete logging framework - we add this only here (and not in the
94114
root POM) to not tie the projects to one specific framework and make
@@ -118,8 +138,6 @@
118138
<artifactId>log4j-1.2-api</artifactId>
119139
<scope>compile</scope>
120140
</dependency>
121-
122-
123141
</dependencies>
124142

125143

‎fluss-dist/src/main/assemblies/plugins.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,21 @@
2525
<includeBaseDirectory>true</includeBaseDirectory>
2626
<baseDirectory>fluss-${project.version}</baseDirectory>
2727

28+
<dependencySets>
29+
<dependencySet>
30+
<outputDirectory>plugins/paimon</outputDirectory>
31+
<unpack>false</unpack>
32+
<useProjectArtifact>false</useProjectArtifact>
33+
<useProjectAttachments>false</useProjectAttachments>
34+
<useTransitiveDependencies>true</useTransitiveDependencies>
35+
<useTransitiveFiltering>true</useTransitiveFiltering>
36+
<includes>
37+
<include>org.apache.flink:flink-shaded-hadoop-2-uber</include>
38+
</includes>
39+
</dependencySet>
40+
41+
</dependencySets>
42+
2843
<files>
2944
<!-- filesystem -->
3045
<!-- output directory should correspond to the file system *schema* name, i.e., plugins/<schema>/ -->
@@ -63,6 +78,14 @@
6378
<destName>fluss-metrics-jmx-${project.version}.jar</destName>
6479
<fileMode>0644</fileMode>
6580
</file>
81+
82+
<!-- lake formats -->
83+
<file>
84+
<source>../fluss-lake/fluss-lake-paimon/target/fluss-lake-paimon-${project.version}.jar</source>
85+
<outputDirectory>plugins/paimon/</outputDirectory>
86+
<destName>fluss-lake-paimon-${project.version}.jar</destName>
87+
<fileMode>0644</fileMode>
88+
</file>
6689
</files>
6790

6891
</assembly>

‎fluss-lake/fluss-lake-paimon/pom.xml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright (c) 2025 Alibaba Group Holding Ltd.
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License");
6+
~ you may not use this file except in compliance with the License.
7+
~ You may obtain a copy of the License at
8+
~
9+
~ http://www.apache.org/licenses/LICENSE-2.0
10+
~
11+
~ Unless required by applicable law or agreed to in writing, software
12+
~ distributed under the License is distributed on an "AS IS" BASIS,
13+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
~ See the License for the specific language governing permissions and
15+
~ limitations under the License.
16+
-->
17+
18+
<project xmlns="http://maven.apache.org/POM/4.0.0"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
<parent>
23+
<groupId>com.alibaba.fluss</groupId>
24+
<artifactId>fluss-lake</artifactId>
25+
<version>0.7-SNAPSHOT</version>
26+
</parent>
27+
28+
<artifactId>fluss-lake-paimon</artifactId>
29+
<name>Fluss : Lake : Paimon</name>
30+
31+
<packaging>jar</packaging>
32+
33+
<properties>
34+
<paimon.version>1.0.1</paimon.version>
35+
</properties>
36+
37+
<dependencies>
38+
<dependency>
39+
<groupId>com.alibaba.fluss</groupId>
40+
<artifactId>fluss-common</artifactId>
41+
<version>${project.version}</version>
42+
</dependency>
43+
44+
<dependency>
45+
<groupId>org.apache.paimon</groupId>
46+
<artifactId>paimon-bundle</artifactId>
47+
<version>${paimon.version}</version>
48+
</dependency>
49+
50+
<dependency>
51+
<groupId>com.alibaba.fluss</groupId>
52+
<artifactId>fluss-client</artifactId>
53+
<version>${project.version}</version>
54+
<scope>test</scope>
55+
</dependency>
56+
57+
<dependency>
58+
<groupId>com.alibaba.fluss</groupId>
59+
<artifactId>fluss-test-utils</artifactId>
60+
</dependency>
61+
62+
<dependency>
63+
<groupId>org.apache.curator</groupId>
64+
<artifactId>curator-test</artifactId>
65+
<version>${curator.version}</version>
66+
<scope>test</scope>
67+
</dependency>
68+
69+
<dependency>
70+
<groupId>com.alibaba.fluss</groupId>
71+
<artifactId>fluss-server</artifactId>
72+
<version>${project.version}</version>
73+
<scope>test</scope>
74+
</dependency>
75+
76+
<dependency>
77+
<groupId>org.apache.hadoop</groupId>
78+
<artifactId>hadoop-hdfs-client</artifactId>
79+
<version>${fluss.hadoop.version}</version>
80+
<scope>provided</scope>
81+
</dependency>
82+
<dependency>
83+
<groupId>org.apache.hadoop</groupId>
84+
<artifactId>hadoop-common</artifactId>
85+
<scope>provided</scope>
86+
<exclusions>
87+
<exclusion>
88+
<artifactId>avro</artifactId>
89+
<groupId>org.apache.avro</groupId>
90+
</exclusion>
91+
<exclusion>
92+
<artifactId>log4j</artifactId>
93+
<groupId>log4j</groupId>
94+
</exclusion>
95+
<exclusion>
96+
<artifactId>slf4j-log4j12</artifactId>
97+
<groupId>org.slf4j</groupId>
98+
</exclusion>
99+
<exclusion>
100+
<groupId>ch.qos.reload4j</groupId>
101+
<artifactId>reload4j</artifactId>
102+
</exclusion>
103+
<exclusion>
104+
<groupId>org.slf4j</groupId>
105+
<artifactId>slf4j-reload4j</artifactId>
106+
</exclusion>
107+
<exclusion>
108+
<artifactId>jdk.tools</artifactId>
109+
<groupId>jdk.tools</groupId>
110+
</exclusion>
111+
<exclusion>
112+
<artifactId>protobuf-java</artifactId>
113+
<groupId>com.google.protobuf</groupId>
114+
</exclusion>
115+
<exclusion>
116+
<artifactId>commons-io</artifactId>
117+
<groupId>commons-io</groupId>
118+
</exclusion>
119+
</exclusions>
120+
</dependency>
121+
122+
<dependency>
123+
<groupId>com.alibaba.fluss</groupId>
124+
<artifactId>fluss-server</artifactId>
125+
<version>${project.version}</version>
126+
<type>test-jar</type>
127+
<scope>test</scope>
128+
</dependency>
129+
</dependencies>
130+
131+
132+
<build>
133+
<plugins>
134+
<plugin>
135+
<groupId>org.apache.maven.plugins</groupId>
136+
<artifactId>maven-shade-plugin</artifactId>
137+
<executions>
138+
<execution>
139+
<id>shade-fluss</id>
140+
<phase>package</phase>
141+
<goals>
142+
<goal>shade</goal>
143+
</goals>
144+
<configuration>
145+
<artifactSet>
146+
<includes>
147+
<include>org.apache.paimon:paimon-bundle</include>
148+
</includes>
149+
</artifactSet>
150+
</configuration>
151+
</execution>
152+
</executions>
153+
</plugin>
154+
</plugins>
155+
</build>
156+
157+
158+
</project>

0 commit comments

Comments
 (0)