Skip to content

Commit 2b1cce5

Browse files
andybj0228andybj0228
authored
[filesystem] Add HuaweiCloud OBS integration (#1274)
1 parent 60bc74f commit 2b1cce5

24 files changed

+1496
-0
lines changed

‎fluss-dist/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,13 @@
6868
<scope>provided</scope>
6969
</dependency>
7070

71+
<dependency>
72+
<groupId>com.alibaba.fluss</groupId>
73+
<artifactId>fluss-fs-obs</artifactId>
74+
<version>${project.version}</version>
75+
<scope>provided</scope>
76+
</dependency>
77+
7178
<!-- metrics plugin -->
7279
<dependency>
7380
<groupId>com.alibaba.fluss</groupId>

‎fluss-dist/src/main/assemblies/plugins.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@
4545
<files>
4646
<!-- filesystem -->
4747
<!-- output directory should correspond to the file system *schema* name, i.e., plugins/<schema>/ -->
48+
<file>
49+
<source>../fluss-filesystems/fluss-fs-obs/target/fluss-fs-obs-${project.version}.jar</source>
50+
<outputDirectory>plugins/obs/</outputDirectory>
51+
<destName>fluss-fs-obs-${project.version}.jar</destName>
52+
<fileMode>0644</fileMode>
53+
</file>
4854
<file>
4955
<source>../fluss-filesystems/fluss-fs-gs/target/fluss-fs-gs-${project.version}.jar</source>
5056
<outputDirectory>plugins/gs/</outputDirectory>

‎fluss-filesystems/fluss-fs-obs/pom.xml‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
<parent>
25+
<groupId>com.alibaba.fluss</groupId>
26+
<artifactId>fluss-filesystems</artifactId>
27+
<version>0.8-SNAPSHOT</version>
28+
</parent>
29+
30+
<artifactId>fluss-fs-obs</artifactId>
31+
<name>Fluss : FileSystems : OBS FS</name>
32+
33+
<properties>
34+
<fs.obs.sdk.version>3.24.12</fs.obs.sdk.version>
35+
<huaweicloud.sdk.iam.version>3.1.87</huaweicloud.sdk.iam.version>
36+
<hadoop.huaweicloud.version>3.4.0</hadoop.huaweicloud.version>
37+
</properties>
38+
39+
<dependencies>
40+
<dependency>
41+
<groupId>com.alibaba.fluss</groupId>
42+
<artifactId>fluss-common</artifactId>
43+
<version>${project.version}</version>
44+
<scope>provided</scope>
45+
</dependency>
46+
47+
<dependency>
48+
<groupId>com.alibaba.fluss</groupId>
49+
<artifactId>fluss-fs-hadoop-shaded</artifactId>
50+
<version>${project.version}</version>
51+
</dependency>
52+
53+
<dependency>
54+
<groupId>com.alibaba.fluss</groupId>
55+
<artifactId>fluss-fs-hadoop</artifactId>
56+
<version>${project.version}</version>
57+
<exclusions>
58+
<exclusion>
59+
<groupId>org.apache.hadoop</groupId>
60+
<artifactId>hadoop-hdfs-client</artifactId>
61+
</exclusion>
62+
</exclusions>
63+
</dependency>
64+
65+
<dependency>
66+
<groupId>org.apache.hadoop</groupId>
67+
<artifactId>hadoop-huaweicloud</artifactId>
68+
<version>${hadoop.huaweicloud.version}</version>
69+
<exclusions>
70+
<exclusion>
71+
<groupId>com.huaweicloud</groupId>
72+
<artifactId>esdk-obs-java-bundle</artifactId>
73+
</exclusion>
74+
<exclusion>
75+
<groupId>org.apache.hadoop</groupId>
76+
<artifactId>hadoop-common</artifactId>
77+
</exclusion>
78+
<exclusion>
79+
<groupId>ch.qos.reload4j</groupId>
80+
<artifactId>reload4j</artifactId>
81+
</exclusion>
82+
<exclusion>
83+
<groupId>org.slf4j</groupId>
84+
<artifactId>slf4j-reload4j</artifactId>
85+
</exclusion>
86+
</exclusions>
87+
</dependency>
88+
89+
<dependency>
90+
<groupId>com.huaweicloud</groupId>
91+
<artifactId>esdk-obs-java</artifactId>
92+
<version>${fs.obs.sdk.version}</version>
93+
<exclusions>
94+
<exclusion>
95+
<groupId>javax.xml.bind</groupId>
96+
<artifactId>jaxb-api</artifactId>
97+
</exclusion>
98+
</exclusions>
99+
</dependency>
100+
101+
<dependency>
102+
<groupId>com.huaweicloud.sdk</groupId>
103+
<artifactId>huaweicloud-sdk-iam</artifactId>
104+
<version>${huaweicloud.sdk.iam.version}</version>
105+
<exclusions>
106+
<exclusion>
107+
<groupId>org.bouncycastle</groupId>
108+
<artifactId>bcprov-jdk18on</artifactId>
109+
</exclusion>
110+
<exclusion>
111+
<groupId>org.openeuler</groupId>
112+
<artifactId>bgmprovider</artifactId>
113+
</exclusion>
114+
</exclusions>
115+
</dependency>
116+
117+
<dependency>
118+
<!-- Hadoop requires jaxb-api for javax.xml.bind.JAXBException -->
119+
<groupId>javax.xml.bind</groupId>
120+
<artifactId>jaxb-api</artifactId>
121+
<version>${jaxb.api.version}</version>
122+
<!-- packaged as an optional dependency that is only accessible on Java 11+ -->
123+
<scope>provided</scope>
124+
</dependency>
125+
126+
<dependency>
127+
<groupId>com.alibaba.fluss</groupId>
128+
<artifactId>fluss-test-utils</artifactId>
129+
</dependency>
130+
<!-- for the behavior test suite -->
131+
<dependency>
132+
<groupId>com.alibaba.fluss</groupId>
133+
<artifactId>fluss-common</artifactId>
134+
<version>${project.version}</version>
135+
<scope>test</scope>
136+
<type>test-jar</type>
137+
</dependency>
138+
</dependencies>
139+
140+
<build>
141+
<plugins>
142+
<plugin>
143+
<groupId>org.apache.maven.plugins</groupId>
144+
<artifactId>maven-jar-plugin</artifactId>
145+
<executions>
146+
<execution>
147+
<goals>
148+
<goal>jar</goal>
149+
</goals>
150+
</execution>
151+
</executions>
152+
<configuration>
153+
<archive>
154+
<manifestEntries>
155+
<!-- jaxb-api is packaged as an optional dependency that is only accessible on Java 11 -->
156+
<Multi-Release>true</Multi-Release>
157+
</manifestEntries>
158+
</archive>
159+
</configuration>
160+
</plugin>
161+
162+
<plugin>
163+
<groupId>org.apache.maven.plugins</groupId>
164+
<artifactId>maven-dependency-plugin</artifactId>
165+
<executions>
166+
<execution>
167+
<id>copy-javax-jars</id>
168+
<phase>process-resources</phase>
169+
<goals>
170+
<goal>copy</goal>
171+
</goals>
172+
</execution>
173+
</executions>
174+
<configuration>
175+
<artifactItems>
176+
<artifactItem>
177+
<groupId>javax.xml.bind</groupId>
178+
<artifactId>jaxb-api</artifactId>
179+
<version>${jaxb.api.version}</version>
180+
<type>jar</type>
181+
<overWrite>true</overWrite>
182+
</artifactItem>
183+
</artifactItems>
184+
<outputDirectory>${project.build.directory}/temporary</outputDirectory>
185+
</configuration>
186+
</plugin>
187+
188+
<plugin>
189+
<groupId>org.apache.maven.plugins</groupId>
190+
<artifactId>maven-antrun-plugin</artifactId>
191+
<executions>
192+
<execution>
193+
<id>unpack-javax-libraries</id>
194+
<phase>process-resources</phase>
195+
<goals>
196+
<goal>run</goal>
197+
</goals>
198+
<configuration>
199+
<target>
200+
<echo message="unpacking javax jars"/>
201+
<unzip dest="${project.build.directory}/classes/META-INF/versions/11">
202+
<fileset dir="${project.build.directory}/temporary">
203+
<include name="*"/>
204+
</fileset>
205+
</unzip>
206+
</target>
207+
</configuration>
208+
</execution>
209+
</executions>
210+
</plugin>
211+
212+
<plugin>
213+
<groupId>org.apache.maven.plugins</groupId>
214+
<artifactId>maven-shade-plugin</artifactId>
215+
<executions>
216+
<execution>
217+
<id>shade-fluss</id>
218+
<phase>package</phase>
219+
<goals>
220+
<goal>shade</goal>
221+
</goals>
222+
<configuration>
223+
<artifactSet>
224+
<includes>
225+
<include>*:*</include>
226+
</includes>
227+
<excludes>
228+
<exclude>javax.servlet:servlet-api</exclude>
229+
<exclude>xmlenc:xmlenc</exclude>
230+
</excludes>
231+
</artifactSet>
232+
<filters>
233+
<filter>
234+
<artifact>*</artifact>
235+
<excludes>
236+
<exclude>.gitkeep</exclude>
237+
<exclude>mime.types</exclude>
238+
<exclude>mozilla/**</exclude>
239+
<exclude>LICENSE.txt</exclude>
240+
<exclude>license/LICENSE*</exclude>
241+
<exclude>okhttp3/internal/publicsuffix/NOTICE</exclude>
242+
<exclude>NOTICE</exclude>
243+
</excludes>
244+
</filter>
245+
</filters>
246+
<relocations>
247+
<relocation>
248+
<pattern>org.apache.commons</pattern>
249+
<shadedPattern>com.alibaba.fluss.shaded.org.apache.commons</shadedPattern>
250+
</relocation>
251+
</relocations>
252+
</configuration>
253+
</execution>
254+
</executions>
255+
</plugin>
256+
</plugins>
257+
</build>
258+
259+
</project>

‎fluss-filesystems/fluss-fs-obs/src/main/java/com/alibaba/fluss/fs/obs/OBSFileSystem.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.alibaba.fluss.fs.obs;
19+
20+
import com.alibaba.fluss.fs.hdfs.HadoopFileSystem;
21+
import com.alibaba.fluss.fs.obs.token.OBSSecurityTokenProvider;
22+
import com.alibaba.fluss.fs.token.ObtainedSecurityToken;
23+
24+
import org.apache.hadoop.conf.Configuration;
25+
import org.apache.hadoop.fs.FileSystem;
26+
27+
import java.io.IOException;
28+
29+
/**
30+
* A {@link FileSystem} for HuaweiCloud OBS that wraps an {@link HadoopFileSystem}, but overwrite
31+
* method to generate access security token.
32+
*/
33+
class OBSFileSystem extends HadoopFileSystem {
34+
35+
private final Configuration conf;
36+
private volatile OBSSecurityTokenProvider obsSecurityTokenProvider;
37+
private final String scheme;
38+
39+
OBSFileSystem(FileSystem hadoopFileSystem, String scheme, Configuration conf) {
40+
super(hadoopFileSystem);
41+
this.scheme = scheme;
42+
this.conf = conf;
43+
}
44+
45+
@Override
46+
public ObtainedSecurityToken obtainSecurityToken() throws IOException {
47+
try {
48+
mayCreateSecurityTokenProvider();
49+
return obsSecurityTokenProvider.obtainSecurityToken(scheme);
50+
} catch (Exception e) {
51+
throw new IOException(e);
52+
}
53+
}
54+
55+
private void mayCreateSecurityTokenProvider() throws IOException {
56+
if (obsSecurityTokenProvider == null) {
57+
synchronized (this) {
58+
if (obsSecurityTokenProvider == null) {
59+
obsSecurityTokenProvider = new OBSSecurityTokenProvider(conf);
60+
}
61+
}
62+
}
63+
}
64+
}

0 commit comments

Comments
 (0)