Skip to content

Commit 0698e8a

Browse files
wojiaodoubaowojiaodoubao
authored
HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Implementation. (#7294)
Contributed by: ZhengHu, SunXin, XianyinXin, Rascal Wu, FangBo, Yuanzhihuan. Co-authored-by: lijinglun <[email protected]>
1 parent 6cc743e commit 0698e8a

File tree

111 files changed

+14075
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

111 files changed

+14075
-0
lines changed

‎hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml

+11
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,17 @@
124124
<groupId>org.apache.hadoop</groupId>
125125
<artifactId>hadoop-huaweicloud</artifactId>
126126
<scope>compile</scope>
127+
<exclusions>
128+
<exclusion>
129+
<groupId>com.squareup.okhttp3</groupId>
130+
<artifactId>okhttp</artifactId>
131+
</exclusion>
132+
</exclusions>
133+
</dependency>
134+
<dependency>
135+
<groupId>org.apache.hadoop</groupId>
136+
<artifactId>hadoop-tos</artifactId>
137+
<scope>compile</scope>
127138
</dependency>
128139
</dependencies>
129140
</project>

‎hadoop-cloud-storage-project/hadoop-tos/dev-support/findbugs-exclude.xml

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<FindBugsFilter>
18+
<Match>
19+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectOutputStream"/>
20+
<or>
21+
<Bug pattern="IS2_INCONSISTENT_SYNC" />
22+
<Bug pattern="NP_NULL_ON_SOME_PATH" />
23+
</or>
24+
</Match>
25+
<Match>
26+
<Class name="org.apache.hadoop.fs.tosfs.conf.TosKeys"/>
27+
<Bug pattern="MS_MUTABLE_ARRAY" />
28+
</Match>
29+
<Match>
30+
<Class name="org.apache.hadoop.fs.tosfs.object.Constants"/>
31+
<Bug pattern="MS_MUTABLE_ARRAY" />
32+
</Match>
33+
<Match>
34+
<Class name="org.apache.hadoop.fs.tosfs.RawFileStatus"/>
35+
<or>
36+
<Bug pattern="EI_EXPOSE_REP" />
37+
<Bug pattern="EI_EXPOSE_REP2" />
38+
</or>
39+
</Match>
40+
<Match>
41+
<Class name="org.apache.hadoop.fs.tosfs.TosChecksum"/>
42+
<or>
43+
<Bug pattern="EI_EXPOSE_REP" />
44+
<Bug pattern="EI_EXPOSE_REP2" />
45+
</or>
46+
</Match>
47+
<Match>
48+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectContent"/>
49+
<or>
50+
<Bug pattern="EI_EXPOSE_REP" />
51+
<Bug pattern="EI_EXPOSE_REP2" />
52+
</or>
53+
</Match>
54+
<Match>
55+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectInfo"/>
56+
<or>
57+
<Bug pattern="EI_EXPOSE_REP" />
58+
<Bug pattern="EI_EXPOSE_REP2" />
59+
</or>
60+
</Match>
61+
<Match>
62+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectMultiRangeInputStream"/>
63+
<or>
64+
<Bug pattern="EI_EXPOSE_REP" />
65+
<Bug pattern="EI_EXPOSE_REP2" />
66+
</or>
67+
</Match>
68+
<Match>
69+
<Class name="org.apache.hadoop.fs.tosfs.object.ObjectRangeInputStream"/>
70+
<or>
71+
<Bug pattern="EI_EXPOSE_REP" />
72+
<Bug pattern="EI_EXPOSE_REP2" />
73+
</or>
74+
</Match>
75+
<Match>
76+
<Class name="org.apache.hadoop.fs.tosfs.object.tos.GetObjectOutput"/>
77+
<or>
78+
<Bug pattern="EI_EXPOSE_REP" />
79+
<Bug pattern="EI_EXPOSE_REP2" />
80+
</or>
81+
</Match>
82+
<Match>
83+
<Class name="org.apache.hadoop.fs.tosfs.object.tos.ChainTOSInputStream"/>
84+
<Bug pattern="EI_EXPOSE_REP" />
85+
</Match>
86+
<Match>
87+
<Bug pattern="ICAST_QUESTIONABLE_UNSIGNED_RIGHT_SHIFT" />
88+
</Match>
89+
</FindBugsFilter>

‎hadoop-cloud-storage-project/hadoop-tos/pom.xml

+165
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
18+
<modelVersion>4.0.0</modelVersion>
19+
<parent>
20+
<groupId>org.apache.hadoop</groupId>
21+
<artifactId>hadoop-project</artifactId>
22+
<version>3.5.0-SNAPSHOT</version>
23+
<relativePath>../../hadoop-project</relativePath>
24+
</parent>
25+
26+
<artifactId>hadoop-tos</artifactId>
27+
<version>3.5.0-SNAPSHOT</version>
28+
<name>Apache Hadoop Volcano Engine Services support</name>
29+
<description>
30+
This module contains code to support integration with Volcano Engine TOS.
31+
It also declares the dependencies needed to work with Volcano Engine services.
32+
</description>
33+
<packaging>jar</packaging>
34+
35+
<properties>
36+
<file.encoding>UTF-8</file.encoding>
37+
<ve-tos-java-sdk.version>2.8.7</ve-tos-java-sdk.version>
38+
</properties>
39+
40+
<dependencies>
41+
<dependency>
42+
<groupId>org.apache.hadoop</groupId>
43+
<artifactId>hadoop-common</artifactId>
44+
<scope>provided</scope>
45+
</dependency>
46+
<dependency>
47+
<groupId>org.apache.hadoop</groupId>
48+
<artifactId>hadoop-mapreduce-client-core</artifactId>
49+
<scope>provided</scope>
50+
</dependency>
51+
<dependency>
52+
<groupId>com.volcengine</groupId>
53+
<artifactId>ve-tos-java-sdk</artifactId>
54+
<version>${ve-tos-java-sdk.version}</version>
55+
<exclusions>
56+
<exclusion>
57+
<groupId>org.jetbrains.kotlin</groupId>
58+
<artifactId>kotlin-stdlib-common</artifactId>
59+
</exclusion>
60+
</exclusions>
61+
</dependency>
62+
<dependency>
63+
<groupId>org.jetbrains.kotlin</groupId>
64+
<artifactId>kotlin-stdlib-common</artifactId>
65+
<version>1.6.20</version>
66+
</dependency>
67+
68+
<!-- Test dependencies -->
69+
<dependency>
70+
<groupId>org.apache.hadoop</groupId>
71+
<artifactId>hadoop-common</artifactId>
72+
<scope>test</scope>
73+
<type>test-jar</type>
74+
</dependency>
75+
<!-- Artifacts needed to bring up a Mini MR Yarn cluster-->
76+
<dependency>
77+
<groupId>org.apache.hadoop</groupId>
78+
<artifactId>hadoop-mapreduce-examples</artifactId>
79+
<scope>test</scope>
80+
</dependency>
81+
<dependency>
82+
<groupId>org.apache.hadoop</groupId>
83+
<artifactId>hadoop-minicluster</artifactId>
84+
<scope>test</scope>
85+
</dependency>
86+
<dependency>
87+
<groupId>org.apache.hadoop</groupId>
88+
<artifactId>hadoop-distcp</artifactId>
89+
<version>${hadoop.version}</version>
90+
<scope>test</scope>
91+
</dependency>
92+
<dependency>
93+
<groupId>org.apache.hadoop</groupId>
94+
<artifactId>hadoop-distcp</artifactId>
95+
<version>${hadoop.version}</version>
96+
<scope>test</scope>
97+
<type>test-jar</type>
98+
</dependency>
99+
100+
<dependency>
101+
<groupId>org.assertj</groupId>
102+
<artifactId>assertj-core</artifactId>
103+
<scope>test</scope>
104+
</dependency>
105+
<dependency>
106+
<groupId>junit</groupId>
107+
<artifactId>junit</artifactId>
108+
<scope>test</scope>
109+
</dependency>
110+
<dependency>
111+
<groupId>org.mockito</groupId>
112+
<artifactId>mockito-core</artifactId>
113+
<version>4.11.0</version>
114+
<scope>test</scope>
115+
</dependency>
116+
</dependencies>
117+
118+
<build>
119+
<plugins>
120+
<plugin>
121+
<groupId>org.apache.maven.plugins</groupId>
122+
<artifactId>maven-surefire-plugin</artifactId>
123+
<version>2.21.0</version>
124+
<configuration>
125+
<parallel>classes</parallel>
126+
<threadCount>1</threadCount>
127+
<perCoreThreadCount>true</perCoreThreadCount>
128+
<rerunFailingTestsCount>2</rerunFailingTestsCount>
129+
<forkCount>8</forkCount>
130+
<argLine>-Xmx2048m</argLine>
131+
</configuration>
132+
</plugin>
133+
<plugin>
134+
<artifactId>maven-assembly-plugin</artifactId>
135+
<configuration>
136+
<finalName>hadoop-tos-bundle-${project.version}</finalName>
137+
<skipAssembly>false</skipAssembly>
138+
<appendAssemblyId>false</appendAssemblyId>
139+
<descriptors>
140+
<descriptor>src/main/assembly/hadoop-tos.xml</descriptor>
141+
</descriptors>
142+
</configuration>
143+
<executions>
144+
<execution>
145+
<id>make-assembly</id>
146+
<phase>package</phase>
147+
<goals>
148+
<goal>single</goal>
149+
</goals>
150+
</execution>
151+
</executions>
152+
</plugin>
153+
<plugin>
154+
<groupId>com.github.spotbugs</groupId>
155+
<artifactId>spotbugs-maven-plugin</artifactId>
156+
<configuration>
157+
<xmlOutput>true</xmlOutput>
158+
<excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
159+
</excludeFilterFile>
160+
<effort>Max</effort>
161+
</configuration>
162+
</plugin>
163+
</plugins>
164+
</build>
165+
</project>

‎hadoop-cloud-storage-project/hadoop-tos/src/main/assembly/hadoop-tos.xml

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?xml version="1.0"?>
2+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
5+
<!--
6+
/**
7+
* Licensed to the Apache Software Foundation (ASF) under one
8+
* or more contributor license agreements. See the NOTICE file
9+
* distributed with this work for additional information
10+
* regarding copyright ownership. The ASF licenses this file
11+
* to you under the Apache License, Version 2.0 (the
12+
* "License"); you may not use this file except in compliance
13+
* with the License. You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*/
23+
-->
24+
25+
<id>tos</id>
26+
<formats>
27+
<format>tar.gz</format>
28+
</formats>
29+
<dependencySets>
30+
<dependencySet>
31+
<outputDirectory>lib</outputDirectory>
32+
</dependencySet>
33+
</dependencySets>
34+
<fileSets>
35+
<fileSet>
36+
<directory>${project.build.directory}</directory>
37+
<outputDirectory>/</outputDirectory>
38+
<includes>
39+
<include>*.jar</include>
40+
</includes>
41+
</fileSet>
42+
</fileSets>
43+
</assembly>

‎hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFS.java

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.tosfs;
20+
21+
import org.apache.hadoop.conf.Configuration;
22+
import org.apache.hadoop.fs.DelegateToFileSystem;
23+
24+
import java.io.IOException;
25+
import java.net.URI;
26+
import java.net.URISyntaxException;
27+
28+
/**
29+
* The implementation class of the raw AbstractFileSystem. If you want to use object storage as
30+
* YARN’s resource storage dir via the fs.defaultFS configuration property in Hadoop’s
31+
* core-site.xml, you should add this configuration to Hadoop's core-site.xml.
32+
* <pre>
33+
* fs.AbstractFileSystem.{scheme}.impl=io.proton.fs.RawFS.
34+
* </pre>
35+
*/
36+
public class RawFS extends DelegateToFileSystem {
37+
private static final int TOS_DEFAULT_PORT = -1;
38+
39+
public RawFS(URI uri, Configuration conf) throws IOException, URISyntaxException {
40+
super(uri, new RawFileSystem(), conf, uri.getScheme(), false);
41+
}
42+
43+
@Override
44+
public int getUriDefaultPort() {
45+
return TOS_DEFAULT_PORT;
46+
}
47+
}

0 commit comments

Comments
 (0)