Skip to content

Commit 223a3c4

Browse files
beryllwberyllw
authored
[filesystems/hdfs] Publish Bundled HDFS Artifact (#1522)
1 parent 60363bd commit 223a3c4

File tree

31 files changed

+367
-223
lines changed

31 files changed

+367
-223
lines changed

‎fluss-dist/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
<!-- filesystem plugin -->
4343
<dependency>
4444
<groupId>org.apache.fluss</groupId>
45-
<artifactId>fluss-fs-hadoop</artifactId>
45+
<artifactId>fluss-fs-hdfs</artifactId>
4646
<version>${project.version}</version>
4747
<scope>provided</scope>
4848
</dependency>

‎fluss-dist/src/main/assemblies/plugins.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@
4646
<!-- filesystem -->
4747
<!-- output directory should correspond to the file system *schema* name, i.e., plugins/<schema>/ -->
4848
<file>
49-
<source>../fluss-filesystems/fluss-fs-hadoop/target/fluss-fs-hadoop-${project.version}-with-dependencies.jar</source>
49+
<source>../fluss-filesystems/fluss-fs-hdfs/target/fluss-fs-hdfs-${project.version}.jar</source>
5050
<outputDirectory>plugins/hdfs/</outputDirectory>
51-
<destName>fluss-fs-hadoop-${project.version}.jar</destName>
51+
<destName>fluss-fs-hdfs-${project.version}.jar</destName>
5252
<fileMode>0644</fileMode>
5353
</file>
5454

‎fluss-filesystems/fluss-fs-gs/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -185,12 +185,6 @@
185185
<groupId>org.apache.fluss</groupId>
186186
<artifactId>fluss-fs-hadoop</artifactId>
187187
<version>${project.version}</version>
188-
<exclusions>
189-
<exclusion>
190-
<groupId>org.apache.hadoop</groupId>
191-
<artifactId>hadoop-hdfs-client</artifactId>
192-
</exclusion>
193-
</exclusions>
194188
</dependency>
195189

196190
<dependency>

‎fluss-filesystems/fluss-fs-gs/src/main/java/org/apache/fluss/fs/gs/GSFileSystem.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@
1919

2020
import org.apache.fluss.fs.FileSystem;
2121
import org.apache.fluss.fs.hdfs.HadoopFileSystem;
22+
import org.apache.fluss.fs.token.ObtainedSecurityToken;
2223

2324
import org.apache.hadoop.conf.Configuration;
2425

26+
import java.io.IOException;
27+
2528
// TODO: implement obtainSecurityToken to enable clients access the Google File System
2629
// through the FileSystem security token receiver API.
2730
/**
@@ -48,4 +51,9 @@ public GSFileSystem(
4851
this.scheme = scheme;
4952
this.conf = conf;
5053
}
54+
55+
@Override
56+
public ObtainedSecurityToken obtainSecurityToken() throws IOException {
57+
throw new UnsupportedOperationException("Not impl.");
58+
}
5159
}

‎fluss-filesystems/fluss-fs-hadoop/pom.xml‎

Lines changed: 4 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -43,165 +43,24 @@
4343
<groupId>org.apache.fluss</groupId>
4444
<artifactId>fluss-fs-hadoop-shaded</artifactId>
4545
<version>${project.version}</version>
46-
</dependency>
47-
48-
<dependency>
49-
<groupId>org.apache.hadoop</groupId>
50-
<artifactId>hadoop-hdfs</artifactId>
51-
<!-- to keep same to the hadoop version of fluss-fs-hadoop-shaded -->
52-
<version>${fs.hadoopshaded.version}</version>
5346
<scope>provided</scope>
54-
<exclusions>
55-
<exclusion>
56-
<groupId>ch.qos.reload4j</groupId>
57-
<artifactId>reload4j</artifactId>
58-
</exclusion>
59-
<exclusion>
60-
<groupId>org.slf4j</groupId>
61-
<artifactId>slf4j-reload4j</artifactId>
62-
</exclusion>
63-
</exclusions>
64-
</dependency>
65-
66-
<dependency>
67-
<groupId>org.apache.hadoop</groupId>
68-
<artifactId>hadoop-hdfs-client</artifactId>
69-
<version>${fs.hadoopshaded.version}</version>
70-
<exclusions>
71-
<exclusion>
72-
<groupId>ch.qos.reload4j</groupId>
73-
<artifactId>reload4j</artifactId>
74-
</exclusion>
75-
<exclusion>
76-
<groupId>org.slf4j</groupId>
77-
<artifactId>slf4j-reload4j</artifactId>
78-
</exclusion>
79-
</exclusions>
8047
</dependency>
8148

8249
<!-- for the behavior test suite -->
83-
<dependency>
84-
<groupId>org.apache.fluss</groupId>
85-
<artifactId>fluss-common</artifactId>
86-
<version>${project.version}</version>
87-
<scope>test</scope>
88-
<type>test-jar</type>
89-
</dependency>
90-
9150
<dependency>
9251
<groupId>org.apache.fluss</groupId>
9352
<artifactId>fluss-test-utils</artifactId>
9453
</dependency>
95-
96-
<!-- for the HDFS mini cluster test suite -->
97-
<dependency>
98-
<groupId>org.apache.hadoop</groupId>
99-
<artifactId>hadoop-hdfs</artifactId>
100-
<version>${fluss.hadoop.version}</version>
101-
<scope>test</scope>
102-
<type>test-jar</type>
103-
<exclusions>
104-
<exclusion>
105-
<groupId>log4j</groupId>
106-
<artifactId>log4j</artifactId>
107-
</exclusion>
108-
<exclusion>
109-
<groupId>ch.qos.reload4j</groupId>
110-
<artifactId>reload4j</artifactId>
111-
</exclusion>
112-
<exclusion>
113-
<groupId>org.slf4j</groupId>
114-
<artifactId>slf4j-reload4j</artifactId>
115-
</exclusion>
116-
</exclusions>
117-
</dependency>
118-
119-
<dependency>
120-
<groupId>org.apache.hadoop</groupId>
121-
<artifactId>hadoop-common</artifactId>
122-
<scope>test</scope>
123-
<type>test-jar</type>
124-
<version>${fs.hadoopshaded.version}</version><!--$NO-MVN-MAN-VER$-->
125-
<exclusions>
126-
<exclusion>
127-
<groupId>log4j</groupId>
128-
<artifactId>log4j</artifactId>
129-
</exclusion>
130-
<exclusion>
131-
<groupId>org.slf4j</groupId>
132-
<artifactId>slf4j-log4j12</artifactId>
133-
</exclusion>
134-
<exclusion>
135-
<!-- This dependency is no longer shipped with the JDK since Java 9.-->
136-
<groupId>jdk.tools</groupId>
137-
<artifactId>jdk.tools</artifactId>
138-
</exclusion>
139-
<exclusion>
140-
<groupId>ch.qos.reload4j</groupId>
141-
<artifactId>reload4j</artifactId>
142-
</exclusion>
143-
<exclusion>
144-
<groupId>org.slf4j</groupId>
145-
<artifactId>slf4j-reload4j</artifactId>
146-
</exclusion>
147-
</exclusions>
148-
</dependency>
149-
150-
<!-- it's needed to init a hadoop test cluster -->
151-
<!-- https://mvnrepository.com/artifact/org.hamcrest/hamcrest -->
152-
<dependency>
153-
<groupId>org.hamcrest</groupId>
154-
<artifactId>hamcrest</artifactId>
155-
<version>2.2</version>
156-
<scope>test</scope>
157-
</dependency>
158-
15954
</dependencies>
16055

16156
<build>
16257
<plugins>
16358
<plugin>
16459
<groupId>org.apache.maven.plugins</groupId>
165-
<artifactId>maven-shade-plugin</artifactId>
166-
<executions>
167-
<execution>
168-
<id>shade-fluss</id>
169-
<phase>package</phase>
170-
<goals>
171-
<goal>shade</goal>
172-
</goals>
173-
<configuration>
174-
<shadedArtifactAttached>true</shadedArtifactAttached>
175-
<shadedClassifierName>with-dependencies</shadedClassifierName>
176-
<artifactSet>
177-
<includes>
178-
<include>*:*</include>
179-
</includes>
180-
<excludes>
181-
<exclude>javax.servlet:servlet-api</exclude>
182-
<exclude>xmlenc:xmlenc</exclude>
183-
</excludes>
184-
</artifactSet>
185-
<filters>
186-
<filter>
187-
<artifact>*</artifact>
188-
<excludes>
189-
<exclude>.gitkeep</exclude>
190-
<exclude>mime.types</exclude>
191-
<exclude>mozilla/**</exclude>
192-
<exclude>okhttp3/internal/publicsuffix/NOTICE</exclude>
193-
</excludes>
194-
</filter>
195-
</filters>
196-
<relocations>
197-
<relocation>
198-
<pattern>org.apache.commons</pattern>
199-
<shadedPattern>org.apache.fluss.shaded.org.apache.commons</shadedPattern>
200-
</relocation>
201-
</relocations>
202-
</configuration>
203-
</execution>
204-
</executions>
60+
<artifactId>maven-deploy-plugin</artifactId>
61+
<configuration>
62+
<skip>true</skip>
63+
</configuration>
20564
</plugin>
20665
</plugins>
20766
</build>

‎fluss-filesystems/fluss-fs-hadoop/src/main/java/org/apache/fluss/fs/hdfs/HadoopFileSystem.java‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,11 @@
2020
import org.apache.fluss.fs.FileStatus;
2121
import org.apache.fluss.fs.FileSystem;
2222
import org.apache.fluss.fs.FsPath;
23-
import org.apache.fluss.fs.token.ObtainedSecurityToken;
2423

2524
import org.apache.hadoop.fs.Path;
2625

2726
import java.io.IOException;
2827
import java.net.URI;
29-
import java.util.Collections;
3028

3129
import static org.apache.fluss.utils.Preconditions.checkNotNull;
3230

@@ -35,13 +33,10 @@
3533
* additional information regarding copyright ownership. */
3634

3735
/**
38-
* A {@link FileSystem} that wraps an {@link org.apache.hadoop.fs.FileSystem Hadoop File System}.
36+
* An abstract {@link FileSystem} implementation that wraps a {@link org.apache.hadoop.fs.FileSystem
37+
* Hadoop File System}.
3938
*/
40-
public class HadoopFileSystem extends FileSystem {
41-
42-
private static final ObtainedSecurityToken TOKEN =
43-
new ObtainedSecurityToken(
44-
HadoopFsPlugin.SCHEME, new byte[0], null, Collections.emptyMap());
39+
public abstract class HadoopFileSystem extends FileSystem {
4540

4641
/** The wrapped Hadoop File System. */
4742
private final org.apache.hadoop.fs.FileSystem fs;
@@ -60,11 +55,6 @@ public HadoopFileSystem(org.apache.hadoop.fs.FileSystem hadoopFileSystem) {
6055
// file system methods
6156
// ------------------------------------------------------------------------
6257

63-
@Override
64-
public ObtainedSecurityToken obtainSecurityToken() throws IOException {
65-
return TOKEN;
66-
}
67-
6858
@Override
6959
public URI getUri() {
7060
return fs.getUri();

0 commit comments

Comments
 (0)