Bump Hadoop to 3.4.0

nastra · nastra · commit e911e997d0ef · 2025-11-21T17:54:49.000+01:00
diff --git a/fluss-filesystems/fluss-fs-gs/src/main/resources/META-INF/NOTICE b/fluss-filesystems/fluss-fs-gs/src/main/resources/META-INF/NOTICE
@@ -81,9 +81,9 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.commons:commons-text:1.4
 - org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1
 - org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1
-- org.apache.hadoop:hadoop-annotations:3.3.4
-- org.apache.hadoop:hadoop-auth:3.3.4
-- org.apache.hadoop:hadoop-common:3.3.4
+- org.apache.hadoop:hadoop-annotations:3.4.0
+- org.apache.hadoop:hadoop-auth:3.4.0
+- org.apache.hadoop:hadoop-common:3.4.0
 - org.apache.httpcomponents:httpclient:4.5.13
 - org.apache.httpcomponents:httpcore:4.4.13
 - org.apache.kerby:kerb-core:1.0.1
diff --git a/fluss-filesystems/fluss-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE b/fluss-filesystems/fluss-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE
@@ -24,9 +24,9 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.commons:commons-text:1.4
 - org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1
 - org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1
-- org.apache.hadoop:hadoop-annotations:3.3.4
-- org.apache.hadoop:hadoop-auth:3.3.4
-- org.apache.hadoop:hadoop-common:3.3.4
+- org.apache.hadoop:hadoop-annotations:3.4.0
+- org.apache.hadoop:hadoop-auth:3.4.0
+- org.apache.hadoop:hadoop-common:3.4.0
 - org.apache.kerby:kerb-core:1.0.1
 - org.apache.kerby:kerby-asn1:1.0.1
 - org.apache.kerby:kerby-pkix:1.0.1
diff --git a/fluss-filesystems/fluss-fs-hdfs/src/main/resources/META-INF/NOTICE b/fluss-filesystems/fluss-fs-hdfs/src/main/resources/META-INF/NOTICE
@@ -9,11 +9,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - com.fasterxml.jackson.core:jackson-annotations:2.15.3
 - com.fasterxml.jackson.core:jackson-core:2.15.3
 - com.fasterxml.jackson.core:jackson-databind:2.15.3
-- com.squareup.okhttp3:okhttp:4.9.3
-- com.squareup.okio:okio:2.8.0
-- org.apache.hadoop:hadoop-hdfs-client:3.3.4
-- org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
-- org.jetbrains.kotlin:kotlin-stdlib:1.4.10
+- org.apache.hadoop:hadoop-hdfs-client:3.4.0
 
 This project bundles the following dependencies under BSD-3 License (https://opensource.org/licenses/BSD-3-Clause).
 See bundled license files for details.
diff --git a/fluss-filesystems/fluss-fs-obs/src/main/resources/META-INF/NOTICE b/fluss-filesystems/fluss-fs-obs/src/main/resources/META-INF/NOTICE
@@ -8,14 +8,14 @@ This project bundles the following dependencies under the Apache Software Licens
 
 - com.huaweicloud:esdk-obs-java:3.24.12
 - com.huaweicloud.sdk:huaweicloud-sdk-iam:3.1.87
-- com.fasterxml.woodstox:woodstox-core:5.3.0
+- com.fasterxml.woodstox:woodstox-core:5.4.0
 - com.google.guava:guava:11.0.2
 - com.jamesmurty.utils:java-xmlbuilder:0.4
 - commons-beanutils:commons-beanutils:1.9.4
 - commons-collections:commons-collections:3.2.2
 - commons-configuration:commons-configuration:1.6
 - commons-digester:commons-digester:1.8
-- commons-io:commons-io:2.5
+- commons-io:commons-io:2.14
 - commons-lang:commons-lang:2.6
 - commons-logging:commons-logging:1.1.3
 - org.apache.hadoop:hadoop-huaweicloud:3.4.0
@@ -26,9 +26,9 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.directory.api:api-util:1.0.0-M20
 - org.apache.directory.server:apacheds-i18n:2.0.0-M15
 - org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15
-- org.apache.hadoop:hadoop-annotations:2.10.2
-- org.apache.hadoop:hadoop-auth:2.10.2
-- org.apache.hadoop:hadoop-common:2.10.2
+- org.apache.hadoop:hadoop-annotations:3.4.0
+- org.apache.hadoop:hadoop-auth:3.4.0
+- org.apache.hadoop:hadoop-common:3.4.0
 - org.apache.htrace:htrace-core4:4.1.0-incubating
 - org.codehaus.jackson:jackson-core-asl:1.9.13
 - org.codehaus.jackson:jackson-mapper-asl:1.9.13
diff --git a/fluss-filesystems/fluss-fs-oss/src/main/resources/META-INF/NOTICE b/fluss-filesystems/fluss-fs-oss/src/main/resources/META-INF/NOTICE
@@ -10,43 +10,61 @@ This project bundles the following dependencies under the Apache Software Licens
 - com.aliyun:aliyun-java-sdk-core:4.5.10
 - com.aliyun:aliyun-java-sdk-kms:2.11.0
 - com.aliyun:aliyun-java-sdk-ram:3.1.0
-- com.fasterxml.woodstox:woodstox-core:5.3.0
+- com.fasterxml.jackson.core:jackson-annotations:2.15.3
+- com.fasterxml.jackson.core:jackson-core:2.15.3
+- com.fasterxml.jackson.core:jackson-databind:2.15.3
+- com.fasterxml.woodstox:woodstox-core:5.4.0
+- com.github.pjfanning:jersey-json:1.20
 - com.google.code.gson:gson:2.8.6
-- com.google.guava:guava:11.0.2
-- com.jamesmurty.utils:java-xmlbuilder:0.4
+- com.google.guava:failureaccess:1.0
+- com.google.guava:guava:27.0-jre
+- com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
+- com.google.j2objc:j2objc-annotations:1.1
+- com.google.re2j:re2j:1.1
+- com.sun.xml.bind:jaxb-impl:2.2.3-1
 - commons-beanutils:commons-beanutils:1.9.4
 - commons-collections:commons-collections:3.2.2
-- commons-configuration:commons-configuration:1.6
-- commons-digester:commons-digester:1.8
-- commons-io:commons-io:2.5
-- commons-lang:commons-lang:2.6
+- commons-io:commons-io:2.14.0
 - commons-codec:commons-codec:1.11
-- commons-logging:commons-logging:1.1.3
+- commons-logging:commons-logging:1.2
+- dnsjava:dnsjava:3.4.0
+- io.dropwizard.metrics:metrics-core:3.2.4
+- io.netty:netty-buffer:4.1.100.Final
+- io.netty:netty-codec:4.1.100.Final
+- io.netty:netty-common:4.1.100.Final
+- io.netty:netty-handler:4.1.100.Final
+- io.netty:netty-resolver:4.1.100.Final
+- io.netty:netty-transport-classes-epoll:4.1.100.Final
+- io.netty:netty-transport-native-epoll:4.1.100.Final
+- io.netty:netty-transport-native-unix-common:4.1.100.Final
+- io.netty:netty-transport:4.1.100.Final
 - io.opentracing:opentracing-api:0.33.0
 - io.opentracing:opentracing-noop:0.33.0
 - io.opentracing:opentracing-util:0.33.0
-- org.apache.hadoop:hadoop-aliyun:3.3.4
+- jakarta.activation:jakarta.activation-api:1.2.1
+- org.apache.hadoop:hadoop-aliyun:3.4.0
+- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.2.0
+- org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_21:1.2.0
 - org.apache.httpcomponents:httpclient:4.5.13
 - org.apache.httpcomponents:httpcore:4.4.13
+- org.apache.kerby:kerb-core:2.0.3
+- org.apache.kerby:kerby-asn1:2.0.3
+- org.apache.kerby:kerby-pkix:2.0.3
+- org.apache.kerby:kerby-util:2.0.3
+- org.bouncycastle:bcprov-jdk15on:1.70
 - org.codehaus.jettison:jettison:1.5.4
 - org.ini4j:ini4j:0.5.4
-- net.java.dev.jets3t:jets3t:0.9.0
-- org.apache.commons:commons-compress:1.21
+- org.apache.commons:commons-compress:1.24.0
+- org.apache.commons:commons-configuration2:2.8.0
+- org.apache.commons:commons-text:1.10.0
 - org.apache.commons:commons-lang3:3.18.0
-- org.apache.directory.api:api-asn1-api:1.0.0-M20
-- org.apache.directory.api:api-util:1.0.0-M20
-- org.apache.directory.server:apacheds-i18n:2.0.0-M15
-- org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15
-- org.apache.hadoop:hadoop-annotations:2.10.2
-- org.apache.hadoop:hadoop-auth:2.10.2
-- org.apache.hadoop:hadoop-common:2.10.2
-- org.apache.htrace:htrace-core4:4.1.0-incubating
-- org.codehaus.jackson:jackson-core-asl:1.9.13
-- org.codehaus.jackson:jackson-mapper-asl:1.9.13
+- org.apache.hadoop:hadoop-annotations:3.4.0
+- org.apache.hadoop:hadoop-auth:3.4.0
+- org.apache.hadoop:hadoop-common:3.4.0
+- org.checkerframework:checker-qual:2.5.2
+- org.codehaus.mojo:animal-sniffer-annotations:1.17
 - org.codehaus.woodstox:stax2-api:4.2.1
-- org.mortbay.jetty:jetty-sslengine:6.1.26
-- org.mortbay.jetty:jetty-util:6.1.26
-- org.mortbay.jetty:jetty:6.1.26
+- org.xerial.snappy:snappy-java:1.1.10.4
 
 The binary distribution of this product bundles these dependencies under the Eclipse Public License - v 2.0 (https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt)
 See bundled license files for details.
diff --git a/fluss-filesystems/fluss-fs-s3/src/main/resources/META-INF/NOTICE b/fluss-filesystems/fluss-fs-s3/src/main/resources/META-INF/NOTICE
@@ -13,41 +13,57 @@ This project bundles the following dependencies under the Apache Software Licens
 - com.fasterxml.jackson.core:jackson-core:2.15.3
 - com.fasterxml.jackson.core:jackson-databind:2.15.3
 - com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.15.3
-- com.fasterxml.woodstox:woodstox-core:5.3.0
+- com.fasterxml.woodstox:woodstox-core:5.4.0
+- com.github.pjfanning:jersey-json:1.20
 - com.google.guava:failureaccess:1.0
 - com.google.guava:guava:27.0-jre
 - com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 - com.google.j2objc:j2objc-annotations:1.1
+- com.sun.xml.bind:jaxb-impl:2.2.3-1
 - commons-beanutils:commons-beanutils:1.9.4
 - commons-codec:commons-codec:1.15
 - commons-collections:commons-collections:3.2.2
-- commons-io:commons-io:2.8.0
+- commons-io:commons-io:2.14.0
 - commons-logging:commons-logging:1.1.3
+- io.dropwizard.metrics:metrics-core:3.2.4
+- io.netty:netty-buffer:4.1.100.Final
+- io.netty:netty-codec:4.1.100.Final
+- io.netty:netty-common:4.1.100.Final
+- io.netty:netty-handler:4.1.100.Final
+- io.netty:netty-resolver:4.1.100.Final
+- io.netty:netty-transport-classes-epoll:4.1.100.Final
+- io.netty:netty-transport-native-epoll:4.1.100.Final
+- io.netty:netty-transport-native-unix-common:4.1.100.Final
+- io.netty:netty-transport:4.1.100.Final
 - joda-time:joda-time:2.8.1
-- org.apache.commons:commons-compress:1.21
-- org.apache.commons:commons-configuration2:2.1.1
+- org.apache.commons:commons-compress:1.24.0
+- org.apache.commons:commons-configuration2:2.8.0
 - org.apache.commons:commons-lang3:3.18.0
-- org.apache.commons:commons-text:1.4
-- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1
-- org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1
-- org.apache.hadoop:hadoop-annotations:3.3.4
-- org.apache.hadoop:hadoop-auth:3.3.4
-- org.apache.hadoop:hadoop-aws:3.3.4
-- org.apache.hadoop:hadoop-common:3.3.4
+- org.apache.commons:commons-text:1.10.0
+- org.apache.commons:commons-configuration2:2.8.0
+- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.2.0
+- org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.2.0
+- org.apache.hadoop:hadoop-annotations:3.4.0
+- org.apache.hadoop:hadoop-auth:3.4.0
+- org.apache.hadoop:hadoop-aws:3.4.0
+- org.apache.hadoop:hadoop-common:3.4.0
 - org.apache.httpcomponents:httpclient:4.5.13
 - org.apache.httpcomponents:httpcore:4.4.13
-- org.apache.kerby:kerb-core:1.0.1
-- org.apache.kerby:kerby-asn1:1.0.1
-- org.apache.kerby:kerby-pkix:1.0.1
-- org.apache.kerby:kerby-util:1.0.1
-- org.wildfly.openssl:wildfly-openssl:1.0.7.Final
+- org.apache.kerby:kerb-core:2.0.3
+- org.apache.kerby:kerby-asn1:2.0.3
+- org.apache.kerby:kerby-pkix:2.0.3
+- org.apache.kerby:kerby-util:2.0.3
+- org.bouncycastle:bcprov-jdk15on:1.70
+- org.codehaus.jettison:jettison:1.5.4
+- org.wildfly.openssl:wildfly-openssl:1.1.3.Final
 - org.xerial.snappy:snappy-java:1.1.10.4
+- software.amazon.awssdk:bundle:2.23.19
 - software.amazon.ion:ion-java:1.0.2
 
 This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause).
 See bundled license files for details.
 
-- dnsjava:dnsjava:2.1.7
+- dnsjava:dnsjava:3.4.0
 
 This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
 See bundled license files for details.
diff --git a/fluss-filesystems/pom.xml b/fluss-filesystems/pom.xml
@@ -40,7 +40,7 @@
     <packaging>pom</packaging>
 
     <properties>
-        <fs.hadoopshaded.version>3.3.4</fs.hadoopshaded.version>
+        <fs.hadoopshaded.version>3.4.0</fs.hadoopshaded.version>
         <commons.beanutils.version>1.9.4</commons.beanutils.version>
     </properties>
 
diff --git a/fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/NOTICE b/fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/NOTICE
@@ -24,7 +24,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.orc:orc-core:nohive:1.9.5
 - org.apache.orc:orc-shims:1.9.5
 - org.apache.commons:commons-lang3:3.18.0
-- org.apache.commons:commons-compress:1.21
+- org.apache.commons:commons-compress:1.24.0
 - org.apache.httpcomponents.client5:httpclient5:5.4.3
 - org.apache.httpcomponents.core5:httpcore5:5.3.4
 - org.apache.httpcomponents.core5:httpcore5-h2:5.3.4
diff --git a/fluss-lake/fluss-lake-lance/src/main/resources/META-INF/NOTICE b/fluss-lake/fluss-lake-lance/src/main/resources/META-INF/NOTICE
@@ -22,7 +22,7 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.arrow:arrow-vector:15.0.0
 - org.questdb:jar-jni:1.1.1
 - com.lancedb:lance-core:0.33.0
-- commons-codec:commons-codec:1.4
+- commons-codec:commons-codec:1.15
 - com.google.flatbuffers:flatbuffers-java:23.5.26
 
 This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
diff --git a/pom.xml b/pom.xml
@@ -91,7 +91,7 @@
         <paimon.version>1.2.0</paimon.version>
         <iceberg.version>1.9.1</iceberg.version>
 
-        <fluss.hadoop.version>2.10.2</fluss.hadoop.version>
+        <fluss.hadoop.version>3.4.0</fluss.hadoop.version>
         <frocksdb.version>6.20.3-ververica-2.0</frocksdb.version>
         <slf4j.version>1.7.36</slf4j.version>
         <log4j.version>2.17.1</log4j.version>
diff --git a/website/docs/maintenance/filesystems/hdfs.md b/website/docs/maintenance/filesystems/hdfs.md
@@ -56,7 +56,7 @@ fluss.hadoop.hadoop.security.kerberos.ticket.cache.path: /tmp/krb5cc_1000
 
 #### Use Machine Hadoop Environment Configuration
 
-Fluss includes bundled Hadoop libraries with version 3.3.4 for deploying Fluss in machine without Hadoop installed. 
+Fluss includes bundled Hadoop libraries with version 3.4.0 for deploying Fluss in machine without Hadoop installed. 
 For most use cases, these work perfectly. However, you should configure your machine's native Hadoop environment if:
 1. Your HDFS uses kerberos security
 2. You need to avoid version conflicts between Fluss's bundled hadoop libraries and your HDFS cluster
