diff --git a/pom.xml b/pom.xml
index 2bda824..5e8d2fd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,7 +10,7 @@
com.facebook.presto.hadoop
hadoop-apache2
- 2.7.4-6-SNAPSHOT
+ 2.9.2-1-SNAPSHOT
hadoop-apache2
Shaded version of Apache Hadoop 2.x for Presto
@@ -41,8 +41,8 @@
UTF-8
1.7
com.facebook.presto.hadoop.\$internal
- 1.7.13
- 2.7.4
+ 1.7.25
+ 2.9.2
diff --git a/src/main/java/com/facebook/presto/hadoop/HadoopNative.java b/src/main/java/com/facebook/presto/hadoop/HadoopNative.java
index 40c942b..ca351f2 100644
--- a/src/main/java/com/facebook/presto/hadoop/HadoopNative.java
+++ b/src/main/java/com/facebook/presto/hadoop/HadoopNative.java
@@ -25,6 +25,8 @@
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
import static org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses;
@@ -59,11 +61,23 @@ public static synchronized void requireHadoopNative()
}
}
+ public static List> getCodecs(Configuration conf)
+ {
+ // Skip zstd for now, we rely on it being pre-installed rather than included in presto-hadoop-apache2
+ List> codecs = new ArrayList<>();
+ for (Class extends CompressionCodec> clazz : getCodecClasses(conf)) {
+ if (clazz.getName() != "org.apache.hadoop.io.compress.ZStandardCodec") {
+ codecs.add(clazz);
+ }
+ }
+ return codecs;
+ }
+
private static void loadAllCodecs()
{
Configuration conf = new Configuration();
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
- for (Class extends CompressionCodec> clazz : getCodecClasses(conf)) {
+ for (Class extends CompressionCodec> clazz : getCodecs(conf)) {
CompressionCodec codec = factory.getCodecByClassName(clazz.getName());
if (codec == null) {
throw new RuntimeException("failed to load codec: " + clazz.getName());
diff --git a/src/main/resources/nativelib/Linux-amd64/libhadoop.so b/src/main/resources/nativelib/Linux-amd64/libhadoop.so
index cc02d67..cf0d2b1 100755
Binary files a/src/main/resources/nativelib/Linux-amd64/libhadoop.so and b/src/main/resources/nativelib/Linux-amd64/libhadoop.so differ
diff --git a/src/main/resources/nativelib/Mac_OS_X-x86_64/libhadoop.dylib b/src/main/resources/nativelib/Mac_OS_X-x86_64/libhadoop.dylib
index 8077d2f..8501674 100644
Binary files a/src/main/resources/nativelib/Mac_OS_X-x86_64/libhadoop.dylib and b/src/main/resources/nativelib/Mac_OS_X-x86_64/libhadoop.dylib differ
diff --git a/src/test/java/com/facebook/presto/hadoop/TestHadoopNative.java b/src/test/java/com/facebook/presto/hadoop/TestHadoopNative.java
index 5f4e8a9..509a794 100644
--- a/src/test/java/com/facebook/presto/hadoop/TestHadoopNative.java
+++ b/src/test/java/com/facebook/presto/hadoop/TestHadoopNative.java
@@ -16,7 +16,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
import org.apache.hadoop.util.NativeCodeLoader;
import org.testng.annotations.Test;
@@ -27,8 +26,8 @@
import java.io.InputStream;
import java.io.OutputStream;
+import static com.facebook.presto.hadoop.HadoopNative.getCodecs;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@@ -43,7 +42,6 @@ public void testNative()
assertTrue(NativeCodeLoader.isNativeCodeLoaded());
assertTrue(NativeCodeLoader.buildSupportsSnappy());
assertTrue(ZlibFactory.isNativeZlibLoaded(new Configuration()));
- assertTrue(Bzip2Factory.isNativeBzip2Loaded(new Configuration()));
}
@Test
@@ -54,7 +52,7 @@ public void testCodecRoundTrip()
Configuration conf = new Configuration();
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
- for (Class extends CompressionCodec> clazz : getCodecClasses(conf)) {
+ for (Class extends CompressionCodec> clazz : getCodecs(conf)) {
CompressionCodec codec = factory.getCodecByClassName(clazz.getName());
assertNotNull(codec, clazz.getName());
@@ -70,7 +68,6 @@ private static byte[] compress(CompressionCodec codec, byte[] input)
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try (OutputStream out = codec.createOutputStream(bytes)) {
out.write(input);
- out.close();
}
return bytes.toByteArray();
}