diff --git a/pom.xml b/pom.xml index 2bda824..5e8d2fd 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto.hadoop hadoop-apache2 - 2.7.4-6-SNAPSHOT + 2.9.2-1-SNAPSHOT hadoop-apache2 Shaded version of Apache Hadoop 2.x for Presto @@ -41,8 +41,8 @@ UTF-8 1.7 com.facebook.presto.hadoop.\$internal - 1.7.13 - 2.7.4 + 1.7.25 + 2.9.2 diff --git a/src/main/java/com/facebook/presto/hadoop/HadoopNative.java b/src/main/java/com/facebook/presto/hadoop/HadoopNative.java index 40c942b..ca351f2 100644 --- a/src/main/java/com/facebook/presto/hadoop/HadoopNative.java +++ b/src/main/java/com/facebook/presto/hadoop/HadoopNative.java @@ -25,6 +25,8 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; import static org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses; @@ -59,11 +61,23 @@ public static synchronized void requireHadoopNative() } } + public static List> getCodecs(Configuration conf) + { + // Skip zstd for now, we rely on it being pre-installed rather than included in presto-hadoop-apache2 + List> codecs = new ArrayList<>(); + for (Class clazz : getCodecClasses(conf)) { + if (clazz.getName() != "org.apache.hadoop.io.compress.ZStandardCodec") { + codecs.add(clazz); + } + } + return codecs; + } + private static void loadAllCodecs() { Configuration conf = new Configuration(); CompressionCodecFactory factory = new CompressionCodecFactory(conf); - for (Class clazz : getCodecClasses(conf)) { + for (Class clazz : getCodecs(conf)) { CompressionCodec codec = factory.getCodecByClassName(clazz.getName()); if (codec == null) { throw new RuntimeException("failed to load codec: " + clazz.getName()); diff --git a/src/main/resources/nativelib/Linux-amd64/libhadoop.so b/src/main/resources/nativelib/Linux-amd64/libhadoop.so index cc02d67..cf0d2b1 100755 Binary files a/src/main/resources/nativelib/Linux-amd64/libhadoop.so and b/src/main/resources/nativelib/Linux-amd64/libhadoop.so differ diff --git a/src/main/resources/nativelib/Mac_OS_X-x86_64/libhadoop.dylib b/src/main/resources/nativelib/Mac_OS_X-x86_64/libhadoop.dylib index 8077d2f..8501674 100644 Binary files a/src/main/resources/nativelib/Mac_OS_X-x86_64/libhadoop.dylib and b/src/main/resources/nativelib/Mac_OS_X-x86_64/libhadoop.dylib differ diff --git a/src/test/java/com/facebook/presto/hadoop/TestHadoopNative.java b/src/test/java/com/facebook/presto/hadoop/TestHadoopNative.java index 5f4e8a9..509a794 100644 --- a/src/test/java/com/facebook/presto/hadoop/TestHadoopNative.java +++ b/src/test/java/com/facebook/presto/hadoop/TestHadoopNative.java @@ -16,7 +16,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; -import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; import org.apache.hadoop.io.compress.zlib.ZlibFactory; import org.apache.hadoop.util.NativeCodeLoader; import org.testng.annotations.Test; @@ -27,8 +26,8 @@ import java.io.InputStream; import java.io.OutputStream; +import static com.facebook.presto.hadoop.HadoopNative.getCodecs; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.io.compress.CompressionCodecFactory.getCodecClasses; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; @@ -43,7 +42,6 @@ public void testNative() assertTrue(NativeCodeLoader.isNativeCodeLoaded()); assertTrue(NativeCodeLoader.buildSupportsSnappy()); assertTrue(ZlibFactory.isNativeZlibLoaded(new Configuration())); - assertTrue(Bzip2Factory.isNativeBzip2Loaded(new Configuration())); } @Test @@ -54,7 +52,7 @@ public void testCodecRoundTrip() Configuration conf = new Configuration(); CompressionCodecFactory factory = new CompressionCodecFactory(conf); - for (Class clazz : getCodecClasses(conf)) { + for (Class clazz : getCodecs(conf)) { CompressionCodec codec = factory.getCodecByClassName(clazz.getName()); assertNotNull(codec, clazz.getName()); @@ -70,7 +68,6 @@ private static byte[] compress(CompressionCodec codec, byte[] input) ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (OutputStream out = codec.createOutputStream(bytes)) { out.write(input); - out.close(); } return bytes.toByteArray(); }