Use pure rust lz4_flex

HC support from PSeitz/lz4_flex#191

Author: yujincheng08

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "selinux"]
 	path = native/src/external/selinux
 	url = https://github.com/topjohnwu/selinux.git
-[submodule "lz4"]
-	path = native/src/external/lz4
-	url = https://github.com/lz4/lz4.git
 [submodule "libcxx"]
 	path = native/src/external/libcxx
 	url = https://github.com/topjohnwu/libcxx.git

native/src/Android.mk

@@ -79,7 +79,6 @@ include $(CLEAR_VARS)
 LOCAL_MODULE := magiskboot
 LOCAL_STATIC_LIBRARIES := \
     libbase \
-    liblz4 \
     libboot-rs
 
 LOCAL_SRC_FILES := \

native/src/Cargo.lock

@@ -34,7 +34,7 @@ quick-protobuf = "0.8.1"
 flate2 = { version = "1.1.5", default-features = false }
 bzip2 = "0.6.1"
 zopfli = "0.8.3"
-lz4 = "1.28.1"
+lz4_flex = { git = "https://github.com/PSeitz/lz4_flex.git", rev = "1c08f39e1ab52f2824763e0a57f2aa925f392167", default-features = false, features = ["hc", "std", "frame"] }
 lzma-rust2 = { version = "0.15.4", default-features = false }
 nix = "0.30.1"
 bitflags = "2.10.0"
@@ -53,7 +53,6 @@ der = "0.8.0-rc.10"
 [patch.crates-io]
 pb-rs = { git = "https://github.com/topjohnwu/quick-protobuf.git" }
 quick-protobuf = { git = "https://github.com/topjohnwu/quick-protobuf.git" }
-lz4-sys = { path = "external/lz4-sys" }
 
 [workspace.lints.clippy]
 unwrap_used = "deny"

native/src/Cargo.toml

@@ -34,6 +34,6 @@ bytemuck = { workspace = true, features = ["derive", "min_const_generics"] }
 num-traits = { workspace = true }
 flate2 = { workspace = true, features = ["zlib-rs"] }
 bzip2 = { workspace = true }
-lz4 = { workspace = true }
+lz4_flex = { workspace = true, features = ["hc", "std", "frame"]}
 lzma-rust2 = { workspace = true, features = ["xz", "std", "encoder", "optimization"] }
 zopfli = { workspace = true, features = ["gzip"] }

native/src/boot/Cargo.toml

@@ -7,11 +7,9 @@ use bzip2::write::BzEncoder;
 use flate2::Compression as GzCompression;
 use flate2::read::MultiGzDecoder;
 use flate2::write::GzEncoder;
-use lz4::block::CompressionMode;
-use lz4::liblz4::BlockChecksum;
-use lz4::{
-    BlockMode, BlockSize, ContentChecksum, Decoder as LZ4FrameDecoder, Encoder as LZ4FrameEncoder,
-    EncoderBuilder as LZ4FrameEncoderBuilder,
+use lz4_flex::frame::{
+    BlockSize as LZ4BlockSize, FrameDecoder as LZ4FrameDecoder, FrameEncoder as LZ4FrameEncoder,
+    FrameInfo as LZ4FrameInfo,
 };
 use lzma_rust2::{CheckType, LzmaOptions, LzmaReader, LzmaWriter, XzOptions, XzReader, XzWriter};
 use std::cmp::min;
@@ -51,9 +49,7 @@ impl<W: Write> WriteFinish<W> for BufWriter<ZopFliEncoder<W>> {
 
 impl<W: Write> WriteFinish<W> for LZ4FrameEncoder<W> {
     fn finish(self: Box<Self>) -> std::io::Result<W> {
-        let (w, r) = Self::finish(*self);
-        r?;
-        Ok(w)
+        Ok(Self::finish(*self)?)
     }
 }
 
@@ -65,40 +61,31 @@ impl<W: Write> WriteFinish<W> for LZ4FrameEncoder<W> {
 // LZ4BlockEncoder
 
 const LZ4_BLOCK_SIZE: usize = 0x800000;
-const LZ4HC_CLEVEL_MAX: i32 = 12;
+const LZ4HC_CLEVEL_MAX: u8 = 12;
 const LZ4_MAGIC: u32 = 0x184c2102;
 
 struct LZ4BlockEncoder<W: Write> {
     write: W,
     chunker: Chunker,
-    out_buf: Box<[u8]>,
     total: u32,
     is_lg: bool,
 }
 
 impl<W: Write> LZ4BlockEncoder<W> {
     fn new(write: W, is_lg: bool) -> Self {
-        let out_sz = lz4::block::compress_bound(LZ4_BLOCK_SIZE).unwrap_or(LZ4_BLOCK_SIZE);
         LZ4BlockEncoder {
             write,
             chunker: Chunker::new(LZ4_BLOCK_SIZE),
-            // SAFETY: all bytes will be initialized before it is used
-            out_buf: unsafe { Box::new_uninit_slice(out_sz).assume_init() },
             total: 0,
             is_lg,
         }
     }
 
-    fn encode_block(write: &mut W, out_buf: &mut [u8], chunk: &[u8]) -> std::io::Result<()> {
-        let compressed_size = lz4::block::compress_to_buffer(
-            chunk,
-            Some(CompressionMode::HIGHCOMPRESSION(LZ4HC_CLEVEL_MAX)),
-            false,
-            out_buf,
-        )?;
-        let block_size = compressed_size as u32;
+    fn encode_block(write: &mut W, chunk: &[u8]) -> std::io::Result<()> {
+        let compressed = lz4_flex::block::compress_hc_to_vec(chunk, LZ4HC_CLEVEL_MAX);
+        let block_size = compressed.len() as u32;
         write.write_pod(&block_size)?;
-        write.write_all(&out_buf[..compressed_size])
+        write.write_all(&compressed)
     }
 }
 
@@ -123,7 +110,7 @@ impl<W: Write> Write for LZ4BlockEncoder<W> {
             let (b, chunk) = self.chunker.add_data(buf);
             buf = b;
             if let Some(chunk) = chunk {
-                Self::encode_block(&mut self.write, &mut self.out_buf, chunk)?;
+                Self::encode_block(&mut self.write, chunk)?;
             }
         }
         Ok(())
@@ -134,7 +121,7 @@ impl<W: Write> WriteFinish<W> for LZ4BlockEncoder<W> {
     fn finish(mut self: Box<Self>) -> std::io::Result<W> {
         let chunk = self.chunker.get_available();
         if !chunk.is_empty() {
-            Self::encode_block(&mut self.write, &mut self.out_buf, chunk)?;
+            Self::encode_block(&mut self.write, chunk)?;
         }
         if self.is_lg {
             self.write.write_pod(&self.total)?;
@@ -155,10 +142,12 @@ struct LZ4BlockDecoder<R: Read> {
 
 impl<R: Read> LZ4BlockDecoder<R> {
     fn new(read: R) -> Self {
-        let compressed_sz = lz4::block::compress_bound(LZ4_BLOCK_SIZE).unwrap_or(LZ4_BLOCK_SIZE);
         Self {
             read,
-            in_buf: unsafe { Box::new_uninit_slice(compressed_sz).assume_init() },
+            in_buf: unsafe {
+                Box::new_uninit_slice(lz4_flex::block::get_maximum_output_size(LZ4_BLOCK_SIZE))
+                    .assume_init()
+            },
             out_buf: unsafe { Box::new_uninit_slice(LZ4_BLOCK_SIZE).assume_init() },
             out_len: 0,
             out_pos: 0,
@@ -200,11 +189,8 @@ impl<R: Read> Read for LZ4BlockDecoder<R> {
                 }
             }
 
-            self.out_len = lz4::block::decompress_to_buffer(
-                compressed_block,
-                Some(LZ4_BLOCK_SIZE as i32),
-                &mut self.out_buf,
-            )?;
+            self.out_len = lz4_flex::block::decompress_into(compressed_block, &mut self.out_buf)
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
             self.out_pos = 0;
         }
         let copy_len = min(buf.len(), self.out_len - self.out_pos);
@@ -232,17 +218,14 @@ pub fn get_encoder<'a, W: Write + 'a>(
             None,
         )?),
         FileFormat::BZIP2 => Box::new(BzEncoder::new(w, BzCompression::best())),
-        FileFormat::LZ4 => {
-            let encoder = LZ4FrameEncoderBuilder::new()
-                .block_size(BlockSize::Max4MB)
-                .block_mode(BlockMode::Independent)
-                .checksum(ContentChecksum::ChecksumEnabled)
-                .block_checksum(BlockChecksum::BlockChecksumEnabled)
-                .level(9)
-                .auto_flush(true)
-                .build(w)?;
-            Box::new(encoder)
-        }
+        FileFormat::LZ4 => Box::new(LZ4FrameEncoder::with_compression_level(
+            LZ4FrameInfo::new()
+                .block_size(LZ4BlockSize::Max4MB)
+                .block_checksums(true)
+                .content_checksum(true),
+            w,
+            LZ4HC_CLEVEL_MAX,
+        )),
         FileFormat::LZ4_LEGACY => Box::new(LZ4BlockEncoder::new(w, false)),
         FileFormat::LZ4_LG => Box::new(LZ4BlockEncoder::new(w, true)),
         FileFormat::ZOPFLI => {
@@ -267,7 +250,7 @@ pub fn get_decoder<'a, R: Read + 'a>(
         FileFormat::XZ => Box::new(XzReader::new(r, true)),
         FileFormat::LZMA => Box::new(LzmaReader::new_mem_limit(r, u32::MAX, None)?),
         FileFormat::BZIP2 => Box::new(BzDecoder::new(r)),
-        FileFormat::LZ4 => Box::new(LZ4FrameDecoder::new(r)?),
+        FileFormat::LZ4 => Box::new(LZ4FrameDecoder::new(r)),
         FileFormat::LZ4_LG | FileFormat::LZ4_LEGACY => Box::new(LZ4BlockDecoder::new(r)),
         FileFormat::ZOPFLI | FileFormat::GZIP => Box::new(MultiGzDecoder::new(r)),
         _ => unreachable!(),

native/src/boot/compress.rs

@@ -11,18 +11,6 @@ LOCAL_SRC_FILES := \
     xz-embedded/xz_dec_stream.c
 include $(BUILD_STATIC_LIBRARY)
 
-# liblz4.a
-include $(CLEAR_VARS)
-LOCAL_MODULE := liblz4
-LOCAL_C_INCLUDES := $(LOCAL_PATH)/lz4/lib
-LOCAL_EXPORT_C_INCLUDES := $(LOCAL_C_INCLUDES)
-LOCAL_SRC_FILES := \
-    lz4/lib/lz4.c \
-    lz4/lib/lz4frame.c \
-    lz4/lib/lz4hc.c \
-    lz4/lib/xxhash.c
-include $(BUILD_STATIC_LIBRARY)
-
 SE_PATH := $(LOCAL_PATH)/selinux
 
 # libsepol.a