hdfs + hdrs

Author: zuston

.github/workflows/build.yaml

@@ -40,7 +40,7 @@ jobs:
         working-directory: ./
         run: cargo fmt --check
 
-  with-default:
+  default:
     runs-on: ubuntu-latest
     steps:
       - name: Install Protoc
@@ -70,6 +70,21 @@ jobs:
         working-directory: ./
         run: cargo test --verbose --features memory-prof -- --test-threads=1
 
+  aarch64-default:
+    runs-on: ubuntu-22.04-arm
+    steps:
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.2"
+      - uses: actions/checkout@v3
+      - name: Build
+        working-directory: ./
+        run: cargo build --release
+      - name: Unit tests
+        working-directory: ./
+        run: cargo test --verbose -- --test-threads=1
+
   with-hdrs:
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -48,7 +48,7 @@ memory-prof = ["jemalloc", "tikv-jemallocator/profiling", "allocator-analysis"]
 
 hdfs = ["dep:hdfs-native"]
 
-hdrs = ["hdfs", "dep:hdrs"]
+hdrs = ["dep:hdrs"]
 
 allocator-analysis = ["dep:cap"]
 

src/store/hadoop/mod.rs

@@ -1,9 +1,9 @@
-#[cfg(feature = "hdfs")]
+#[cfg(any(feature = "hdfs", feature = "hdrs"))]
 mod hdfs_native;
 #[cfg(feature = "hdrs")]
 mod hdrs;
 
-#[cfg(feature = "hdfs")]
+#[cfg(any(feature = "hdfs", feature = "hdrs"))]
 use crate::store::hadoop::hdfs_native::HdfsNativeClient;
 #[cfg(feature = "hdrs")]
 use crate::store::hadoop::hdrs::HdrsClient;
@@ -51,7 +51,7 @@ pub(crate) trait HdfsDelegator: Send + Sync {
     }
 }
 
-#[cfg(feature = "hdfs")]
+#[cfg(any(feature = "hdfs", feature = "hdrs"))]
 pub fn get_hdfs_delegator(
     root: &str,
     configs: HashMap<String, String>,

src/store/hybrid.rs

@@ -29,7 +29,7 @@ use crate::metric::{
     TOTAL_MEMORY_SPILL_TO_LOCALFILE,
 };
 use crate::readable_size::ReadableSize;
-#[cfg(feature = "hdfs")]
+#[cfg(any(feature = "hdfs", feature = "hdrs"))]
 use crate::store::hdfs::HdfsStore;
 use crate::store::localfile::LocalFileStore;
 use crate::store::memory::MemoryStore;
@@ -73,7 +73,7 @@ impl PersistentStore for LocalFileStore {
     }
 }
 
-#[cfg(feature = "hdfs")]
+#[cfg(any(feature = "hdfs", feature = "hdrs"))]
 impl PersistentStore for HdfsStore {
     fn as_any(&self) -> &dyn Any {
         self
@@ -127,12 +127,12 @@ impl HybridStore {
         }
 
         if StorageType::contains_hdfs(&store_type) {
-            #[cfg(not(feature = "hdfs"))]
+            #[cfg(not(all(feature = "hdfs", feature = "hdrs")))]
             panic!("The binary is not compiled with feature of hdfs! So the storage type can't involve hdfs.");
 
-            #[cfg(feature = "hdfs")]
+            #[cfg(any(feature = "hdfs", feature = "hdrs"))]
             let hdfs_store = HdfsStore::from(config.hdfs_store.unwrap(), &runtime_manager);
-            #[cfg(feature = "hdfs")]
+            #[cfg(any(feature = "hdfs", feature = "hdrs"))]
             persistent_stores.push_back(Box::new(hdfs_store));
         }
 
@@ -202,10 +202,10 @@ impl HybridStore {
 
     #[allow(unused)]
     fn is_hdfs(&self, store: &dyn Any) -> bool {
-        #[cfg(feature = "hdfs")]
+        #[cfg(any(feature = "hdfs", feature = "hdrs"))]
         return store.is::<HdfsStore>();
 
-        #[cfg(not(feature = "hdfs"))]
+        #[cfg(not(all(feature = "hdfs", feature = "hdrs")))]
         false
     }
 

src/store/mod.rs

@@ -17,7 +17,7 @@
 
 pub mod alignment;
 mod hadoop;
-#[cfg(feature = "hdfs")]
+#[cfg(any(feature = "hdfs", feature = "hdrs"))]
 pub mod hdfs;
 pub mod hybrid;
 pub mod local;

src/store/spill/spill_test.rs

@@ -193,7 +193,7 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(feature = "hdfs")]
+    #[cfg(any(feature = "hdfs", feature = "hdrs"))]
     async fn test_single_buffer_spill() {
         GAUGE_MEMORY_SPILL_IN_QUEUE_BYTES.set(0);
 
@@ -240,7 +240,7 @@ mod tests {
     }
 
     #[tokio::test]
-    #[cfg(feature = "hdfs")]
+    #[cfg(any(feature = "hdfs", feature = "hdrs"))]
     async fn test_localfile_disk_unhealthy() {
         GAUGE_MEMORY_SPILL_IN_QUEUE_BYTES.set(0);