feat(hf): add support for deleting files

Author: kszucs

core/services/huggingface/src/backend.rs

@@ -22,6 +22,7 @@ use log::debug;
 use super::HUGGINGFACE_SCHEME;
 use super::config::HfConfig;
 use super::core::HfCore;
+use super::deleter::HfDeleter;
 use super::lister::HfLister;
 use super::reader::HfReader;
 use super::uri::{HfRepo, RepoType};
@@ -180,6 +181,7 @@ impl Builder for HfBuilder {
                     stat: true,
                     read: true,
                     write: token.is_some(),
+                    delete: token.is_some(),
                     list: true,
                     list_with_recursive: true,
                     shared: true,
@@ -212,7 +214,7 @@ impl Access for HfBackend {
     type Reader = HfReader;
     type Writer = oio::OneShotWriter<HfWriter>;
     type Lister = oio::PageLister<HfLister>;
-    type Deleter = ();
+    type Deleter = oio::OneShotDeleter<HfDeleter>;
 
     fn info(&self) -> Arc<AccessorInfo> {
         self.core.info.clone()
@@ -243,6 +245,13 @@ impl Access for HfBackend {
         let writer = HfWriter::new(&self.core, path, args);
         Ok((RpWrite::default(), oio::OneShotWriter::new(writer)))
     }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(HfDeleter::new(self.core.clone())),
+        ))
+    }
 }
 
 #[cfg(test)]

core/services/huggingface/src/core.rs

@@ -21,6 +21,7 @@ use std::sync::Arc;
 use bytes::Buf;
 use bytes::Bytes;
 use http::Request;
+use http::StatusCode;
 use http::header;
 use serde::Deserialize;
 
@@ -63,13 +64,20 @@ pub(super) struct LfsFile {
     pub size: u64,
 }
 
+#[derive(Debug, serde::Serialize)]
+pub(super) struct DeletedFile {
+    pub path: String,
+}
+
 #[derive(serde::Serialize)]
 pub(super) struct MixedCommitPayload {
     pub summary: String,
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub files: Vec<CommitFile>,
     #[serde(rename = "lfsFiles", skip_serializing_if = "Vec::is_empty")]
     pub lfs_files: Vec<LfsFile>,
+    #[serde(rename = "deletedFiles", skip_serializing_if = "Vec::is_empty")]
+    pub deleted_files: Vec<DeletedFile>,
 }
 
 // API response types
@@ -381,46 +389,37 @@ impl HfCore {
         Ok(resp)
     }
 
-    /// Commit uploaded files to the repository.
+    /// Commit file changes (uploads and/or deletions) to the repository.
     pub(super) async fn commit_files(
         &self,
         regular_files: Vec<CommitFile>,
         lfs_files: Vec<LfsFile>,
-    ) -> Result<http::Response<Buffer>> {
+        deleted_files: Vec<DeletedFile>,
+    ) -> Result<()> {
         let _token = self.token.as_deref().ok_or_else(|| {
             Error::new(
                 ErrorKind::PermissionDenied,
-                "token is required for write operations",
+                "token is required for commit operations",
             )
             .with_operation("commit")
         })?;
 
-        let mut summary_paths = Vec::new();
-        for file in &regular_files {
-            summary_paths.push(file.path.clone());
-        }
-        for file in &lfs_files {
-            summary_paths.push(file.path.clone());
-        }
-
-        let summary = if summary_paths.len() == 1 {
-            format!("Upload {} via OpenDAL", summary_paths[0])
-        } else {
-            format!("Upload {} files via OpenDAL", summary_paths.len())
-        };
-
-        let client = self.info.http_client();
-        // Use the first file's path to determine the commit URL
-        let first_path = summary_paths
+        let first_path = regular_files
             .first()
+            .map(|f| f.path.as_str())
+            .or_else(|| lfs_files.first().map(|f| f.path.as_str()))
+            .or_else(|| deleted_files.first().map(|f| f.path.as_str()))
             .ok_or_else(|| Error::new(ErrorKind::Unexpected, "no files to commit"))?;
+
+        let client = self.info.http_client();
         let uri = self.repo.uri(&self.root, first_path);
         let url = uri.commit_url(&self.endpoint);
 
         let payload = MixedCommitPayload {
-            summary,
+            summary: "Commit via OpenDAL".to_string(),
             files: regular_files,
             lfs_files,
+            deleted_files,
         };
 
         let json_body = serde_json::to_vec(&payload).map_err(new_json_serialize_error)?;
@@ -432,7 +431,11 @@ impl HfCore {
             .body(Buffer::from(json_body))
             .map_err(new_request_build_error)?;
 
-        client.send(req).await
+        let resp = client.send(req).await?;
+        match resp.status() {
+            StatusCode::OK | StatusCode::CREATED => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
     }
 }
 

core/services/huggingface/src/deleter.rs

@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::{DeletedFile, HfCore};
+use opendal_core::raw::*;
+use opendal_core::*;
+
+pub struct HfDeleter {
+    core: Arc<HfCore>,
+}
+
+impl HfDeleter {
+    pub fn new(core: Arc<HfCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for HfDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let deleted = vec![DeletedFile { path }];
+        match self.core.commit_files(vec![], vec![], deleted).await {
+            Ok(()) => Ok(()),
+            Err(err) if err.kind() == ErrorKind::NotFound => Ok(()),
+            Err(err) => Err(err),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::core::HfCore;
+    use super::super::uri::{HfRepo, RepoType};
+    use super::super::writer::HfWriter;
+    use super::*;
+    use oio::OneShotDelete;
+    use oio::OneShotWrite;
+
+    fn testing_core() -> HfCore {
+        let repo_id = std::env::var("HF_OPENDAL_DATASET").expect("HF_OPENDAL_DATASET must be set");
+
+        let info = AccessorInfo::default();
+        info.set_scheme("huggingface")
+            .set_native_capability(Capability {
+                write: true,
+                delete: true,
+                ..Default::default()
+            });
+
+        HfCore {
+            info: info.into(),
+            repo: HfRepo::new(RepoType::Dataset, repo_id, Some("main".to_string())),
+            root: "/".to_string(),
+            token: std::env::var("HF_OPENDAL_TOKEN").ok(),
+            endpoint: "https://huggingface.co".to_string(),
+            #[cfg(feature = "xet")]
+            xet_enabled: false,
+        }
+    }
+
+    #[tokio::test]
+    #[ignore]
+    async fn test_delete_once() {
+        let core = Arc::new(testing_core());
+
+        // First write a file so we have something to delete
+        let writer = HfWriter::new(&core, "delete-test.txt", OpWrite::default());
+        writer
+            .write_once(Buffer::from("temporary content"))
+            .await
+            .expect("write should succeed");
+
+        // Now delete it
+        let deleter = HfDeleter::new(core);
+        deleter
+            .delete_once("delete-test.txt".to_string(), OpDelete::default())
+            .await
+            .expect("delete should succeed");
+    }
+
+    #[tokio::test]
+    #[ignore]
+    async fn test_delete_nonexistent() {
+        let core = Arc::new(testing_core());
+
+        let deleter = HfDeleter::new(core);
+        deleter
+            .delete_once("nonexistent-file.txt".to_string(), OpDelete::default())
+            .await
+            .expect("deleting nonexistent file should succeed");
+    }
+}

core/services/huggingface/src/docs.md

@@ -11,7 +11,7 @@ This service can be used to:
 - [x] stat
 - [x] read
 - [x] write
-- [ ] delete
+- [x] delete
 - [x] list
 - [ ] copy
 - [ ] rename

core/services/huggingface/src/lib.rs

@@ -28,6 +28,7 @@ pub fn register_huggingface_service(registry: &opendal_core::OperatorRegistry) {
 mod backend;
 mod config;
 mod core;
+mod deleter;
 mod error;
 mod lister;
 mod reader;

core/services/huggingface/src/writer.rs

@@ -18,12 +18,11 @@
 use std::sync::Arc;
 
 use base64::Engine;
-use http::StatusCode;
 use sha2::{Digest, Sha256};
 
-use super::core::{CommitFile, HfCore, LfsFile, PreuploadFile};
+use super::core::{CommitFile, HfCore, PreuploadFile};
 #[cfg(feature = "xet")]
-use super::core::{XetTokenRefresher, map_xet_error};
+use super::core::{LfsFile, XetTokenRefresher, map_xet_error};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -181,12 +180,10 @@ impl HfWriter {
         // Commit the files
         let regular_files: Vec<_> = commit_file.into_iter().collect();
         let lfs_files: Vec<_> = lfs_file.into_iter().collect();
-        let resp = self.core.commit_files(regular_files, lfs_files).await?;
-
-        match resp.status() {
-            StatusCode::OK | StatusCode::CREATED => Ok(Metadata::default()),
-            _ => Err(super::error::parse_error(resp)),
-        }
+        self.core
+            .commit_files(regular_files, lfs_files, vec![])
+            .await?;
+        Ok(Metadata::default())
     }
 }
 
@@ -236,16 +233,9 @@ mod tests {
             .await
             .expect("upload should succeed");
 
-        let resp = core
-            .commit_files(vec![commit_file], vec![])
+        core.commit_files(vec![commit_file], vec![], vec![])
             .await
             .expect("commit should succeed");
-
-        assert!(
-            resp.status() == StatusCode::OK || resp.status() == StatusCode::CREATED,
-            "expected OK or CREATED status, got {}",
-            resp.status()
-        );
     }
 
     #[tokio::test]
@@ -288,16 +278,9 @@ mod tests {
             .await
             .expect("xet upload should succeed");
 
-        let resp = core
-            .commit_files(vec![], vec![lfs_file])
+        core.commit_files(vec![], vec![lfs_file], vec![])
             .await
             .expect("commit should succeed");
-
-        assert!(
-            resp.status() == StatusCode::OK || resp.status() == StatusCode::CREATED,
-            "expected OK or CREATED status, got {}",
-            resp.status()
-        );
     }
 
     #[cfg(feature = "xet")]