Merge pull request #1728 from satya-bodapati/trunk

PXB-3643 [trunk] : Improve xbcloud delete to support Hierarchical Namespaces

Author: satya-bodapati

storage/innobase/xtrabackup/src/xbcloud/CMakeLists.txt

@@ -45,6 +45,7 @@ MYSQL_ADD_EXECUTABLE(xbcloud
   xbcloud.cc
   ../xbstream_read.cc
   http.cc
+  object_store.cc
   azure.cc
   s3.cc
   s3_ec2.cc

storage/innobase/xtrabackup/src/xbcloud/azure.cc

@@ -40,7 +40,7 @@ const std::string AZURE_DATE_HEADER = "x-ms-date";
 const std::string AZURE_VERSION_HEADER = "x-ms-version";
 const std::string AZURE_BLOB_TYPE_HEADER = "x-ms-blob-type";
 const std::string AZURE_STORAGE_CLASS_HEADER = "x-ms-access-tier";
-const std::string AZURE_VERSION_DATE = "2020-06-12";
+const std::string AZURE_VERSION_DATE = "2020-10-02";
 const std::string AZURE_DEVELOPMENT_HOST = "127.0.0.1:10000";
 const std::string AZURE_HOST = ".blob.core.windows.net";
 
@@ -531,9 +531,12 @@ Azure_client::Azure_client(const Http_client *client,
       storage_account, access_key, development_storage, storage_class));
 }
 
-bool Azure_client::list_objects_with_prefix(const std::string &container,
-                                            const std::string &prefix,
-                                            std::vector<std::string> &objects) {
+// Common helper function for listing objects - handles pagination and XML
+// parsing
+template <typename ProcessBlob>
+bool Azure_client::list_objects_common(const std::string &container,
+                                       const std::string &prefix,
+                                       ProcessBlob &&process_blob) {
   bool truncated = true;
   std::string next_marker;
 
@@ -602,20 +605,67 @@ bool Azure_client::list_objects_with_prefix(const std::string &container,
 
     auto node = blobs_node->first_node("Blob");
     while (node != nullptr) {
-      auto name = node->first_node("Name");
-      if (name == nullptr) {
-        msg_ts(
-            "%s: Failed to parse list container result. Cannot find object "
-            "name.\n",
-            my_progname);
-        return false;
+      if (!process_blob(node)) {
+        return false;  // Processing failed
       }
-      objects.push_back(name->value());
       node = node->next_sibling("Blob");
     }
   }
 
   return true;
 }
 
+bool Azure_client::list_objects_with_prefix(const std::string &container,
+                                            const std::string &prefix,
+                                            std::vector<std::string> &objects) {
+  return list_objects_common(
+      container, prefix, [&objects](rapidxml::xml_node<> *node) {
+        auto name = node->first_node("Name");
+        if (name == nullptr) {
+          msg_ts(
+              "%s: Failed to parse list container result. Cannot find object "
+              "name.\n",
+              my_progname);
+          return false;
+        }
+        objects.push_back(name->value());
+        return true;
+      });
+}
+
+bool Azure_client::list_objects_files_and_dirs(const std::string &container,
+                                               const std::string &prefix,
+                                               std::vector<std::string> &files,
+                                               std::vector<std::string> &dirs) {
+  return list_objects_common(
+      container, prefix, [&](rapidxml::xml_node<> *node) {
+        auto name = node->first_node("Name");
+        if (name == nullptr) {
+          msg_ts(
+              "%s: Failed to parse list container result. Cannot find object "
+              "name.\n",
+              my_progname);
+          return false;
+        }
+
+        // HNS returns directories explicitly via the ResourceType property.
+        bool is_directory = false;
+        auto properties_node = node->first_node("Properties");
+        if (properties_node) {
+          auto type_node = properties_node->first_node("ResourceType");
+          if (type_node && type_node->value() &&
+              strcmp(type_node->value(), "directory") == 0) {
+            is_directory = true;
+          }
+        }
+
+        if (is_directory) {
+          dirs.push_back(name->value());
+        } else {
+          files.push_back(name->value());
+        }
+        return true;
+      });
+}
+
 }  // namespace xbcloud

storage/innobase/xtrabackup/src/xbcloud/azure.h

@@ -107,6 +107,14 @@ class Azure_client {
       Event_handler *h, Azure_client::async_download_callback_t callback,
       CURLcode rc, const Http_connection *conn, ulong count);
 
+  // Common helper function for listing objects - handles pagination and XML
+  // parsing ProcessBlob is a callable that takes (rapidxml::xml_node<>* node)
+  // and returns bool Returns false to stop processing, true to continue
+  template <typename ProcessBlob>
+  bool list_objects_common(const std::string &container,
+                           const std::string &prefix,
+                           ProcessBlob &&process_blob);
+
  public:
   Azure_client(const Http_client *client, const std::string &storage_account,
                const std::string &access_key, bool development_storage,
@@ -147,6 +155,22 @@ class Azure_client {
                                 const std::string &prefix,
                                 std::vector<std::string> &objects);
 
+  /**
+   * List objects under a prefix and split them into files and directories.
+   *
+   * For HNS-enabled containers, directory entries are returned explicitly.
+   *
+   * @param container Container name.
+   * @param prefix Prefix to list.
+   * @param files Output list of file objects.
+   * @param dirs Output list of directory objects.
+   * @return true on success, false on error.
+   */
+  bool list_objects_files_and_dirs(const std::string &container,
+                                   const std::string &prefix,
+                                   std::vector<std::string> &files,
+                                   std::vector<std::string> &dirs);
+
   ulong get_max_retries() { return max_retries; }
 
   ulong get_max_backoff() { return max_backoff; }
@@ -235,6 +259,13 @@ class Azure_object_store : public Object_store {
                                       bool &success) override {
     return azure_client.download_object(container, name, success);
   }
+  virtual bool list_objects_files_and_dirs(
+      const std::string &container, const std::string &directory,
+      std::vector<std::string> &files,
+      std::vector<std::string> &dirs) override {
+    return azure_client.list_objects_files_and_dirs(container, directory + "/",
+                                                    files, dirs);
+  }
 };
 }  // namespace xbcloud
 

storage/innobase/xtrabackup/src/xbcloud/object_store.cc

@@ -0,0 +1,41 @@
+/******************************************************
+Copyright (c) 2026 Percona LLC and/or its affiliates.
+
+Object Store interface.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include "xbcloud/object_store.h"
+
+namespace xbcloud {
+
+/**
+ * Default implementation that lists objects and treats all as files.
+ *
+ * @param container Container/bucket name.
+ * @param directory Directory prefix to list.
+ * @param files Output list of file objects.
+ * @param dirs Output list of directory objects (unused here).
+ * @return true on success, false on error.
+ */
+bool Object_store::list_objects_files_and_dirs(const std::string &container,
+                                               const std::string &directory,
+                                               std::vector<std::string> &files,
+                                               std::vector<std::string> &dirs) {
+  return list_objects_in_directory(container, directory, files);
+}
+
+}  // namespace xbcloud

storage/innobase/xtrabackup/src/xbcloud/object_store.h

@@ -56,6 +56,21 @@ class Object_store {
   virtual Http_buffer download_object(const std::string &container,
                                       const std::string &name,
                                       bool &success) = 0;
+  /**
+   * List objects under a directory prefix and split them into files and dirs.
+   *
+   * Default implementation treats all returned objects as files.
+   *
+   * @param container Container/bucket name.
+   * @param directory Directory prefix to list.
+   * @param files Output list of file objects.
+   * @param dirs Output list of directory objects (may be empty).
+   * @return true on success, false on error.
+   */
+  virtual bool list_objects_files_and_dirs(const std::string &container,
+                                           const std::string &directory,
+                                           std::vector<std::string> &files,
+                                           std::vector<std::string> &dirs);
   virtual ~Object_store() {}
 };
 

storage/innobase/xtrabackup/src/xbcloud/xbcloud.cc

@@ -1046,16 +1046,27 @@ bool chunk_name_to_file_name(const std::string &chunk_name,
   return true;
 }
 
+/**
+ * Delete a backup directory from object storage.
+ *
+ * @param store Object store implementation.
+ * @param container Container/bucket name.
+ * @param backup_name Backup directory to delete.
+ * @return true on success, false on error.
+ */
 bool xbcloud_delete(Object_store *store, const std::string &container,
                     const std::string &backup_name) {
-  std::vector<std::string> object_list;
+  std::vector<std::string> files;
+  std::vector<std::string> dirs;
 
-  if (!store->list_objects_in_directory(container, backup_name, object_list)) {
+  // First pass lists files/dirs to validate existence and drive deletion.
+  if (!store->list_objects_files_and_dirs(container, backup_name, files,
+                                          dirs)) {
     msg_ts("%s: Delete failed. Cannot list %s.\n", my_progname,
            backup_name.c_str());
     return false;
   }
-  if (object_list.empty()) {
+  if (files.empty() && dirs.empty()) {
     msg_ts("%s: error: backup named %s doesn't exists!\n", my_progname,
            backup_name.c_str());
     return false;
@@ -1069,7 +1080,7 @@ bool xbcloud_delete(Object_store *store, const std::string &container,
   auto thread = h.run();
 
   bool error = false;
-  for (const auto &obj : object_list) {
+  for (const auto &obj : files) {
     std::string file_name;
     my_off_t idx;
     if (error) break;
@@ -1091,6 +1102,8 @@ bool xbcloud_delete(Object_store *store, const std::string &container,
                   }
                 },
                 std::placeholders::_1, obj, &error))) {
+      h.stop();
+      thread.join();
       return false;
     }
   }
@@ -1100,11 +1113,31 @@ bool xbcloud_delete(Object_store *store, const std::string &container,
 
   if (error) {
     msg_ts("%s: Delete failed.\n", my_progname);
-  } else {
-    msg_ts("%s: Delete completed.\n", my_progname);
+    return false;
+  }
+
+  if (!dirs.empty()) {
+    std::sort(dirs.begin(), dirs.end(), std::greater<std::string>());
+    for (const auto &d : dirs) {
+      msg_ts("%s: Deleting directory %s.\n", my_progname, d.c_str());
+      if (!store->delete_object(container, d)) {
+        msg_ts("%s: Delete failed. Cannot delete directory %s.\n", my_progname,
+               d.c_str());
+        return false;
+      }
+    }
+
+    // Delete the root directory of the backup
+    msg_ts("%s: Deleting directory %s.\n", my_progname, backup_name.c_str());
+    if (!store->delete_object(container, backup_name)) {
+      msg_ts("%s: Warning: Failed to delete root directory %s.\n", my_progname,
+             backup_name.c_str());
+    }
   }
 
-  return !error;
+  msg_ts("%s: Delete completed.\n", my_progname);
+
+  return true;
 }
 
 void download_func(download_thread_ctxt_t &cntx) {