[BugFix] fix bundle file vacuum when Fe downgrade (backport #67849) (#67887)

Signed-off-by: luohaha <18810541851@163.com>
Co-authored-by: Yixin Luo <18810541851@163.com>

Author: mergify[bot]

be/src/storage/lake/async_file_deleter.h

@@ -120,6 +120,11 @@ class AsyncBundleFileDeleter : public AsyncFileDeleter {
 
     bool is_empty() const { return _pending_files.empty(); }
 
+    void clear() {
+        _pending_files.clear();
+        _delay_delete_files.clear();
+    }
+
 private:
     // file to shared count.
     std::unordered_map<std::string, uint32_t> _pending_files;

be/src/storage/lake/vacuum.cpp

@@ -471,6 +471,13 @@ static Status vacuum_tablet_metadata(TabletManager* tablet_mgr, std::string_view
         final_vacuum_version = std::min(final_vacuum_version, tablet_vacuumed_version);
         max_vacuum_version = std::max(max_vacuum_version, tablet_vacuumed_version);
     }
+    if (!bundle_file_deleter.is_empty() && !enable_file_bundling) {
+        // If the enable_file_bundling flag is not set in the vacuum request issued by the FE,
+        // it indicates that the request does not contain the complete tablet information for the partition.
+        // Due to this lack of sufficient information, bundle files cannot be deleted.
+        // It may happen when downgrade FE from a version that supports file bundling to a version that does not.
+        bundle_file_deleter.clear();
+    }
     // delete bundle files
     if (max_vacuum_version > 0 && !bundle_file_deleter.is_empty()) {
         RETURN_IF_ERROR(collect_alive_bundle_files(tablet_mgr, tablet_infos, max_vacuum_version, root_dir,

be/test/storage/lake/vacuum_test.cpp

@@ -2943,6 +2943,36 @@ TEST_P(LakeVacuumTest, test_vacuum_shared_data_files) {
     tablet_metas_v4[601] = *t601_v4;
     ASSERT_OK(_tablet_mgr->put_bundle_tablet_metadata(tablet_metas_v4));
 
+    // enable_file_bundling is false, shared files will not be deleted.
+    {
+        VacuumRequest request;
+        VacuumResponse response;
+        request.set_delete_txn_log(false);
+        request.add_tablet_ids(600);
+        request.add_tablet_ids(601);
+        TabletInfoPB tablet_info1;
+        tablet_info1.set_tablet_id(600);
+        tablet_info1.set_min_version(3);
+        TabletInfoPB tablet_info2;
+        tablet_info2.set_tablet_id(601);
+        tablet_info2.set_min_version(3);
+        request.add_tablet_infos()->CopyFrom(tablet_info1);
+        request.add_tablet_infos()->CopyFrom(tablet_info2);
+        request.set_min_retain_version(4);
+        request.set_grace_timestamp(grace_timestamp + 1);
+        request.set_min_active_txn_id(12345);
+        // If enable_file_bundling is false, we need to clear the bundle_file_deleter.
+        request.set_enable_file_bundling(false);
+        vacuum(_tablet_mgr.get(), request, &response);
+        ASSERT_TRUE(response.has_status());
+        EXPECT_EQ(0, response.status().status_code()) << response.status().error_msgs(0);
+
+        EXPECT_TRUE(file_exist("00000000000259e4_27dc159f-6bfc-4a3a-9d9c-c97c10bb2e1d.dat"));
+        EXPECT_TRUE(file_exist("00000000000259e4_27dc159f-6bfc-4a3a-9d9c-c97c10bb2e1e.dat"));
+        EXPECT_TRUE(file_exist("00000000000259e4_a542395a-bff5-48a7-a3a7-2ed05691b58c.dat"));
+        EXPECT_TRUE(file_exist("00000000000259e4_a542395a-bff5-48a7-a3a7-2ed05691b58d.dat"));
+    }
+
     // shared files will be deleted.
     {
         VacuumRequest request;