[Enhancement] update persistent index size statistic when do major compaction (backport #62195) (#62200)

Signed-off-by: luohaha <18810541851@163.com>
Co-authored-by: Yixin Luo <18810541851@163.com>

Author: mergify[bot]

be/src/storage/persistent_index.cpp

@@ -5069,7 +5069,8 @@ Status PersistentIndex::TEST_major_compaction(PersistentIndexMetaPB& index_meta)
 // 1. load current l2 vec
 // 2. merge l2 files to new l2 file
 // 3. modify PersistentIndexMetaPB and make this step atomic.
-Status PersistentIndex::major_compaction(DataDir* data_dir, int64_t tablet_id, std::shared_timed_mutex* mutex) {
+Status PersistentIndex::major_compaction(DataDir* data_dir, int64_t tablet_id, std::shared_timed_mutex* mutex,
+                                         IOStat* stat) {
     if (_cancel_major_compaction) {
         return Status::InternalError("cancel major compaction");
     }
@@ -5128,6 +5129,9 @@ Status PersistentIndex::major_compaction(DataDir* data_dir, int64_t tablet_id, s
                 l0_version, _l1_version,
                 _l2_versions.size() > 0 ? _l2_versions[0] : EditVersionWithMerge(INT64_MAX, INT64_MAX, true)));
         _calc_memory_usage();
+        if (stat != nullptr) {
+            stat->total_file_size = (_l0 ? _l0->file_size() : 0) + _l1_l2_file_size();
+        }
     }
     (void)_delete_major_compaction_tmp_index_file();
     return Status::OK();

be/src/storage/persistent_index.h

@@ -783,7 +783,8 @@ class PersistentIndex {
     // just for unit test
     bool has_bf() { return _l1_vec.empty() ? false : _l1_vec[0]->has_bf(); }
 
-    Status major_compaction(DataDir* data_dir, int64_t tablet_id, std::shared_timed_mutex* mutex);
+    Status major_compaction(DataDir* data_dir, int64_t tablet_id, std::shared_timed_mutex* mutex,
+                            IOStat* stat = nullptr);
 
     Status TEST_major_compaction(PersistentIndexMetaPB& index_meta);
 

be/src/storage/primary_index.cpp

@@ -1572,15 +1572,16 @@ std::unique_ptr<PrimaryIndex> TEST_create_primary_index(const Schema& pk_schema)
     return std::make_unique<PrimaryIndex>(pk_schema);
 }
 
-Status PrimaryIndex::major_compaction(DataDir* data_dir, int64_t tablet_id, std::shared_timed_mutex* mutex) {
+Status PrimaryIndex::major_compaction(DataDir* data_dir, int64_t tablet_id, std::shared_timed_mutex* mutex,
+                                      IOStat* stat) {
     // `_persistent_index` could be reset when call `unload()`, so we need to fetch reference first.
     std::shared_ptr<PersistentIndex> pindex;
     {
         std::lock_guard<std::mutex> lg(_lock);
         pindex = _persistent_index;
     }
     if (pindex != nullptr) {
-        return pindex->major_compaction(data_dir, tablet_id, mutex);
+        return pindex->major_compaction(data_dir, tablet_id, mutex, stat);
     } else {
         return Status::OK();
     }

be/src/storage/primary_index.h

@@ -131,7 +131,8 @@ class PrimaryIndex {
 
     Status on_commited();
 
-    Status major_compaction(DataDir* data_dir, int64_t tablet_id, std::shared_timed_mutex* mutex);
+    Status major_compaction(DataDir* data_dir, int64_t tablet_id, std::shared_timed_mutex* mutex,
+                            IOStat* stat = nullptr);
 
     Status abort();
 

be/src/storage/tablet_updates.cpp

@@ -4773,10 +4773,12 @@ Status TabletUpdates::pk_index_major_compaction() {
         }
     });
     manager->index_cache().update_object_size(index_entry, index.memory_usage());
-    st = index.major_compaction(_tablet.data_dir(), _tablet.tablet_id(), _tablet.updates()->get_index_lock());
+    IOStat stat;
+    st = index.major_compaction(_tablet.data_dir(), _tablet.tablet_id(), _tablet.updates()->get_index_lock(), &stat);
     if (st.ok()) {
         // reset score after major compaction finish
         _pk_index_write_amp_score.store(0.0);
+        _extra_file_size_cache.pindex_size.store(stat.total_file_size);
     }
     return st;
 }