address comments

Author: JacksonYao287

src/lib/homestore_backend/gc_manager.cpp

@@ -1,20 +1,95 @@
 #include <homestore/btree/btree_req.hpp>
 #include <homestore/btree/btree_kv.hpp>
+#include <homestore/meta_service.hpp>
 
 #include "hs_homeobject.hpp"
 namespace homeobject {
 
 /* GCManager */
 
 GCManager::GCManager(std::shared_ptr< HeapChunkSelector > chunk_selector, HSHomeObject* homeobject) :
-        m_chunk_selector{chunk_selector}, m_hs_home_object{homeobject} {}
+        m_chunk_selector{chunk_selector}, m_hs_home_object{homeobject} {
+    homestore::HomeStore::instance()->meta_service().register_handler(
+        GCManager::_gc_actor_meta_name,
+        [this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
+            on_gc_actor_meta_blk_found(std::move(buf), voidptr_cast(mblk));
+        },
+        nullptr, true);
+
+    homestore::HomeStore::instance()->meta_service().register_handler(
+        GCManager::_gc_reserved_chunk_meta_name,
+        [this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
+            on_reserved_chunk_meta_blk_found(std::move(buf), voidptr_cast(mblk));
+        },
+        nullptr,
+        // we need to recover reserved chunk before after we recover gc actor
+        true, std::optional< homestore::meta_subtype_vec_t >({GCManager::_gc_actor_meta_name}));
+
+    homestore::HomeStore::instance()->meta_service().register_handler(
+        GCManager::_gc_task_meta_name,
+        [this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
+            on_gc_task_meta_blk_found(std::move(buf), voidptr_cast(mblk));
+        },
+        nullptr, true, std::optional< homestore::meta_subtype_vec_t >({GCManager::_gc_reserved_chunk_meta_name}));
+}
+
+void GCManager::on_gc_task_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie) {
+    homestore::superblk< GCManager::gc_task_superblk > gc_task_sb(GCManager::_gc_task_meta_name);
+    const auto pending_gc_task = gc_task_sb.load(buf, meta_cookie);
+
+    // if a gc_task_super blk is found, we can make sure that all the valid data in move_from_chunk has been copied to
+    // move_to_chunk, and all the blob -> (new pba) have been written to the gc index table. Now, what we need to do is
+    // just updating blob indexes in pg index table according to the blob indexes in gc index table.
+
+    // pg_index_table: [pg_id, shard_id, blob_id] -> old pba
+    // gc_index_table: [move_to_chunk_id, pg_id, shard_id, blob_id] -> new pba
+
+    // we need to find all keys with the prefix of move_to_chunk_id in gc index table, and update the corrsponding
+    // keys(same pg_id + shard_id + blob_id) in pg index table with the new pba.
+    auto pdev_id = m_chunk_selector->get_extend_vchunk(pending_gc_task->move_from_chunk)->get_pdev_id();
+    auto gc_actor = get_pdev_gc_actor(pdev_id);
+    RELEASE_ASSERT(gc_actor, "can not get gc actor for pdev {}!", pdev_id);
+    gc_actor->handle_recovered_gc_task(pending_gc_task);
+
+    // delete gc task meta blk, so that it will not be recovered again if restart
+    gc_task_sb.destroy();
+}
+
+void GCManager::on_gc_actor_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie) {
+    homestore::superblk< GCManager::gc_actor_superblk > gc_actor_sb(GCManager::_gc_actor_meta_name);
+    gc_actor_sb.load(buf, meta_cookie);
+    auto pdev_id = gc_actor_sb->pdev_id;
+    auto index_table_uuid = gc_actor_sb->index_table_uuid;
+
+    auto gc_index_table = m_hs_home_object->get_gc_index_table(boost::uuids::to_string(index_table_uuid));
+
+    RELEASE_ASSERT(gc_index_table, "can not get gc index table for pdev {} with uuid {}!", pdev_id,
+                   boost::uuids::to_string(index_table_uuid));
+
+    // create a gc actor for this pdev if not exists
+    auto gc_actor = try_create_pdev_gc_actor(pdev_id, gc_index_table);
+    RELEASE_ASSERT(gc_actor, "can not get gc actor for pdev {}!", pdev_id);
+}
+
+void GCManager::on_reserved_chunk_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie) {
+    homestore::superblk< GCManager::gc_reserved_chunk_superblk > reserved_chunk_sb(
+        GCManager::_gc_reserved_chunk_meta_name);
+    reserved_chunk_sb.load(buf, meta_cookie);
+    auto chunk_id = reserved_chunk_sb->chunk_id;
+    auto pdev_id = m_chunk_selector->get_extend_vchunk(chunk_id)->get_pdev_id();
+    auto gc_actor = get_pdev_gc_actor(pdev_id);
+    RELEASE_ASSERT(gc_actor, "can not get gc actor for pdev {}!", pdev_id);
+    gc_actor->add_reserved_chunk(chunk_id);
+    // mark a reserved chunk as gc state, so that it will not be selected as a gc candidate
+    m_chunk_selector->try_mark_chunk_to_gc_state(chunk_id, true /* force */);
+}
 
 GCManager::~GCManager() { stop(); }
 
 void GCManager::start() {
     for (const auto& [pdev_id, gc_actor] : m_pdev_gc_actors) {
         gc_actor->start();
-        LOGERROR("start gc actor for pdev={}", pdev_id);
+        LOGINFO("start gc actor for pdev={}", pdev_id);
     }
 
     m_gc_timer_hdl = iomanager.schedule_global_timer(
@@ -72,12 +147,22 @@ std::shared_ptr< GCManager::pdev_gc_actor > GCManager::get_pdev_gc_actor(uint32_
     return it->second;
 }
 
-bool GCManager::is_eligible_for_gc(std::shared_ptr< HeapChunkSelector::ExtendedVChunk > chunk) {
+bool GCManager::is_eligible_for_gc(chunk_id_t chunk_id) {
+    auto chunk = m_chunk_selector->get_extend_vchunk(chunk_id);
+
     // 1 if the chunk state is inuse, it is occupied by a open shard, so it can not be selected and we don't need gc it.
     // 2 if the chunk state is gc, it means this chunk is being gc, or this is a reserved chunk, so we don't need gc it.
-    if (chunk->m_state != ChunkState::AVAILABLE) return false;
+    if (chunk->m_state != ChunkState::AVAILABLE) {
+        LOGINFO("chunk_id={} state is {}, not eligible for gc", chunk_id, chunk->m_state)
+        return false;
+    }
     // it does not belong to any pg, so we don't need to gc it.
-    if (!chunk->m_pg_id.has_value()) return false;
+    if (!chunk->m_pg_id.has_value()) {
+        LOGINFO("chunk_id={} belongs to no pg, not eligible for gc", chunk_id)
+        return false;
+    }
+
+    LOGINFO("chunk_id={} is eligible for gc, belongs to pg {}", chunk_id, chunk->m_pg_id.value());
 
     auto defrag_blk_num = chunk->get_defrag_nblks();
     auto total_blk_num = chunk->get_total_blks();
@@ -88,21 +173,21 @@ bool GCManager::is_eligible_for_gc(std::shared_ptr< HeapChunkSelector::ExtendedV
 }
 
 void GCManager::scan_chunks_for_gc() {
-    // in every iteration, we will select at most 2 * RESERVED_CHUNK_NUM_PER_PDEV gc tasks
-    auto max_task_num = 2 * (RESERVED_CHUNK_NUM_PER_PDEV - RESERVED_CHUNK_NUM_DEDICATED_FOR_EGC);
-
-    for (const auto& [_, chunk] : m_chunk_selector->get_all_chunks()) {
-        if (is_eligible_for_gc(chunk)) {
-            auto pdev_id = chunk->get_pdev_id();
-            auto it = m_pdev_gc_actors.find(pdev_id);
-            if (it != m_pdev_gc_actors.end()) {
-                auto& actor = it->second;
-                auto chunk_id = chunk->get_chunk_id();
+    for (const auto& [pdev_id, chunks] : m_chunk_selector->get_pdev_chunks()) {
+        // in every iteration, we will select at most 2 * RESERVED_CHUNK_NUM_PER_PDEV gc tasks
+        auto max_task_num = 2 * (RESERVED_CHUNK_NUM_PER_PDEV - RESERVED_CHUNK_NUM_DEDICATED_FOR_EGC);
+        auto it = m_pdev_gc_actors.find(pdev_id);
+        RELEASE_ASSERT(it != m_pdev_gc_actors.end(), "can not find gc actor for pdev_id {} when scanning chunks for gc",
+                       pdev_id);
+        auto& actor = it->second;
+
+        for (const auto& chunk_id : chunks) {
+            if (is_eligible_for_gc(chunk_id)) {
                 auto future = actor->add_gc_task(static_cast< uint8_t >(task_priority::normal), chunk_id);
                 if (future.isReady()) {
                     // future is not expected to be ready immediately. if it is ready here, it probably means failing to
                     // add gc task. then we try to add one more.
-                    LOGWARN("failed to add gc task for chunk_id={} on pdev_id={}, return value= {}", chunk_id, pdev_id,
+                    LOGWARN("failed to add gc task for chunk_id={} on pdev_id={}, return value={}", chunk_id, pdev_id,
                             future.value());
                 } else if (0 == --max_task_num) {
                     LOGINFO("reached max gc task limit for pdev_id={}, stopping further gc task submissions", pdev_id);
@@ -168,12 +253,12 @@ folly::SemiFuture< bool > GCManager::pdev_gc_actor::add_gc_task(uint8_t priority
 
         if (sisl_unlikely(priority == static_cast< uint8_t >(task_priority::emergent))) {
             m_egc_executor->add([this, priority, move_from_chunk, promise = std::move(promise)]() mutable {
-                LOGERROR("start gc task : move_from_chunk_id={}, priority={}", move_from_chunk, priority);
+                LOGINFO("start emergent gc task : move_from_chunk_id={}, priority={}", move_from_chunk, priority);
                 process_gc_task(move_from_chunk, priority, std::move(promise));
             });
         } else {
             m_gc_executor->add([this, priority, move_from_chunk, promise = std::move(promise)]() mutable {
-                LOGERROR("start gc task : move_from_chunk_id={}, priority={}", move_from_chunk, priority);
+                LOGINFO("start gc task : move_from_chunk_id={}, priority={}", move_from_chunk, priority);
                 process_gc_task(move_from_chunk, priority, std::move(promise));
             });
         }
@@ -256,8 +341,8 @@ bool GCManager::pdev_gc_actor::copy_valid_data(chunk_id_t move_from_chunk, chunk
 
 void GCManager::pdev_gc_actor::purge_reserved_chunk(chunk_id_t chunk) {
     auto vchunk = m_chunk_selector->get_extend_vchunk(chunk);
-    RELEASE_ASSERT(!vchunk->m_pg_id.has_value(), "chunk_id={} is expected to a reserved chunk, and not belong to a pg",
-                   chunk);
+    RELEASE_ASSERT(!vchunk->m_pg_id.has_value(),
+                   "chunk_id={} is expected to be a reserved chunk, and not belong to a pg", chunk);
     vchunk->reset(); // reset the chunk to make sure it is empty
 
     // clear all the entries of this chunk in the gc index table
@@ -277,22 +362,8 @@ void GCManager::pdev_gc_actor::purge_reserved_chunk(chunk_id_t chunk) {
 
 void GCManager::pdev_gc_actor::process_gc_task(chunk_id_t move_from_chunk, uint8_t priority,
                                                folly::Promise< bool > task) {
-    auto move_from_vchunk = m_chunk_selector->get_extend_vchunk(move_from_chunk);
-
     LOGINFO("start process gc task for move_from_chunk={} with priority={} ", move_from_chunk, priority);
 
-    // we need to select the move_from_chunk out of the per pg chunk heap in chunk selector if it is a gc task with
-    // normal priority. for the task with emergent priority, it is already selected since it is now used for an open
-    // shard.
-
-    if (!GCManager::is_eligible_for_gc(move_from_vchunk)) {
-        LOGWARN("move_from_chunk={} is not eligible for gc, so we can not process the gc task", move_from_chunk);
-        task.setValue(false);
-        return;
-    }
-
-    LOGINFO("move_from_chunk={} belongs to pg {} ", move_from_chunk, move_from_vchunk->m_pg_id.value());
-
     // make chunk to gc state, so that it can be select for creating shard
     auto succeed = m_chunk_selector->try_mark_chunk_to_gc_state(
         move_from_chunk, priority == static_cast< uint8_t >(task_priority::emergent) /* force */);
@@ -353,7 +424,7 @@ void GCManager::pdev_gc_actor::process_gc_task(chunk_id_t move_from_chunk, uint8
 
 GCManager::pdev_gc_actor::~pdev_gc_actor() {
     stop();
-    LOGINFO("pdev gc actor for pdev_id={} is destroyed", m_pdev_id);
+    LOGINFO("gc actor for pdev_id={} is destroyed", m_pdev_id);
 }
 
 /* RateLimiter */

src/lib/homestore_backend/gc_manager.hpp

@@ -173,16 +173,20 @@ class GCManager {
      */
     std::shared_ptr< pdev_gc_actor > try_create_pdev_gc_actor(uint32_t pdev_id,
                                                               std::shared_ptr< GCBlobIndexTable > index_table);
-    std::shared_ptr< pdev_gc_actor > get_pdev_gc_actor(uint32_t pdev_id);
 
-    static bool is_eligible_for_gc(std::shared_ptr< HeapChunkSelector::ExtendedVChunk > chunk);
+    bool is_eligible_for_gc(chunk_id_t chunk_id);
 
     void start();
     void stop();
 
 private:
     void scan_chunks_for_gc();
 
+    void on_gc_task_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie);
+    void on_gc_actor_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie);
+    void on_reserved_chunk_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie);
+    std::shared_ptr< pdev_gc_actor > get_pdev_gc_actor(uint32_t pdev_id);
+
 private:
     std::shared_ptr< HeapChunkSelector > m_chunk_selector;
     folly::ConcurrentHashMap< uint32_t, std::shared_ptr< pdev_gc_actor > > m_pdev_gc_actors;

src/lib/homestore_backend/heap_chunk_selector.cpp

@@ -310,11 +310,11 @@ bool HeapChunkSelector::recover_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_
     return true;
 }
 
-void HeapChunkSelector::recover_per_dev_chunk_heap() {
+void HeapChunkSelector::build_pdev_available_chunk_heap() {
     std::unique_lock lock_guard(m_chunk_selector_mtx);
     for (auto [p_chunk_id, chunk] : m_chunks) {
-        // if selected for pg, not add to pdev.
-        bool add_to_heap = !chunk->m_pg_id.has_value();
+        // if selected for pg, or it is marked as GC state(reserved chunk), not add to pdev.
+        bool add_to_heap = !chunk->m_pg_id.has_value() && chunk->m_state != ChunkState::GC;
         add_chunk_internal(p_chunk_id, add_to_heap);
     }
 }
@@ -438,85 +438,18 @@ uint64_t HeapChunkSelector::total_blks(uint32_t dev_id) const {
     return it->second->m_total_blks;
 }
 
-std::list< uint32_t > HeapChunkSelector::get_pdev_ids() const {
-    std::list< uint32_t > pdev_ids;
-    for (const auto& [pdev_id, _] : m_per_dev_heap) {
-        pdev_ids.emplace_back(pdev_id);
+std::unordered_map< uint32_t, std::vector< homestore::chunk_num_t > > HeapChunkSelector::get_pdev_chunks() const {
+    std::unordered_map< uint32_t, std::vector< homestore::chunk_num_t > > pdev_chunks;
+    for (const auto& [_, EXvchunk] : m_chunks) {
+        auto pdev_id = EXvchunk->get_pdev_id();
+        pdev_chunks.try_emplace(pdev_id, std::vector< homestore::chunk_num_t >());
+        pdev_chunks[pdev_id].emplace_back(EXvchunk->get_chunk_id());
     }
-    return pdev_ids;
-}
-
-std::shared_ptr< HeapChunkSelector::ExtendedVChunk > HeapChunkSelector::select_empty_chunk_from_pdev(uint32_t pdev_id) {
-    std::unique_lock lock_guard(m_chunk_selector_mtx);
-    auto it = m_per_dev_heap.find(pdev_id);
-    if (it == m_per_dev_heap.end()) {
-        LOGWARNMOD(homeobject, "No pdev heap found for pdev {}", pdev_id);
-        return nullptr;
-    }
-    auto pdev_heap = it->second;
-    std::unique_lock lock(pdev_heap->mtx);
-    if (pdev_heap->m_heap.empty()) {
-        LOGWARNMOD(homeobject, "No available chunk found for pdev {}", pdev_id);
-        return nullptr;
-    }
-    auto chunk = pdev_heap->m_heap.top();
-    pdev_heap->m_heap.pop();
-    pdev_heap->available_blk_count -= chunk->available_blks();
-    RELEASE_ASSERT(chunk->available_blks() == chunk->get_total_blks(),
-                   "the selected chunk should be empty, chunk_id={}, pdev_id={}", chunk->get_chunk_id(), pdev_id);
-    RELEASE_ASSERT(!chunk->m_pg_id.has_value(), "chunk {} is selected from pdev heap, but it has a pg_id {}!",
-                   chunk->get_chunk_id(), chunk->m_pg_id.value());
-
-    return chunk;
-}
-
-std::shared_ptr< HeapChunkSelector::ExtendedVChunk >
-HeapChunkSelector::select_specific_chunk_from_pdev(uint32_t pdev_id, const chunk_num_t chunk_id) {
-    std::unique_lock lock_guard(m_chunk_selector_mtx);
-    auto it = m_per_dev_heap.find(pdev_id);
-    if (it == m_per_dev_heap.end()) {
-        LOGWARNMOD(homeobject, "No pdev heap found for pdev {}", pdev_id);
-        return nullptr;
-    }
-    auto pdev_heap = it->second;
-    std::shared_ptr< HeapChunkSelector::ExtendedVChunk > EVchunk;
-    std::list< std::shared_ptr< HeapChunkSelector::ExtendedVChunk > > EVchunks;
-
-    std::unique_lock lock(pdev_heap->mtx);
-    for (; pdev_heap->m_heap.empty();) {
-        EVchunk = pdev_heap->m_heap.top();
-        pdev_heap->m_heap.pop();
-        if (EVchunk->get_chunk_id() == chunk_id) {
-            break;
-        } else {
-            EVchunks.emplace_back(EVchunk);
-        }
-    }
-
-    for (const auto& chunk : EVchunks) {
-        pdev_heap->m_heap.emplace(chunk);
-    }
-
-    if (!EVchunk) {
-        LOGWARNMOD(homeobject, "No available chunk found for pdev {}, chunk_id {}", pdev_id, chunk_id);
-        return nullptr;
-    }
-
-    pdev_heap->available_blk_count -= EVchunk->available_blks();
-
-    RELEASE_ASSERT(!EVchunk->m_pg_id.has_value(), "chunk {} is selected from pdev heap, but it has a pg_id {}!",
-                   EVchunk->get_chunk_id(), EVchunk->m_pg_id.value());
-
-    return EVchunk;
-}
-
-const std::unordered_map< homestore::chunk_num_t, homestore::cshared< HeapChunkSelector::ExtendedVChunk > >&
-HeapChunkSelector::get_all_chunks() const {
-    return m_chunks;
+    return pdev_chunks;
 }
 
 homestore::cshared< HeapChunkSelector::ExtendedVChunk >
-HeapChunkSelector::get_extend_vchunk(const chunk_num_t chunk_id) const {
+HeapChunkSelector::get_extend_vchunk(const homestore::chunk_num_t chunk_id) const {
     auto it = m_chunks.find(chunk_id);
     if (it != m_chunks.end()) { return it->second; }
     return nullptr;

src/lib/homestore_backend/heap_chunk_selector.h

@@ -10,6 +10,7 @@
 
 #include <queue>
 #include <vector>
+#include <unordered_set>
 #include <mutex>
 #include <functional>
 #include <atomic>
@@ -128,7 +129,7 @@ class HeapChunkSelector : public homestore::ChunkSelector {
     bool recover_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids);
 
     // this should be called after all pg meta blk recovered
-    void recover_per_dev_chunk_heap();
+    void build_pdev_available_chunk_heap();
 
     // this should be called after ShardManager is initialized and get all the open shards
     bool recover_pg_chunks_states(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excluding_v_chunk_ids);
@@ -193,16 +194,7 @@ class HeapChunkSelector : public homestore::ChunkSelector {
     /**
      * @brief Returns all the pdev ids that managed by this chunk selector.
      */
-    std::list< uint32_t > get_pdev_ids() const;
-
-    /**
-     * @brief select an available chunk from the given pdev.
-     */
-    std::shared_ptr< ExtendedVChunk > select_empty_chunk_from_pdev(uint32_t pdev_id);
-
-    std::shared_ptr< ExtendedVChunk > select_specific_chunk_from_pdev(uint32_t pdev_id, const chunk_num_t chunk_id);
-
-    const std::unordered_map< chunk_num_t, homestore::cshared< ExtendedVChunk > >& get_all_chunks() const;
+    std::unordered_map< uint32_t, std::vector< chunk_num_t > > get_pdev_chunks() const;
 
     homestore::cshared< ExtendedVChunk > get_extend_vchunk(const chunk_num_t chunk_id) const;