implement gc thread model

Author: JacksonYao287

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "2.3.11"
+    version = "2.3.12"
 
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeReplication"
@@ -48,9 +48,9 @@ def build_requirements(self):
         self.test_requires("gtest/1.14.0")
 
     def requirements(self):
-        self.requires("sisl/[^12.2]@oss/master", transitive_headers=True)
+        self.requires("sisl/[^12.3]@oss/master", transitive_headers=True)
         self.requires("homestore/[>=6.9.0]@oss/master")
-        self.requires("iomgr/[^11.3]@oss/master")
+        self.requires("iomgr/[^11.4]@oss/master")
         self.requires("lz4/1.9.4", override=True)
         self.requires("openssl/3.3.1", override=True)
 

src/lib/blob_route.hpp

@@ -5,7 +5,7 @@
 
 #include <boost/functional/hash.hpp>
 
-#include "homeobject/common.hpp"
+#include "homeobject_impl.hpp"
 
 namespace homeobject {
 
@@ -19,11 +19,37 @@ struct BlobRoute {
     auto operator<=>(BlobRoute const&) const = default;
     sisl::blob to_blob() const { return sisl::blob{uintptr_cast(const_cast< BlobRoute* >(this)), sizeof(*this)}; }
 };
+
+// used for gc to quickly identify all the blob in the move_to chunk
+struct BlobRouteByChunk {
+    // homestore::chunk_num_t == uint16_t
+    uint16_t chunk;
+    shard_id_t shard;
+    blob_id_t blob;
+    auto operator<=>(BlobRouteByChunk const&) const = default;
+    sisl::blob to_blob() const {
+        return sisl::blob{uintptr_cast(const_cast< BlobRouteByChunk* >(this)), sizeof(*this)};
+    }
+};
 #pragma pack()
 
 } // namespace homeobject
 
 namespace fmt {
+template <>
+struct formatter< homeobject::BlobRouteByChunk > {
+    template < typename ParseContext >
+    constexpr auto parse(ParseContext& ctx) {
+        return ctx.begin();
+    }
+
+    template < typename FormatContext >
+    auto format(homeobject::BlobRouteByChunk const& r, FormatContext& ctx) {
+        return format_to(ctx.out(), "{:04x}:{:04x}:{:012x}:{:016x}", r.chunk, (r.shard >> homeobject::shard_width),
+                         (r.shard & homeobject::shard_mask), r.blob);
+    }
+};
+
 template <>
 struct formatter< homeobject::BlobRoute > {
     template < typename ParseContext >
@@ -37,6 +63,7 @@ struct formatter< homeobject::BlobRoute > {
                          (r.shard & homeobject::shard_mask), r.blob);
     }
 };
+
 } // namespace fmt
 
 template <>
@@ -45,3 +72,10 @@ struct std::hash< homeobject::BlobRoute > {
         return boost::hash_value< homeobject::blob_id_t >(std::make_pair(r.shard, r.blob));
     }
 };
+
+template <>
+struct std::hash< homeobject::BlobRouteByChunk > {
+    std::size_t operator()(homeobject::BlobRouteByChunk const& r) const noexcept {
+        return boost::hash_value< homeobject::blob_id_t >(std::make_pair(r.shard, r.blob));
+    }
+};

src/lib/homestore_backend/CMakeLists.txt

@@ -29,6 +29,7 @@ target_sources("${PROJECT_NAME}_homestore" PRIVATE
     replication_state_machine.cpp
     hs_cp_callbacks.cpp
     hs_http_manager.cpp
+    gc_manager.cpp
     $<TARGET_OBJECTS:${PROJECT_NAME}_core>
 )
 target_link_libraries("${PROJECT_NAME}_homestore" PUBLIC
@@ -114,3 +115,11 @@ add_test(NAME HomestoreResyncTestWithLeaderRestart
         --override_config homestore_config.generic.repl_dev_cleanup_interval_sec=5
         --override_config homestore_config.consensus.max_grpc_message_size:138412032
         --gtest_filter=HomeObjectFixture.RestartLeader*)
+
+add_executable(homestore_test_gc)
+target_sources(homestore_test_gc PRIVATE $<TARGET_OBJECTS:homestore_tests_gc>)
+target_link_libraries(homestore_test_gc PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
+add_test(NAME HomestoreTestGC COMMAND homestore_test_pg -csv error --executor immediate --config_path ./
+        --override_config homestore_config.consensus.snapshot_freq_distance:0
+        --override_config homestore_config.consensus.max_grpc_message_size:138412032)
+