Make chunk_size to uint64_t to avoid overflow and support chunk size greater equal than 4GB

Author: shosseinimotlagh

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "7.3.3"
+    version = "8.3.3"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

src/include/homestore/blkdata_service.hpp

@@ -72,7 +72,7 @@ class BlkDataService {
      * @param chunk_size The size of chunks to use for the virtual device, in bytes.
      */
     void create_vdev(uint64_t size, HSDevType devType, uint32_t blk_size, blk_allocator_type_t alloc_type,
-                     chunk_selector_type_t chunk_sel_type, uint32_t num_chunks, uint32_t chunk_size);
+                     chunk_selector_type_t chunk_sel_type, uint32_t num_chunks, uint64_t chunk_size);
 
     /**
      * @brief Opens a virtual device with the specified virtual device information.

src/include/homestore/index_service.hpp

@@ -63,7 +63,7 @@ class IndexService {
     // Creates the vdev that is needed to initialize the device
     void create_vdev(uint64_t size, HSDevType devType, uint32_t num_chunks,
                      chunk_selector_type_t chunk_sel_type = chunk_selector_type_t::ROUND_ROBIN,
-                     uint32_t chunk_size = 0);
+                     uint64_t chunk_size = 0);
     // Open the existing vdev which is represnted by the vdev_info_block
     shared< VirtualDev > open_vdev(const vdev_info& vb, bool load_existing);
     std::shared_ptr< ChunkSelector > get_chunk_selector() { return m_custom_chunk_selector; };

src/include/homestore/logstore_service.hpp

@@ -160,7 +160,7 @@ class LogStoreService {
      */
     void device_truncate();
 
-    folly::Future< std::error_code > create_vdev(uint64_t size, HSDevType devType, uint32_t chunk_size);
+    folly::Future< std::error_code > create_vdev(uint64_t size, HSDevType devType, uint64_t chunk_size);
     std::shared_ptr< VirtualDev > open_vdev(const vdev_info& vinfo, bool load_existing);
     std::shared_ptr< JournalVirtualDev > get_vdev() const { return m_logdev_vdev; }
     std::vector< std::shared_ptr< LogDev > > get_all_logdevs();

src/lib/blkalloc/bitmap_blk_allocator.cpp

@@ -59,6 +59,7 @@ void BitmapBlkAllocator::cp_flush(CP*) {
 
     if (m_is_disk_bm_dirty.load()) {
         sisl::byte_array bitmap_buf = acquire_underlying_buffer();
+        BLKALLOC_LOG(INFO, "Flushing bitmap (name: {} id: {}) size = {}", get_name(), m_chunk_id, bitmap_buf->size());
         if (m_meta_blk_cookie) {
             meta_service().update_sub_sb(bitmap_buf->cbytes(), bitmap_buf->size(), m_meta_blk_cookie);
         } else {

src/lib/blkdata_svc/blkdata_service.cpp

@@ -39,7 +39,7 @@ BlkDataService::~BlkDataService() = default;
 
 // first-time boot path
 void BlkDataService::create_vdev(uint64_t size, HSDevType devType, uint32_t blk_size, blk_allocator_type_t alloc_type,
-                                 chunk_selector_type_t chunk_sel_type, uint32_t num_chunks, uint32_t chunk_size) {
+                                 chunk_selector_type_t chunk_sel_type, uint32_t num_chunks, uint64_t chunk_size) {
     hs_vdev_context vdev_ctx;
     vdev_ctx.type = hs_vdev_type_t::DATA_VDEV;
 

src/lib/device/device.h

@@ -43,18 +43,18 @@ struct vdev_info {
     uint32_t num_mirrors{0};                   // 12: Total number of mirrors
     uint32_t blk_size{0};                      // 16: IO block size for this vdev
     uint32_t num_primary_chunks{0};            // 20: number of primary chunks
-    uint32_t chunk_size{0};                    // 24: chunk size used in vdev.
-    vdev_size_type_t size_type{};              // 28: Whether its a static or dynamic type.
-    uint8_t slot_allocated{0};                 // 29: Is this current slot allocated
-    uint8_t failed{0};                         // 30: set to true if disk is replaced
-    uint8_t hs_dev_type{0};                    // 31: PDev dev type (as in fast or data)
-    uint8_t multi_pdev_choice{0};              // 32: Choice when multiple pdevs are present (vdev_multi_pdev_opts_t)
-    char name[max_name_len];                   // 33: Name of the vdev
-    uint16_t checksum{0};                      // 97: Checksum of this entire Block
-    uint8_t alloc_type;                        // 98: Allocator type of this vdev
-    uint8_t chunk_sel_type;                    // 99: Chunk Selector type of this vdev_id
-    uint8_t use_slab_allocator{0};             // 100: Use slab allocator for this vdev
-    uint8_t padding[154]{};                    // 101: Padding to make it 256 bytes
+    uint64_t chunk_size{0};                    // 24: chunk size used in vdev.
+    vdev_size_type_t size_type{};              // 32: Whether its a static or dynamic type.
+    uint8_t slot_allocated{0};                 // 33: Is this current slot allocated
+    uint8_t failed{0};                         // 34: set to true if disk is replaced
+    uint8_t hs_dev_type{0};                    // 35: PDev dev type (as in fast or data)
+    uint8_t multi_pdev_choice{0};              // 36: Choice when multiple pdevs are present (vdev_multi_pdev_opts_t)
+    char name[max_name_len];                   // 37: Name of the vdev
+    uint16_t checksum{0};                      // 101: Checksum of this entire Block
+    uint8_t alloc_type;                        // 102: Allocator type of this vdev
+    uint8_t chunk_sel_type;                    // 103: Chunk Selector type of this vdev_id
+    uint8_t use_slab_allocator{0};             // 104: Use slab allocator for this vdev
+    uint8_t padding[150]{};                    // 105: Padding to make it 256 bytes
     uint8_t user_private[user_private_size]{}; // 128: User sepcific information
 
     uint32_t get_vdev_id() const { return vdev_id; }

src/lib/device/device_manager.cpp

@@ -415,7 +415,7 @@ void DeviceManager::compose_vparam(uint64_t vdev_id, vdev_parameters& vparam, st
     }
 
     // Based on the min chunk size, we calculate the max number of chunks that can be created in each target pdev
-    uint32_t min_chunk_size = hs_super_blk::min_chunk_size(vparam.dev_type);
+    uint64_t min_chunk_size = hs_super_blk::min_chunk_size(vparam.dev_type);
     // FIXME: it is possible that each vdev is less than max_num_chunks, but total is more than MAX_CHUNKS_IN_SYSTEM.
     // uint32 convert is safe as it only overflow when vdev size > 64PB with 16MB min_chunk_size.
     uint32_t max_num_chunks = std::min(uint32_t(vparam.vdev_size / min_chunk_size), hs_super_blk::MAX_CHUNKS_IN_SYSTEM);

src/lib/device/hs_super_blk.h

@@ -207,7 +207,7 @@ class hs_super_blk {
         // for a device with 100MB and min_chunk_size = 16MB, we should get 6 chunks, not 7.
         return dinfo.dev_size / min_c_size;
     }
-    static uint32_t min_chunk_size(HSDevType dtype) {
+    static uint64_t min_chunk_size(HSDevType dtype) {
         uint64_t min_chunk_size = (dtype == HSDevType::Fast) ? MIN_CHUNK_SIZE_FAST_DEVICE : MIN_CHUNK_SIZE_DATA_DEVICE;
 #ifdef _PRERELEASE
         auto chunk_size = iomgr_flip::instance()->get_test_flip< long >("set_minimum_chunk_size");

src/lib/index/index_service.cpp

@@ -45,7 +45,7 @@ IndexService::IndexService(std::unique_ptr< IndexServiceCallbacks > cbs, shared<
 }
 
 void IndexService::create_vdev(uint64_t size, HSDevType devType, uint32_t num_chunks,
-                               chunk_selector_type_t chunk_sel_type, uint32_t chunk_size) {
+                               chunk_selector_type_t chunk_sel_type, uint64_t chunk_size) {
     auto const atomic_page_size = hs()->device_mgr()->atomic_page_size(devType);
     hs_vdev_context vdev_ctx;
     vdev_ctx.type = hs_vdev_type_t::INDEX_VDEV;