Back the hash table with huge pages on create and attach

Summary:
Backs the access-container hash-table segments (kShmHashTableName, kShmChainedItemHashTableName) with HugeTLB pages when the cache is configured via enableHugePages().

To avoid a footgun where a per-AccessConfig page size could silently diverge from the cache-wide enableHugePages() setting, pageSize is removed entirely from AccessConfig (ChainedHashTable::Config).

Persistence is normal-page only, so PersistenceManager now rejects a huge-page config outright.

Reviewed By: AlnisM

Differential Revision: D113626131

fbshipit-source-id: 32ff4d5fafe6aa9d55c9967cbc822381ca460b59

Author: rlyerly

cachelib/allocator/CacheAllocator.h

@@ -2255,6 +2255,14 @@ class CacheAllocator : public CacheBase {
                   std::chrono::seconds timeout = std::chrono::seconds{0});
 
   ShmSegmentOpts createShmCacheOpts();
+
+  // Builds the shm options for an access-container (hash table) segment,
+  ShmSegmentOpts createShmAccessOpts();
+
+  // Validates and applies a requested huge page size (bytes; 0 == normal) onto
+  // opts, throwing if the kernel does not support the size.
+  void applyHugePageOpts(ShmSegmentOpts& opts, const PageSize& pageSize) const;
+
   std::unique_ptr<MemoryAllocator> createNewMemoryAllocator();
   std::unique_ptr<MemoryAllocator> restoreMemoryAllocator();
   std::unique_ptr<CCacheManager> restoreCCacheManager();
@@ -2805,23 +2813,38 @@ ShmSegmentOpts CacheAllocator<CacheTrait>::createShmCacheOpts() {
     throw std::invalid_argument("CacheLib only supports a single memory tier");
   }
   opts.memBindNumaNodes = config_.memoryTierConfigs[0].getMemBind();
+  applyHugePageOpts(opts, config_.hugePageSize);
+  return opts;
+}
 
-  if (config_.hugePageSize.isHugePage()) {
-    if (!PageSize::supportedHugePageSizes().contains(
-            config_.hugePageSize.getPageSize())) {
-      throw std::invalid_argument(fmt::format(
-          "Requested huge page size {} is not supported by the kernel",
-          config_.hugePageSize.getPageSize()));
-    } else if (config_.hugePageMountDir.empty() && config_.usePosixShm) {
-      throw std::invalid_argument(
-          "Requested huge pages for POSIX shared memory regions but didn't "
-          "supply a hugetlbfs mount");
-    }
-    opts.pageSize = config_.hugePageSize;
-  }
+template <typename CacheTrait>
+ShmSegmentOpts CacheAllocator<CacheTrait>::createShmAccessOpts() {
+  ShmSegmentOpts opts;
+  applyHugePageOpts(opts, config_.hugePageSize);
   return opts;
 }
 
+template <typename CacheTrait>
+void CacheAllocator<CacheTrait>::applyHugePageOpts(
+    ShmSegmentOpts& opts, const PageSize& pageSize) const {
+  if (!pageSize.isHugePage()) {
+    return;
+  } else if (!PageSize::supportedHugePageSizes().contains(
+                 pageSize.getPageSize())) {
+    throw std::invalid_argument(fmt::format(
+        "Requested huge page size {} is not supported by the kernel",
+        pageSize.getPageSize()));
+  } else if (config_.hugePageMountDir.empty() && config_.usePosixShm) {
+    throw std::invalid_argument(
+        "Requested huge pages for POSIX shared memory regions but didn't "
+        "supply a hugetlbfs mount");
+  }
+  opts.pageSize = pageSize;
+  // The segment is attached at a page-aligned address, so its alignment must be
+  // at least the huge page size or the kernel rejects the mapping
+  opts.alignment = std::max(opts.alignment, pageSize.getPageSize());
+}
+
 template <typename CacheTrait>
 std::unique_ptr<MemoryAllocator>
 CacheAllocator<CacheTrait>::createNewMemoryAllocator() {
@@ -3029,17 +3052,18 @@ CacheAllocator<CacheTrait>::initAccessContainer(InitMemType type,
                 name,
                 AccessContainer::getRequiredSize(config.getNumBuckets()),
                 nullptr,
-                ShmSegmentOpts(config.getPageSize()))
+                createShmAccessOpts())
             .addr,
         compressor_,
         [this](Item* it) -> WriteHandle { return acquire(it); });
   } else if (type == InitMemType::kMemAttach) {
     return std::make_unique<AccessContainer>(
         deserializer_->deserialize<AccessSerializationType>(),
         config,
-        shmManager_->attachShm(name),
+        shmManager_->attachShm(name, nullptr, createShmAccessOpts()),
         compressor_,
-        [this](Item* it) -> WriteHandle { return acquire(it); });
+        [this](Item* it) -> WriteHandle { return acquire(it); },
+        config_.hugePageSize);
   }
 
   // Invalid type

cachelib/allocator/ChainedHashTable.h

@@ -239,26 +239,15 @@ class ChainedHashTable {
 
     // @param bucketsPower number of buckets in base 2 logarithm
     // @param locksPower number of locks in base 2 logarithm
-    // @param pageSize page size
-    Config(unsigned int bucketsPower,
-           unsigned int locksPower,
-           PageSize pageSize = PageSize())
-        : Config(bucketsPower,
-                 locksPower,
-                 std::make_shared<MurmurHash2>(),
-                 std::move(pageSize)) {}
+    Config(unsigned int bucketsPower, unsigned int locksPower)
+        : Config(bucketsPower, locksPower, std::make_shared<MurmurHash2>()) {}
 
     // @param bucketsPower number of buckets in base 2 logarithm
     // @param locksPower number of locks in base 2 logarithm
     // @param hasher the key hash function
-    // @param pageSize page size
-    Config(unsigned int bucketsPower,
-           unsigned int locksPower,
-           Hasher hasher,
-           PageSize pageSize = PageSize())
+    Config(unsigned int bucketsPower, unsigned int locksPower, Hasher hasher)
         : bucketsPower_(bucketsPower),
           locksPower_(locksPower),
-          pageSize_(std::move(pageSize)),
           hasher_(std::move(hasher)) {
       if (bucketsPower_ > kMaxBucketPower || locksPower_ > kMaxLockPower ||
           locksPower_ > bucketsPower_) {
@@ -314,8 +303,6 @@ class ChainedHashTable {
       return configMap;
     }
 
-    const PageSize& getPageSize() const { return pageSize_; }
-
    private:
     // 4 billion buckets should be good enough for everyone.
     static constexpr unsigned int kMaxBucketPower = 32;
@@ -330,8 +317,6 @@ class ChainedHashTable {
     // total number of locks for the hashtable expressed as a power of two.
     unsigned int locksPower_{5};
 
-    PageSize pageSize_{};
-
     Hasher hasher_ = std::make_shared<MurmurHash2>();
   };
 
@@ -392,6 +377,8 @@ class ChainedHashTable {
     // @param memSegment  shared memory segment for the hash table
     // @param compressor  object used to compress/decompress node pointers
     // @param hm          the functor that creates a Handle from T*
+    // @param hugePageSize page size the segment was backed with (bytes), used
+    //        to validate the mapped segment size.
     //
     // @throw std::invalid argument if the bucket power in new config does not
     //        match the previous state or the size of the memSegment does not
@@ -400,7 +387,8 @@ class ChainedHashTable {
               const Config& newConfig,
               ShmAddr memSegment,
               const PtrCompressor& compressor,
-              HandleMaker hm = kDefaultHandleMaker);
+              HandleMaker hm = kDefaultHandleMaker,
+              PageSize hugePageSize = PageSize());
 
     // restore hash table from previous state. This only works when the
     // hash table memory is managed by the user.
@@ -411,6 +399,8 @@ class ChainedHashTable {
     // @param nBytes      size of memory allocation pointed to by memStart
     // @param compressor  object used to compress/decompress node pointers
     // @param hm          the functor that creates a Handle from T*
+    // @param hugePageSize page size the segment was backed with (bytes), used
+    //        to validate the mapped segment size.
     //
     // @throw std::invalid argument if the bucket power in new config does not
     //        match the previous state or the size of the memSegment does not
@@ -420,7 +410,8 @@ class ChainedHashTable {
               void* memStart,
               size_t nBytes,
               const PtrCompressor& compressor,
-              HandleMaker hm = kDefaultHandleMaker);
+              HandleMaker hm = kDefaultHandleMaker,
+              PageSize hugePageSize = PageSize());
 
     Container(const Container&) = delete;
     Container& operator=(const Container&) = delete;
@@ -1024,13 +1015,15 @@ ChainedHashTable::Container<T, HookPtr, LockT>::Container(
     const Config& config,
     ShmAddr memSegment,
     const PtrCompressor& compressor,
-    HandleMaker hm)
+    HandleMaker hm,
+    PageSize hugePageSize)
     : Container(object,
                 config,
                 memSegment.addr,
                 memSegment.size,
                 compressor,
-                std::move(hm)) {}
+                std::move(hm),
+                hugePageSize) {}
 
 template <typename T,
           typename ChainedHashTable::Hook<T> T::* HookPtr,
@@ -1041,7 +1034,8 @@ ChainedHashTable::Container<T, HookPtr, LockT>::Container(
     void* memStart,
     size_t nBytes,
     const PtrCompressor& compressor,
-    HandleMaker hm)
+    HandleMaker hm,
+    PageSize hugePageSize)
     : config_{config},
       handleMaker_(std::move(hm)),
       ht_{config_.getNumBuckets(), memStart, compressor, config_.getHasher(),
@@ -1058,8 +1052,7 @@ ChainedHashTable::Container<T, HookPtr, LockT>::Container(
 
   // Take page alignment into consideration when comparing the size of the
   // shared memory and the size of the hashtable.
-  if (nBytes !=
-      util::getAlignedSize(ht_.size(), config_.getPageSize().getPageSize())) {
+  if (nBytes != util::getAlignedSize(ht_.size(), hugePageSize.getPageSize())) {
     throw std::invalid_argument(
         fmt::format("Hashtable size not compatible. old = {}, new = {}",
                     ht_.size(),

cachelib/persistence/PersistenceManager.h

@@ -93,7 +93,6 @@ class PersistenceStreamWriter {
  *  - isNvmCacheEncryption
  *  - isNvmCacheTruncateAllocSize
  *  - accessConfig.numBuckets
- *  - accessConfig.pageSize
  *  - chainedItemAccessConfig.numBuckets
  *  - nvmConfig.navyConfig.FileName
  *  - nvmConfig.navyConfig.RaidPaths
@@ -117,9 +116,8 @@ class PersistenceManager {
     // requires plumbing page sizes: saveShm() would need to attach the source
     // at its real page size, and restoreCache() would need to create segments
     // at the destination's configured page size (and hugetlbfs mount).
-    CACHELIB_CHECK_THROW(config.accessConfig.getPageSize().getPageSize() ==
-                             PageSize::systemPageSize(),
-                         "Only default PageSize is supported to persist");
+    CACHELIB_CHECK_THROW(!config.hugePageSize.isHugePage(),
+                         "Huge pages are not supported to persist");
 
     if (config.nvmConfig.has_value()) {
       const auto& navyConfig = config.nvmConfig->navyConfig;

cachelib/persistence/tests/PersistenceManagerTest.cpp

@@ -86,12 +86,11 @@ TEST_F(PersistenceManagerTest, testConfigChange) {
   {
     // change cache config
     auto config = cache_.config_;
-    // non-default page size is not allowed
-    config.setAccessConfig(
-        Cache::AccessConfig{10, 10, PageSize(PageSize::kHugePageSize2MB)});
+    // huge pages are not allowed
+    config.enableHugePages(PageSize(PageSize::kHugePageSize2MB));
     ASSERT_THROW_WITH_MSG(PersistenceManager manager(config),
                           std::invalid_argument,
-                          "Only default PageSize is supported to persist");
+                          "Huge pages are not supported to persist");
   }
 }
 

website/docs/Cache_Library_User_Guides/Cross_Host_Cache_Persistence.md

@@ -16,7 +16,7 @@ shared infra, but this is not a persistent storage that serves like databases.
 1. Create a persistence cache with [Cache Persistence](Cache_persistence) user guide.
 2. Shutdown cache successfully.
 3. Only POSIX is supported.
-4. Only default PageSize is supported.
+4. Huge pages are not supported.
 
 ## Implement Stream Reader and Writer APIs
 
@@ -152,11 +152,10 @@ incompatible.
 5.  isNvmCacheEncryption
 6.  isNvmCacheTruncateAllocSize
 7.  accessConfig.numBuckets
-8.  accessConfig.pageSize
-9.  chainedItemAccessConfig.numBuckets
-10.  nvmConfig.navyConfig.getFileName
-11.  nvmConfig.navyConfig.getRaidPaths
-12.  nvmConfig.navyConfig.getFileSize
+8.  chainedItemAccessConfig.numBuckets
+9.  nvmConfig.navyConfig.getFileName
+10.  nvmConfig.navyConfig.getRaidPaths
+11.  nvmConfig.navyConfig.getFileSize
 
 ## Simple Example