Skip to content

Commit f9e8274

Browse files
byrnedjbyrnedj
committed
batch eviction / promotion
1 parent fd9606e commit f9e8274

14 files changed

+920
-312
lines changed

‎cachelib/allocator/CacheAllocator-inl.h

+531-105
Large diffs are not rendered by default.

‎cachelib/allocator/CacheAllocator.h

+120-183
Large diffs are not rendered by default.

‎cachelib/allocator/MM2Q-inl.h

+33-8
Original file line numberDiff line numberDiff line change
@@ -227,16 +227,41 @@ bool MM2Q::Container<T, HookPtr>::add(T& node) noexcept {
227227
if (node.isInMMContainer()) {
228228
return false;
229229
}
230+
addNodeLocked(node, currTime);
231+
return true;
232+
});
233+
}
230234

231-
markHot(node);
232-
unmarkCold(node);
233-
unmarkTail(node);
234-
lru_.getList(LruType::Hot).linkAtHead(node);
235-
rebalance();
235+
// adds the node to the list assuming not in
236+
// container and holding container lock
237+
template <typename T, MM2Q::Hook<T> T::*HookPtr>
238+
void MM2Q::Container<T, HookPtr>::addNodeLocked(T& node, const Time& currTime) {
239+
XDCHECK(!node.isInMMContainer());
240+
markHot(node);
241+
unmarkCold(node);
242+
unmarkTail(node);
243+
lru_.getList(LruType::Hot).linkAtHead(node);
244+
rebalance();
245+
246+
node.markInMMContainer();
247+
setUpdateTime(node, currTime);
248+
}
236249

237-
node.markInMMContainer();
238-
setUpdateTime(node, currTime);
239-
return true;
250+
template <typename T, MM2Q::Hook<T> T::*HookPtr>
251+
template <typename It>
252+
uint32_t MM2Q::Container<T, HookPtr>::addBatch(It begin, It end) noexcept {
253+
const auto currTime = static_cast<Time>(util::getCurrentTimeSec());
254+
return lruMutex_->lock_combine([this, begin, end, currTime]() {
255+
uint32_t i = 0;
256+
for (auto itr = begin; itr != end; itr++) {
257+
T* node = *itr;
258+
if (node->isInMMContainer()) {
259+
return i;
260+
}
261+
addNodeLocked(*node,currTime);
262+
i++;
263+
}
264+
return i;
240265
});
241266
}
242267

‎cachelib/allocator/MM2Q.h

+12
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,18 @@ class MM2Q {
463463
// is unchanged.
464464
bool add(T& node) noexcept;
465465

466+
// helper function to add the node under the container lock
467+
void addNodeLocked(T& node, const Time& currTime);
468+
469+
// adds the given nodes into the container and marks each as being present in
470+
// the container. The nodes are added to the head of the lru.
471+
//
472+
// @param vector of nodes The nodes to be added to the container.
473+
// @return number of nodes added - it is up to user to verify all
474+
// expected nodes have been added.
475+
template <typename It>
476+
uint32_t addBatch(It begin, It end) noexcept;
477+
466478
// removes the node from the lru and sets it previous and next to nullptr.
467479
//
468480
// @param node The node to be removed from the container.

‎cachelib/allocator/MMLru-inl.h

+37-12
Original file line numberDiff line numberDiff line change
@@ -193,24 +193,50 @@ void MMLru::Container<T, HookPtr>::updateLruInsertionPoint() noexcept {
193193
template <typename T, MMLru::Hook<T> T::*HookPtr>
194194
bool MMLru::Container<T, HookPtr>::add(T& node) noexcept {
195195
const auto currTime = static_cast<Time>(util::getCurrentTimeSec());
196-
197196
return lruMutex_->lock_combine([this, &node, currTime]() {
198197
if (node.isInMMContainer()) {
199198
return false;
200199
}
201-
if (config_.lruInsertionPointSpec == 0 || insertionPoint_ == nullptr) {
202-
lru_.linkAtHead(node);
203-
} else {
204-
lru_.insertBefore(*insertionPoint_, node);
205-
}
206-
node.markInMMContainer();
207-
setUpdateTime(node, currTime);
208-
unmarkAccessed(node);
209-
updateLruInsertionPoint();
200+
addNodeLocked(node,currTime);
210201
return true;
211202
});
212203
}
213204

205+
template <typename T, MMLru::Hook<T> T::*HookPtr>
206+
void MMLru::Container<T, HookPtr>::addNodeLocked(T& node, const Time& currTime) {
207+
XDCHECK(!node.isInMMContainer());
208+
if (config_.lruInsertionPointSpec == 0 || insertionPoint_ == nullptr) {
209+
lru_.linkAtHead(node);
210+
} else {
211+
lru_.insertBefore(*insertionPoint_, node);
212+
}
213+
node.markInMMContainer();
214+
setUpdateTime(node, currTime);
215+
unmarkAccessed(node);
216+
updateLruInsertionPoint();
217+
}
218+
219+
template <typename T, MMLru::Hook<T> T::*HookPtr>
220+
template <typename It>
221+
uint32_t MMLru::Container<T, HookPtr>::addBatch(It begin, It end) noexcept {
222+
const auto currTime = static_cast<Time>(util::getCurrentTimeSec());
223+
return lruMutex_->lock_combine([this, begin, end, currTime]() {
224+
uint32_t i = 0;
225+
for (auto itr = begin; itr != end; ++itr) {
226+
T* node = *itr;
227+
XDCHECK(!node->isInMMContainer());
228+
if (node->isInMMContainer()) {
229+
throw std::runtime_error(
230+
folly::sformat("Was not able to add all new items, failed item {}",
231+
node->toString()));
232+
}
233+
addNodeLocked(*node,currTime);
234+
i++;
235+
}
236+
return i;
237+
});
238+
}
239+
214240
template <typename T, MMLru::Hook<T> T::*HookPtr>
215241
typename MMLru::Container<T, HookPtr>::LockedIterator
216242
MMLru::Container<T, HookPtr>::getEvictionIterator() const noexcept {
@@ -231,8 +257,7 @@ void MMLru::Container<T, HookPtr>::withEvictionIterator(F&& fun) {
231257

232258
template <typename T, MMLru::Hook<T> T::*HookPtr>
233259
template <typename F>
234-
void
235-
MMLru::Container<T, HookPtr>::withPromotionIterator(F&& fun) {
260+
void MMLru::Container<T, HookPtr>::withPromotionIterator(F&& fun) {
236261
if (config_.useCombinedLockForIterators) {
237262
lruMutex_->lock_combine([this, &fun]() { fun(Iterator{lru_.begin()}); });
238263
} else {

‎cachelib/allocator/MMLru.h

+12
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,18 @@ class MMLru {
339339
// is unchanged.
340340
bool add(T& node) noexcept;
341341

342+
// helper function to add the node under the container lock
343+
void addNodeLocked(T& node, const Time& currTime);
344+
345+
// adds the given nodes into the container and marks each as being present in
346+
// the container. The nodes are added to the head of the lru.
347+
//
348+
// @param vector of nodes The nodes to be added to the container.
349+
// @return number of nodes added - it is up to user to verify all
350+
// expected nodes have been added.
351+
template <typename It>
352+
uint32_t addBatch(It begin, It end) noexcept;
353+
342354
// removes the node from the lru and sets it previous and next to nullptr.
343355
//
344356
// @param node The node to be removed from the container.

‎cachelib/allocator/MMTinyLFU-inl.h

+25-1
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,15 @@ bool MMTinyLFU::Container<T, HookPtr>::add(T& node) noexcept {
182182
if (node.isInMMContainer()) {
183183
return false;
184184
}
185+
addNodeLocked(node, currTime);
186+
return true;
187+
}
185188

189+
// adds the node to the list assuming not in
190+
// container and holding container lock
191+
template <typename T, MMTinyLFU::Hook<T> T::*HookPtr>
192+
void MMTinyLFU::Container<T, HookPtr>::addNodeLocked(T& node, const Time &currTime) {
193+
XDCHECK(!node.isInMMContainer());
186194
auto& tinyLru = lru_.getList(LruType::Tiny);
187195
tinyLru.linkAtHead(node);
188196
markTiny(node);
@@ -210,7 +218,23 @@ bool MMTinyLFU::Container<T, HookPtr>::add(T& node) noexcept {
210218
node.markInMMContainer();
211219
setUpdateTime(node, currTime);
212220
unmarkAccessed(node);
213-
return true;
221+
}
222+
223+
template <typename T, MMTinyLFU::Hook<T> T::*HookPtr>
224+
template <typename It>
225+
uint32_t MMTinyLFU::Container<T, HookPtr>::addBatch(It begin, It end) noexcept {
226+
const auto currTime = static_cast<Time>(util::getCurrentTimeSec());
227+
LockHolder l(lruMutex_);
228+
uint32_t i = 0;
229+
for (auto itr = begin; itr != end; itr++) {
230+
T* node = *itr;
231+
if (node->isInMMContainer()) {
232+
return i;
233+
}
234+
addNodeLocked(*node, currTime);
235+
i++;
236+
}
237+
return i;
214238
}
215239

216240
template <typename T, MMTinyLFU::Hook<T> T::*HookPtr>

‎cachelib/allocator/MMTinyLFU.h

+12
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,18 @@ class MMTinyLFU {
327327
// if the node was already in the contianer. On error state of node
328328
// is unchanged.
329329
bool add(T& node) noexcept;
330+
331+
// helper function to add the node under the container lock
332+
void addNodeLocked(T& node, const Time& currTime);
333+
334+
// adds the given nodes into the container and marks each as being present in
335+
// the container. The nodes are added to the head of the lru.
336+
//
337+
// @param vector of nodes The nodes to be added to the container.
338+
// @return number of nodes added - it is up to user to verify all
339+
// expected nodes have been added.
340+
template <typename It>
341+
uint32_t addBatch(It begin, It end) noexcept;
330342

331343
// removes the node from the lru and sets it previous and next to nullptr.
332344
//

‎cachelib/allocator/memory/AllocationClass.cpp

+40
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,26 @@ void* AllocationClass::addSlabAndAllocate(Slab* slab) {
140140
});
141141
}
142142

143+
std::vector<void*> AllocationClass::addSlabAndAllocateBatch(Slab* slab, uint64_t batch) {
144+
XDCHECK_NE(nullptr, slab);
145+
std::vector<void*> allocs;
146+
allocs.reserve(batch);
147+
lock_->lock_combine([this, slab, batch, &allocs]() {
148+
addSlabLocked(slab);
149+
uint64_t total = 0;
150+
while (total < batch) {
151+
void *alloc = allocateLocked();
152+
if (alloc != nullptr) {
153+
allocs.push_back(alloc);
154+
total++;
155+
} else {
156+
break;
157+
}
158+
}
159+
});
160+
return allocs;
161+
}
162+
143163
void* AllocationClass::allocateFromCurrentSlabLocked() noexcept {
144164
XDCHECK(canAllocateFromCurrentSlabLocked());
145165
void* ret = currSlab_->memoryAtOffset(currOffset_);
@@ -159,6 +179,26 @@ void* AllocationClass::allocate() {
159179
return lock_->lock_combine([this]() -> void* { return allocateLocked(); });
160180
}
161181

182+
std::vector<void*> AllocationClass::allocateBatch(uint64_t batch) {
183+
std::vector<void*> allocs;
184+
if (!canAllocate_) {
185+
return allocs;
186+
}
187+
lock_->lock_combine([this, &allocs, batch]() {
188+
uint64_t total = 0;
189+
while (total < batch) {
190+
void *alloc = allocateLocked();
191+
if (alloc != nullptr) {
192+
allocs.push_back(alloc);
193+
total++;
194+
} else {
195+
break;
196+
}
197+
}
198+
});
199+
return allocs;
200+
}
201+
162202
void* AllocationClass::allocateLocked() {
163203
// fast path for case when the cache is mostly full.
164204
if (freedAllocations_.empty() && freeSlabs_.empty() &&

‎cachelib/allocator/memory/AllocationClass.h

+2
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ class AllocationClass {
116116
// don't have any free memory. The caller will have to add a slab
117117
// to this slab class to make further allocations out of it.
118118
void* allocate();
119+
std::vector<void*> allocateBatch(uint64_t batch);
119120

120121
// @param ctx release context for the slab owning this alloc
121122
// @param memory memory to check
@@ -227,6 +228,7 @@ class AllocationClass {
227228
// @param slab a new slab to be added. This can NOT be nullptr.
228229
// @return new allocation. This cannot fail.
229230
void* addSlabAndAllocate(Slab* slab);
231+
std::vector<void*> addSlabAndAllocateBatch(Slab* slab, uint64_t batch);
230232

231233
// Releasing a slab is a two step process.
232234
// 1. Mark a slab for release, by calling `startSlabRelease`.

‎cachelib/allocator/memory/MemoryAllocator.cpp

+10
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,16 @@ void* MemoryAllocator::allocate(PoolId id, uint32_t size) {
7171
return mp.allocate(size);
7272
}
7373

74+
void* MemoryAllocator::allocateByCid(PoolId id, ClassId cid) {
75+
auto& mp = memoryPoolManager_.getPoolById(id);
76+
return mp.allocateByCid(cid);
77+
}
78+
79+
std::vector<void*> MemoryAllocator::allocateByCidBatch(PoolId id, ClassId cid, uint64_t batch) {
80+
auto& mp = memoryPoolManager_.getPoolById(id);
81+
return mp.allocateByCidBatch(cid, batch);
82+
}
83+
7484
void* MemoryAllocator::allocateZeroedSlab(PoolId id) {
7585
if (!config_.enableZeroedSlabAllocs) {
7686
throw std::logic_error("Zeroed Slab allcoation is not enabled");

‎cachelib/allocator/memory/MemoryAllocator.h

+2
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,8 @@ class MemoryAllocator {
167167
// @throw std::invalid_argument if the poolId is invalid or the size is
168168
// invalid.
169169
void* allocate(PoolId id, uint32_t size);
170+
void* allocateByCid(PoolId id, ClassId cid);
171+
std::vector<void*> allocateByCidBatch(PoolId id, ClassId cid, uint64_t batch);
170172

171173
// Allocate a zeroed Slab
172174
//

0 commit comments

Comments
 (0)