cachebench crashes when run kv test with FDP enabled #412
Description
Describe the bug
cachebench crashes when it runs kv test with FDP enabled
To Reproduce
Steps to reproduce the behavior:
- in cachelib git repo, checkout to almost latest 07c7275, and build
- enable FDP in one disk with 8 RU in nvme0n1
- add "navyQDepth": 1,
"navyEnableIoUring": true,
"deviceEnableFDP": true
in default config_kvcache.json - run kv test with the modified config_kvcache.json
Expected behavior
cachebench runs or ends normally .
Screenshots
when it runs, it always prints like
10:46:05 768.93M ops completed. 22536690 items in cache. 41133208 items in nvm cache. 657 items evicted from nvm cache. Hit Ratio 46.63% (RAM 44.98%, NVM 2.99%).
NVMe Status:Successful Completion: The command completed without error(0)
10:46:08 770.56M ops completed. 22538305 items in cache. 41291482 items in nvm cache. 680 items evicted from nvm cache. Hit Ratio 46.77% (RAM 45.20%, NVM 2.87%).
NVMe Status:Successful Completion: The command completed without error(0)
10:46:12 772.12M ops completed. 22534337 items in cache. 41358786 items in nvm cache. 691 items evicted from nvm cache. Hit Ratio 79.29% (RAM 78.63%, NVM 3.09%).
NVMe Status:Successful Completion: The command completed without error(0)
10:46:15 773.63M ops completed. 22534622 items in cache. 41366585 items in nvm cache. 692 items evicted from nvm cache. Hit Ratio 98.06% (RAM 97.97%, NVM 4.00%).
E0818 10:46:17.991082 133810 RegionManager.cpp:182] Failed to assign buffers. All buffers(2) are being used
NVMe Status:Successful Completion: The command completed without error(0)
10:46:18 775.24M ops completed. 22539823 items in cache. 41512748 items in nvm cache. 706 items evicted from nvm cache. Hit Ratio 47.27% (RAM 45.51%, NVM 3.22%).
NVMe Status:Successful Completion: The command completed without error(0)
10:46:21 776.89M ops completed. 22562769 items in cache. 41632004 items in nvm cache. 715 items evicted from nvm cache. Hit Ratio 53.14% (RAM 51.56%, NVM 3.27%).
NVMe Status:Successful Completion: The command completed without error(0)
and when it crashes:
F0814 18:22:16.500877 32199 NvmCache.h:702] Check failed: res.second 18:22:16 [10/1819]
*** Aborted at 1755166936 (Unix time, try 'date -d @1755166936') ***
*** Signal 6 (SIGABRT) (0x7d50) received by PID 32080 (pthread TID 0x74f5edffb6c0) (linux TID 32199) (maybe from PID 32080, UID 0) (code: -6), stack trace: ***
@ 0000000000c81a17 folly::symbolizer::(anonymous namespace)::innerSignalHandler(int, siginfo_t*, void*)
/home/dahui/Documents/CacheLib/cachelib/external/folly/folly/debugging/symbolizer/SignalHandler.cpp:453
@ 0000000000c81afe folly::symbolizer::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
/home/dahui/Documents/CacheLib/cachelib/external/folly/folly/debugging/symbolizer/SignalHandler.cpp:474
@ 000000000004532f (unknown)
@ 000000000009eb2c pthread_kill
@ 000000000004527d gsignal
@ 00000000000288fe abort
@ 000000000109a277 folly::LogCategory::admitMessage(folly::LogMessage const&) const
/home/dahui/Documents/CacheLib/cachelib/external/folly/folly/logging/LogCategory.cpp:71
@ 00000000010bdd00 folly::LogStreamProcessor::logNow()
/home/dahui/Documents/CacheLib/cachelib/external/folly/folly/logging/LogStreamProcessor.cpp:190
@ 00000000010bde71 folly::LogStreamVoidify<true>::operator&(std::ostream&)
/home/dahui/Documents/CacheLib/cachelib/external/folly/folly/logging/LogStreamProcessor.cpp:222
@ 0000000000714b79 facebook::cachelib::NvmCache<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::find(facebook::cachelib::HashedKey)
/home/dahui/Documents/CacheLib/cachelib/../cachelib/allocator/nvmcache/NvmCache.h:702
-> /home/dahui/Documents/CacheLib/cachelib/allocator/CacheAllocatorLruCache.cpp
@ 00000000006de036 facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait>::find(facebook::cachelib::KAllocation::Key)
/home/dahui/Documents/CacheLib/cachelib/../cachelib/allocator/CacheAllocator.h:4278
-> /home/dahui/Documents/CacheLib/cachelib/allocator/CacheAllocatorLruCache.cpp
@ 00000000003deefd facebook::cachelib::cachebench::Cache<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::find(facebook::cachelib::KAllocation::Key)::{lambda()#1}::operator()() const
/home/dahui/Documents/CacheLib/cachelib/../cachelib/cachebench/cache/Cache.h:970
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000003df026 facebook::cachelib::cachebench::Cache<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::find(facebook::cachelib::KAllocation::Key)
/home/dahui/Documents/CacheLib/cachelib/../cachelib/cachebench/cache/Cache.h:981
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000003c5348 facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::stressByDiscreteDistribution(facebook::cachelib::cachebench::ThroughputStats&)
/home/dahui/Documents/CacheLib/cachelib/../cachelib/cachebench/runner/CacheStressor.h:361
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000003ade18 facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator()() const::{lambda()#1}::operator()() const
/home/dahui/Documents/CacheLib/cachelib/../cachelib/cachebench/runner/CacheStressor.h:155
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000004c82cd void std::__invoke_impl<void, facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator()() const::{lambda()#1}>(std::__invoke_other, facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator()() const::{lambda()#1}&&)
/usr/include/c++/13/bits/invoke.h:61
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000004c7de1 std::__invoke_result<facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator()() const::{lambda()#1}>::type std::__invoke<facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator()() const::{lambda()#1}>(facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator()() const::{lambda()#1}&&)
/usr/include/c++/13/bits/invoke.h:96
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000004c790b void std::thread::_Invoker<std::tuple<facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator()() const::{lambda()
#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>)
/usr/include/c++/13/bits/std_thread.h:292
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000004c766b std::thread::_Invoker<std::tuple<facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator()() const::{lambda()#1}>
>::operator()()
/usr/include/c++/13/bits/std_thread.h:299
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000004c7453 std::thread::_State_impl<std::thread::_Invoker<std::tuple<facebook::cachelib::cachebench::CacheStressor<facebook::cachelib::CacheAllocator<facebook::cachelib::LruCacheTrait> >::start()::{lambda()#1}::operator(
)() const::{lambda()#1}> > >::_M_run()
/usr/include/c++/13/bits/std_thread.h:244
-> /home/dahui/Documents/CacheLib/cachelib/cachebench/runner/Stressor.cpp
@ 00000000000ecdb3 (unknown)
@ 000000000009caa3 (unknown)
Desktop (please complete the following information):
- OS: ubuntu 24.04