fix: fixes async warmup deadlock (#41995)

issue: #41993

also updated cachinglayer metrics

Signed-off-by: Buqian Zheng <[email protected]>

Author: zhengbuqian

internal/core/src/cachinglayer/CacheSlot.h

@@ -90,8 +90,10 @@ class CacheSlot final : public std::enable_shared_from_this<CacheSlot<CellT>> {
                 SemiInlineGet(PinCells(std::move(cids)));
                 break;
             case CacheWarmupPolicy::CacheWarmupPolicy_Async:
+                // PinCells submits tasks to middle priority thread pool, thus here we submit to
+                // low priority thread pool to avoid dead lock.
                 auto& pool = milvus::ThreadPools::GetThreadPool(
-                    milvus::ThreadPoolPriority::MIDDLE);
+                    milvus::ThreadPoolPriority::LOW);
                 pool.Submit([this, cids = std::move(cids)]() mutable {
                     SemiInlineGet(PinCells(std::move(cids)));
                 });

internal/core/src/cachinglayer/Utils.h

@@ -304,19 +304,24 @@ cache_op_result_count_miss(StorageType storage_type) {
 }
 
 inline prometheus::Counter&
-cache_eviction_count(StorageType storage_type) {
+cache_cell_eviction_count(StorageType storage_type) {
     switch (storage_type) {
         case StorageType::MEMORY:
-            return monitor::internal_cache_eviction_count_memory;
+            return monitor::internal_cache_cell_eviction_count_memory;
         case StorageType::DISK:
-            return monitor::internal_cache_eviction_count_disk;
+            return monitor::internal_cache_cell_eviction_count_disk;
         case StorageType::MIXED:
-            return monitor::internal_cache_eviction_count_mixed;
+            return monitor::internal_cache_cell_eviction_count_mixed;
         default:
             PanicInfo(ErrorCode::UnexpectedError, "Unknown StorageType");
     }
 }
 
+inline prometheus::Counter&
+cache_eviction_event_count() {
+    return monitor::internal_cache_eviction_event_count_all;
+}
+
 inline prometheus::Histogram&
 cache_item_lifetime_seconds(StorageType storage_type) {
     switch (storage_type) {

internal/core/src/cachinglayer/lrucache/DList.cpp

@@ -160,9 +160,10 @@ DList::tryEvict(const ResourceUsage& expected_eviction,
             usageInfo(actively_pinned));
     }
 
+    internal::cache_eviction_event_count().Increment();
     for (auto* list_node : to_evict) {
         auto size = list_node->size();
-        internal::cache_eviction_count(size.storage_type()).Increment();
+        internal::cache_cell_eviction_count(size.storage_type()).Increment();
         popItem(list_node);
         list_node->clear_data();
         used_memory_ -= size;

internal/core/src/monitor/prometheus_client.cpp

@@ -454,18 +454,25 @@ DEFINE_PROMETHEUS_GAUGE(internal_cache_capacity_bytes_mixed,
                         cacheMixedLabel);
 
 // Eviction count and resource size
-DEFINE_PROMETHEUS_COUNTER_FAMILY(internal_cache_eviction_count,
-                                 "[cpp]cache eviction count");
-DEFINE_PROMETHEUS_COUNTER(internal_cache_eviction_count_memory,
-                          internal_cache_eviction_count,
+DEFINE_PROMETHEUS_COUNTER_FAMILY(internal_cache_cell_eviction_count,
+                                 "[cpp]cache cell eviction count");
+DEFINE_PROMETHEUS_COUNTER(internal_cache_cell_eviction_count_memory,
+                          internal_cache_cell_eviction_count,
                           cacheMemoryLabel);
-DEFINE_PROMETHEUS_COUNTER(internal_cache_eviction_count_disk,
-                          internal_cache_eviction_count,
+DEFINE_PROMETHEUS_COUNTER(internal_cache_cell_eviction_count_disk,
+                          internal_cache_cell_eviction_count,
                           cacheDiskLabel);
-DEFINE_PROMETHEUS_COUNTER(internal_cache_eviction_count_mixed,
-                          internal_cache_eviction_count,
+DEFINE_PROMETHEUS_COUNTER(internal_cache_cell_eviction_count_mixed,
+                          internal_cache_cell_eviction_count,
                           cacheMixedLabel);
 
+// Eviction event count
+DEFINE_PROMETHEUS_COUNTER_FAMILY(internal_cache_eviction_event_count,
+                                 "[cpp]cache eviction event count");
+DEFINE_PROMETHEUS_COUNTER(internal_cache_eviction_event_count_all,
+                          internal_cache_eviction_event_count,
+                          {});
+
 DEFINE_PROMETHEUS_COUNTER_FAMILY(internal_cache_evicted_bytes,
                                  "[cpp]total bytes evicted from cache");
 DEFINE_PROMETHEUS_COUNTER(internal_cache_evicted_bytes_memory,

internal/core/src/monitor/prometheus_client.h

@@ -197,15 +197,18 @@ DECLARE_PROMETHEUS_GAUGE_FAMILY(internal_cache_capacity_bytes);
 DECLARE_PROMETHEUS_GAUGE(internal_cache_capacity_bytes_memory);
 DECLARE_PROMETHEUS_GAUGE(internal_cache_capacity_bytes_disk);
 
-DECLARE_PROMETHEUS_COUNTER_FAMILY(internal_cache_eviction_count);
-DECLARE_PROMETHEUS_COUNTER(internal_cache_eviction_count_memory);
-DECLARE_PROMETHEUS_COUNTER(internal_cache_eviction_count_disk);
-DECLARE_PROMETHEUS_COUNTER(internal_cache_eviction_count_mixed);
+DECLARE_PROMETHEUS_COUNTER_FAMILY(internal_cache_cell_eviction_count);
+DECLARE_PROMETHEUS_COUNTER(internal_cache_cell_eviction_count_memory);
+DECLARE_PROMETHEUS_COUNTER(internal_cache_cell_eviction_count_disk);
+DECLARE_PROMETHEUS_COUNTER(internal_cache_cell_eviction_count_mixed);
 
 DECLARE_PROMETHEUS_COUNTER_FAMILY(internal_cache_evicted_bytes);
 DECLARE_PROMETHEUS_COUNTER(internal_cache_evicted_bytes_memory);
 DECLARE_PROMETHEUS_COUNTER(internal_cache_evicted_bytes_disk);
 
+DECLARE_PROMETHEUS_COUNTER_FAMILY(internal_cache_eviction_event_count);
+DECLARE_PROMETHEUS_COUNTER(internal_cache_eviction_event_count_all);
+
 DECLARE_PROMETHEUS_HISTOGRAM_FAMILY(internal_cache_item_lifetime_seconds);
 DECLARE_PROMETHEUS_HISTOGRAM(internal_cache_item_lifetime_seconds_memory);
 DECLARE_PROMETHEUS_HISTOGRAM(internal_cache_item_lifetime_seconds_disk);

internal/core/src/storage/ThreadPool.h

@@ -131,31 +131,4 @@ class ThreadPool {
     std::string name_;
 };
 
-class Worker {
- private:
-    int id_;
-    ThreadPool* pool_;
-
- public:
-    Worker(ThreadPool* pool, const int id) : pool_(pool), id_(id) {
-    }
-
-    void
-    operator()() {
-        std::function<void()> func;
-        bool dequeue;
-        while (!pool_->shutdown_) {
-            std::unique_lock<std::mutex> lock(pool_->mutex_);
-            if (pool_->work_queue_.empty()) {
-                pool_->condition_lock_.wait(lock);
-            }
-            dequeue = pool_->work_queue_.dequeue(func);
-            lock.unlock();
-            if (dequeue) {
-                func();
-            }
-        }
-    }
-};
-
 }  // namespace milvus