Merge remote-tracking branch 'origin/master_modular_genai' into xp/enable_release_weights

Author: xipingyan

src/cpp/src/module_genai/module_base.cpp

@@ -164,6 +164,15 @@ bool IBaseModule::check_bool_param(const std::string& param_name, const bool& de
     return default_value;
 }
 
+// PipelineDesc implementation
+PipelineDesc::PipelineDesc() : m_resource_cache(std::make_unique<PipelineResourceCache>()) {}
+
+PipelineDesc::~PipelineDesc() = default;
+
+PipelineResourceCache& PipelineDesc::get_resource_cache() {
+    return *m_resource_cache;
+}
+
 }  // namespace module
 }  // namespace genai
 }  // namespace ov

src/cpp/src/module_genai/module_desc.hpp

@@ -3,6 +3,11 @@
 
 #pragma once
 
+#include <mutex>
+#include <any>
+#include <functional>
+#include <memory>
+
 #include "module_genai/module_data_type.hpp"
 #include "module_genai/module_print_config.hpp"
 #include "module_genai/module_type.hpp"
@@ -52,14 +57,56 @@ class IBaseModuleDesc {
 // map: module name -> module desc
 using PipelineModulesDesc = std::unordered_map<std::string, IBaseModuleDesc::PTR>;
 
+// Pipeline-scoped resource cache for sharing compiled models across modules
+// Resources are automatically released when PipelineDesc is destroyed
+class PipelineResourceCache {
+public:
+    PipelineResourceCache() = default;
+    ~PipelineResourceCache() = default;
+
+    // Non-copyable
+    PipelineResourceCache(const PipelineResourceCache&) = delete;
+    PipelineResourceCache& operator=(const PipelineResourceCache&) = delete;
+
+    // Get or create a resource with the given key
+    // Returns {resource, was_cached} - the bool indicates whether the resource was already in the cache
+    template<typename T>
+    std::pair<std::shared_ptr<T>, bool> get_or_create(
+        std::size_t cache_key,
+        std::function<std::shared_ptr<T>()> creator) {
+        std::lock_guard<std::mutex> lock(m_mutex);
+        auto it = m_cache.find(cache_key);
+        if (it != m_cache.end()) {
+            auto typed = std::any_cast<std::shared_ptr<T>>(&it->second);
+            if (!typed) {
+                OPENVINO_THROW("PipelineResourceCache: cache key collision - entry exists with a different type for key ",
+                               cache_key);
+            }
+            return {*typed, true};
+        }
+        auto resource = creator();
+        m_cache[cache_key] = resource;
+        return {resource, false};
+    }
+
+    void clear() {
+        std::lock_guard<std::mutex> lock(m_mutex);
+        m_cache.clear();
+    }
+
+private:
+    std::mutex m_mutex;
+    std::unordered_map<std::size_t, std::any> m_cache;
+};
+
 class PipelineDesc {
 protected:
-    PipelineDesc() = default;
+    PipelineDesc();
     PipelineDesc(const PipelineDesc&) = delete;
     PipelineDesc& operator=(const PipelineDesc&) = delete;
 
 public:
-    ~PipelineDesc() = default;
+    ~PipelineDesc();
     // global_context;
     std::string model_type;
 
@@ -75,13 +122,18 @@ class PipelineDesc {
         return m_models_map;
     }
 
+    // Pipeline-scoped resource cache for modules to share resources
+    // Resources are released when PipelineDesc is destroyed
+    PipelineResourceCache& get_resource_cache();
+
     using PTR = std::shared_ptr<PipelineDesc>;
     static PTR create() {
         return std::shared_ptr<PipelineDesc>(new PipelineDesc());
     }
 
 private:
     ConfigModelsMap m_models_map;
+    std::unique_ptr<PipelineResourceCache> m_resource_cache;
 };
 
 }  // namespace module

src/cpp/src/module_genai/modules/md_clip_text_encoder.cpp

@@ -89,14 +89,11 @@ bool ClipTextEncoderModule::initialize() {
     // Create cache key hash from model XML content and device
     std::size_t cache_key = compute_cache_key(text_encoder_path, device);
 
-    // Check if resources already exist in cache
-    bool is_cached = ClipTextEncoderResourceCache::instance().exists(cache_key);
-    if (is_cached) {
-        GENAI_INFO("ClipTextEncoderModule: Reusing cached model for: " + root_dir.string() + " on device: " + device);
-    }
+    // Get pipeline-scoped resource cache (resources released when pipeline is destroyed)
+    auto& resource_cache = pipeline_desc->get_resource_cache();
 
-    // Get or create shared resources
-    m_shared_resources = ClipTextEncoderResourceCache::instance().get_or_create(
+    // Get or create shared resources (cache hit detected atomically under the same lock)
+    auto [shared_resources, was_cached] = resource_cache.get_or_create<ClipTextEncoderSharedResources>(
         cache_key,
         [this, &root_dir, &device, &text_encoder_path, model_type]() -> std::shared_ptr<ClipTextEncoderSharedResources> {
             GENAI_INFO("ClipTextEncoderModule: Loading model from: " + root_dir.string() + " to device: " + device);
@@ -148,6 +145,11 @@ bool ClipTextEncoderModule::initialize() {
             return resources;
         });
 
+    m_shared_resources = shared_resources;
+    if (was_cached) {
+        GENAI_INFO("ClipTextEncoderModule: Reusing cached model for: " + root_dir.string() + " on device: " + device);
+    }
+
     if (!m_shared_resources) {
         GENAI_ERR("ClipTextEncoderModule[" + module_desc->name + "]: Failed to initialize shared resources");
         return false;

src/cpp/src/module_genai/modules/md_clip_text_encoder.hpp

@@ -4,8 +4,6 @@
 #pragma once
 
 #include <yaml-cpp/yaml.h>
-#include <mutex>
-#include <unordered_map>
 #include <fstream>
 #include <filesystem>
 
@@ -50,51 +48,14 @@ inline std::size_t compute_cache_key(const std::filesystem::path& model_xml_path
     return std::hash<std::string_view>{}(std::string_view(content.data(), content.size()));
 }
 
-// Global cache for shared resources
-class ClipTextEncoderResourceCache {
-public:
-    static ClipTextEncoderResourceCache& instance() {
-        static ClipTextEncoderResourceCache cache;
-        return cache;
-    }
-
-    std::shared_ptr<ClipTextEncoderSharedResources> get_or_create(
-        std::size_t cache_key,
-        std::function<std::shared_ptr<ClipTextEncoderSharedResources>()> creator) {
-        std::lock_guard<std::mutex> lock(m_mutex);
-        auto it = m_cache.find(cache_key);
-        if (it != m_cache.end()) {
-            return it->second;
-        }
-        auto resources = creator();
-        m_cache[cache_key] = resources;
-        return resources;
-    }
-
-    bool exists(std::size_t cache_key) {
-        std::lock_guard<std::mutex> lock(m_mutex);
-        return m_cache.find(cache_key) != m_cache.end();
-    }
-
-    void clear() {
-        std::lock_guard<std::mutex> lock(m_mutex);
-        m_cache.clear();
-    }
-
-private:
-    ClipTextEncoderResourceCache() = default;
-    std::mutex m_mutex;
-    std::unordered_map<std::size_t, std::shared_ptr<ClipTextEncoderSharedResources>> m_cache;
-};
-
 class ClipTextEncoderModule : public IBaseModule {
     DeclareModuleConstructor(ClipTextEncoderModule);
 
 private:
     bool initialize();
     bool do_classifier_free_guidance(float guidance_scale);
 
-    // Shared resources (shared across modules with same model_path)
+    // Shared resources (shared across modules with same model_path within the same pipeline)
     std::shared_ptr<ClipTextEncoderSharedResources> m_shared_resources;
 
     // Per-instance infer request (each module has its own)

src/cpp/src/module_genai/modules/md_denoiser_loop/class.cpp

@@ -113,7 +113,7 @@ bool DenoiserLoopModule::initialize() {
     }
 
     if (m_splitted_model) {
-        m_splitted_model_infer = CSplittedModelInfer::create(transformer_model_path,
+        m_splitted_model_infer = CSplittedModelInfer::create(transformer_model_path.string(),
                                                              module_desc->device,
                                                              m_dynamic_load_weights,
                                                              properties);