[GPU] Optimize node's memdeps to reduce memory footprints. (#29237)

[GPU] Reduce memory footprint by optimizing node's memdeps. 
std::unordered_set<size_t> was originally used as program_node's
memory_dependency and primitive_inst's runtime_memory_dependency for a
better memory pool performance efficiency, however they are less memory
efficient. This optimization takes advantage that runtime memdeps (of
pritimive_inst instances) is initialized from compile stage memdeps (of
program_node instances) and only appends to it during some runtime skip
passes.

### Details:
 - [x] Change memdeps set from size_t to uint32_t 
- [x] Reserve unordered_set memory in Serializer to reduce memory
overhead of unordered_set when importing from cache_dir
- [x] Reduce unnecessary memory dependencies, such as for constant nodes
- [x] Reduce memory dependencies of ReadValue nodes when they are
optimized out by reusing Variable's memory.
- [x] Split "initial memory dependencies" and "runtime" ones to reduce
overheads in program_node and primitive_instance

### Tickets:
 - *CVS-160820, CVS-163673*

Author: ceciliapeng2011

src/plugins/intel_gpu/include/intel_gpu/graph/serialization/set_serializer.hpp

@@ -55,7 +55,7 @@ class Serializer<BufferType, std::unordered_set<T>, typename std::enable_if<std:
     static void load(BufferType& buffer, std::unordered_set<T>& set) {
         typename std::unordered_set<T>::size_type set_size = 0UL;
         buffer >> set_size;
-
+        if (set.empty()) set.reserve(set_size);
         for (long unsigned int i = 0; i < set_size; i++) {
             T el;
             buffer >> el;

src/plugins/intel_gpu/include/intel_gpu/runtime/memory_pool.hpp

@@ -26,6 +26,47 @@ class engine;
 using primitive_id = std::string;
 using memory_ptr = std::shared_ptr<memory>;
 
+template<typename Key, typename Hash = std::hash<Key>, typename KeyEqual = std::equal_to<Key>>
+class memory_restricter {
+    private:
+        const std::unordered_set<Key, Hash, KeyEqual>* set1;  // Const reference to immutable set
+        std::unordered_set<Key, Hash, KeyEqual> set2;         // Internal mutable set
+
+    public:
+        memory_restricter() : set1(nullptr) {};
+
+        // Constructor to initialize with a const reference for set1
+        explicit memory_restricter(const std::unordered_set<Key, Hash, KeyEqual>* externalSet)
+            : set1(externalSet) {}
+
+        // Insert into set2 (set1 is read-only)
+        void insert(const Key& key) {
+            if (set1->find(key) == set1->end())
+                set2.insert(key);
+        }
+
+        // Check existence in either set
+        bool contains(const Key& key) const {
+            return set1->find(key) != set1->end() || set2.find(key) != set2.end();
+        }
+
+        // Total size of both sets
+        size_t size() const {
+            return set1->size() + set2.size();
+        }
+
+        // Check if both sets are empty
+        bool empty() const {
+            return set1->empty() && set2.empty();
+        }
+
+        // Iterate over both sets
+        void for_each(void(*func)(const Key&)) const {
+            for (const auto& key : set1) func(key);
+            for (const auto& key : set2) func(key);
+        }
+}; // end of memory_restricter
+
 struct memory_user {
     size_t _unique_id;
     uint32_t _network_id;
@@ -112,7 +153,7 @@ struct padded_pool_comparer {
 
 class memory_pool {
     memory_ptr alloc_memory(const layout& layout, allocation_type type, bool reset = true);
-    static bool has_conflict(const memory_set&, const std::unordered_set<size_t>&, uint32_t network_id);
+    static bool has_conflict(const memory_set&, const memory_restricter<uint32_t>&);
 
     std::multimap<uint64_t, memory_record> _non_padded_pool;
     std::map<layout, std::list<memory_record>, padded_pool_comparer> _padded_pool;
@@ -127,7 +168,7 @@ class memory_pool {
                           const primitive_id& id,
                           size_t unique_id,
                           uint32_t network_id,
-                          const std::unordered_set<size_t>& restrictions,
+                          const memory_restricter<uint32_t>& restrictions,
                           allocation_type type,
                           bool reusable = true,
                           bool reset = true,
@@ -137,21 +178,16 @@ class memory_pool {
                                         const primitive_id& prim_id,
                                         size_t unique_id,
                                         uint32_t network_id,
-                                        const std::unordered_set<size_t>&,
+                                        const memory_restricter<uint32_t>&,
                                         allocation_type type,
                                         bool reset = true,
                                         bool is_dynamic = false);
     memory_ptr get_from_padded_pool(const layout& layout,
                                     const primitive_id& prim_id,
                                     size_t unique_id,
                                     uint32_t network_id,
-                                    const std::unordered_set<size_t>& restrictions,
+                                    const memory_restricter<uint32_t>& restrictions,
                                     allocation_type type);
-    memory_ptr get_from_across_networks_pool(const layout& layout,
-                                             const primitive_id& id,
-                                             size_t unique_id,
-                                             uint32_t network_id,
-                                             allocation_type type);
     void clear_pool_for_network(uint32_t network_id);
     void release_memory(memory* memory, const size_t& unique_id, primitive_id prim_id, uint32_t network_id);
 

src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_quantization.cpp

@@ -288,10 +288,10 @@ void prepare_quantization::prepare_scale_shift_opt(program &p, quantize_node& qu
     p.add_connection(in_shift_node, new_quantize_node);
     p.add_connection(out_scale_node, new_quantize_node);
     p.add_connection(out_shift_node, new_quantize_node);
-    new_quantize_node.add_memory_dependency(in_scale_node.get_unique_id());
-    new_quantize_node.add_memory_dependency(in_shift_node.get_unique_id());
-    new_quantize_node.add_memory_dependency(out_scale_node.get_unique_id());
-    new_quantize_node.add_memory_dependency(out_shift_node.get_unique_id());
+    new_quantize_node.add_memory_dependency(in_scale_node);
+    new_quantize_node.add_memory_dependency(in_shift_node);
+    new_quantize_node.add_memory_dependency(out_scale_node);
+    new_quantize_node.add_memory_dependency(out_shift_node);
     p.get_processing_order().insert(&new_quantize_node, &in_shift_node);
     p.get_processing_order().insert(&new_quantize_node, &in_scale_node);
     p.get_processing_order().insert(&new_quantize_node, &out_shift_node);

src/plugins/intel_gpu/src/graph/graph_optimizer/skipped_branch_memory_dependencies.cpp

@@ -19,7 +19,7 @@ void skipped_branch_memory_dependencies::run(program& p) {
     while (itrB != processing_order.end()) {
         auto& nodeB = *itrB;
         auto itrA = ++itrB;
-        if (nodeB->is_constant())
+        if (!nodeB->may_use_mempool())
             continue;
         if (nodeB->get_users().size() == 0)
             continue;

src/plugins/intel_gpu/src/graph/include/pass_manager.h

@@ -322,10 +322,10 @@ class memory_dependency_pass : public base_pass {
 
         if ((!dep->can_be_optimized() || !dep->is_runtime_skippable()) && ((node->can_be_optimized() && !node->is_runtime_skippable())
             || !dep->can_be_optimized())) {
-            node->add_memory_dependency(static_cast<int32_t>(dep->get_unique_id()));
+            node->add_memory_dependency(*dep);
         } else {
             if (node->is_runtime_skippable() || dep->is_runtime_skippable() || dep->can_be_optimized()) {
-                node->add_memory_dependency(static_cast<int32_t>(dep->get_unique_id()));
+                node->add_memory_dependency(*dep);
             }
 
             for (const auto& subdep : dep->get_dependencies()) {

src/plugins/intel_gpu/src/graph/include/primitive_inst.h

@@ -222,7 +222,7 @@ class primitive_inst {
         _users = _network.get_primitives(users);
     }
 
-    const std::unordered_set<size_t>& get_runtime_memory_dependencies() const { return _runtime_memory_dependencies; }
+    const memory_restricter<uint32_t>& get_runtime_memory_dependencies() const { return _runtime_memory_dependencies; }
 
     const kernel_impl_params* get_impl_params() const { return _impl_params.get(); }
     // return pointer to const to prevent arbitrary 'execute' call -> use primitive_inst.execute() instead
@@ -307,7 +307,7 @@ class primitive_inst {
                                        memory_pool& pool,
                                        const program_node& _node,
                                        const kernel_impl_params& impl_params,
-                                       const std::unordered_set<size_t>& memory_dependencies,
+                                       const memory_restricter<uint32_t>& memory_dependencies,
                                        uint32_t net_id,
                                        bool is_internal,
                                        size_t idx = 0,
@@ -379,7 +379,7 @@ class primitive_inst {
     std::vector<primitive_inst*> _exec_deps;
 
     // List of primitive ids that this primitive can't share memory buffers with
-    std::unordered_set<size_t> _runtime_memory_dependencies;
+    memory_restricter<uint32_t> _runtime_memory_dependencies;
 
     // This is sub-network generated on demand to execute unfused primitives sequence instead of single fused primitive
     // Needed for dynamic path only, as fusion in some cases may be illegal, but it can't be checked on program build phase,

src/plugins/intel_gpu/src/graph/include/program_node.h

@@ -12,6 +12,7 @@
 #include "intel_gpu/graph/fused_primitive_desc.hpp"
 #include "intel_gpu/graph/kernel_impl_params.hpp"
 #include "intel_gpu/primitives/reorder.hpp"
+#include "intel_gpu/primitives/read_value.hpp"
 #include "intel_gpu/runtime/utils.hpp"
 
 #include <set>
@@ -207,9 +208,15 @@ struct program_node {
     size_t get_dependency_index(const program_node& node) const;
     size_t get_user_index(const program_node& node) const;
 
-    std::unordered_set<size_t> get_memory_dependencies() const;
-    void add_memory_dependency(size_t);
+    const std::unordered_set<uint32_t>& get_memory_dependencies() const;
+
     void add_memory_dependency(std::vector<size_t>);
+    void add_memory_dependency(const program_node& node);
+
+    // At least the following scenarios are not allocating from memory pool:
+    // 1. constant nodes
+    // 2. read_value nodes that are optimized out to reuse from Variables.
+    bool may_use_mempool() const { return !(is_constant() || (is_type<read_value>() && optimized)); }
 
     template <class PType>
     bool have_user_with_type() const {
@@ -497,7 +504,7 @@ struct program_node {
     std::list<program_node*> users;
 
     // list of primitives that can reuse same memory buffers due to execution order conflicts
-    std::unordered_set<size_t> memory_dependencies;
+    std::unordered_set<uint32_t> memory_dependencies;
 
     impl_types impl_type = impl_types::any;
     impl_types forced_impl_type = impl_types::any;

src/plugins/intel_gpu/src/graph/primitive_inst.cpp

@@ -157,7 +157,7 @@ static memory::ptr get_memory_from_pool(engine& _engine,
                                 const layout& layout,
                                 allocation_type type,
                                 bool reusable_across_network,
-                                const std::unordered_set<size_t>& memory_dependencies,
+                                const memory_restricter<uint32_t>& memory_dependencies,
                                 bool reset = true,
                                 memory* curr_memory = nullptr) {
     OPENVINO_ASSERT(!layout.is_dynamic() || layout.has_upper_bound(),
@@ -1340,7 +1340,7 @@ void primitive_inst::do_runtime_skip_reorder() {
                     update_memory_dependencies = [&](std::vector<primitive_inst*> users) {
                         for (auto& user : users) {
                             GPU_DEBUG_TRACE_DETAIL << "[do runtime skip reorder] add " << id() << " to restriction list of " << user->id() << std::endl;
-                            user->_runtime_memory_dependencies.insert(get_node().get_unique_id());
+                            user->_runtime_memory_dependencies.insert(static_cast<uint32_t>(get_node().get_unique_id()));
                             if (user->can_be_optimized())
                                 update_memory_dependencies(user->get_user_insts());
                         }
@@ -2085,7 +2085,7 @@ primitive_inst::primitive_inst(network & network, program_node const& node, bool
     , _use_shared_kernels(node.get_program().get_config().get_enable_kernels_reuse())
     , _impl_params(node.get_kernel_impl_params())
     , _impl(node.get_selected_impl() ? node.get_selected_impl()->clone() : nullptr)
-    , _runtime_memory_dependencies(node.get_memory_dependencies())
+    , _runtime_memory_dependencies(&node.get_memory_dependencies())
     , _outputs({})
     , _reordered_weights_cache(network.get_weights_cache_capacity())
     , _is_dynamic(node.is_dynamic())
@@ -2390,7 +2390,7 @@ memory::ptr primitive_inst::allocate_output(engine& _engine,
                                             memory_pool& pool,
                                             const program_node& _node,
                                             const kernel_impl_params& impl_params,
-                                            const std::unordered_set<size_t>& memory_dependencies,
+                                            const memory_restricter<uint32_t>& memory_dependencies,
                                             uint32_t net_id,
                                             bool is_internal,
                                             size_t idx,

src/plugins/intel_gpu/src/graph/program.cpp

@@ -748,7 +748,7 @@ void program::prepare_memory_dependencies() {
     if (!_config.get_enable_memory_pool())
         return;
     for (auto& node : get_processing_order()) {
-        node->add_memory_dependency(node->get_unique_id());
+        node->add_memory_dependency(*node);
     }
     apply_opt_pass<basic_memory_dependencies>();
     apply_opt_pass<skipped_branch_memory_dependencies>();
@@ -766,7 +766,7 @@ std::string program::get_memory_dependencies_string() const {
                          .append("(unique_id:")
                          .append(std::to_string(node->get_unique_id()))
                          .append(") restricted list: ");
-        for (auto it : node->get_memory_dependencies())
+        for (const auto& it : node->get_memory_dependencies())
             mem_dep = mem_dep.append(std::to_string(it)).append(",");
         mem_dep = mem_dep.append("\n");
     }
@@ -1715,7 +1715,7 @@ std::pair<int64_t, int64_t> program::get_estimated_device_mem_usage() {
                                                                       pool,
                                                                       *node,
                                                                       *node->get_kernel_impl_params(),
-                                                                      node->get_memory_dependencies(),
+                                                                      memory_restricter<uint32_t>(&node->get_memory_dependencies()),
                                                                       0,
                                                                       false,
                                                                       0,

src/plugins/intel_gpu/src/graph/program_node.cpp

@@ -195,12 +195,17 @@ void program_node::remove_dependency(size_t idx) {
     dependencies.erase(dependencies.begin() + idx);
 }
 
-std::unordered_set<size_t> program_node::get_memory_dependencies() const { return memory_dependencies; }
-
-void program_node::add_memory_dependency(size_t prim) { memory_dependencies.insert(prim); }
+const std::unordered_set<uint32_t>& program_node::get_memory_dependencies() const { return memory_dependencies; }
 
 void program_node::add_memory_dependency(std::vector<size_t> prim_list) {
-    memory_dependencies.insert(prim_list.begin(), prim_list.end());
+    for (size_t val : prim_list) {
+        memory_dependencies.insert(static_cast<uint32_t>(val));
+    }
+}
+
+void program_node::add_memory_dependency(const program_node& dep) {
+    if (dep.may_use_mempool() && may_use_mempool())
+        memory_dependencies.insert(static_cast<uint32_t>(dep.get_unique_id()));
 }
 
 std::unique_ptr<json_composite> program_node::desc_to_json() const {