Use common api for all internal bo creations (#9373)

Signed-off-by: Rahul Bramandlapalli <rbramand@amd.com>

Author: rbramand-xilinx

src/runtime_src/core/common/api/bo_int.h

@@ -24,18 +24,19 @@ get_offset(const xrt::bo& bo);
 // enum for different buffer use flags
 // This is for internal use only
 enum class use_type {
-  debug       = XRT_BO_USE_DEBUG,       // debug data
-  kmd         = XRT_BO_USE_KMD,         // shared with kernel mode driver
-  dtrace      = XRT_BO_USE_DTRACE,      // dynamic trace data
-  log         = XRT_BO_USE_LOG,         // logging info
-  debug_queue = XRT_BO_USE_DEBUG_QUEUE, // debug queue data
-  uc_debug    = XRT_BO_USE_UC_DEBUG,    // microblaze debug data
-  host_only   = XRT_BO_USE_HOST_ONLY,   // system memory buffer
-  instruction = XRT_BO_USE_INSTRUCTION, // instruction (instructon buffer)
-  preemption  = XRT_BO_USE_PREEMPTION,  // preemption data
-  scratch_pad = XRT_BO_USE_SCRATCH_PAD, // scratch pad data
-  pdi         = XRT_BO_USE_PDI,         // PDI data
-  ctrlpkt     = XRT_BO_USE_CTRLPKT      // control packet
+  debug            = XRT_BO_USE_DEBUG,            // debug data
+  kmd              = XRT_BO_USE_KMD,              // shared with kernel mode driver
+  dtrace           = XRT_BO_USE_DTRACE,           // dynamic trace data
+  log              = XRT_BO_USE_LOG,              // logging info
+  debug_queue      = XRT_BO_USE_DEBUG_QUEUE,      // debug queue data
+  uc_debug         = XRT_BO_USE_UC_DEBUG,         // microblaze debug data
+  host_only        = XRT_BO_USE_HOST_ONLY,        // system memory buffer
+  instruction      = XRT_BO_USE_INSTRUCTION,      // instruction (instructon buffer)
+  preemption       = XRT_BO_USE_PREEMPTION,       // preemption data
+  scratch_pad      = XRT_BO_USE_SCRATCH_PAD,      // scratch pad data
+  ctrl_scratch_pad = XRT_BO_USE_CTRL_SCRATCH_PAD, // ctrl scratchpad data
+  pdi              = XRT_BO_USE_PDI,              // PDI data
+  ctrlpkt          = XRT_BO_USE_CTRLPKT           // control packet
 };
 
 // create_bo() - Create a buffer object within a device for specific use case

src/runtime_src/core/common/api/xrt_bo.cpp

@@ -1711,7 +1711,7 @@ compose_internal_bo_flags(use_type type)
   case use_type::instruction:
   case use_type::preemption:
   case use_type::pdi:
-  case use_type::scratch_pad:
+  case use_type::ctrl_scratch_pad:
     // client use case, create buffer in sram
     flags.flags = XRT_BO_FLAGS_CACHEABLE;
     break;
@@ -1720,6 +1720,7 @@ compose_internal_bo_flags(use_type type)
   case use_type::host_only:
   case use_type::uc_debug:
   case use_type::log:
+  case use_type::scratch_pad:
     flags.flags = XRT_BO_FLAGS_HOST_ONLY;
     break;
   default:

src/runtime_src/core/common/api/xrt_hw_context.cpp

@@ -77,8 +77,8 @@ class hw_context_impl : public std::enable_shared_from_this<hw_context_impl>
                            size_t size_per_uc,
                            size_t num_uc)
     {
-      auto bo = xrt_core::bo_int::
-        create_bo(device, (size_per_uc * num_uc), xrt_core::bo_int::use_type::log);
+      auto bo = xrt_core::bo_int::create_bo(
+          device, (size_per_uc * num_uc), xrt_core::bo_int::use_type::log);
 
       // Log buffers first 8 bytes are used for metadata
       // So make sure for each uC metadata bytes are initialized with
@@ -465,8 +465,9 @@ class hw_context_impl : public std::enable_shared_from_this<hw_context_impl>
     std::call_once(m_scratchpad_init_flag, [this, size_per_col] () {
       try {
         // create scratchpad memory buffer using this context
-        m_scratchpad_buf = xrt::ext::bo{xrt::hw_context(get_shared_ptr()),
-                                        size_per_col * m_partition_size};
+        auto buf_size = size_per_col * m_partition_size;
+        m_scratchpad_buf = xrt_core::bo_int::create_bo(
+            m_core_device, buf_size, xrt_core::bo_int::use_type::scratch_pad);
       }
       catch (...) { /*do nothing*/ }
     });

src/runtime_src/core/common/api/xrt_kernel.cpp

@@ -1278,6 +1278,8 @@ class argument
   { return arg.type; }
 };
 
+namespace xbi = xrt_core::bo_int;
+
 // class buffer_cache - A thread safe cache to hold pre created buffers
 //
 // This class maintains a fixed size pool of reusable buffers to
@@ -1290,6 +1292,7 @@ class buffer_cache
   xrt::hw_context m_hw_ctx;
   std::vector<uint8_t> m_buf_data;
   const std::size_t m_max_cache_size;
+  xbi::use_type m_use_flag;
 
   mutable std::mutex m_mutex;
   mutable std::vector<xrt::bo> m_bo_cache;
@@ -1303,7 +1306,7 @@ class buffer_cache
   }
 
   static std::vector<xrt::bo>
-  initialize_bo_cache(const xrt::hw_context& hwctx, size_t cache_size, size_t bo_size)
+  initialize_bo_cache(const xrt::hw_context& hwctx, size_t cache_size, size_t bo_size, xbi::use_type flag)
   {
     if (!cache_size)
       return {};
@@ -1312,17 +1315,18 @@ class buffer_cache
     cache.reserve(cache_size);
 
     for (size_t i = 0; i < cache_size; ++i)
-      cache.push_back(xrt::ext::bo(hwctx, bo_size));
+      cache.push_back(xbi::create_bo(hwctx, bo_size, flag));
 
     return cache;
   }
 
 public:
-  buffer_cache(xrt::hw_context ctx, std::vector<uint8_t>&& data, size_t max_size)
+  buffer_cache(xrt::hw_context ctx, std::vector<uint8_t>&& data, size_t max_size, xbi::use_type flag)
     : m_hw_ctx(std::move(ctx))
     , m_buf_data(std::move(data))
     , m_max_cache_size(max_size)
-    , m_bo_cache{initialize_bo_cache(m_hw_ctx, m_max_cache_size, m_buf_data.size())}
+    , m_use_flag(flag)
+    , m_bo_cache{initialize_bo_cache(m_hw_ctx, m_max_cache_size, m_buf_data.size(), m_use_flag)}
   {}
 
   xrt::bo
@@ -1338,7 +1342,7 @@ class buffer_cache
     }
 
     if (!bo)
-      bo = xrt::ext::bo(m_hw_ctx, m_buf_data.size()); // create new bo if cache is empty
+      bo = xbi::create_bo(m_hw_ctx, m_buf_data.size(), m_use_flag); // create new bo if cache is empty
 
     fill_bo_with_data(bo, m_buf_data);
     return bo;
@@ -1360,6 +1364,7 @@ class buffer_cache
 } // namespace
 
 namespace xrt {
+
 // struct kernel_impl - The internals of an xrtKernelHandle
 //
 // An single object of kernel_type can be shared with multiple
@@ -1648,7 +1653,8 @@ class kernel_impl : public std::enable_shared_from_this<kernel_impl>
     size_t max_pool_size = std::max(pool_memory_size / ctrlpkt_buf_size, min_pool_size);
 
     // Create and return buffer_cache with calculated pool size
-    return std::make_unique<buffer_cache>(ctx, std::move(ctrlpkt_data), max_pool_size);
+    // use ctrlpkt flag while creating buffers
+    return std::make_unique<buffer_cache>(ctx, std::move(ctrlpkt_data), max_pool_size, xbi::use_type::ctrlpkt);
   }
 
 public:

src/runtime_src/core/include/xrt/detail/xrt_mem.h

@@ -148,7 +148,10 @@ struct xcl_bo_flags
  * hold instructions for firmware.
  *
  * XRT_BO_USE_SCRATCH_PAD indicates that the buffer will be used as
- * scratch pad memory for firmware.
+ * scratch pad memory to store L2 memory at time of preemption.
+ * 
+ * XRT_BO_USE_CTRL_SCRATCH_PAD indicates that the buffer will be used
+ * to store/retrieve control state information during model execution.
  *
  * XRT_BO_USE_PDI indicates that the buffer will be used to hold
  * PDI data for firmware.
@@ -161,19 +164,20 @@ struct xcl_bo_flags
 // constexpr and using NOLINT block to supress clng-tidy warnings
 
 // NOLINTBEGIN
-#define XRT_BO_USE_UNUSED      0
-#define XRT_BO_USE_DEBUG       1
-#define XRT_BO_USE_KMD         2
-#define XRT_BO_USE_DTRACE      3
-#define XRT_BO_USE_LOG         4
-#define XRT_BO_USE_DEBUG_QUEUE 5
-#define XRT_BO_USE_UC_DEBUG    6
-#define XRT_BO_USE_PREEMPTION  7
-#define XRT_BO_USE_HOST_ONLY   8
-#define XRT_BO_USE_INSTRUCTION 9
-#define XRT_BO_USE_SCRATCH_PAD 10
-#define XRT_BO_USE_PDI         11
-#define XRT_BO_USE_CTRLPKT     12
+#define XRT_BO_USE_UNUSED           0
+#define XRT_BO_USE_DEBUG            1
+#define XRT_BO_USE_KMD              2
+#define XRT_BO_USE_DTRACE           3
+#define XRT_BO_USE_LOG              4
+#define XRT_BO_USE_DEBUG_QUEUE      5
+#define XRT_BO_USE_UC_DEBUG         6
+#define XRT_BO_USE_PREEMPTION       7
+#define XRT_BO_USE_HOST_ONLY        8
+#define XRT_BO_USE_INSTRUCTION      9
+#define XRT_BO_USE_SCRATCH_PAD      10
+#define XRT_BO_USE_CTRL_SCRATCH_PAD 11
+#define XRT_BO_USE_PDI              12
+#define XRT_BO_USE_CTRLPKT          13
 // NOLINTEND
 
 /**