Skip to content

Commit 50efd4e

Browse files
tdavidcltdavidcl
authored
[Backends] adding DeviceBuffer (#490)
* [Backends] use DeviceScheduler instead of queue to init USM buffer * [Backends] rename usmbuffer to DeviceBuffer * [Backends] implement memory & event handling for device buffer * [Backends] group memory handling in templates * [Backends] add some error messages to event handling * [Backends][Test] clean DeviceBuffer test * [Backends][Test] add missing license * [Backends] remove SourceLocation form DeviceBuffer * [Backends] use a rust like constructor and document USMPtrHolder * [Backends] format and improve doc of USMPtrHolder * [Backends] start writing doc for the BufferEventHandler * [Backends] document BufferEventHandler * [Backends] document memoryHandler.hpp * [Backends] document DeviceBuffer * [Backends] format DeviceBuffer.hpp
1 parent f5dc06e commit 50efd4e

15 files changed

Lines changed: 895 additions & 283 deletions

‎src/shambackends/CMakeLists.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ project(Shambackends CXX C)
55
set(Sources
66
src/comm/details/CommunicationBufferImpl.cpp
77
src/comm/CommunicationBuffer.cpp
8-
src/usmbuffer.cpp
8+
src/details/BufferEventHandler.cpp
9+
src/details/memoryHandle.cpp
10+
src/USMPtrHolder.cpp
11+
src/DeviceBuffer.cpp
912
src/Device.cpp
1013
src/DeviceContext.cpp
1114
src/DeviceQueue.cpp

‎src/shambackends/include/shambackends/DeviceBuffer.hpp‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// -------------------------------------------------------//
2+
//
3+
// SHAMROCK code for hydrodynamics
4+
// Copyright(C) 2021-2023 Timothée David--Cléris <timothee.david--cleris@ens-lyon.fr>
5+
// Licensed under CeCILL 2.1 License, see LICENSE for more information
6+
//
7+
// -------------------------------------------------------//
8+
9+
#pragma once
10+
11+
/**
12+
* @file DeviceBuffer.hpp
13+
* @author Timothée David--Cléris (timothee.david--cleris@ens-lyon.fr)
14+
* @brief
15+
*
16+
*/
17+
18+
#include "shambackends/DeviceScheduler.hpp"
19+
#include "shambackends/USMPtrHolder.hpp"
20+
#include "shambackends/details/BufferEventHandler.hpp"
21+
#include "shambackends/details/memoryHandle.hpp"
22+
23+
#include <memory>
24+
25+
namespace sham {
26+
27+
/**
28+
* @brief A buffer allocated in USM (Unified Shared Memory)
29+
*
30+
* @tparam T The type of the buffer's elements
31+
* @tparam target The USM target where the buffer is allocated (host, device, shared)
32+
*/
33+
template<class T, USMKindTarget target = device>
34+
class DeviceBuffer {
35+
36+
public:
37+
/**
38+
* @brief Construct a new Device Buffer object
39+
*
40+
* @param sz The size of the buffer in number of elements
41+
* @param dev_sched A shared pointer to the Device Scheduler
42+
*
43+
* This constructor creates a new Device Buffer object with the given size.
44+
* It allocates the buffer as USM memory and stores the USM pointer and the
45+
* size in the respective member variables.
46+
*/
47+
DeviceBuffer(size_t sz, std::shared_ptr<DeviceScheduler> dev_sched)
48+
: hold(details::create_usm_ptr<target>(sz * sizeof(T), dev_sched)), size(sz) {}
49+
50+
/**
51+
* @brief Deleted copy constructor
52+
*/
53+
DeviceBuffer(const DeviceBuffer &other) = delete;
54+
55+
/**
56+
* @brief Deleted copy assignment operator
57+
*/
58+
DeviceBuffer &operator=(const DeviceBuffer &other) = delete;
59+
60+
/**
61+
* @brief Destructor for DeviceBuffer
62+
*
63+
* This destructor releases the USM pointer and event handler
64+
* by transfering them back to the memory handler
65+
*/
66+
~DeviceBuffer() {
67+
// give the ptr holder and event handler to the memory handler
68+
details::release_usm_ptr(std::move(hold), std::move(events_hndl));
69+
}
70+
71+
/**
72+
* @brief Get a read-only pointer to the buffer's data.
73+
*
74+
* This function returns a const pointer to the buffer's data. The
75+
* pointer is locked for reading and the event handler is updated to
76+
* reflect the read access.
77+
*
78+
* @param depends_list A vector of SYCL events to wait for before
79+
* accessing the buffer.
80+
* @return A const pointer to the buffer's data.
81+
*/
82+
[[nodiscard]] inline const T *get_read_access(std::vector<sycl::event> &depends_list) {
83+
events_hndl.read_access(depends_list);
84+
return hold.template ptr_cast<T>();
85+
}
86+
87+
/**
88+
* @brief Get a read-write pointer to the buffer's data
89+
*
90+
* This function returns a pointer to the buffer's data. The event handler is updated to
91+
* reflect the write access.
92+
*
93+
* @param depends_list A vector of SYCL events to wait for before
94+
* accessing the buffer.
95+
* @return A pointer to the buffer's data.
96+
*/
97+
[[nodiscard]] inline T *get_write_access(std::vector<sycl::event> &depends_list) {
98+
events_hndl.write_access(depends_list);
99+
return hold.template ptr_cast<T>();
100+
}
101+
102+
/**
103+
* @brief Complete the event state of the buffer.
104+
*
105+
* This function complete the event state of the buffer by registering the
106+
* event resulting of the last queried access
107+
*
108+
* @param e The SYCL event resulting of the queried access.
109+
*/
110+
void complete_event_state(sycl::event e) { events_hndl.complete_state(e); }
111+
112+
/**
113+
* @brief Gets the Device scheduler corresponding to the held allocation
114+
*
115+
* @return The Device scheduler
116+
*/
117+
[[nodiscard]] inline DeviceScheduler &get_dev_scheduler() const {
118+
return hold.get_dev_scheduler();
119+
}
120+
121+
/**
122+
* @brief Gets the number of elements in the buffer
123+
*
124+
* @return The number of elements in the buffer
125+
*/
126+
[[nodiscard]] inline size_t get_size() const { return size; }
127+
128+
/**
129+
* @brief Gets the size of the buffer in bytes
130+
*
131+
* @return The size of the buffer in bytes
132+
*/
133+
[[nodiscard]] inline size_t get_bytesize() const { return hold.get_bytesize(); }
134+
135+
private:
136+
/**
137+
* @brief The USM pointer holder
138+
*/
139+
USMPtrHolder<target> hold;
140+
141+
/**
142+
* @brief The number of elements in the buffer
143+
*/
144+
size_t size = 0;
145+
146+
/**
147+
* @brief Event handler for the buffer
148+
*
149+
* This event handler keeps track of the events associated with read and write
150+
* accesses to the buffer. It is used to ensure that the buffer is not accessed
151+
* before the data is in a complete state.
152+
*/
153+
details::BufferEventHandler events_hndl;
154+
};
155+
156+
} // namespace sham

‎src/shambackends/include/shambackends/USMPtrHolder.hpp‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
// -------------------------------------------------------//
2+
//
3+
// SHAMROCK code for hydrodynamics
4+
// Copyright(C) 2021-2023 Timothée David--Cléris <timothee.david--cleris@ens-lyon.fr>
5+
// Licensed under CeCILL 2.1 License, see LICENSE for more information
6+
//
7+
// -------------------------------------------------------//
8+
9+
#pragma once
10+
11+
/**
12+
* @file USMPtrHolder.hpp
13+
* @author Timothée David--Cléris (timothee.david--cleris@ens-lyon.fr)
14+
* @brief This file contains the declaration of the USMPtrHolder class.
15+
*
16+
* The USMPtrHolder class is a smart pointer that manages the memory allocated
17+
* using SYCL unified shared memory (USM). It provides a way to safely allocate, use,
18+
* and deallocate memory in USM.
19+
*/
20+
21+
#include "shambackends/DeviceScheduler.hpp"
22+
#include <memory>
23+
#include <utility>
24+
25+
namespace sham {
26+
27+
/**
28+
* @brief Enum listing the different types of USM pointers allocations
29+
*
30+
* - Device USM pointers are allocated on the device's memory, and can only be accessed by the
31+
* device.
32+
*
33+
* - Shared USM pointers are allocated on the host's memory, and can be accessed by both the
34+
* host and the device. (May induce implicit communications between the host and the device)
35+
*
36+
* - Host USM pointers are allocated on the host's memory, and can only be accessed by the host.
37+
*/
38+
enum USMKindTarget {
39+
device, ///< Device memory
40+
shared, ///< Shared memory
41+
host ///< Host memory
42+
};
43+
44+
/**
45+
* @brief Class for holding a USM pointer
46+
*
47+
* This class is a simple RAII wrapper around a USM (Unified Shared Memory) pointer.
48+
* It is a move-only class that manages the lifetime of the USM buffer.
49+
*
50+
* The USM buffer can be either a device, shared or host buffer, depending on the
51+
* template parameter `target`.
52+
*
53+
* The move constructor and move assignment operator are deleted to prevent
54+
* accidental copies of the class.
55+
*/
56+
template<USMKindTarget target>
57+
class USMPtrHolder {
58+
59+
void *usm_ptr = nullptr; ///< The USM buffer pointer
60+
size_t size = 0; ///< The size of the USM buffer
61+
std::shared_ptr<DeviceScheduler>
62+
dev_sched; ///< The SYCL queue used to allocate/free the USM buffer
63+
64+
USMPtrHolder(void *usm_ptr, size_t size, std::shared_ptr<DeviceScheduler> dev_sched)
65+
: usm_ptr(usm_ptr), size(size), dev_sched(std::move(dev_sched)) {}
66+
67+
public:
68+
void free_ptr(); ///< Free the held pointer
69+
70+
/**
71+
* @brief Create a USM pointer holder
72+
*
73+
* Allocate a USM buffer of the given size using the provided SYCL queue.
74+
* The USM buffer can be either a device, shared or host buffer,
75+
* depending on the template parameter `target`.
76+
*
77+
* @param sz The size of the USM buffer to be allocated
78+
* @param dev_sched The Device Scheduler used to allocate/free the USM buffer
79+
*
80+
* @return A USMPtrHolder instance wrapping the allocated USM buffer
81+
*/
82+
static USMPtrHolder create(size_t sz, std::shared_ptr<DeviceScheduler> dev_sched);
83+
84+
/**
85+
* @brief USM pointer holder destructor
86+
*
87+
* Frees the USM pointer if not equall to nullptr
88+
*/
89+
~USMPtrHolder();
90+
91+
/**
92+
* @brief Deleted copy constructor
93+
*/
94+
USMPtrHolder(const USMPtrHolder &other) = delete;
95+
96+
/**
97+
* @brief Move constructor
98+
*
99+
* Moves the contents of the other USMPtrHolder into this one, leaving the other
100+
* one with a nullptr USM pointer, which disable the destructor.
101+
*
102+
* @param other The USMPtrHolder to be moved from
103+
*/
104+
USMPtrHolder(USMPtrHolder &&other) noexcept
105+
: usm_ptr(std::exchange(other.usm_ptr, nullptr)), size(other.size),
106+
dev_sched(other.dev_sched) {}
107+
108+
/**
109+
* @brief Deleted copy assignment operator
110+
*/
111+
USMPtrHolder &operator=(const USMPtrHolder &other) = delete;
112+
113+
/**
114+
* @brief Move assignment operator
115+
*
116+
* Moves the contents of the other USMPtrHolder into this one, leaving the other
117+
* one in a valid but unspecified state. The other USMPtrHolder will not free the
118+
* USM buffer on destruction.
119+
*
120+
* @param other The USMPtrHolder to be moved from
121+
*/
122+
USMPtrHolder &operator=(USMPtrHolder &&other) noexcept {
123+
dev_sched = other.dev_sched;
124+
size = other.size;
125+
std::swap(usm_ptr, other.usm_ptr);
126+
return *this;
127+
}
128+
129+
/**
130+
* @brief Cast the USM pointer to the given type
131+
*
132+
* @tparam T The type to cast the USM buffer pointer to
133+
* @return The casted USM pointer
134+
*/
135+
template<class T>
136+
inline T *ptr_cast() const {
137+
return reinterpret_cast<T *>(usm_ptr);
138+
}
139+
140+
/**
141+
* @brief Get the raw pointer of the USM allocation
142+
*
143+
* This method returns the raw pointer to the USM allocation. The caller must
144+
* be careful with the type and the usage of the returned pointer.
145+
*
146+
* @return The raw pointer of the USM allocation
147+
*/
148+
[[nodiscard]] inline void *get_raw_ptr() const { return usm_ptr; }
149+
150+
/**
151+
* @brief Get the size of the USM allocation (in byte)
152+
*
153+
* @return The size of the USM allocation (in byte)
154+
*/
155+
[[nodiscard]] inline size_t get_bytesize() const { return size; }
156+
157+
/**
158+
* @brief Get the SYCL context used for allocation/freeing the USM buffer
159+
*
160+
* @return The SYCL context used for allocation/freeing the USM buffer
161+
*/
162+
[[nodiscard]] inline DeviceScheduler &get_dev_scheduler() const { return *dev_sched; }
163+
};
164+
165+
} // namespace sham

0 commit comments

Comments
 (0)