Fix thread local realm client allocator usage

This was actually sharing one allocator for all clients because the cached TLS handle is stored in a single instance of the allocator, rather than the intended usage where each client has a handle via its dynamic buffer. Oops!

Making the allocator itself thread local fixes it but leads to every created thread allocating, even if they aren't going to serve clients. The easy solution is to add a mode to require an explicit thread_enter() call to tell the allocator that you intend to use it on the current thread. Solved!

Author: Chaosvex

src/libs/core/allocators/include/allocators/TLSBlockAllocator.h

@@ -23,15 +23,18 @@ namespace ember::allocators {
 
 struct SafeEntrant {};
 struct NoRefCounting {};
+struct ImplicitInit {};
 
 struct UnsafeEntrant : SafeEntrant {};
-struct RefCounting : NoRefCounting {};
+struct RefCounting   : NoRefCounting {};
+struct ExplicitInit  : ImplicitInit {};
 
 template<typename _ty,
 	std::size_t _elements,
 	std::derived_from<NoRefCounting> RefCountPolicy = NoRefCounting,
 	std::derived_from<SafeEntrant> EntrantPolicy = SafeEntrant,
-	std::derived_from<NoPageLock> PageLockPolicy = NoPageLock
+	std::derived_from<NoPageLock> PageLockPolicy = NoPageLock,
+	std::derived_from<ImplicitInit> InitPolicy = ImplicitInit
 >
 class TLSBlockAllocator final {
 	using AllocatorType = BlockAllocator<_ty, PageLockPolicy>;
@@ -75,7 +78,9 @@ class TLSBlockAllocator final {
 #endif
 
 	TLSBlockAllocator(std::string_view tag = {}) : tag_(tag) {
-		thread_enter();
+		if constexpr(std::is_same_v<InitPolicy, ImplicitInit>) {
+			thread_enter();
+		}
 	}
 
 	/*

src/realm/ClientAllocator.h

@@ -25,7 +25,8 @@ using ClientAllocator = allocators::TLSBlockAllocator<
 	PREALLOCATED_CLIENTS_PER_THREAD,
 	allocators::NoRefCounting,
 	allocators::UnsafeEntrant,
-	PageLockPolicy
+	PageLockPolicy,
+	allocators::ExplicitInit
 >;
 
 } // realm, ember

src/realm/ClientBuilder.h

@@ -20,15 +20,17 @@ namespace ember::realm {
 
 class ClientBuilder {
 	constexpr static std::string_view allocator_tag { "realm_client_create" };
+	static inline thread_local ClientAllocator allocator_ { allocator_tag };
 
-	mutable ClientAllocator allocator_;
 	ClientHandlerBuilder ch_builder_;
 	ClientConnectionBuilder cc_builder_;
 	EventDispatcher& dispatcher_;
 	thread::ServicePool& pool_;
 	log::Logger& logger_;
 
 	unique_client_ptr make_unique_client(tcp_socket socket, std::size_t index) const {
+		allocator_.thread_enter();
+
 		return unique_client_ptr(allocator_.allocate(
 			std::move(socket), index, dispatcher_, logger_, ch_builder_, cc_builder_
 		), ClientDeleter(allocator_, pool_.get(index)));
@@ -37,8 +39,7 @@ class ClientBuilder {
 public:
 	ClientBuilder(ClientHandlerBuilder ch_builder, ClientConnectionBuilder cc_builder,
 	              EventDispatcher& dispatcher, thread::ServicePool& pool, log::Logger& logger)
-		: allocator_(allocator_tag)
-		, ch_builder_(ch_builder)
+		: ch_builder_(ch_builder)
 		, cc_builder_(cc_builder)
 		, dispatcher_(dispatcher)
 		, pool_(pool)