Skip to content

Commit 0aa0618

Browse files
spikehmeta-codesync[bot]
spikeh
authored and
meta-codesync[bot]
committed
IoUringBufferProvider: add buffer provided dispatch
Summary: Add `IoUringBufferProvider` to dispatch to either `IoUringProvidedBufferRing` or `IoUringDynamicProvidedBufferRing`. Add `IoUringProvidedBufferRingMode` to `IoUringOptions`, defaulting to the fixed implementation. Reviewed By: jlhe97 Differential Revision: D116735819 fbshipit-source-id: 72aae4f932aef32162d3f2296634514f73047a78
1 parent 907ef46 commit 0aa0618

9 files changed

Lines changed: 267 additions & 35 deletions

File tree

‎third-party/folly/src/folly/io/async/AsyncIoUringSocket.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class AsyncDetachFdCallback {
5454

5555
namespace folly {
5656
class IoUringBackend;
57-
class IoUringProvidedBufferRing;
57+
class IoUringBufferProvider;
5858

5959
class AsyncIoUringSocket : public AsyncSocketTransport {
6060
public:
@@ -259,14 +259,14 @@ class AsyncIoUringSocket : public AsyncSocketTransport {
259259
void unregisterFd();
260260
void readProcessSubmit(
261261
struct io_uring_sqe* sqe,
262-
IoUringProvidedBufferRing* bufferProvider,
262+
IoUringBufferProvider* bufferProvider,
263263
size_t* maxSize,
264-
IoUringProvidedBufferRing* usedBufferProvider) noexcept;
264+
IoUringBufferProvider* usedBufferProvider) noexcept;
265265
void readCallback(
266266
int res,
267267
uint32_t flags,
268268
size_t maxSize,
269-
IoUringProvidedBufferRing* bufferProvider) noexcept;
269+
IoUringBufferProvider* bufferProvider) noexcept;
270270
void allowReads();
271271
void previousReadDone();
272272
void processWriteQueue() noexcept;
@@ -347,7 +347,7 @@ class AsyncIoUringSocket : public AsyncSocketTransport {
347347

348348
bool isEOF(const io_uring_cqe* cqe) noexcept;
349349

350-
IoUringProvidedBufferRing* lastUsedBufferProvider_;
350+
IoUringBufferProvider* lastUsedBufferProvider_;
351351
ReadCallback* readCallback_ = nullptr;
352352
AsyncIoUringSocket* parent_;
353353
size_t maxSize_;

‎third-party/folly/src/folly/io/async/CMakeLists.txt‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ folly_add_library(
544544
folly_function
545545
folly_io_async_async_base
546546
folly_io_async_delayed_destruction
547-
folly_io_async_io_uring_provided_buffer_ring
547+
folly_io_async_io_uring_buffer_provider
548548
folly_io_async_io_uring_zero_copy_buffer_pool
549549
folly_io_async_liburing
550550
folly_io_iobuf
@@ -572,6 +572,17 @@ folly_add_library(
572572
folly_io_async_async_base
573573
)
574574

575+
folly_add_library(
576+
NAME io_uring_buffer_provider
577+
SRCS
578+
IoUringBufferProvider.cpp
579+
HEADERS
580+
IoUringBufferProvider.h
581+
EXPORTED_DEPS
582+
folly_io_async_io_uring_dynamic_provided_buffer_ring
583+
folly_io_async_io_uring_provided_buffer_ring
584+
)
585+
575586
folly_add_library(
576587
NAME io_uring_connect
577588
SRCS

‎third-party/folly/src/folly/io/async/IoUringBackend.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,18 +1035,22 @@ void IoUringBackend::initSubmissionLinked() {
10351035

10361036
if (options_.initialProvidedBuffersCount) {
10371037
try {
1038-
IoUringProvidedBufferRing::Options options = {
1038+
IoUringBufferProvider::Options options = {
10391039
.gid = nextBufferProviderGid(),
10401040
.bufferCount = options_.initialProvidedBuffersCount,
10411041
.bufferSize = options_.initialProvidedBuffersEachSize,
10421042
.useIncrementalBuffers = options_.enableIncrementalBuffers,
10431043
};
10441044
for (size_t i = 0; i < options_.providedBufRings; i++) {
10451045
bufferProviders_.push_back(
1046-
IoUringProvidedBufferRing::create(this->ioRingPtr(), options));
1046+
IoUringBufferProvider::create(
1047+
this->ioRingPtr(),
1048+
options_.providedBufferRingMode ==
1049+
IoUringOptions::ProvidedBufferRingMode::Dynamic,
1050+
options));
10471051
options.gid = nextBufferProviderGid();
10481052
}
1049-
} catch (const IoUringProvidedBufferRing::LibUringCallError& ex) {
1053+
} catch (const IoUringBufferProvider::LibUringCallError& ex) {
10501054
LOG(ERROR) << folly::to<std::string>(
10511055
"failed to make provided buffer ring, buffer count: ",
10521056
options_.initialProvidedBuffersCount,

‎third-party/folly/src/folly/io/async/IoUringBackend.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@
4040
#include <folly/io/async/EventBase.h>
4141
#include <folly/io/async/EventBaseBackendBase.h>
4242
#include <folly/io/async/IoUringBase.h>
43+
#include <folly/io/async/IoUringBufferProvider.h>
4344
#include <folly/io/async/IoUringOptions.h>
44-
#include <folly/io/async/IoUringProvidedBufferRing.h>
4545
#include <folly/io/async/IoUringZeroCopyBufferPool.h>
4646
#include <folly/io/async/Liburing.h>
4747
#include <folly/portability/Asm.h>
@@ -208,7 +208,7 @@ class IoUringBackend : public EventBaseBackendBase {
208208
void cancel(IoSqeBase* sqe);
209209

210210
// built in buffer provider
211-
IoUringProvidedBufferRing* bufferProvider() {
211+
IoUringBufferProvider* bufferProvider() {
212212
return bufferProviders_
213213
[bufferProviderIdx_++ & (bufferProviders_.size() - 1)]
214214
.get();
@@ -226,7 +226,7 @@ class IoUringBackend : public EventBaseBackendBase {
226226

227227
struct IoUringStats {
228228
IoUringZeroCopyBufferPool::Stats zcrx;
229-
IoUringProvidedBufferRing::Stats providedBuffer;
229+
IoUringBufferProvider::Stats providedBuffer;
230230

231231
auto operator<=>(const IoUringStats&) const = default;
232232
};
@@ -238,7 +238,7 @@ class IoUringBackend : public EventBaseBackendBase {
238238
}
239239

240240
if (hasBufferProvider()) {
241-
IoUringProvidedBufferRing* bufProvider =
241+
IoUringBufferProvider* bufProvider =
242242
bufferProviders_[bufferProviderIdx_ & (bufferProviders_.size() - 1)]
243243
.get();
244244
bufProvider->getStats(stats.providedBuffer);
@@ -477,7 +477,7 @@ class IoUringBackend : public EventBaseBackendBase {
477477
void prepRecvmsgMultishot(io_uring_sqe* sqe, int fd, msghdr* msg) noexcept {
478478
CHECK(sqe);
479479
::io_uring_prep_recvmsg_multishot(sqe, fd, msg, MSG_TRUNC);
480-
if (IoUringProvidedBufferRing* bp = backend_->bufferProvider()) {
480+
if (auto* bp = backend_->bufferProvider()) {
481481
sqe->buf_group = bp->gid();
482482
sqe->flags |= IOSQE_BUFFER_SELECT;
483483
}
@@ -867,7 +867,7 @@ class IoUringBackend : public EventBaseBackendBase {
867867
// submit
868868
IoSqeBaseList submitList_;
869869
uint16_t bufferProviderGidNext_{0};
870-
std::vector<IoUringProvidedBufferRing::UniquePtr> bufferProviders_;
870+
std::vector<std::unique_ptr<IoUringBufferProvider>> bufferProviders_;
871871
uint64_t bufferProviderIdx_{0};
872872
IoUringZeroCopyBufferPool::UniquePtr zcBufferPool_;
873873

‎third-party/folly/src/folly/io/async/IoUringBufferProvider.cpp‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include <folly/io/async/IoUringBufferProvider.h>
18+
19+
#if FOLLY_HAS_LIBURING
20+
21+
namespace folly {
22+
23+
namespace {
24+
25+
template <typename Ring>
26+
typename Ring::UniquePtr createRing(
27+
io_uring* ioRingPtr, const IoUringBufferProvider::Options& options) {
28+
return Ring::create(
29+
ioRingPtr,
30+
{
31+
.gid = options.gid,
32+
.bufferCount = options.bufferCount,
33+
.bufferSize = options.bufferSize,
34+
.useIncrementalBuffers = options.useIncrementalBuffers,
35+
});
36+
}
37+
38+
} // namespace
39+
40+
std::unique_ptr<IoUringBufferProvider> IoUringBufferProvider::create(
41+
io_uring* ioRingPtr, bool useDynamicRing, Options options) {
42+
try {
43+
if (useDynamicRing) {
44+
return std::unique_ptr<IoUringBufferProvider>(new IoUringBufferProvider(
45+
Ring{createRing<IoUringDynamicProvidedBufferRing>(
46+
ioRingPtr, options)}));
47+
}
48+
return std::unique_ptr<IoUringBufferProvider>(new IoUringBufferProvider(
49+
Ring{createRing<IoUringProvidedBufferRing>(ioRingPtr, options)}));
50+
} catch (const IoUringProvidedBufferRing::LibUringCallError& ex) {
51+
throw LibUringCallError(ex.what());
52+
} catch (const IoUringDynamicProvidedBufferRing::LibUringCallError& ex) {
53+
throw LibUringCallError(ex.what());
54+
}
55+
}
56+
57+
void IoUringBufferProvider::enobuf() noexcept {
58+
std::visit([](auto& ring) { ring->enobuf(); }, ring_);
59+
}
60+
61+
uint32_t IoUringBufferProvider::getAndResetEnobufCount() noexcept {
62+
return std::visit(
63+
[](auto& ring) { return ring->getAndResetEnobufCount(); }, ring_);
64+
}
65+
66+
std::unique_ptr<IOBuf> IoUringBufferProvider::getIoBuf(
67+
uint16_t startBufId, size_t totalLength, bool hasMore) noexcept {
68+
return std::visit(
69+
[&](auto& ring) {
70+
return ring->getIoBuf(startBufId, totalLength, hasMore);
71+
},
72+
ring_);
73+
}
74+
75+
std::unique_ptr<IOBuf> IoUringBufferProvider::getIoBuf(
76+
const struct io_uring_cqe* cqe) noexcept {
77+
return std::visit([&](auto& ring) { return ring->getIoBuf(cqe); }, ring_);
78+
}
79+
80+
uint32_t IoUringBufferProvider::count() const noexcept {
81+
return std::visit([](const auto& ring) { return ring->count(); }, ring_);
82+
}
83+
84+
bool IoUringBufferProvider::available() const noexcept {
85+
return std::visit([](const auto& ring) { return ring->available(); }, ring_);
86+
}
87+
88+
size_t IoUringBufferProvider::sizePerBuffer() const noexcept {
89+
return std::visit(
90+
[](const auto& ring) { return ring->sizePerBuffer(); }, ring_);
91+
}
92+
93+
uint16_t IoUringBufferProvider::gid() const noexcept {
94+
return std::visit([](const auto& ring) { return ring->gid(); }, ring_);
95+
}
96+
97+
int IoUringBufferProvider::getUtilPct() const noexcept {
98+
return std::visit([](const auto& ring) { return ring->getUtilPct(); }, ring_);
99+
}
100+
101+
uint16_t IoUringBufferProvider::areaCount() const noexcept {
102+
auto* ring = std::get_if<IoUringDynamicProvidedBufferRing::UniquePtr>(&ring_);
103+
if (!ring) {
104+
return 1;
105+
}
106+
return (*ring)->areaCount();
107+
}
108+
109+
} // namespace folly
110+
111+
#endif

‎third-party/folly/src/folly/io/async/IoUringBufferProvider.h‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#pragma once
18+
19+
#include <variant>
20+
21+
#include <folly/io/async/IoUringDynamicProvidedBufferRing.h>
22+
#include <folly/io/async/IoUringProvidedBufferRing.h>
23+
24+
#if FOLLY_HAS_LIBURING
25+
26+
namespace folly {
27+
28+
class IoUringBufferProvider {
29+
public:
30+
class LibUringCallError : public std::runtime_error {
31+
public:
32+
using std::runtime_error::runtime_error;
33+
};
34+
35+
struct Options {
36+
uint16_t gid{0};
37+
uint32_t bufferCount{0};
38+
uint32_t bufferSize{0};
39+
bool useIncrementalBuffers{false};
40+
};
41+
42+
struct Stats {
43+
uint32_t enobufCount{0};
44+
int utilPct{-1};
45+
uint16_t areaCount{0};
46+
47+
auto operator<=>(const Stats&) const = default;
48+
};
49+
50+
static std::unique_ptr<IoUringBufferProvider> create(
51+
io_uring* ioRingPtr, bool useDynamicRing, Options options);
52+
53+
void enobuf() noexcept;
54+
uint32_t getAndResetEnobufCount() noexcept;
55+
56+
std::unique_ptr<IOBuf> getIoBuf(
57+
uint16_t startBufId, size_t totalLength, bool hasMore) noexcept;
58+
std::unique_ptr<IOBuf> getIoBuf(const struct io_uring_cqe* cqe) noexcept;
59+
60+
uint32_t count() const noexcept;
61+
bool available() const noexcept;
62+
size_t sizePerBuffer() const noexcept;
63+
uint16_t gid() const noexcept;
64+
int getUtilPct() const noexcept;
65+
uint16_t areaCount() const noexcept;
66+
67+
void getStats(Stats& stats) noexcept {
68+
stats.enobufCount = getAndResetEnobufCount();
69+
stats.utilPct = getUtilPct();
70+
stats.areaCount = areaCount();
71+
}
72+
73+
private:
74+
using Ring = std::variant<
75+
IoUringProvidedBufferRing::UniquePtr,
76+
IoUringDynamicProvidedBufferRing::UniquePtr>;
77+
78+
explicit IoUringBufferProvider(Ring ring) : ring_(std::move(ring)) {}
79+
80+
Ring ring_;
81+
};
82+
83+
} // namespace folly
84+
85+
#endif

‎third-party/folly/src/folly/io/async/IoUringOptions.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@ using SrcPortForQueueIdCallback = folly::Function<int(
4747
uint16_t maxPort)>;
4848

4949
struct IoUringOptions {
50+
enum class ProvidedBufferRingMode {
51+
Fixed,
52+
Dynamic,
53+
};
54+
5055
enum Flags {
5156
POLL_SQ = 0x1,
5257
POLL_CQ = 0x2,
@@ -150,6 +155,11 @@ struct IoUringOptions {
150155
return *this;
151156
}
152157

158+
IoUringOptions& setProvidedBufferRingMode(ProvidedBufferRingMode mode) {
159+
providedBufferRingMode = mode;
160+
return *this;
161+
}
162+
153163
IoUringOptions& setProvidedBufRings(uint32_t v) {
154164
if (!folly::isPowTwo(v)) {
155165
throw std::runtime_error(
@@ -289,6 +299,7 @@ struct IoUringOptions {
289299
uint32_t initialProvidedBuffersCount{0};
290300
uint32_t initialProvidedBuffersEachSize{0};
291301
uint32_t providedBufRings{1};
302+
ProvidedBufferRingMode providedBufferRingMode{ProvidedBufferRingMode::Fixed};
292303
bool providedBufUseBundles{false};
293304

294305
uint32_t flags{0};

‎third-party/folly/src/folly/io/async/IoUringRecv.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ class IoUringRecvHandle::RecvRequest
198198
DestructorGuard handleGuard_;
199199
bool useBundles_{false};
200200

201-
IoUringProvidedBufferRing* bufferRing_{nullptr};
201+
IoUringBufferProvider* bufferRing_{nullptr};
202202
IoUringZeroCopyBufferPool* bufferPool_{nullptr};
203203
size_t recvLen_{0};
204204
std::unique_ptr<IOBuf> fallbackBuffer_;

0 commit comments

Comments
 (0)