Skip to content

Commit f538975

Browse files
ballard26ballard26
committed
core: add label to io_threaded_fallbacks to categorize operations
1 parent 4a3406c commit f538975

File tree

4 files changed

+77
-39
lines changed

4 files changed

+77
-39
lines changed

‎src/core/file.cc

+9-9
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ posix_file_impl::stat() noexcept {
214214
struct stat st;
215215
auto ret = ::fstat(fd, &st);
216216
return wrap_syscall(ret, st);
217-
}).then([] (syscall_result_extra<struct stat> ret) {
217+
}, submit_reason::file_operation).then([] (syscall_result_extra<struct stat> ret) {
218218
ret.throw_if_error();
219219
return make_ready_future<struct stat>(ret.extra);
220220
});
@@ -224,7 +224,7 @@ future<>
224224
posix_file_impl::truncate(uint64_t length) noexcept {
225225
return engine()._thread_pool->submit<syscall_result<int>>([this, length] {
226226
return wrap_syscall<int>(::ftruncate(_fd, length));
227-
}).then([] (syscall_result<int> sr) {
227+
}, submit_reason::file_operation).then([] (syscall_result<int> sr) {
228228
sr.throw_if_error();
229229
return make_ready_future<>();
230230
});
@@ -234,7 +234,7 @@ future<int>
234234
posix_file_impl::ioctl(uint64_t cmd, void* argp) noexcept {
235235
return engine()._thread_pool->submit<syscall_result<int>>([this, cmd, argp] () mutable {
236236
return wrap_syscall<int>(::ioctl(_fd, cmd, argp));
237-
}).then([] (syscall_result<int> sr) {
237+
}, submit_reason::file_operation).then([] (syscall_result<int> sr) {
238238
sr.throw_if_error();
239239
// Some ioctls require to return a positive integer back.
240240
return make_ready_future<int>(sr.result);
@@ -255,7 +255,7 @@ future<int>
255255
posix_file_impl::fcntl(int op, uintptr_t arg) noexcept {
256256
return engine()._thread_pool->submit<syscall_result<int>>([this, op, arg] () mutable {
257257
return wrap_syscall<int>(::fcntl(_fd, op, arg));
258-
}).then([] (syscall_result<int> sr) {
258+
}, submit_reason::file_operation).then([] (syscall_result<int> sr) {
259259
sr.throw_if_error();
260260
// Some fcntls require to return a positive integer back.
261261
return make_ready_future<int>(sr.result);
@@ -277,7 +277,7 @@ posix_file_impl::discard(uint64_t offset, uint64_t length) noexcept {
277277
return engine()._thread_pool->submit<syscall_result<int>>([this, offset, length] () mutable {
278278
return wrap_syscall<int>(::fallocate(_fd, FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE,
279279
offset, length));
280-
}).then([] (syscall_result<int> sr) {
280+
}, submit_reason::file_operation).then([] (syscall_result<int> sr) {
281281
sr.throw_if_error();
282282
return make_ready_future<>();
283283
});
@@ -298,7 +298,7 @@ posix_file_impl::allocate(uint64_t position, uint64_t length) noexcept {
298298
supported = false; // Racy, but harmless. At most we issue an extra call or two.
299299
}
300300
return wrap_syscall<int>(ret);
301-
}).then([] (syscall_result<int> sr) {
301+
}, submit_reason::file_operation).then([] (syscall_result<int> sr) {
302302
sr.throw_if_error();
303303
return make_ready_future<>();
304304
});
@@ -340,7 +340,7 @@ posix_file_impl::close() noexcept {
340340
try {
341341
return engine()._thread_pool->submit<syscall_result<int>>([fd] {
342342
return wrap_syscall<int>(::close(fd));
343-
});
343+
}, submit_reason::file_operation);
344344
} catch (...) {
345345
report_exception("Running ::close() in reactor thread, submission failed with exception", std::current_exception());
346346
return make_ready_future<syscall_result<int>>(wrap_syscall<int>(::close(fd)));
@@ -362,7 +362,7 @@ blockdev_file_impl::size() noexcept {
362362
uint64_t size;
363363
int ret = ::ioctl(_fd, BLKGETSIZE64, &size);
364364
return wrap_syscall(ret, size);
365-
}).then([] (syscall_result_extra<uint64_t> ret) {
365+
}, submit_reason::file_operation).then([] (syscall_result_extra<uint64_t> ret) {
366366
ret.throw_if_error();
367367
return make_ready_future<uint64_t>(ret.extra);
368368
});
@@ -662,7 +662,7 @@ blockdev_file_impl::discard(uint64_t offset, uint64_t length) noexcept {
662662
return engine()._thread_pool->submit<syscall_result<int>>([this, offset, length] () mutable {
663663
uint64_t range[2] { offset, length };
664664
return wrap_syscall<int>(::ioctl(_fd, BLKDISCARD, &range));
665-
}).then([] (syscall_result<int> sr) {
665+
}, submit_reason::file_operation).then([] (syscall_result<int> sr) {
666666
sr.throw_if_error();
667667
return make_ready_future<>();
668668
});

‎src/core/reactor.cc

+35-24
Original file line numberDiff line numberDiff line change
@@ -2013,7 +2013,7 @@ reactor::open_file_dma(std::string_view nameref, open_flags flags, file_open_opt
20132013
}
20142014
close_fd.cancel();
20152015
return wrap_syscall(fd, st);
2016-
}).then([&options, name = std::move(name), &open_flags] (syscall_result_extra<struct stat> sr) {
2016+
}, submit_reason::file_operation).then([&options, name = std::move(name), &open_flags] (syscall_result_extra<struct stat> sr) {
20172017
sr.throw_fs_exception_if_error("open failed", name);
20182018
return make_file_impl(sr.result, options, open_flags, sr.extra);
20192019
}).then([] (shared_ptr<file_impl> impl) {
@@ -2028,7 +2028,7 @@ reactor::remove_file(std::string_view pathname) noexcept {
20282028
return futurize_invoke([this, pathname] {
20292029
return _thread_pool->submit<syscall_result<int>>([pathname = sstring(pathname)] {
20302030
return wrap_syscall<int>(::remove(pathname.c_str()));
2031-
}).then([pathname = sstring(pathname)] (syscall_result<int> sr) {
2031+
}, submit_reason::file_operation).then([pathname = sstring(pathname)] (syscall_result<int> sr) {
20322032
sr.throw_fs_exception_if_error("remove failed", pathname);
20332033
return make_ready_future<>();
20342034
});
@@ -2041,7 +2041,8 @@ reactor::rename_file(std::string_view old_pathname, std::string_view new_pathnam
20412041
return futurize_invoke([this, old_pathname, new_pathname] {
20422042
return _thread_pool->submit<syscall_result<int>>([old_pathname = sstring(old_pathname), new_pathname = sstring(new_pathname)] {
20432043
return wrap_syscall<int>(::rename(old_pathname.c_str(), new_pathname.c_str()));
2044-
}).then([old_pathname = sstring(old_pathname), new_pathname = sstring(new_pathname)] (syscall_result<int> sr) {
2044+
}, submit_reason::file_operation
2045+
).then([old_pathname = sstring(old_pathname), new_pathname = sstring(new_pathname)] (syscall_result<int> sr) {
20452046
sr.throw_fs_exception_if_error("rename failed", old_pathname, new_pathname);
20462047
return make_ready_future<>();
20472048
});
@@ -2054,7 +2055,7 @@ reactor::link_file(std::string_view oldpath, std::string_view newpath) noexcept
20542055
return futurize_invoke([this, oldpath, newpath] {
20552056
return _thread_pool->submit<syscall_result<int>>([oldpath = sstring(oldpath), newpath = sstring(newpath)] {
20562057
return wrap_syscall<int>(::link(oldpath.c_str(), newpath.c_str()));
2057-
}).then([oldpath = sstring(oldpath), newpath = sstring(newpath)] (syscall_result<int> sr) {
2058+
}, submit_reason::file_operation).then([oldpath = sstring(oldpath), newpath = sstring(newpath)] (syscall_result<int> sr) {
20582059
sr.throw_fs_exception_if_error("link failed", oldpath, newpath);
20592060
return make_ready_future<>();
20602061
});
@@ -2068,7 +2069,7 @@ reactor::chmod(std::string_view name, file_permissions permissions) noexcept {
20682069
return futurize_invoke([name, mode, this] {
20692070
return _thread_pool->submit<syscall_result<int>>([name = sstring(name), mode] {
20702071
return wrap_syscall<int>(::chmod(name.c_str(), mode));
2071-
}).then([name = sstring(name), mode] (syscall_result<int> sr) {
2072+
}, submit_reason::file_operation).then([name = sstring(name), mode] (syscall_result<int> sr) {
20722073
if (sr.result == -1) {
20732074
auto reason = format("chmod(0{:o}) failed", mode);
20742075
sr.throw_fs_exception(reason, fs::path(name));
@@ -2112,7 +2113,7 @@ reactor::file_type(std::string_view name, follow_symlink follow) noexcept {
21122113
auto stat_syscall = follow ? stat : lstat;
21132114
auto ret = stat_syscall(name.c_str(), &st);
21142115
return wrap_syscall(ret, st);
2115-
}).then([name = sstring(name)] (syscall_result_extra<struct stat> sr) {
2116+
}, submit_reason::file_operation).then([name = sstring(name)] (syscall_result_extra<struct stat> sr) {
21162117
if (long(sr.result) == -1) {
21172118
if (sr.error != ENOENT && sr.error != ENOTDIR) {
21182119
sr.throw_fs_exception_if_error("stat failed", name);
@@ -2142,7 +2143,7 @@ future<size_t> reactor::read_directory(int fd, char* buffer, size_t buffer_size)
21422143
return _thread_pool->submit<syscall_result<long>>([fd, buffer, buffer_size] () {
21432144
auto ret = ::syscall(__NR_getdents64, fd, reinterpret_cast<linux_dirent64*>(buffer), buffer_size);
21442145
return wrap_syscall(ret);
2145-
}).then([] (syscall_result<long> ret) {
2146+
}, submit_reason::file_operation).then([] (syscall_result<long> ret) {
21462147
ret.throw_if_error();
21472148
return make_ready_future<size_t>(ret.result);
21482149
});
@@ -2155,7 +2156,7 @@ reactor::inotify_add_watch(int fd, std::string_view path, uint32_t flags) {
21552156
return _thread_pool->submit<syscall_result<int>>([fd, path = sstring(path), flags] {
21562157
auto ret = ::inotify_add_watch(fd, path.c_str(), flags);
21572158
return wrap_syscall(ret);
2158-
}).then([] (syscall_result<int> ret) {
2159+
}, submit_reason::file_operation).then([] (syscall_result<int> ret) {
21592160
ret.throw_if_error();
21602161
return make_ready_future<int>(ret.result);
21612162
});
@@ -2257,7 +2258,7 @@ reactor::spawn(std::string_view pathname,
22572258
return wrap_syscall<int>(::posix_spawn(&child_pid, pathname.c_str(), &actions, &attr,
22582259
const_cast<char* const *>(argv.data()),
22592260
const_cast<char* const *>(env.data())));
2260-
});
2261+
}, submit_reason::process_operation);
22612262
}).finally([&actions, &attr] {
22622263
posix_spawn_file_actions_destroy(&actions);
22632264
posix_spawnattr_destroy(&attr);
@@ -2295,7 +2296,7 @@ static auto next_waitpid_timeout(std::chrono::milliseconds this_timeout) {
22952296
future<int> reactor::waitpid(pid_t pid) {
22962297
return _thread_pool->submit<syscall_result<int>>([pid] {
22972298
return wrap_syscall<int>(syscall(__NR_pidfd_open, pid, O_NONBLOCK));
2298-
}).then([pid, this] (syscall_result<int> pidfd) {
2299+
}, submit_reason::process_operation).then([pid, this] (syscall_result<int> pidfd) {
22992300
if (pidfd.result == -1) {
23002301
// pidfd_open() was introduced in linux 5.3, so the pidfd.error could be ENOSYS on
23012302
// older kernels. But it could be other error like EMFILE or ENFILE. anyway, we
@@ -2308,7 +2309,7 @@ future<int> reactor::waitpid(pid_t pid) {
23082309
&wait_timeout] {
23092310
return _thread_pool->submit<syscall_result<pid_t>>([pid, &wstatus] {
23102311
return wrap_syscall<pid_t>(::waitpid(pid, &wstatus, WNOHANG));
2311-
}).then([&wstatus, &wait_timeout] (syscall_result<pid_t> ret) mutable {
2312+
}, submit_reason::process_operation).then([&wstatus, &wait_timeout] (syscall_result<pid_t> ret) mutable {
23122313
if (ret.result == 0) {
23132314
wait_timeout = next_waitpid_timeout(wait_timeout);
23142315
return ::seastar::sleep(wait_timeout).then([] {
@@ -2328,7 +2329,7 @@ future<int> reactor::waitpid(pid_t pid) {
23282329
return pidfd.readable().then([pid, &wstatus, this] {
23292330
return _thread_pool->submit<syscall_result<pid_t>>([pid, &wstatus] {
23302331
return wrap_syscall<pid_t>(::waitpid(pid, &wstatus, WNOHANG));
2331-
});
2332+
}, submit_reason::process_operation);
23322333
}).then([&wstatus] (syscall_result<pid_t> ret) {
23332334
ret.throw_if_error();
23342335
assert(ret.result > 0);
@@ -2353,7 +2354,7 @@ reactor::file_stat(std::string_view pathname, follow_symlink follow) noexcept {
23532354
auto stat_syscall = follow ? stat : lstat;
23542355
auto ret = stat_syscall(pathname.c_str(), &st);
23552356
return wrap_syscall(ret, st);
2356-
}).then([pathname = sstring(pathname)] (syscall_result_extra<struct stat> sr) {
2357+
}, submit_reason::file_operation).then([pathname = sstring(pathname)] (syscall_result_extra<struct stat> sr) {
23572358
sr.throw_fs_exception_if_error("stat failed", pathname);
23582359
struct stat& st = sr.extra;
23592360
stat_data sd;
@@ -2391,7 +2392,7 @@ reactor::file_accessible(std::string_view pathname, access_flags flags) noexcept
23912392
auto aflags = std::underlying_type_t<access_flags>(flags);
23922393
auto ret = ::access(pathname.c_str(), aflags);
23932394
return wrap_syscall(ret);
2394-
}).then([pathname = sstring(pathname), flags] (syscall_result<int> sr) {
2395+
}, submit_reason::file_operation).then([pathname = sstring(pathname), flags] (syscall_result<int> sr) {
23952396
if (sr.result < 0) {
23962397
if ((sr.error == ENOENT && flags == access_flags::exists) ||
23972398
(sr.error == EACCES && flags != access_flags::exists)) {
@@ -2413,7 +2414,7 @@ reactor::file_system_at(std::string_view pathname) noexcept {
24132414
struct statfs st;
24142415
auto ret = statfs(pathname.c_str(), &st);
24152416
return wrap_syscall(ret, st);
2416-
}).then([pathname = sstring(pathname)] (syscall_result_extra<struct statfs> sr) {
2417+
}, submit_reason::file_operation).then([pathname = sstring(pathname)] (syscall_result_extra<struct statfs> sr) {
24172418
static std::unordered_map<long int, fs_type> type_mapper = {
24182419
{ internal::fs_magic::xfs, fs_type::xfs },
24192420
{ internal::fs_magic::ext2, fs_type::ext2 },
@@ -2440,7 +2441,7 @@ reactor::fstatfs(int fd) noexcept {
24402441
struct statfs st;
24412442
auto ret = ::fstatfs(fd, &st);
24422443
return wrap_syscall(ret, st);
2443-
}).then([] (syscall_result_extra<struct statfs> sr) {
2444+
}, submit_reason::file_operation).then([] (syscall_result_extra<struct statfs> sr) {
24442445
sr.throw_if_error();
24452446
struct statfs st = sr.extra;
24462447
return make_ready_future<struct statfs>(std::move(st));
@@ -2455,7 +2456,7 @@ reactor::statvfs(std::string_view pathname) noexcept {
24552456
struct statvfs st;
24562457
auto ret = ::statvfs(pathname.c_str(), &st);
24572458
return wrap_syscall(ret, st);
2458-
}).then([pathname = sstring(pathname)] (syscall_result_extra<struct statvfs> sr) {
2459+
}, submit_reason::file_operation).then([pathname = sstring(pathname)] (syscall_result_extra<struct statvfs> sr) {
24592460
sr.throw_fs_exception_if_error("statvfs failed", pathname);
24602461
struct statvfs st = sr.extra;
24612462
return make_ready_future<struct statvfs>(std::move(st));
@@ -2479,7 +2480,7 @@ reactor::open_directory(std::string_view name) noexcept {
24792480
}
24802481
}
24812482
return wrap_syscall(fd, st);
2482-
}).then([name = sstring(name), oflags] (syscall_result_extra<struct stat> sr) {
2483+
}, submit_reason::file_operation).then([name = sstring(name), oflags] (syscall_result_extra<struct stat> sr) {
24832484
sr.throw_fs_exception_if_error("open failed", name);
24842485
return make_file_impl(sr.result, file_open_options(), oflags, sr.extra);
24852486
}).then([] (shared_ptr<file_impl> file_impl) {
@@ -2495,7 +2496,7 @@ reactor::make_directory(std::string_view name, file_permissions permissions) noe
24952496
return _thread_pool->submit<syscall_result<int>>([name = sstring(name), permissions] {
24962497
auto mode = static_cast<mode_t>(permissions);
24972498
return wrap_syscall<int>(::mkdir(name.c_str(), mode));
2498-
}).then([name = sstring(name)] (syscall_result<int> sr) {
2499+
}, submit_reason::file_operation).then([name = sstring(name)] (syscall_result<int> sr) {
24992500
sr.throw_fs_exception_if_error("mkdir failed", name);
25002501
});
25012502
});
@@ -2508,7 +2509,7 @@ reactor::touch_directory(std::string_view name, file_permissions permissions) no
25082509
return _thread_pool->submit<syscall_result<int>>([name = sstring(name), permissions] {
25092510
auto mode = static_cast<mode_t>(permissions);
25102511
return wrap_syscall<int>(::mkdir(name.c_str(), mode));
2511-
}).then([name = sstring(name)] (syscall_result<int> sr) {
2512+
}, submit_reason::file_operation).then([name = sstring(name)] (syscall_result<int> sr) {
25122513
if (sr.result == -1 && sr.error != EEXIST) {
25132514
sr.throw_fs_exception("mkdir failed", fs::path(name));
25142515
}
@@ -2553,7 +2554,7 @@ reactor::fdatasync(int fd) noexcept {
25532554
}
25542555
return _thread_pool->submit<syscall_result<int>>([fd] {
25552556
return wrap_syscall<int>(::fdatasync(fd));
2556-
}).then([] (syscall_result<int> sr) {
2557+
}, submit_reason::file_operation).then([] (syscall_result<int> sr) {
25572558
sr.throw_if_error();
25582559
return make_ready_future<>();
25592560
});
@@ -2700,6 +2701,12 @@ void reactor::register_metrics() {
27002701

27012702
namespace sm = seastar::metrics;
27022703

2704+
auto io_fallback_counter = [this](const sstring& reason_str, submit_reason r) {
2705+
static auto reason_label = sm::label("reason");
2706+
return sm::make_counter("io_threaded_fallbacks", std::bind(&thread_pool::count, _thread_pool.get(), r),
2707+
sm::description("Total number of io-threaded-fallbacks operations"), { reason_label(reason_str), });
2708+
};
2709+
27032710
_metric_groups.add_group("reactor", {
27042711
sm::make_gauge("tasks_pending", std::bind(&reactor::pending_task_count, this), sm::description("Number of pending tasks in the queue")),
27052712
// total_operations value:DERIVE:0:U
@@ -2725,9 +2732,13 @@ void reactor::register_metrics() {
27252732
// total_operations value:DERIVE:0:U
27262733
sm::make_counter("fsyncs", _fsyncs, sm::description("Total number of fsync operations")),
27272734
// total_operations value:DERIVE:0:U
2728-
sm::make_counter("io_threaded_fallbacks", std::bind(&thread_pool::operation_count, _thread_pool.get()),
2729-
sm::description("Total number of io-threaded-fallbacks operations")),
2730-
2735+
io_fallback_counter("aio_fallback", submit_reason::aio_fallback),
2736+
// total_operations value:DERIVE:0:U
2737+
io_fallback_counter("file_operation", submit_reason::file_operation),
2738+
// total_operations value:DERIVE:0:U
2739+
io_fallback_counter("process_operation", submit_reason::process_operation),
2740+
// total_operations value:DERIVE:0:U
2741+
io_fallback_counter("unknown", submit_reason::unknown),
27312742
});
27322743

27332744
_metric_groups.add_group("memory", {

‎src/core/reactor_backend.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,7 @@ void aio_storage_context::schedule_retry() {
260260
return _r._thread_pool->submit<syscall_result<int>>([this] () mutable {
261261
auto r = io_submit(_io_context, _aio_retries.size(), _aio_retries.data());
262262
return wrap_syscall<int>(r);
263-
}).then_wrapped([this] (future<syscall_result<int>> f) {
263+
}, submit_reason::aio_fallback).then_wrapped([this] (future<syscall_result<int>> f) {
264264
// If submit failed, just log the error and exit the loop.
265265
// The next call to submit_work will call schedule_retry again.
266266
if (f.failed()) {

0 commit comments

Comments
 (0)