Skip to content

fuse: make fuse max-write/max-read params tunable #4501

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: devel
Choose a base branch
from
Open

fuse: make fuse max-write/max-read params tunable #4501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions contrib/fuse-include/fuse_kernel.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,8 @@ struct fuse_file_lock {
#define FUSE_MAX_PAGES (1 << 22)
#define FUSE_CACHE_SYMLINKS (1 << 23)

#define FUSE_MAX_MAX_PAGES 256

/**
* CUSE INIT request/reply flags
*
Expand Down
41 changes: 41 additions & 0 deletions glusterfsd/src/glusterfsd.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,10 @@ static struct argp_option gf_options[] = {
{"fuse-handle-copy_file_range", ARGP_FUSE_HANDLE_COPY_FILE_RANGE, "BOOL",
OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
"enable the handler of the FUSE_COPY_FILE_RANGE message"},
{"fuse-max-write", ARGP_FUSE_MAX_WRITE_KEY, "INTEGER", OPTION_ARG_OPTIONAL,
"set fuse max-write bytes"},
{"fuse-max-read", ARGP_FUSE_MAX_READ_KEY, "INTEGER", OPTION_ARG_OPTIONAL,
"set fuse max-read bytes"},
{0, 0, 0, 0, "Miscellaneous Options:"},
{
0,
Expand Down Expand Up @@ -566,6 +570,16 @@ set_fuse_mount_options(glusterfs_ctx_t *ctx, dict_t *options)
break;
}

if (cmd_args->fuse_max_write) {
DICT_SET_VAL(dict_set_uint32, options, "fuse-max-write",
cmd_args->fuse_max_write, glusterfsd_msg_3);
}

if (cmd_args->fuse_max_read) {
DICT_SET_VAL(dict_set_uint32, options, "fuse-max-read",
cmd_args->fuse_max_read, glusterfsd_msg_3);
}

if (cmd_args->fs_display_name) {
DICT_SET_VAL(dict_set_dynstr, options, "fs-display-name",
cmd_args->fs_display_name, glusterfsd_msg_3);
Expand Down Expand Up @@ -1463,6 +1477,33 @@ parse_opts(int key, char *arg, struct argp_state *state)
"unknown fuse handle copy_file_range setting \"%s\"",
arg);
break;
case ARGP_FUSE_MAX_WRITE_KEY:
if (gf_string2uint32(arg, &cmd_args->fuse_max_write)) {
argp_failure(state, -1, 0, "unknown fuse max-write option %s",
arg);
} else if ((cmd_args->fuse_max_write < 4096) ||
(cmd_args->fuse_max_write > 1048576)) {
argp_failure(state, -1, 0,
"Invalid fuse max-write bytes %s. "
"Valid range: [\"4096, 1048576\"]",
arg);
}

break;
case ARGP_FUSE_MAX_READ_KEY:
if (gf_string2uint32(arg, &cmd_args->fuse_max_read)) {
argp_failure(state, -1, 0, "unknown fuse max-read option %s",
arg);
} else if ((cmd_args->fuse_max_read < 4096) ||
(cmd_args->fuse_max_read > 1048576)) {
argp_failure(state, -1, 0,
"Invalid fuse max-read bytes %s. "
"Valid range: [\"4096, 1048576\"]",
arg);
}

break;

}
return 0;
}
Expand Down
2 changes: 2 additions & 0 deletions glusterfsd/src/glusterfsd.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ enum argp_option_keys {
ARGP_FUSE_INODE_TABLESIZE_KEY = 198,
ARGP_FUSE_SETLK_HANDLE_INTERRUPT_KEY = 199,
ARGP_FUSE_HANDLE_COPY_FILE_RANGE = 200,
ARGP_FUSE_MAX_WRITE_KEY = 201,
ARGP_FUSE_MAX_READ_KEY = 202,
};

int
Expand Down
2 changes: 2 additions & 0 deletions libglusterfs/src/glusterfs/glusterfs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,8 @@ struct _cmd_args {
int fuse_auto_inval;

uint32_t fuse_dev_eperm_ratelimit_ns;
uint32_t fuse_max_write;
uint32_t fuse_max_read;

bool global_threading;
bool brick_mux;
Expand Down
42 changes: 36 additions & 6 deletions xlators/mount/fuse/src/fuse-bridge.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5154,6 +5154,15 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
fino.max_readahead = 1 << 17;
fino.max_write = 1 << 17;
fino.flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS;
#if FUSE_KERNEL_MINOR_VERSION >= 28
if (fini->minor >= 28) {
fino.max_readahead = priv->fuse_max_read;
fino.max_write = priv->fuse_max_write;
fino.max_pages = max(priv->fuse_max_write, priv->fuse_max_read) / sysconf(_SC_PAGESIZE);
if (fino.max_pages == FUSE_MAX_MAX_PAGES)
fino.flags |= FUSE_MAX_PAGES;
}
#endif
#if FUSE_KERNEL_MINOR_VERSION >= 17
if (fini->minor >= 17)
fino.flags |= FUSE_FLOCK_LOCKS;
Expand Down Expand Up @@ -6090,14 +6099,12 @@ fuse_thread_proc(void *data)
struct pollfd pfd[2] = {{
0,
}};
uint32_t psize;

this = data;
priv = this->private;

THIS = this;

psize = ((struct iobuf_pool *)this->ctx->iobuf_pool)->default_page_size;
priv->msg0_len_p = &msg0_size;

for (;;) {
Expand Down Expand Up @@ -6153,7 +6160,8 @@ fuse_thread_proc(void *data)
size from 'fuse', which is as of today 128KB. If we bring in
support for higher block sizes support, then we should be
changing this one too */
iobuf = iobuf_get(this->ctx->iobuf_pool);
iobuf = iobuf_get2(this->ctx->iobuf_pool,
max(priv->fuse_max_write, priv->fuse_max_read));

/* Add extra 512 byte to the first iov so that it can
* accommodate "ordinary" non-write requests. It's not
Expand All @@ -6177,7 +6185,7 @@ fuse_thread_proc(void *data)
iov_in[1].iov_base = iobuf->ptr;

iov_in[0].iov_len = msg0_size;
iov_in[1].iov_len = psize;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you still need psize?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you still need psize?

Thanks, indeed no longer need for psize. This draft needs further improvements and is still under development and test.

I will push my updates as soon as possible.

iov_in[1].iov_len = max(priv->fuse_max_write, priv->fuse_max_read);

res = sys_readv(priv->fd, iov_in, 2);

Expand Down Expand Up @@ -6910,6 +6918,12 @@ init(xlator_t *this_xl)
GF_OPTION_INIT("fuse-dev-eperm-ratelimit-ns",
priv->fuse_dev_eperm_ratelimit_ns, uint32, cleanup_exit);

GF_OPTION_INIT("fuse-max-write", priv->fuse_max_write, uint32,
cleanup_exit);

GF_OPTION_INIT("fuse-max-read", priv->fuse_max_read, uint32,
cleanup_exit);

/* user has set only background-qlen, not congestion-threshold,
use the fuse kernel driver formula to set congestion. ie, 75% */
if (dict_get(this_xl->options, "background-qlen") &&
Expand Down Expand Up @@ -6954,11 +6968,11 @@ init(xlator_t *this_xl)
goto cleanup_exit;
}

gf_asprintf(&mnt_args, "%s%s%s%sallow_other,max_read=131072",
gf_asprintf(&mnt_args, "%s%s%s%sallow_other,max_read=%lu",
priv->acl ? "" : "default_permissions,",
priv->read_only ? "ro," : "",
priv->fuse_mountopts ? priv->fuse_mountopts : "",
priv->fuse_mountopts ? "," : "");
priv->fuse_mountopts ? "," : "", priv->fuse_max_read);
if (!mnt_args)
goto cleanup_exit;

Expand Down Expand Up @@ -7198,6 +7212,22 @@ struct volume_options options[] = {
.max = 64,
.description = "Sets fuse reader thread count.",
},
{
.key = {"fuse-max-write"},
.type = GF_OPTION_TYPE_INT,
.default_value = "131072",
.min = 4096,
.max = 1048576,
.description = "Sets fuse max-write bytes.",
},
{
.key = {"fuse-max-read"},
.type = GF_OPTION_TYPE_INT,
.default_value = "131072",
.min = 4096,
.max = 1048576,
.description = "Sets fuse max-read bytes.",
},
{
.key = {"kernel-writeback-cache"},
.type = GF_OPTION_TYPE_BOOL,
Expand Down
2 changes: 2 additions & 0 deletions xlators/mount/fuse/src/fuse-bridge.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,8 @@ struct fuse_private {
uint32_t inode_table_size;
uint32_t invalidate_limit;
uint32_t fuse_dev_eperm_ratelimit_ns;
uint32_t fuse_max_write;
uint32_t fuse_max_read;

/* counters for fusdev errnos */
uint8_t fusedev_errno_cnt[FUSEDEV_EMAXPLUS];
Expand Down
14 changes: 14 additions & 0 deletions xlators/mount/fuse/utils/mount.glusterfs.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,14 @@ start_glusterfs ()
cmd_line=$(echo "$cmd_line --fuse-handle-copy_file_range=$fuse_handle_copy_file_range");
fi

if [ -n "$fuse_max_write" ]; then
cmd_line=$(echo "$cmd_line --fuse-max-write=$fuse_max_write");
fi

if [ -n "$fuse_max_read" ]; then
cmd_line=$(echo "$cmd_line --fuse-max-read=$fuse_max_read");
fi

if [ -n "$process_name" ]; then
cmd_line=$(echo "$cmd_line --process-name fuse.$process_name");
else
Expand Down Expand Up @@ -641,6 +649,12 @@ with_options()
"fuse-dev-eperm-ratelimit-ns")
fuse_dev_eperm_ratelimit_ns=$value
;;
"fuse-max-write")
fuse_max_write=$value
;;
"fuse-max-read")
fuse_max_read=$value
;;
"context"|"fscontext"|"defcontext"|"rootcontext")
# standard SElinux mount options to pass to the kernel
[ -z "$fuse_mountopts" ] || fuse_mountopts="$fuse_mountopts,"
Expand Down