From 6728779c2d2a02f738a79e3b279a513c52730423 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Mon, 13 Oct 2025 13:42:07 -0500 Subject: [PATCH] Rework config, centralize fuse config --- libfuse/include/fuse_cfg.hpp | 38 +++ libfuse/include/fuse_common.h | 5 - libfuse/include/fuse_config.hpp | 31 --- libfuse/include/fuse_msgbuf.hpp | 10 +- {src => libfuse/include}/int_types.h | 0 libfuse/lib/fuse.cpp | 75 +---- libfuse/lib/fuse_cfg.cpp | 21 ++ libfuse/lib/fuse_config.cpp | 73 ----- libfuse/lib/fuse_i.h | 5 - libfuse/lib/fuse_loop.cpp | 10 +- libfuse/lib/fuse_lowlevel.cpp | 162 ++++------- libfuse/lib/fuse_msgbuf.cpp | 31 ++- libfuse/lib/mount_generic.c | 65 ++--- mkdocs/docs/config/options.md | 24 +- mkdocs/docs/faq/configuration_and_policies.md | 92 +++++-- mkdocs/docs/remote_filesystems.md | 10 +- mkdocs/mkdocs.yml | 4 +- src/config.cpp | 257 ++++++++++-------- src/config.hpp | 53 ++-- src/fuse_create.cpp | 6 +- src/fuse_init.cpp | 6 +- src/fuse_open.cpp | 10 +- src/option_parser.cpp | 129 +++++---- src/tofrom_ref.hpp | 39 +++ src/tofrom_string.hpp | 2 + src/tofrom_wrapper.hpp | 2 +- 26 files changed, 554 insertions(+), 606 deletions(-) create mode 100644 libfuse/include/fuse_cfg.hpp delete mode 100644 libfuse/include/fuse_config.hpp rename {src => libfuse/include}/int_types.h (100%) create mode 100644 libfuse/lib/fuse_cfg.cpp delete mode 100644 libfuse/lib/fuse_config.cpp create mode 100644 src/tofrom_ref.hpp diff --git a/libfuse/include/fuse_cfg.hpp b/libfuse/include/fuse_cfg.hpp new file mode 100644 index 00000000..21296c12 --- /dev/null +++ b/libfuse/include/fuse_cfg.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include "int_types.h" + +#include +#include + +#define FUSE_CFG_INVALID_ID -1 +#define FUSE_CFG_INVALID_UMASK -1 + + +struct fuse_cfg_t +{ + s64 uid = FUSE_CFG_INVALID_ID; + bool valid_uid() const; + + s64 gid = FUSE_CFG_INVALID_ID; + bool valid_gid() const; + + s64 umask = FUSE_CFG_INVALID_UMASK; + bool valid_umask() const; + + s64 remember_nodes = 0; + + bool debug = false; + + int max_background = 0; + int congestion_threshold = 0; + u32 max_pages = 32; + int passthrough_max_stack_depth = 1; + + int read_thread_count = 0; + int process_thread_count = -1; + int process_thread_queue_depth = 2; + std::string pin_threads = "false"; +}; + +extern fuse_cfg_t fuse_cfg; diff --git a/libfuse/include/fuse_common.h b/libfuse/include/fuse_common.h index 1cef8f9d..78cbbce1 100644 --- a/libfuse/include/fuse_common.h +++ b/libfuse/include/fuse_common.h @@ -139,13 +139,8 @@ struct fuse_conn_info { unsigned proto_major; unsigned proto_minor; - unsigned max_write; - unsigned max_readahead; uint64_t capable; uint64_t want; - unsigned max_background; - unsigned congestion_threshold; - uint16_t max_pages; }; struct fuse_session; diff --git a/libfuse/include/fuse_config.hpp b/libfuse/include/fuse_config.hpp deleted file mode 100644 index 39c604d6..00000000 --- a/libfuse/include/fuse_config.hpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - ISC License - - Copyright (c) 2023, Antonio SJ Musumeci - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ - -#pragma once - -#include - -int fuse_config_get_read_thread_count(); -int fuse_config_get_process_thread_count(); -int fuse_config_get_process_thread_queue_depth(); -std::string fuse_config_get_pin_threads(); - -void fuse_config_set_read_thread_count(int const); -void fuse_config_set_process_thread_count(int const); -void fuse_config_set_process_thread_queue_depth(int const); -void fuse_config_set_pin_threads(std::string const); diff --git a/libfuse/include/fuse_msgbuf.hpp b/libfuse/include/fuse_msgbuf.hpp index 07215e57..8872d302 100644 --- a/libfuse/include/fuse_msgbuf.hpp +++ b/libfuse/include/fuse_msgbuf.hpp @@ -18,11 +18,10 @@ #pragma once +#include "int_types.h" #include "fuse_msgbuf_t.h" -#include "extern_c.h" - -EXTERN_C_BEGIN +u32 msgbuf_get_pagesize(); void msgbuf_set_bufsize(const uint32_t size); uint64_t msgbuf_get_bufsize(); @@ -35,8 +34,3 @@ void msgbuf_gc_10percent(); uint64_t msgbuf_alloc_count(); uint64_t msgbuf_avail_count(); - -void msgbuf_page_align(fuse_msgbuf_t *msgbuf); -void msgbuf_write_align(fuse_msgbuf_t *msgbuf); - -EXTERN_C_END diff --git a/src/int_types.h b/libfuse/include/int_types.h similarity index 100% rename from src/int_types.h rename to libfuse/include/int_types.h diff --git a/libfuse/lib/fuse.cpp b/libfuse/lib/fuse.cpp index 21dac2a8..554e7b9e 100644 --- a/libfuse/lib/fuse.cpp +++ b/libfuse/lib/fuse.cpp @@ -19,6 +19,7 @@ #include "mutex.hpp" #include "node.hpp" +#include "fuse_cfg.hpp" #include "fuse_req.h" #include "fuse_dirents.hpp" #include "fuse_i.h" @@ -70,16 +71,6 @@ static int g_LOG_METRICS = 0; struct fuse_config { - unsigned int uid; - unsigned int gid; - unsigned int umask; - int remember; - int debug; - int nogc; - int set_mode; - int set_uid; - int set_gid; - int help; }; struct lock_queue_element @@ -620,7 +611,7 @@ inline int remember_nodes() { - return (f.conf.remember > 0); + return (fuse_cfg.remember_nodes > 0); } static @@ -703,7 +694,7 @@ find_node(uint64_t parent, goto out_err; node->nodeid = generate_nodeid(&f.nodeid_gen); - if(f.conf.remember) + if(fuse_cfg.remember_nodes) inc_nlookup(node); if(hash_name(node,parent,name) == -1) @@ -1264,12 +1255,12 @@ void set_stat(uint64_t nodeid, struct stat *stbuf) { - if(f.conf.set_mode) - stbuf->st_mode = (stbuf->st_mode & S_IFMT) | (0777 & ~f.conf.umask); - if(f.conf.set_uid) - stbuf->st_uid = f.conf.uid; - if(f.conf.set_gid) - stbuf->st_gid = f.conf.gid; + if(fuse_cfg.valid_uid()) + stbuf->st_uid = fuse_cfg.uid; + if(fuse_cfg.valid_gid()) + stbuf->st_gid = fuse_cfg.gid; + if(fuse_cfg.valid_umask()) + stbuf->st_mode = (stbuf->st_mode & S_IFMT) | (0777 & ~fuse_cfg.umask); } static @@ -3449,7 +3440,7 @@ fuse_prune_some_remembered_nodes(int *offset_) checked++; age = (now - fn->time); - if(f.conf.remember > age) + if(fuse_cfg.remember_nodes > age) break; assert(fn->node->nlookup == 1); @@ -3597,38 +3588,9 @@ enum static const struct fuse_opt fuse_lib_opts[] = { - FUSE_OPT_KEY("-h", KEY_HELP), - FUSE_OPT_KEY("--help", KEY_HELP), - FUSE_OPT_KEY("debug", FUSE_OPT_KEY_KEEP), - FUSE_OPT_KEY("-d", FUSE_OPT_KEY_KEEP), - FUSE_LIB_OPT("debug", debug,1), - FUSE_LIB_OPT("-d", debug,1), - FUSE_LIB_OPT("nogc", nogc,1), - FUSE_LIB_OPT("umask=", set_mode,1), - FUSE_LIB_OPT("umask=%o", umask,0), - FUSE_LIB_OPT("uid=", set_uid,1), - FUSE_LIB_OPT("uid=%d", uid,0), - FUSE_LIB_OPT("gid=", set_gid,1), - FUSE_LIB_OPT("gid=%d", gid,0), - FUSE_LIB_OPT("noforget", remember,-1), - FUSE_LIB_OPT("remember=%u", remember,0), FUSE_OPT_END }; -static void fuse_lib_help(void) -{ - fprintf(stderr, - " -o umask=M set file permissions (octal)\n" - " -o uid=N set file owner\n" - " -o gid=N set file group\n" - " -o noforget never forget cached inodes\n" - " -o remember=T remember cached inodes for T seconds (0s)\n" - " -o threads=NUM number of worker threads. 0 = autodetect.\n" - " Negative values autodetect then divide by\n" - " absolute value. default = 0\n" - "\n"); -} - static int fuse_lib_opt_proc(void *data, @@ -3636,24 +3598,9 @@ fuse_lib_opt_proc(void *data, int key, struct fuse_args *outargs) { - (void)arg; (void)outargs; - - if(key == KEY_HELP) - { - struct fuse_config *conf = (struct fuse_config *)data; - fuse_lib_help(); - conf->help = 1; - } - return 1; } -int -fuse_is_lib_option(const char *opt) -{ - return fuse_lowlevel_is_lib_option(opt) || fuse_opt_match(fuse_lib_opts,opt); -} - static int node_table_init(struct node_table *t) @@ -3872,7 +3819,7 @@ fuse_new(struct fuse_chan *ch, if(fuse_opt_parse(args,&f.conf,fuse_lib_opts,fuse_lib_opt_proc) == -1) goto out_free_fs; - g_LOG_METRICS = f.conf.debug; + g_LOG_METRICS = fuse_cfg.debug; f.se = fuse_lowlevel_new_common(args,&llop,sizeof(llop),&f); if(f.se == NULL) diff --git a/libfuse/lib/fuse_cfg.cpp b/libfuse/lib/fuse_cfg.cpp new file mode 100644 index 00000000..01340252 --- /dev/null +++ b/libfuse/lib/fuse_cfg.cpp @@ -0,0 +1,21 @@ +#include "fuse_cfg.hpp" + +fuse_cfg_t fuse_cfg; + +bool +fuse_cfg_t::valid_uid() const +{ + return (uid != FUSE_CFG_INVALID_ID); +} + +bool +fuse_cfg_t::valid_gid() const +{ + return (gid != FUSE_CFG_INVALID_ID); +} + +bool +fuse_cfg_t::valid_umask() const +{ + return (umask != FUSE_CFG_INVALID_UMASK); +} diff --git a/libfuse/lib/fuse_config.cpp b/libfuse/lib/fuse_config.cpp deleted file mode 100644 index 5add4153..00000000 --- a/libfuse/lib/fuse_config.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - ISC License - - Copyright (c) 2023, Antonio SJ Musumeci - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ - -#include - -static int g_READ_THREAD_COUNT = -1; -static int g_PROCESS_THREAD_COUNT = -1; -static int g_PROCESS_THREAD_QUEUE_DEPTH = -1; -static std::string g_PIN_THREADS = {}; - - -int -fuse_config_get_read_thread_count() -{ - return g_READ_THREAD_COUNT; -} - -void -fuse_config_set_read_thread_count(int const v_) -{ - g_READ_THREAD_COUNT = v_; -} - -int -fuse_config_get_process_thread_count() -{ - return g_PROCESS_THREAD_COUNT; -} - -void -fuse_config_set_process_thread_count(int const v_) -{ - g_PROCESS_THREAD_COUNT = v_; -} - -int -fuse_config_get_process_thread_queue_depth() -{ - return g_PROCESS_THREAD_QUEUE_DEPTH; -} - -void -fuse_config_set_process_thread_queue_depth(int const v_) -{ - g_PROCESS_THREAD_QUEUE_DEPTH = v_; -} - -std::string -fuse_config_get_pin_threads() -{ - return g_PIN_THREADS; -} - -void -fuse_config_set_pin_threads(std::string const v_) -{ - g_PIN_THREADS = v_; -} diff --git a/libfuse/lib/fuse_i.h b/libfuse/lib/fuse_i.h index f1b4fa01..53d6896d 100644 --- a/libfuse/lib/fuse_i.h +++ b/libfuse/lib/fuse_i.h @@ -45,11 +45,6 @@ struct fuse_notify_req struct fuse_ll { - unsigned passthrough_max_stack_depth; - int debug; - int no_remote_posix_lock; - int no_remote_flock; - int big_writes; struct fuse_lowlevel_ops op; void *userdata; uid_t owner; diff --git a/libfuse/lib/fuse_loop.cpp b/libfuse/lib/fuse_loop.cpp index 5bfe85c9..599bd77f 100644 --- a/libfuse/lib/fuse_loop.cpp +++ b/libfuse/lib/fuse_loop.cpp @@ -14,7 +14,7 @@ #include "fuse_kernel.h" #include "fuse_lowlevel.h" -#include "fuse_config.hpp" +#include "fuse_cfg.hpp" #include "fuse_msgbuf.hpp" #include "fuse_ll.hpp" @@ -336,10 +336,10 @@ fuse_loop_mt(struct fuse *f_) fuse_populate_maintenance_thread(f_); res = fuse_session_loop_mt(fuse_get_session(), - fuse_config_get_read_thread_count(), - fuse_config_get_process_thread_count(), - fuse_config_get_process_thread_queue_depth(), - fuse_config_get_pin_threads()); + fuse_cfg.read_thread_count, + fuse_cfg.process_thread_count, + fuse_cfg.process_thread_queue_depth, + fuse_cfg.pin_threads); MaintenanceThread::stop(); diff --git a/libfuse/lib/fuse_lowlevel.cpp b/libfuse/lib/fuse_lowlevel.cpp index 2efb1554..3a352415 100644 --- a/libfuse/lib/fuse_lowlevel.cpp +++ b/libfuse/lib/fuse_lowlevel.cpp @@ -14,6 +14,7 @@ #include "lfmp.h" #include "debug.hpp" +#include "fuse_cfg.hpp" #include "fuse_i.h" #include "fuse_kernel.h" #include "fuse_msgbuf.hpp" @@ -43,7 +44,7 @@ #define OFFSET_MAX 0x7fffffffffffffffLL #define container_of(ptr, type, member) ({ \ - const decltype( ((type*)0)->member ) *__mptr = (ptr); \ + const decltype( ((type*)0)->member ) *__mptr = (ptr); \ (type *)( (char*)__mptr - offsetof(type,member) );}) static size_t pagesize; @@ -1117,11 +1118,17 @@ do_init(fuse_req_t *req, struct fuse_in_header *hdr_) { struct fuse_init_out outarg = {0}; - struct fuse_init_in *arg = (struct fuse_init_in *) &hdr_[1]; + struct fuse_init_in *arg = (struct fuse_init_in *)&hdr_[1]; struct fuse_ll *f = req->f; - size_t bufsize = fuse_chan_bufsize(req->ch); + size_t bufsize; uint64_t inargflags; uint64_t outargflags; + u32 max_write; + u32 max_readahead; + + max_write = UINT_MAX; + max_readahead = UINT_MAX; + bufsize = fuse_chan_bufsize(req->ch); inargflags = 0; outargflags = 0; @@ -1158,8 +1165,8 @@ do_init(fuse_req_t *req, if(inargflags & FUSE_INIT_EXT) inargflags |= (((uint64_t)arg->flags2) << 32); - if(arg->max_readahead < f->conn.max_readahead) - f->conn.max_readahead = arg->max_readahead; + if(arg->max_readahead < max_readahead) + max_readahead = arg->max_readahead; if(inargflags & FUSE_ASYNC_READ) f->conn.capable |= FUSE_CAP_ASYNC_READ; @@ -1209,15 +1216,15 @@ do_init(fuse_req_t *req, else { f->conn.want &= ~FUSE_CAP_ASYNC_READ; - f->conn.max_readahead = 0; + max_readahead = 0; } if(req->f->conn.proto_minor >= 18) f->conn.capable |= FUSE_CAP_IOCTL_DIR; - if(f->op.getlk && f->op.setlk && !f->no_remote_posix_lock) + if(f->op.getlk && f->op.setlk) f->conn.want |= FUSE_CAP_POSIX_LOCKS; - if(f->op.flock && !f->no_remote_flock) + if(f->op.flock) f->conn.want |= FUSE_CAP_FLOCK_LOCKS; if(bufsize < FUSE_MIN_READ_BUFFER) @@ -1228,8 +1235,8 @@ do_init(fuse_req_t *req, } bufsize -= pagesize; - if(bufsize < f->conn.max_write) - f->conn.max_write = bufsize; + if(bufsize < max_write) + max_write = bufsize; f->got_init = 1; if(f->op.init) @@ -1238,10 +1245,11 @@ do_init(fuse_req_t *req, outargflags = outarg.flags; if((inargflags & FUSE_MAX_PAGES) && (f->conn.want & FUSE_CAP_MAX_PAGES)) { - outargflags |= FUSE_MAX_PAGES; - outarg.max_pages = f->conn.max_pages; + outargflags |= FUSE_MAX_PAGES; + outarg.max_pages = fuse_cfg.max_pages; - msgbuf_set_bufsize(outarg.max_pages + 1); + msgbuf_set_bufsize(outarg.max_pages); + max_write = (msgbuf_get_pagesize() * outarg.max_pages); } if(f->conn.want & FUSE_CAP_ASYNC_READ) @@ -1288,7 +1296,7 @@ do_init(fuse_req_t *req, if(f->conn.want & FUSE_CAP_PASSTHROUGH) { outargflags |= FUSE_PASSTHROUGH; - outarg.max_stack_depth = f->passthrough_max_stack_depth; + outarg.max_stack_depth = fuse_cfg.passthrough_max_stack_depth; } if(inargflags & FUSE_INIT_EXT) @@ -1299,21 +1307,19 @@ do_init(fuse_req_t *req, outarg.flags = outargflags; - outarg.max_readahead = f->conn.max_readahead; - outarg.max_write = f->conn.max_write; + outarg.max_readahead = max_readahead; + outarg.max_write = max_write; if(f->conn.proto_minor >= 13) { - if(f->conn.max_background >= (1 << 16)) - f->conn.max_background = (1 << 16) - 1; - if(f->conn.congestion_threshold > f->conn.max_background) - f->conn.congestion_threshold = f->conn.max_background; - if(!f->conn.congestion_threshold) - { - f->conn.congestion_threshold = f->conn.max_background * 3 / 4; - } - - outarg.max_background = f->conn.max_background; - outarg.congestion_threshold = f->conn.congestion_threshold; + if(fuse_cfg.max_background >= (1 << 16)) + fuse_cfg.max_background = ((1 << 16) - 1); + if(fuse_cfg.congestion_threshold > fuse_cfg.max_background) + fuse_cfg.congestion_threshold = fuse_cfg.max_background; + if(!fuse_cfg.congestion_threshold) + fuse_cfg.congestion_threshold = (fuse_cfg.max_background * 3 / 4); + + outarg.max_background = fuse_cfg.max_background; + outarg.congestion_threshold = fuse_cfg.congestion_threshold; } if(f->conn.proto_minor >= 23) @@ -1780,86 +1786,6 @@ fuse_ll_funcs[FUSE_OPCODE_LEN] = #define FUSE_MAXOPS (sizeof(fuse_ll_funcs) / sizeof(fuse_ll_funcs[0])) -enum { - KEY_HELP, - KEY_VERSION, -}; - -static const struct fuse_opt fuse_ll_opts[] = - { - { "debug", offsetof(struct fuse_ll, debug), 1 }, - { "-d", offsetof(struct fuse_ll, debug), 1 }, - { "max_readahead=%u", offsetof(struct fuse_ll, conn.max_readahead), 0 }, - { "max_background=%u", offsetof(struct fuse_ll, conn.max_background), 0 }, - { "congestion_threshold=%u", - offsetof(struct fuse_ll, conn.congestion_threshold), 0 }, - { "no_remote_lock", offsetof(struct fuse_ll, no_remote_posix_lock), 1}, - { "no_remote_lock", offsetof(struct fuse_ll, no_remote_flock), 1}, - { "no_remote_flock", offsetof(struct fuse_ll, no_remote_flock), 1}, - { "no_remote_posix_lock", offsetof(struct fuse_ll, no_remote_posix_lock), 1}, - { "passthrough-max-stack-depth=%u", offsetof(struct fuse_ll, passthrough_max_stack_depth), 0}, - FUSE_OPT_KEY("max_read=", FUSE_OPT_KEY_DISCARD), - FUSE_OPT_KEY("-h", KEY_HELP), - FUSE_OPT_KEY("--help", KEY_HELP), - FUSE_OPT_KEY("-V", KEY_VERSION), - FUSE_OPT_KEY("--version", KEY_VERSION), - FUSE_OPT_END - }; - -static -void -fuse_ll_version(void) -{ - fprintf(stderr, "using FUSE kernel interface version %i.%i\n", - FUSE_KERNEL_VERSION, FUSE_KERNEL_MINOR_VERSION); -} - -static -void -fuse_ll_help(void) -{ - fprintf(stderr, - " -o max_readahead=N set maximum readahead\n" - " -o max_background=N set number of maximum background requests\n" - " -o congestion_threshold=N set kernel's congestion threshold\n" - " -o no_remote_lock disable remote file locking\n" - " -o no_remote_flock disable remote file locking (BSD)\n" - " -o no_remote_posix_lock disable remove file locking (POSIX)\n" - ); -} - -static -int -fuse_ll_opt_proc(void *data, - const char *arg, - int key, - struct fuse_args *outargs) -{ - (void) data; (void) outargs; - - switch (key) - { - case KEY_HELP: - fuse_ll_help(); - break; - - case KEY_VERSION: - fuse_ll_version(); - break; - - default: - fprintf(stderr, "fuse: unknown option `%s'\n", arg); - } - - return -1; -} - -int -fuse_lowlevel_is_lib_option(const char *opt) -{ - return fuse_opt_match(fuse_ll_opts, opt); -} - static void fuse_ll_destroy(void *data) @@ -2032,6 +1958,23 @@ fuse_ll_buf_process_read_init(struct fuse_session *se_, return; } +static const struct fuse_opt fuse_ll_opts[] = + { + FUSE_OPT_END + }; + +static +int +fuse_ll_opt_proc(void *data_, + const char *arg_, + int key_, + struct fuse_args *outargs_) +{ + fmt::print(stderr, "* ERROR: unknown option '{}'\n", arg_); + + return -1; +} + /* * always call fuse_lowlevel_new_common() internally, to work around a * misfeature in the FreeBSD runtime linker, which links the old @@ -2060,9 +2003,6 @@ fuse_lowlevel_new_common(struct fuse_args *args, goto out; } - f->conn.max_write = UINT_MAX; - f->conn.max_readahead = UINT_MAX; - f->passthrough_max_stack_depth = 2; list_init_nreq(&f->notify_list); f->notify_ctr = 1; mutex_init(&f->lock); @@ -2075,7 +2015,7 @@ fuse_lowlevel_new_common(struct fuse_args *args, goto out_free; } - if(fuse_opt_parse(args, f, fuse_ll_opts, fuse_ll_opt_proc) == -1) + if(fuse_opt_parse(args,NULL,fuse_ll_opts,fuse_ll_opt_proc) == -1) goto out_key_destroy; memcpy(&f->op, op, op_size); diff --git a/libfuse/lib/fuse_msgbuf.cpp b/libfuse/lib/fuse_msgbuf.cpp index 5803fd36..be35ab98 100644 --- a/libfuse/lib/fuse_msgbuf.cpp +++ b/libfuse/lib/fuse_msgbuf.cpp @@ -16,6 +16,7 @@ */ #include "fuse_msgbuf.hpp" + #include "fuse.h" #include "fuse_kernel.h" @@ -42,22 +43,34 @@ msgbuf_get_bufsize() return g_BUFSIZE; } +u32 +msgbuf_get_pagesize() +{ + return g_PAGESIZE; +} + +// +1 page so that it is used for the "header" of the allocation as +// well as to allow for offseting for write requests to be page +// aligned. +1 again for fuse header as the max_pages value is for the +// body. void msgbuf_set_bufsize(const uint32_t size_in_pages_) { - g_BUFSIZE = ((size_in_pages_ + 1) * g_PAGESIZE); + g_BUFSIZE = ((size_in_pages_ + 2) * g_PAGESIZE); } +static void -msgbuf_page_align(fuse_msgbuf_t *msgbuf_) +_msgbuf_page_align(fuse_msgbuf_t *msgbuf_) { msgbuf_->mem = (char*)msgbuf_; msgbuf_->mem += g_PAGESIZE; msgbuf_->size = (g_BUFSIZE - g_PAGESIZE); } +static void -msgbuf_write_align(fuse_msgbuf_t *msgbuf_) +_msgbuf_write_align(fuse_msgbuf_t *msgbuf_) { msgbuf_->mem = (char*)msgbuf_; msgbuf_->mem += g_PAGESIZE; @@ -72,8 +85,8 @@ void _msgbuf_constructor() { g_PAGESIZE = sysconf(_SC_PAGESIZE); - // FUSE_MAX_MAX_PAGES for payload + 1 for message header - msgbuf_set_bufsize(FUSE_DEFAULT_MAX_MAX_PAGES + 1); + + msgbuf_set_bufsize(FUSE_DEFAULT_MAX_MAX_PAGES); } static @@ -86,7 +99,7 @@ _msgbuf_destructor() static void* -page_aligned_malloc(const uint64_t size_) +_page_aligned_malloc(const uint64_t size_) { int rv; void *buf = NULL; @@ -111,7 +124,7 @@ _msgbuf_alloc(msgbuf_setup_func_t setup_func_) { g_MUTEX.unlock(); - msgbuf = (fuse_msgbuf_t*)page_aligned_malloc(g_BUFSIZE); + msgbuf = (fuse_msgbuf_t*)_page_aligned_malloc(g_BUFSIZE); if(msgbuf == NULL) return NULL; @@ -135,13 +148,13 @@ _msgbuf_alloc(msgbuf_setup_func_t setup_func_) fuse_msgbuf_t* msgbuf_alloc() { - return _msgbuf_alloc(msgbuf_write_align); + return _msgbuf_alloc(_msgbuf_write_align); } fuse_msgbuf_t* msgbuf_alloc_page_aligned() { - return _msgbuf_alloc(msgbuf_page_align); + return _msgbuf_alloc(_msgbuf_page_align); } static diff --git a/libfuse/lib/mount_generic.c b/libfuse/lib/mount_generic.c index 06707868..cb233944 100644 --- a/libfuse/lib/mount_generic.c +++ b/libfuse/lib/mount_generic.c @@ -191,36 +191,37 @@ static int fuse_mount_opt_proc(void *data, const char *arg, int key, { struct mount_opts *mo = data; - switch (key) { - case KEY_RO: - arg = "ro"; - /* fall through */ - case KEY_KERN_FLAG: - set_mount_flag(arg, &mo->flags); - return 0; - - case KEY_KERN_OPT: - return fuse_opt_add_opt(&mo->kernel_opts, arg); - - case KEY_FUSERMOUNT_OPT: - return fuse_opt_add_opt_escaped(&mo->fusermount_opts, arg); - - case KEY_SUBTYPE_OPT: - return fuse_opt_add_opt(&mo->subtype_opt, arg); - - case KEY_MTAB_OPT: - return fuse_opt_add_opt(&mo->mtab_opts, arg); - - case KEY_HELP: - mount_help(); - mo->ishelp = 1; - break; - - case KEY_VERSION: - mount_version(); - mo->ishelp = 1; - break; - } + switch (key) + { + case KEY_RO: + arg = "ro"; + /* fall through */ + case KEY_KERN_FLAG: + set_mount_flag(arg, &mo->flags); + return 0; + + case KEY_KERN_OPT: + return fuse_opt_add_opt(&mo->kernel_opts, arg); + + case KEY_FUSERMOUNT_OPT: + return fuse_opt_add_opt_escaped(&mo->fusermount_opts, arg); + + case KEY_SUBTYPE_OPT: + return fuse_opt_add_opt(&mo->subtype_opt, arg); + + case KEY_MTAB_OPT: + return fuse_opt_add_opt(&mo->mtab_opts, arg); + + case KEY_HELP: + mount_help(); + mo->ishelp = 1; + break; + + case KEY_VERSION: + mount_version(); + mo->ishelp = 1; + break; + } return 1; } @@ -542,8 +543,8 @@ int fuse_kern_mount(const char *mountpoint, struct fuse_args *args) memset(&mo, 0, sizeof(mo)); mo.flags = MS_NOSUID | MS_NODEV; - if (args && - fuse_opt_parse(args, &mo, fuse_mount_opts, fuse_mount_opt_proc) == -1) + if(args && + fuse_opt_parse(args, &mo, fuse_mount_opts, fuse_mount_opt_proc) == -1) return -1; res = 0; diff --git a/mkdocs/docs/config/options.md b/mkdocs/docs/config/options.md index 2c794095..3e299a57 100644 --- a/mkdocs/docs/config/options.md +++ b/mkdocs/docs/config/options.md @@ -130,11 +130,11 @@ config file. * **[fuse_msg_size](fuse_msg_size.md)=PAGESIZE**: Set the max number of pages per FUSE message. Only available on Linux >= 4.20 and ignored otherwise. (min: 1; max: 65535; default: "1M") -* **[threads](threads.md)=INT**: Number of threads to use. When used - alone (`process-thread-count=-1`) it sets the number of threads - reading and processing FUSE messages. When used together it sets the - number of threads reading from FUSE. When set to zero it will - attempt to discover and use the number of logical cores. If the +* **[read-thread-count](threads.md)=INT**: Number of threads to + use. When used alone (`process-thread-count=-1`) it sets the number + of threads reading and processing FUSE messages. When used together + it sets the number of threads reading from FUSE. When set to zero it + will attempt to discover and use the number of logical cores. If the thread count is set negative it will look up the number of cores then divide by the absolute value. ie. threads=-2 on an 8 core machine will result in 8 / 2 = 4 threads. There will always be at @@ -142,7 +142,6 @@ config file. `process-thread-count` then it will try to pick reasonable values based on CPU thread count. NOTE: higher number of threads increases parallelism but usually decreases throughput. (default: 0) -* **[read-thread-count](threads.md)=INT**: Alias for `threads`. * **[process-thread-count](threads.md)=INT**: Enables separate thread pool to asynchronously process FUSE requests. In this mode `read-thread-count` refers to the number of threads reading FUSE @@ -153,7 +152,7 @@ config file. time. Meaning the total memory usage of the queues is queue depth multiplied by the number of process threads plus read thread count. 0 sets the depth to the same as the process thread - count. (default: 0) + count. (default: 2) * **[pin-threads](pin-threads.md)=STR**: Selects a strategy to pin threads to CPUs (default: unset) * **[flush-on-close](flush-on-close.md)=never|always|opened-for-write**: @@ -206,15 +205,20 @@ config file. with `cache.files=per-process` (if the process is not in `process-names`) or `cache.files=off`. (Is a kernel feature added in v6.2) (default: true) -* **[passthrough](passthrough.md)**: Enable [FUSE IO +* **[passthrough](passthrough.md)=ENUM**: Enable [FUSE IO passthrough](https://kernelnewbies.org/Linux_6.9#Faster_FUSE_I.2FO) if available. (default: off) -* **gid-cache-expire-timeout**: Number of seconds till supplemental +* **gid-cache-expire-timeout=INT**: Number of seconds till supplemental group data is refreshed in the [GID cache](../known_issues_bugs.md#supplemental-user-groups). (default: 3600) -* **gid-cache-remove-timeout**: Number of seconds to wait till cached +* **gid-cache-remove-timeout=INT**: Number of seconds to wait till cached data is removed due to lack of usage. (default: 43200) +* **remember-nodes=INT**: The number of seconds to keep the internal + representation of a file once the OS tells mergerfs it is no longer + needed. Really only needed for [exporting mergerfs via + NFS](../remote_filesystems.md) (default: 0) +* **noforget**: Effectively sets `remember-nodes` to infinity. **NOTE:** Options are evaluated in the order listed so if the options are **func.rmdir=rand,category.action=ff** the **action** category diff --git a/mkdocs/docs/faq/configuration_and_policies.md b/mkdocs/docs/faq/configuration_and_policies.md index 78fcd429..3583ec56 100644 --- a/mkdocs/docs/faq/configuration_and_policies.md +++ b/mkdocs/docs/faq/configuration_and_policies.md @@ -18,37 +18,71 @@ mergerfs [does not impact](usage_and_functionality.md) the underlying filesystems and can be added or removed without any impact it is extremely easy to test and experiment with different settings. +Additional reading: + +* [Config/Options](../config/options.md) + + +## What create policy should I use? + +If it isn't clear to you then you should use `pfrd` as described in +the [quickstart guide](../quickstart.md). + ## Why is pfrd the default create policy? Originally the default was `epmfs` however it was found to cause -significant confusion among new users who expected mergerfs to always -choose a branch with available space. For mergerfs v2.41.0 it was -decided to change the default to the less restrictive policy `pfrd`. +significant confusion among new users who assumed mergerfs to behave +like other software which always choose a branch with most available +space without any filtering or restrictions. The reason for using +`epmfs` was to allow people to use mergerfs without impacting the +layout of their data across the branches. Keeping the total entropy of +the system in check. This, however, was not understood by users who +did not read the documentation and caused significant support burdens. + +For mergerfs v2.41.0 it was decided to change the default to `pfrd` to +minimize the support burden. + +`pfrd` was chosen because it prioritizes placement to branches based +on free space (percentage wise) without overloading a specific branch +as `mfs`, `lus`, or other policies could when a singular branch has +significantly more free space. Since it is impossible to know the +usage and access patterns of the filesystem spreading out files across +the branches also spreads the load and therefore good for general use +cases making it a good default. + +Additional reading: + +* [functions, categories and policies](../config/functions_categories_policies.md) ## How can I ensure files are collocated on the same branch? Many people like the idea of ensuring related files, such as all the files to a TV show season or songs in an album, are stored on the same -storage device. However, most people have no actual need for this +storage device. However, most people have no technical need for this behavior. 1. If you backup your data it is extremely likely your backup solution - can restore only those files you are missing. -2. Software such as **Sonarr** can manage the downloading and post - processing of bespoke episodes which may be missing in a - season. Either by downloading the episode individually if available - or by downloading a full season. + can restore only those files you are missing. There is no need to + store data in related units to ensure restoring data is easy. +2. Software such as [Sonarr](https://sonarr.tv) can manage the + downloading and post processing of bespoke files / episodes which + may be missing. Either by downloading the episode individually if + available or by downloading a full season. 3. There is no benefit to keeping files collocated with regard to - drive spinup, caching, or other secondary concern. + drive spinup, caching, or other secondary concern. In fact + depending on usage patterns placing the data on the same underlying + device could harm performance. The main use case for wanting collocation is where the branch is going -to be removed from the pool and you wish to have all data from some -logical set on that device. Such as you intend to take a drive out of -the pool to take on a trip and want a whole show on the -drive. However, even in these situations you typically end up needing -to curate the files anyway because it has show A but not show B. +to be removed from the pool and used in isolation temporarily and you +wish to have all data from some logical set on that device. Such as +you intend to take a drive out of the pool to take on a trip and want +a whole show or artist's works on the drive. However, even in these +situations you typically end up needing to curate the files anyway +because it has show A but not show B. In such cases it is more +convient to use a separate devices and sync what you wish to it. All that said you can accomplish collocation to varying degrees using the following methods: @@ -56,33 +90,37 @@ the following methods: 1. Use [mergerfs.consolidate](https://github.com/trapexit/mergerfs-tools/blob/master/src/mergerfs.consolidate) when consolidation is needed. -2. Use a `msp` create policy. +2. Use a `msp` create policy to do a best effort collocation without + needing explicit directory curation. 3. Use `epmfs` or other `ep` create policy and manually create paths - on the branches directly. -4. Use a `ep` `create` policy and `rand` for `mkdir`. + on the branches directly as you need/desire. +4. Use a `ep` `create` policy and `pfrd` or `rand` for `mkdir`. ## How can I balance files across the pool? Similar to collocation there is generally little reason to balance -files. +files across the pool branches. -1. Since prediction of a filesystem's death or loss of data is near - impossible there is little reason to balance in hopes of limiting - data loss. +1. Since prediction of a device's/filesystem's death or loss of data + is near impossible there is little reason to balance in hopes of + limiting data loss. 2. While performance could be impacted by having too much reading or writing happen to singular underlying filesystems balancing won't help unless you have the ability to manage the access patterns to the pool. -3. Over time most configurations will lead to a random distribution of - files across the branches which is effectively "balancing." +3. Over time the churn of adding and removing files will lead to a + random distribution of files across the branches which is + effectively "balancing" if using create policies which do not favor + any particular branch. -If you wish to move files around or balance the pool you can: +All that said if you wish to move files around or balance the pool you +can: 1. Use `rand` or `pfrd` create policies and just use your system as normal. -2. Write simple scripts using rsync or similar to move files around as - you wish. +2. Write scripts using rsync or similar to move files around as you + wish. 3. Use [mergerfs.balance](https://github.com/trapexit/mergerfs-tools/blob/master/src/mergerfs.balance). Keep in mind that this tool is really just an example of how to diff --git a/mkdocs/docs/remote_filesystems.md b/mkdocs/docs/remote_filesystems.md index 6ed9b1ee..a5d51e92 100644 --- a/mkdocs/docs/remote_filesystems.md +++ b/mkdocs/docs/remote_filesystems.md @@ -76,19 +76,21 @@ but if you run into problems it may be worth trying Samba/SMB. **mergerfs settings:** -* `noforget` +* `noforget` or `remember-nodes=SECONDS` * `inodecalc=path-hash` * `lazy-umount-mountpoint=false` -`noforget` is needed because NFS uses the `name_to_handle_at` and +`noforget` (or `remember-nodes` with a large value such as `86400` for +a full day) is needed because NFS uses the `name_to_handle_at` and `open_by_handle_at` functions which allow a program to keep a reference to a file without technically having it open in the typical sense. The problem is that FUSE has no way to know that NFS has a handle that it will later use to open the file again. As a result, it is possible for the kernel to tell mergerfs to forget about the file node and should NFS ever ask for that node's details in the future it -would have nothing to respond with. Keeping nodes around forever is -not ideal but at the moment the only way to manage the situation. +would have nothing to respond with. Keeping nodes around forever (or +however long is set with `remember-nodes`) is not ideal but at the +moment the only way to manage the situation. `inodecalc=path-hash` is needed because NFS is sensitive to out-of-band changes. FUSE doesn't care if a file's inode value changes diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml index 50804255..70bd8571 100644 --- a/mkdocs/mkdocs.yml +++ b/mkdocs/mkdocs.yml @@ -108,9 +108,9 @@ nav: - extended_usage_patterns.md - FAQ: - faq/why_isnt_it_working.md - - faq/reliability_and_scalability.md - - faq/usage_and_functionality.md - faq/configuration_and_policies.md + - faq/usage_and_functionality.md + - faq/reliability_and_scalability.md - faq/compatibility_and_integration.md - faq/recommendations_and_warnings.md - faq/technical_behavior_and_limitations.md diff --git a/src/config.cpp b/src/config.cpp index 3d904ee0..ae9039e3 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -39,109 +39,126 @@ #define MINFREESPACE_DEFAULT (4294967295ULL) -#define IFERT(S) if(S == s_) return true - constexpr static const char CACHE_FILES_PROCESS_NAMES_DEFAULT[] = "rtorrent|" "qbittorrent-nox"; Config cfg; -namespace l -{ - static - bool - readonly(const std::string &s_) - { - IFERT("allow-idmap"); - IFERT("async_read"); - IFERT("branches-mount-timeout"); - IFERT("branches-mount-timeout-fail"); - IFERT("cache.symlinks"); - IFERT("cache.writeback"); - IFERT("direct-io-allow-mmap"); - IFERT("export-support"); - IFERT("fsname"); - IFERT("fuse_msg_size"); - IFERT("kernel-permissions-check"); - IFERT("mount"); - IFERT("nullrw"); - IFERT("pid"); - IFERT("pin-threads"); - IFERT("process-thread-count"); - IFERT("process-thread-queue-depth"); - IFERT("read-thread-count"); - IFERT("readdirplus"); - IFERT("scheduling-priority"); - IFERT("srcmounts"); - IFERT("threads"); - IFERT("version"); - - return false; - } -} - Config::Config() - : allow_idmap(false), - async_read(true), - auto_cache(false), - minfreespace(MINFREESPACE_DEFAULT), - branches(minfreespace), - branches_mount_timeout(0), - branches_mount_timeout_fail(false), - cache_attr(1), - cache_entry(1), - cache_files(CacheFiles::ENUM::OFF), - cache_files_process_names(CACHE_FILES_PROCESS_NAMES_DEFAULT), - cache_negative_entry(0), - cache_readdir(false), - cache_statfs(0), - cache_symlinks(false), - category(func), - direct_io(false), - direct_io_allow_mmap(true), - dropcacheonclose(false), - export_support(true), - flushonclose(FlushOnClose::ENUM::OPENED_FOR_WRITE), - follow_symlinks(FollowSymlinks::ENUM::NEVER), - fsname(), - func(), - fuse_msg_size("1M"), - gid_cache_expire_timeout(60 * 60), - gid_cache_remove_timeout(60 * 60 * 12), - ignorepponrename(false), - inodecalc("hybrid-hash"), - lazy_umount_mountpoint(false), - link_cow(false), - link_exdev(LinkEXDEV::ENUM::PASSTHROUGH), - log_metrics(false), - mountpoint(), - moveonenospc(true), - nfsopenhack(NFSOpenHack::ENUM::OFF), - nullrw(false), - parallel_direct_writes(true), - posix_acl(false), - proxy_ioprio(false), - readahead(0), - readdir("seq"), - readdirplus(false), - rename_exdev(RenameEXDEV::ENUM::PASSTHROUGH), - scheduling_priority(-10), - security_capability(true), - srcmounts(branches), - statfs(StatFS::ENUM::BASE), - statfs_ignore(StatFSIgnore::ENUM::NONE), - symlinkify(false), - symlinkify_timeout(3600), - fuse_read_thread_count(0), - fuse_process_thread_count(-1), - fuse_process_thread_queue_depth(2), - fuse_pin_threads("false"), - version(MERGERFS_VERSION), - writeback_cache(false), - xattr(XAttr::ENUM::PASSTHROUGH), - _initialized(false) + : + allow_idmap(false), + async_read(true), + auto_cache(false), + branches(minfreespace), + branches_mount_timeout(0), + branches_mount_timeout_fail(false), + cache_attr(1), + cache_entry(1), + cache_files(CacheFiles::ENUM::OFF), + cache_files_process_names(CACHE_FILES_PROCESS_NAMES_DEFAULT), + cache_negative_entry(0), + cache_readdir(false), + cache_statfs(0), + cache_symlinks(false), + cache_writeback(false), + category(func), + congestion_threshold(fuse_cfg.congestion_threshold,0), + debug(fuse_cfg.debug,false), + direct_io(false), + direct_io_allow_mmap(true), + dropcacheonclose(false), + export_support(true), + flushonclose(FlushOnClose::ENUM::OPENED_FOR_WRITE), + follow_symlinks(FollowSymlinks::ENUM::NEVER), + fsname(), + func(), + fuse_msg_size("1M"), + gid(fuse_cfg.gid,FUSE_CFG_INVALID_ID), + gid_cache_expire_timeout(60 * 60), + gid_cache_remove_timeout(60 * 60 * 12), + handle_killpriv(true), + handle_killpriv_v2(true), + ignorepponrename(false), + inodecalc("hybrid-hash"), + kernel_cache(false), + kernel_permissions_check(true), + lazy_umount_mountpoint(false), + link_cow(false), + link_exdev(LinkEXDEV::ENUM::PASSTHROUGH), + log_metrics(false), + max_background(fuse_cfg.max_background,0), + minfreespace(MINFREESPACE_DEFAULT), + mountpoint(), + moveonenospc(true), + nfsopenhack(NFSOpenHack::ENUM::OFF), + nullrw(false), + parallel_direct_writes(true), + passthrough(Passthrough::ENUM::OFF), + passthrough_max_stack_depth(fuse_cfg.passthrough_max_stack_depth,1), + pin_threads(fuse_cfg.pin_threads,"false"), + posix_acl(false), + process_thread_count(fuse_cfg.process_thread_count,-1), + process_thread_queue_depth(fuse_cfg.process_thread_queue_depth,2), + proxy_ioprio(false), + read_thread_count(fuse_cfg.read_thread_count,0), + readahead(0), + readdir("seq"), + readdirplus(false), + remember_nodes(fuse_cfg.remember_nodes,0), + rename_exdev(RenameEXDEV::ENUM::PASSTHROUGH), + scheduling_priority(-10), + security_capability(true), + srcmounts(branches), + statfs(StatFS::ENUM::BASE), + statfs_ignore(StatFSIgnore::ENUM::NONE), + symlinkify(false), + symlinkify_timeout(3600), + threads(fuse_cfg.read_thread_count), + uid(fuse_cfg.uid,FUSE_CFG_INVALID_ID), + umask(fuse_cfg.umask,FUSE_CFG_INVALID_UMASK), + version(MERGERFS_VERSION), + xattr(XAttr::ENUM::PASSTHROUGH), + _initialized(false) { + allow_idmap.ro = + async_read.ro = + branches_mount_timeout.ro = + branches_mount_timeout_fail.ro = + cache_symlinks.ro = + cache_writeback.ro = + congestion_threshold.ro = + direct_io_allow_mmap.ro = + export_support.ro = + fsname.ro = + fuse_msg_size.ro = + handle_killpriv.ro = + handle_killpriv_v2.ro = + kernel_permissions_check.ro = + max_background.ro = + mountpoint.ro = + nullrw.ro = + pid.ro = + pin_threads.ro = + process_thread_count.ro = + process_thread_queue_depth.ro = + read_thread_count.ro = + scheduling_priority.ro = + srcmounts.ro = + version.ro = + true; + auto_cache.display = + congestion_threshold.ro = + direct_io.display = + gid.display = + kernel_cache.display = + max_background.ro = + readdirplus.display = + threads.display = + uid.display = + umask.display = + false; + _map["allow-idmap"] = &allow_idmap; _map["async_read"] = &async_read; _map["auto_cache"] = &auto_cache; @@ -156,12 +173,13 @@ Config::Config() _map["cache.readdir"] = &cache_readdir; _map["cache.statfs"] = &cache_statfs; _map["cache.symlinks"] = &cache_symlinks; - _map["cache.writeback"] = &writeback_cache; + _map["cache.writeback"] = &cache_writeback; _map["category.action"] = &category.action; _map["category.create"] = &category.create; _map["category.search"] = &category.search; - _map["direct_io"] = &direct_io; + _map["debug"] = &debug; _map["direct-io-allow-mmap"] = &direct_io_allow_mmap; + _map["direct_io"] = &direct_io; _map["dropcacheonclose"] = &dropcacheonclose; _map["export-support"] = &export_support; _map["flush-on-close"] = &flushonclose; @@ -189,31 +207,37 @@ Config::Config() _map["func.unlink"] = &func.unlink; _map["func.utimens"] = &func.utimens; _map["fuse_msg_size"] = &fuse_msg_size; + _map["gid"] = &gid; _map["gid-cache-expire-timeout"] = &gid_cache_expire_timeout; _map["gid-cache-remove-timeout"] = &gid_cache_remove_timeout; _map["handle-killpriv"] = &handle_killpriv; _map["handle-killpriv-v2"] = &handle_killpriv_v2; _map["ignorepponrename"] = &ignorepponrename; _map["inodecalc"] = &inodecalc; - _map["kernel_cache"] = &kernel_cache; _map["kernel-permissions-check"] = &kernel_permissions_check; + _map["kernel_cache"] = &kernel_cache; _map["lazy-umount-mountpoint"] = &lazy_umount_mountpoint; - _map["link_cow"] = &link_cow; _map["link-exdev"] = &link_exdev; + _map["link_cow"] = &link_cow; _map["log.metrics"] = &log_metrics; _map["minfreespace"] = &minfreespace; _map["mount"] = &mountpoint; _map["moveonenospc"] = &moveonenospc; _map["nfsopenhack"] = &nfsopenhack; _map["nullrw"] = &nullrw; + _map["parallel-direct-writes"] = ¶llel_direct_writes; _map["passthrough"] = &passthrough; + _map["passthrough-max-stack-depth"] = &passthrough_max_stack_depth; _map["pid"] = &pid; - _map["parallel-direct-writes"] = ¶llel_direct_writes; - _map["pin-threads"] = &fuse_pin_threads; + _map["pin-threads"] = &pin_threads; _map["posix_acl"] = &posix_acl; + _map["process-thread-count"] = &process_thread_count; + _map["process-thread-queue-depth"] = &process_thread_queue_depth; _map["proxy-ioprio"] = &proxy_ioprio; + _map["read-thread-count"] = &read_thread_count; _map["readahead"] = &readahead; _map["readdirplus"] = &readdirplus; + _map["remember-nodes"] = &remember_nodes; _map["rename-exdev"] = &rename_exdev; _map["scheduling-priority"] = &scheduling_priority; _map["security_capability"] = &security_capability; @@ -222,10 +246,9 @@ Config::Config() _map["statfs_ignore"] = &statfs_ignore; _map["symlinkify"] = &symlinkify; _map["symlinkify_timeout"] = &symlinkify_timeout; - _map["threads"] = &fuse_read_thread_count; - _map["read-thread-count"] = &fuse_read_thread_count; - _map["process-thread-count"] = &fuse_process_thread_count; - _map["process-thread-queue-depth"] = &fuse_process_thread_queue_depth; + _map["threads"] = &threads; + _map["uid"] = &uid; + _map["umask"] = &umask; _map["version"] = &version; _map["xattr"] = &xattr; } @@ -275,6 +298,9 @@ Config::keys_xattr(std::string &s_) const s_.reserve(1024); for(const auto &[key,val] : _map) { + if(val->display == false) + continue; + s_ += "user.mergerfs."; s_ += key; s_ += '\0'; @@ -289,6 +315,8 @@ Config::keys_listxattr_size() const rv = 0; for(const auto &[key,val] : _map) { + if(val->display == false) + continue; rv += sizeof("user.mergerfs."); rv += key.size(); } @@ -308,6 +336,9 @@ Config::keys_listxattr(char *list_, for(const auto &[key,val] : _map) { + if(val->display == false) + continue; + auto rv = fmt::format_to_n(list,size, "user.mergerfs.{}\0", key); @@ -336,26 +367,18 @@ Config::get(const std::string &key_, } int -Config::set_raw(const std::string &key_, - const std::string &value_) +Config::set(const std::string &key_, + const std::string &value_) { Str2TFStrMap::iterator i; i = _map.find(key_); if(i == _map.end()) return -ENOATTR; - - return i->second->from_string(value_); -} - -int -Config::set(const std::string &key_, - const std::string &value_) -{ - if(_initialized && l::readonly(key_)) + if(_initialized && i->second->ro) return -EROFS; - return set_raw(key_,value_); + return i->second->from_string(value_); } int diff --git a/src/config.hpp b/src/config.hpp index 8f3313d9..ee171bb6 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -16,6 +16,8 @@ #pragma once +#include "fuse_cfg.hpp" +#include "tofrom_ref.hpp" #include "branches.hpp" #include "category.hpp" #include "config_cachefiles.hpp" @@ -55,13 +57,14 @@ #include -typedef ToFromWrapper ConfigBOOL; -typedef ToFromWrapper ConfigUINT64; -typedef ToFromWrapper ConfigS64; -typedef ToFromWrapper ConfigINT; -typedef ToFromWrapper ConfigSTR; -typedef ToFromWrapper ConfigPath; -typedef std::map Str2TFStrMap; +typedef ToFromWrapper ConfigBOOL; +typedef ToFromWrapper ConfigUINT64; +typedef ToFromWrapper ConfigS64; +typedef ToFromWrapper ConfigINT; +typedef ToFromWrapper ConfigSTR; +typedef ToFromWrapper ConfigPath; +typedef std::map Str2TFStrMap; +typedef ROToFromWrapper ConfigROSTR; extern const std::string CONTROLFILE; @@ -86,7 +89,6 @@ public: ConfigBOOL allow_idmap; ConfigBOOL async_read; ConfigBOOL auto_cache; - ConfigUINT64 minfreespace; Branches branches; ConfigUINT64 branches_mount_timeout; ConfigBOOL branches_mount_timeout_fail; @@ -98,7 +100,10 @@ public: ConfigBOOL cache_readdir; ConfigUINT64 cache_statfs; ConfigBOOL cache_symlinks; + ConfigBOOL cache_writeback; Categories category; + TFSRef congestion_threshold; + TFSRef debug; ConfigBOOL direct_io; ConfigBOOL direct_io_allow_mmap; ConfigBOOL dropcacheonclose; @@ -108,30 +113,39 @@ public: ConfigSTR fsname; Funcs func; ConfigPageSize fuse_msg_size; + TFSRef gid; GIDCacheExpireTimeout gid_cache_expire_timeout; GIDCacheRemoveTimeout gid_cache_remove_timeout; - ConfigBOOL handle_killpriv = true; - ConfigBOOL handle_killpriv_v2 = true; + ConfigBOOL handle_killpriv; + ConfigBOOL handle_killpriv_v2; ConfigBOOL ignorepponrename; InodeCalc inodecalc; ConfigBOOL kernel_cache; - ConfigBOOL kernel_permissions_check = true; + ConfigBOOL kernel_permissions_check; ConfigBOOL lazy_umount_mountpoint; ConfigBOOL link_cow; LinkEXDEV link_exdev; LogMetrics log_metrics; + TFSRef max_background; + ConfigUINT64 minfreespace; ConfigPath mountpoint; MoveOnENOSPC moveonenospc; NFSOpenHack nfsopenhack; ConfigBOOL nullrw; - Passthrough passthrough = Passthrough::ENUM::OFF; ConfigBOOL parallel_direct_writes; + Passthrough passthrough; + TFSRef passthrough_max_stack_depth; ConfigGetPid pid; + TFSRef pin_threads; ConfigBOOL posix_acl; + TFSRef process_thread_count; + TFSRef process_thread_queue_depth; ProxyIOPrio proxy_ioprio; + TFSRef read_thread_count; ConfigUINT64 readahead; FUSE::ReadDir readdir; ConfigBOOL readdirplus; + TFSRef remember_nodes; RenameEXDEV rename_exdev; ConfigINT scheduling_priority; ConfigBOOL security_capability; @@ -140,12 +154,10 @@ public: StatFSIgnore statfs_ignore; ConfigBOOL symlinkify; ConfigS64 symlinkify_timeout; - ConfigINT fuse_read_thread_count; - ConfigINT fuse_process_thread_count; - ConfigINT fuse_process_thread_queue_depth; - ConfigSTR fuse_pin_threads; - ConfigSTR version; - ConfigBOOL writeback_cache; + TFSRef threads; + TFSRef uid; + TFSRef umask; + ConfigROSTR version; XAttr xattr; private: @@ -167,7 +179,6 @@ public: public: int get(const std::string &key, std::string *val) const; - int set_raw(const std::string &key, const std::string &val); int set(const std::string &key, const std::string &val); int set(const std::string &kv); @@ -185,10 +196,6 @@ public: private: Str2TFStrMap _map; - -public: - friend class Read; - friend class Write; }; std::ostream& operator<<(std::ostream &s,const Config::ErrVec &ev); diff --git a/src/fuse_create.cpp b/src/fuse_create.cpp index f1be33ee..998c1746 100644 --- a/src/fuse_create.cpp +++ b/src/fuse_create.cpp @@ -46,7 +46,7 @@ */ static void -_tweak_flags_writeback_cache(int *flags_) +_tweak_flags_cache_writeback(int *flags_) { if((*flags_ & O_ACCMODE) == O_WRONLY) *flags_ = ((*flags_ & ~O_ACCMODE) | O_RDWR); @@ -234,8 +234,8 @@ _create_for_insert_lambda(const fuse_req_ctx_t *ctx_, const ugid::Set ugid(ctx_->uid,ctx_->gid); ::_config_to_ffi_flags(cfg,ctx_->pid,ffi_); - if(cfg.writeback_cache) - ::_tweak_flags_writeback_cache(&ffi_->flags); + if(cfg.cache_writeback) + ::_tweak_flags_cache_writeback(&ffi_->flags); ffi_->noflush = !::_calculate_flush(cfg.flushonclose, ffi_->flags); diff --git a/src/fuse_init.cpp b/src/fuse_init.cpp index c3735365..3c7bedb1 100644 --- a/src/fuse_init.cpp +++ b/src/fuse_init.cpp @@ -131,7 +131,7 @@ _want_if_capable_max_pages(fuse_conn_info *conn_, if(::_capable(conn_,FUSE_CAP_MAX_PAGES)) { ::_want(conn_,FUSE_CAP_MAX_PAGES); - conn_->max_pages = cfg_.fuse_msg_size; + fuse_cfg.max_pages = cfg_.fuse_msg_size; SysLog::info("requesting max pages size of {}", (uint64_t)cfg_.fuse_msg_size); } @@ -205,7 +205,7 @@ FUSE::init(fuse_conn_info *conn_) ::_want_if_capable(conn_,FUSE_CAP_PASSTHROUGH); ::_want_if_capable(conn_,FUSE_CAP_POSIX_ACL,&cfg.posix_acl); ::_want_if_capable(conn_,FUSE_CAP_READDIR_PLUS,&cfg.readdirplus); - ::_want_if_capable(conn_,FUSE_CAP_WRITEBACK_CACHE,&cfg.writeback_cache); + ::_want_if_capable(conn_,FUSE_CAP_WRITEBACK_CACHE,&cfg.cache_writeback); ::_want_if_capable(conn_,FUSE_CAP_ALLOW_IDMAP,&cfg.allow_idmap); // ::_want_if_capable(conn_,FUSE_CAP_READDIR_PLUS_AUTO); ::_want_if_capable_max_pages(conn_,cfg); @@ -228,7 +228,7 @@ FUSE::init(fuse_conn_info *conn_) } if((cfg.passthrough != Passthrough::ENUM::OFF) && - (cfg.writeback_cache == true)) + (cfg.cache_writeback == true)) { SysLog::warning("passthrough and cache.writeback are incompatible."); } diff --git a/src/fuse_open.cpp b/src/fuse_open.cpp index a19f41d7..61345b90 100644 --- a/src/fuse_open.cpp +++ b/src/fuse_open.cpp @@ -111,7 +111,7 @@ _nfsopenhack(const fs::path &fullpath_, */ static void -_tweak_flags_writeback_cache(int *flags_) +_tweak_flags_cache_writeback(int *flags_) { if((*flags_ & O_ACCMODE) == O_WRONLY) *flags_ = ((*flags_ & ~O_ACCMODE) | O_RDWR); @@ -291,8 +291,8 @@ _open_for_insert_lambda(const fuse_req_ctx_t *ctx_, ::_config_to_ffi_flags(cfg,ctx_->pid,ffi_); - if(cfg.writeback_cache) - ::_tweak_flags_writeback_cache(&ffi_->flags); + if(cfg.cache_writeback) + ::_tweak_flags_cache_writeback(&ffi_->flags); ffi_->noflush = !::_calculate_flush(cfg.flushonclose, ffi_->flags); @@ -347,8 +347,8 @@ _open_for_update_lambda(const fuse_req_ctx_t *ctx_, ::_config_to_ffi_flags(cfg,ctx_->pid,ffi_); - if(cfg.writeback_cache) - ::_tweak_flags_writeback_cache(&ffi_->flags); + if(cfg.cache_writeback) + ::_tweak_flags_cache_writeback(&ffi_->flags); ffi_->noflush = !::_calculate_flush(cfg.flushonclose, ffi_->flags); diff --git a/src/option_parser.cpp b/src/option_parser.cpp index 58b06858..ee21fd6c 100644 --- a/src/option_parser.cpp +++ b/src/option_parser.cpp @@ -31,7 +31,6 @@ #include "version.hpp" #include "fuse.h" -#include "fuse_config.hpp" #include #include @@ -79,30 +78,19 @@ _set_kv_option(const std::string &key_, static void -_set_fuse_threads(Config &cfg_) +_set_fsname(fuse_args *args_) { - fuse_config_set_read_thread_count(cfg_.fuse_read_thread_count); - fuse_config_set_process_thread_count(cfg_.fuse_process_thread_count); - fuse_config_set_process_thread_queue_depth(cfg_.fuse_process_thread_queue_depth); - fuse_config_set_pin_threads(cfg_.fuse_pin_threads); -} - -static -void -_set_fsname(Config &cfg_, - fuse_args *args_) -{ - if(cfg_.fsname->empty()) + if(cfg.fsname->empty()) { std::vector paths; - cfg_.branches->to_paths(paths); + cfg.branches->to_paths(paths); if(paths.size() > 0) - cfg_.fsname = str::remove_common_prefix_and_join(paths,':'); + cfg.fsname = str::remove_common_prefix_and_join(paths,':'); } - ::_set_kv_option("fsname",cfg_.fsname,args_); + ::_set_kv_option("fsname",cfg.fsname,args_); } static @@ -114,10 +102,9 @@ _set_subtype(fuse_args *args_) static void -_set_default_options(fuse_args *args_, - Config &cfg_) +_set_default_options(fuse_args *args_) { - if(cfg_.kernel_permissions_check) + if(cfg.kernel_permissions_check) ::_set_option("default_permissions",args_); if(geteuid() == 0) @@ -158,8 +145,7 @@ _should_ignore(const std::string &key_) static int -_parse_and_process_kv_arg(Config &cfg_, - Config::ErrVec *errs_, +_parse_and_process_kv_arg(Config::ErrVec *errs_, const std::string &key_, const std::string &val_) { @@ -169,7 +155,7 @@ _parse_and_process_kv_arg(Config &cfg_, rv = 0; if(key == "config") - return ((cfg_.from_file(val_,errs_) < 0) ? 1 : 0); + return ((cfg.from_file(val_,errs_) < 0) ? 1 : 0); ef(key == "attr_timeout") key = "cache.attr"; ef(key == "entry_timeout") @@ -186,13 +172,15 @@ _parse_and_process_kv_arg(Config &cfg_, val = "true"; ef(key == "sync_read" && val.empty()) {key = "async_read", val = "false";} + ef(key == "noforget" && val.empty()) + {key = "remember-nodes", val = "-1";} ef(::_should_ignore(key_)) return 0; - if(cfg_.has_key(key) == false) + if(cfg.has_key(key) == false) return 1; - rv = cfg_.set_raw(key,val); + rv = cfg.set(key,val); if(rv) errs_->push_back({rv,key+'='+val}); @@ -201,9 +189,8 @@ _parse_and_process_kv_arg(Config &cfg_, static int -_process_opt(Config &cfg_, - Config::ErrVec *errs_, - const std::string &arg_) +_process_opt(Config::ErrVec *errs_, + const std::string &arg_) { std::string key; std::string val; @@ -212,20 +199,19 @@ _process_opt(Config &cfg_, key = str::trim(key); val = str::trim(val); - return ::_parse_and_process_kv_arg(cfg_,errs_,key,val); + return ::_parse_and_process_kv_arg(errs_,key,val); } static int -_process_branches(Config &cfg_, - Config::ErrVec *errs_, - const char *arg_) +_process_branches(Config::ErrVec *errs_, + const char *arg_) { int rv; std::string arg; arg = arg_; - rv = cfg_.set_raw("branches",arg); + rv = cfg.set("branches",arg); if(rv) errs_->push_back({rv,"branches="+arg}); @@ -234,15 +220,14 @@ _process_branches(Config &cfg_, static int -_process_mount(Config &cfg_, - Config::ErrVec *errs_, - const char *arg_) +_process_mount(Config::ErrVec *errs_, + const char *arg_) { int rv; std::string arg; arg = arg_; - rv = cfg_.set_raw("mount",arg); + rv = cfg.set("mount",arg); if(rv) errs_->push_back({rv,"mount="+arg}); @@ -251,26 +236,26 @@ _process_mount(Config &cfg_, static void -_postprocess_passthrough(Config &cfg_) +_postprocess_passthrough() { - if(cfg_.passthrough == Passthrough::ENUM::OFF) + if(cfg.passthrough == Passthrough::ENUM::OFF) return; - if(cfg_.cache_files == CacheFiles::ENUM::OFF) + if(cfg.cache_files == CacheFiles::ENUM::OFF) { SysLog::warning("'cache.files' can not be 'off' when using 'passthrough'." " Setting 'cache.files=full'"); - cfg_.cache_files = CacheFiles::ENUM::FULL; + cfg.cache_files = CacheFiles::ENUM::FULL; } - if(cfg_.writeback_cache == true) + if(cfg.cache_writeback == true) { SysLog::warning("'cache.writeback' can not be enabled when using 'passthrough'." " Setting 'cache.writeback=false'"); - cfg_.writeback_cache = false; + cfg.cache_writeback = false; } - if(cfg_.moveonenospc.enabled == true) + if(cfg.moveonenospc.enabled == true) { SysLog::warning("`moveonenospc` will not function when `passthrough` is enabled"); } @@ -297,13 +282,13 @@ _option_processor(void *data_, switch(key_) { case FUSE_OPT_KEY_OPT: - return ::_process_opt(cfg,errs,arg_); + return ::_process_opt(errs,arg_); case FUSE_OPT_KEY_NONOPT: if(cfg.branches->empty()) - return ::_process_branches(cfg,errs,arg_); + return ::_process_branches(errs,arg_); else - return ::_process_mount(cfg,errs,arg_); + return ::_process_mount(errs,arg_); case MERGERFS_OPT_HELP: ::_usage(); @@ -341,15 +326,14 @@ _option_processor(void *data_, static void -_check_for_mount_loop(Config &cfg_, - Config::ErrVec *errs_) +_check_for_mount_loop(Config::ErrVec *errs_) { fs::path mount; std::vector branches; std::error_code ec; - mount = *cfg_.mountpoint; - branches = cfg_.branches->to_paths(); + mount = *cfg.mountpoint; + branches = cfg.branches->to_paths(); for(const auto &branch : branches) { if(std::filesystem::equivalent(branch,mount,ec)) @@ -365,25 +349,35 @@ _check_for_mount_loop(Config &cfg_, static void -_print_warnings(Config &cfg_) +_print_warnings() { - if(cfg_.passthrough != Passthrough::ENUM::OFF) + if(fuse_cfg.uid != FUSE_CFG_INVALID_ID) + SysLog::warning("overwriting 'uid' is untested and unsupported," + " use at your own risk"); + if(fuse_cfg.gid != FUSE_CFG_INVALID_ID) + SysLog::warning("overwriting 'gid' is untested and unsupported," + " use at your own risk"); + if(fuse_cfg.umask != FUSE_CFG_INVALID_UMASK) + SysLog::warning("overwriting 'umask' is untested and unsupported," + " use at your own risk"); + + if(cfg.passthrough != Passthrough::ENUM::OFF) { - if(cfg_.cache_files == CacheFiles::ENUM::OFF) + if(cfg.cache_files == CacheFiles::ENUM::OFF) { SysLog::warning("'cache.files' can not be 'off' when using 'passthrough'." " Setting 'cache.files=auto-full'"); - cfg_.cache_files = CacheFiles::ENUM::AUTO_FULL; + cfg.cache_files = CacheFiles::ENUM::AUTO_FULL; } - if(cfg_.writeback_cache == true) + if(cfg.cache_writeback == true) { SysLog::warning("'cache.writeback' can not be enabled when using 'passthrough'." " Setting 'cache.writeback=false'"); - cfg_.writeback_cache = false; + cfg.cache_writeback = false; } - if(cfg_.moveonenospc.enabled == true) + if(cfg.moveonenospc.enabled == true) { SysLog::warning("`moveonenospc` will not function when `passthrough` is enabled"); } @@ -392,10 +386,10 @@ _print_warnings(Config &cfg_) static void -_cleanup_options(Config &cfg_) +_cleanup_options() { - if(!cfg_.symlinkify) - cfg_.symlinkify_timeout = -1; + if(!cfg.symlinkify) + cfg.symlinkify_timeout = -1; } namespace options @@ -424,14 +418,13 @@ namespace options if(cfg.mountpoint->empty()) errs_->push_back({0,"mountpoint not set"}); - ::_postprocess_passthrough(cfg); - ::_check_for_mount_loop(cfg,errs_); - ::_set_default_options(args_,cfg); - ::_set_fsname(cfg,args_); + ::_postprocess_passthrough(); + ::_check_for_mount_loop(errs_); + ::_set_default_options(args_); + ::_set_fsname(args_); ::_set_subtype(args_); - ::_set_fuse_threads(cfg); - ::_print_warnings(cfg); - ::_cleanup_options(cfg); + ::_print_warnings(); + ::_cleanup_options(); cfg.finish_initializing(); } diff --git a/src/tofrom_ref.hpp b/src/tofrom_ref.hpp new file mode 100644 index 00000000..29e6e956 --- /dev/null +++ b/src/tofrom_ref.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include "to_string.hpp" +#include "from_string.hpp" +#include "tofrom_string.hpp" + + +template +class TFSRef : public ToFromString +{ +public: + int + from_string(const std::string_view s_) final + { + return str::from(s_,&_data); + } + + std::string + to_string(void) const final + { + return str::to(_data); + } + +public: + TFSRef(T &data_) + : _data(data_) + { + } + + TFSRef(T &data_, + const T val_) + : _data(data_) + { + _data = val_; + } + +private: + T &_data; +}; diff --git a/src/tofrom_string.hpp b/src/tofrom_string.hpp index f3f5b36b..557ff72b 100644 --- a/src/tofrom_string.hpp +++ b/src/tofrom_string.hpp @@ -26,4 +26,6 @@ class ToFromString public: virtual std::string to_string() const = 0; virtual int from_string(const std::string_view) = 0; + bool display = true; + bool ro = false; }; diff --git a/src/tofrom_wrapper.hpp b/src/tofrom_wrapper.hpp index 5eb1fe34..335b38ac 100644 --- a/src/tofrom_wrapper.hpp +++ b/src/tofrom_wrapper.hpp @@ -102,7 +102,7 @@ public: int from_string(const std::string_view s_) final { - return -EINVAL; + return -EROFS; } std::string