diff --git a/libfuse/include/fuse_common.h b/libfuse/include/fuse_common.h index 48607d05..ef2109ad 100644 --- a/libfuse/include/fuse_common.h +++ b/libfuse/include/fuse_common.h @@ -37,7 +37,7 @@ #endif #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 -#define FUSE_MAX_MAX_PAGES 256 +#define FUSE_DEFAULT_MAX_MAX_PAGES 256 EXTERN_C_BEGIN diff --git a/libfuse/lib/fuse_msgbuf.cpp b/libfuse/lib/fuse_msgbuf.cpp index 828a4c44..ed44bed8 100644 --- a/libfuse/lib/fuse_msgbuf.cpp +++ b/libfuse/lib/fuse_msgbuf.cpp @@ -73,7 +73,7 @@ msgbuf_constructor() { g_PAGESIZE = sysconf(_SC_PAGESIZE); // FUSE_MAX_MAX_PAGES for payload + 1 for message header - msgbuf_set_bufsize(FUSE_MAX_MAX_PAGES + 1); + msgbuf_set_bufsize(FUSE_DEFAULT_MAX_MAX_PAGES + 1); } static diff --git a/libfuse/lib/helper.c b/libfuse/lib/helper.c index 2e9b9fe2..09681504 100644 --- a/libfuse/lib/helper.c +++ b/libfuse/lib/helper.c @@ -258,7 +258,7 @@ fuse_mount_common(const char *mountpoint_, return NULL; pagesize = sysconf(_SC_PAGESIZE); - bufsize = ((FUSE_MAX_MAX_PAGES + 1) * pagesize); + bufsize = ((FUSE_DEFAULT_MAX_MAX_PAGES + 1) * pagesize); ch = fuse_chan_new(fd,bufsize); if(!ch) diff --git a/mkdocs/docs/config/fuse_msg_size.md b/mkdocs/docs/config/fuse_msg_size.md index d741c467..7f93fe8d 100644 --- a/mkdocs/docs/config/fuse_msg_size.md +++ b/mkdocs/docs/config/fuse_msg_size.md @@ -1,7 +1,8 @@ # fuse_msg_size -* `fuse_msg_size=UINT` -* Defaults to `256` +* `fuse_msg_size=UINT|SIZE` +* Defaults to `1M` +* Performance improvements often peak at about `4M` FUSE applications communicate with the kernel over a special character device: `/dev/fuse`. A large portion of the overhead associated with @@ -16,14 +17,33 @@ halved. In Linux v4.20 a new feature was added allowing the negotiation of the max message size. Since the size is in multiples of [pages](https://en.wikipedia.org/wiki/Page_(computer_memory)) the -feature is called `max_pages`. There is a maximum `max_pages` value of -256 (1MiB) and minimum of 1 (4KiB). The default used by Linux >=4.20, -and hardcoded value used before 4.20, is 32 (128KiB). In mergerfs it's -referred to as fuse_msg_size to make it clear what it impacts and -provide some abstraction. - -Since there should be no downsides to increasing `fuse_msg_size`, -outside a minor increase in RAM usage due to larger message buffers, -mergerfs defaults the value to 256. On kernels before v4.20 the value -has no effect. The reason the value is configurable is to enable -experimentation and benchmarking. +feature is called `max_pages`. In versions of Linux prior to v6.13 +there is a maximum `max_pages` value of 256 (1MiB) and minimum of 1 +(4KiB). In [Linux +v6.13](https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=2b3933b1e0a0a4b758fbc164bb31db0c113a7e2c) +and above the max value supported by the kernel can range from 1 +(4KiB) to 65535 (~256MiB) (assuming a page size of 4KiB.) The default +used by Linux >= 4.20, and hard coded value used before 4.20, is 32 +(128KiB). In mergerfs it is referred to as `fuse_msg_size` to make it +clear what it impacts and provide some abstraction. + +If the `fuse_msg_size` value provided is more than the system wide +maximum mergerfs will attempt to increase the system wide value to keep +the user from needing to set the value using `sysctl`, +`/etc/sysctl.conf`, or `/proc/sys/fs/fuse/max_pages_limit`. + +The main downside to increasing the value is that memory usage will +increase approximately relative to the number of [processing +threads](threads.md) configured. Keep this in mind for systems with +lower amounts of memory like most SBCs. Performance improvements seem +to peak around 4MiB. + +On kernels before v4.20 the option has no effect. On kernels between +v4.20 and v6.13 the max value is 256. On kernels >= v6.13 the maximum +value is 65535. + +Since page size can differ between systems mergerfs can take a value in +bytes and will convert it to the proper number of pages (rounded up). + +NOTE: If you intend to enable `cache.files` you should also set +[readahead](readahead.md) to match `fuse_msg_size`. diff --git a/mkdocs/docs/config/options.md b/mkdocs/docs/config/options.md index 170c1d74..008ebd10 100644 --- a/mkdocs/docs/config/options.md +++ b/mkdocs/docs/config/options.md @@ -8,7 +8,9 @@ These options are the same regardless of whether you use them with the - BOOL = 'true' | 'false' - INT = [MIN_INT,MAX_INT] - UINT = [0,MAX_INT] -- SIZE = 'NNM'; NN = INT, M = 'K' | 'M' | 'G' | 'T' +- SIZE = 'NNM'; NN = INT, M = 'B' | 'K' | 'M' | 'G' | 'T' +- PAGESIZE = UINT (representing number of pages) | SIZE (in bytes + which will be converted to pages) - STR = string (may refer to an enumerated value, see details of argument) - FUNC = filesystem function @@ -109,9 +111,9 @@ These options are the same regardless of whether you use them with the unavailable the kernel will ensure there is at most one pending read request per file handle and will attempt to order requests by offset. (default: true) -- **[fuse_msg_size](fuse_msg_size.md)=UINT**: Set the max number of +- **[fuse_msg_size](fuse_msg_size.md)=PAGESIZE**: Set the max number of pages per FUSE message. Only available on Linux >= 4.20 and ignored - otherwise. (min: 1; max: 256; default: 256) + otherwise. (min: 1; max: 65535; default: "1M") - **[threads](threads.md)=INT**: Number of threads to use. When used alone (`process-thread-count=-1`) it sets the number of threads reading and processing FUSE messages. When used together it sets the diff --git a/mkdocs/docs/config/readahead.md b/mkdocs/docs/config/readahead.md index 691391a6..5e70b859 100644 --- a/mkdocs/docs/config/readahead.md +++ b/mkdocs/docs/config/readahead.md @@ -11,8 +11,9 @@ doesn't mean that is the size used by the kernel for read and writes. Linux has a max read/write size of 2GB. Since the max FUSE message -size is just over 1MB the kernel will break up read and write requests -with buffers larger than that 1MB. +size is just over 1MiB (by default on more recent kernels) the kernel +will break up read and write requests with buffers larger than that +1MiB. When page caching is disabled (`cache.files=off`), besides the kernel breaking up requests with larger buffers, requests are effectively one @@ -35,4 +36,5 @@ a generic feature but there is no standard way to do so mergerfs added this feature to make it easier to set. There is currently no way to set separate values for different -branches through mergerfs. +branches through mergerfs. In fact at some point the feature may be +changed to only set mergerfs' readahead. diff --git a/src/config.cpp b/src/config.cpp index 255c9d79..0a7ac7ef 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -102,7 +102,7 @@ Config::Config() follow_symlinks(FollowSymlinks::ENUM::NEVER), fsname(), func(), - fuse_msg_size(FUSE_MAX_MAX_PAGES), + fuse_msg_size("1M"), ignorepponrename(false), inodecalc("hybrid-hash"), lazy_umount_mountpoint(false), diff --git a/src/config.hpp b/src/config.hpp index c56a303c..9c1e4a6d 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -21,12 +21,13 @@ #include "config_cachefiles.hpp" #include "config_flushonclose.hpp" #include "config_follow_symlinks.hpp" -#include "config_pid.hpp" #include "config_inodecalc.hpp" #include "config_link_exdev.hpp" #include "config_log_metrics.hpp" #include "config_moveonenospc.hpp" #include "config_nfsopenhack.hpp" +#include "config_pagesize.hpp" +#include "config_pid.hpp" #include "config_rename_exdev.hpp" #include "config_set.hpp" #include "config_statfs.hpp" @@ -124,7 +125,7 @@ public: FollowSymlinks follow_symlinks; ConfigSTR fsname; Funcs func; - ConfigUINT64 fuse_msg_size; + ConfigPageSize fuse_msg_size; ConfigBOOL ignorepponrename; InodeCalc inodecalc; ConfigBOOL kernel_cache; diff --git a/src/config_pagesize.cpp b/src/config_pagesize.cpp new file mode 100644 index 00000000..e486efaa --- /dev/null +++ b/src/config_pagesize.cpp @@ -0,0 +1,61 @@ +/* + ISC License + + Copyright (c) 2025, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include "config_pagesize.hpp" + +#include "from_string.hpp" + +#include +#include + +#include + + +ConfigPageSize::ConfigPageSize(const uint64_t v_) + : _v(v_) +{ + +} + +ConfigPageSize::ConfigPageSize(const std::string &s_) +{ + from_string(s_); +} + +std::string +ConfigPageSize::to_string(void) const +{ + return std::to_string(_v); +} + +int +ConfigPageSize::from_string(const std::string &s_) +{ + uint64_t v; + uint64_t pagesize; + + pagesize = sysconf(_SC_PAGESIZE); + + str::from(s_,&v); + if(!std::isalpha(s_.back())) + v *= pagesize; + + _v = ((v + pagesize - 1) / pagesize); + + return 0; +} diff --git a/src/config_pagesize.hpp b/src/config_pagesize.hpp new file mode 100644 index 00000000..5ab78ad6 --- /dev/null +++ b/src/config_pagesize.hpp @@ -0,0 +1,51 @@ +/* + ISC License + + Copyright (c) 2025, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#pragma once + +#include "tofrom_string.hpp" + +#include + +class ConfigPageSize : public ToFromString +{ +private: + uint64_t _v; + +public: + ConfigPageSize(const uint64_t); + ConfigPageSize(const std::string &); + +public: + std::string to_string(void) const final; + int from_string(const std::string &) final; + +public: + ConfigPageSize& + operator=(const uint64_t v_) + { + _v = v_; + + return *this; + } + + operator const uint64_t() const + { + return _v; + } +}; diff --git a/src/from_string.cpp b/src/from_string.cpp index 638c8772..a557b332 100644 --- a/src/from_string.cpp +++ b/src/from_string.cpp @@ -76,6 +76,11 @@ namespace str tmp = ::strtoll(value_.c_str(),&endptr,10); switch(*endptr) { + case 'b': + case 'B': + tmp *= 1ULL; + break; + case 'k': case 'K': tmp *= 1024ULL; diff --git a/src/fuse_init.cpp b/src/fuse_init.cpp index cea9513d..03771457 100644 --- a/src/fuse_init.cpp +++ b/src/fuse_init.cpp @@ -23,7 +23,10 @@ #include "fuse.h" +#include "ghc/filesystem.hpp" + #include +#include namespace l @@ -68,15 +71,67 @@ namespace l *want_ = false; } + #define MAX_FUSE_MSG_SIZE 65535 + static const char MAX_PAGES_LIMIT_FILEPATH[] = "/proc/sys/fs/fuse/max_pages_limit"; + static void want_if_capable_max_pages(fuse_conn_info *conn_, Config::Write &cfg_) { + std::fstream f; + uint64_t max_pages_limit; + + if(ghc::filesystem::exists(MAX_PAGES_LIMIT_FILEPATH)) + { + if(cfg_->fuse_msg_size > MAX_FUSE_MSG_SIZE) + syslog_info("fuse_msg_size > %u: setting it to %u", + MAX_FUSE_MSG_SIZE, + MAX_FUSE_MSG_SIZE); + cfg_->fuse_msg_size = std::min((uint64_t)cfg_->fuse_msg_size, + (uint64_t)MAX_FUSE_MSG_SIZE); + + f.open(MAX_PAGES_LIMIT_FILEPATH,f.in|f.out); + if(f.is_open()) + { + f >> max_pages_limit; + syslog_info("%s currently set to %u", + MAX_PAGES_LIMIT_FILEPATH, + (uint64_t)max_pages_limit); + if(cfg_->fuse_msg_size > max_pages_limit) + { + f.seekp(0); + f << (uint64_t)cfg_->fuse_msg_size; + f.flush(); + syslog_info("%s changed to %u", + MAX_PAGES_LIMIT_FILEPATH, + (uint64_t)cfg_->fuse_msg_size); + } + f.close(); + } + else + { + if(cfg_->fuse_msg_size != FUSE_DEFAULT_MAX_MAX_PAGES) + syslog_info("unable to open %s",MAX_PAGES_LIMIT_FILEPATH); + } + } + else + { + if(cfg_->fuse_msg_size > FUSE_DEFAULT_MAX_MAX_PAGES) + syslog_info("fuse_msg_size request %u > %u: setting it to %u", + (uint64_t)cfg_->fuse_msg_size, + FUSE_DEFAULT_MAX_MAX_PAGES, + FUSE_DEFAULT_MAX_MAX_PAGES); + cfg_->fuse_msg_size = std::min((uint64_t)cfg_->fuse_msg_size, + (uint64_t)FUSE_DEFAULT_MAX_MAX_PAGES); + } + if(l::capable(conn_,FUSE_CAP_MAX_PAGES)) { l::want(conn_,FUSE_CAP_MAX_PAGES); conn_->max_pages = cfg_->fuse_msg_size; + syslog_info("requesting max pages size of %u", + (uint64_t)cfg_->fuse_msg_size); } else {