From 0918dfd117ff1c36621a67019dc64b77ec28bf76 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Sun, 3 Feb 2019 22:38:57 -0500 Subject: [PATCH] make attr, entry, negative_entry cache timeouts runtime configurable --- README.md | 5 ++- libfuse/include/fuse.h | 12 +++++++- libfuse/lib/fuse.c | 44 +++++++++++++++++++++++++-- man/mergerfs.1 | 12 +++++++- src/fuse_getxattr.cpp | 55 +++++++++++++++++++++++++++++++-- src/fuse_listxattr.cpp | 3 ++ src/fuse_setxattr.cpp | 69 ++++++++++++++++++++++++++++++++++++++++++ src/mergerfs.cpp | 2 +- src/num.cpp | 21 +++++++++++++ src/num.hpp | 5 +-- src/option_parser.cpp | 52 +++++++++++++++++++------------ src/option_parser.hpp | 4 +-- 12 files changed, 253 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 31729c38..a114cc5e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ % mergerfs(1) mergerfs user manual % Antonio SJ Musumeci -% 2019-02-17 +% 2019-02-22 # NAME @@ -85,6 +85,9 @@ mergerfs does **not** support the copy-on-write (CoW) behavior found in **aufs** * **category.<category>=<policy>**: Sets policy of all FUSE functions in the provided category. Example: **category.create=mfs** * **cache.open=<int>**: 'open' policy cache timeout in seconds. (default: 0) * **cache.statfs=<int>**: 'statfs' cache timeout in seconds. (default: 0) +* **cache.attr=<int>**: file attribute cache timeout in seconds. (default: 1) +* **cache.entry=<int>**: file name lookup cache timeout in seconds. (default: 1) +* **cache.negative_entry=<int>**: negative file name lookup cache timeout in seconds. (default: 0) **NOTE:** Options are evaluated in the order listed so if the options are **func.rmdir=rand,category.action=ff** the **action** category setting will override the **rmdir** setting. diff --git a/libfuse/include/fuse.h b/libfuse/include/fuse.h index 0b4d7f22..a9c3d9d5 100644 --- a/libfuse/include/fuse.h +++ b/libfuse/include/fuse.h @@ -701,7 +701,17 @@ int fuse_loop(struct fuse *f); */ void fuse_exit(struct fuse *f); -int fuse_config_num_threads(const struct fuse *f); +void fuse_config_set_entry_timeout(struct fuse *fuse_, + const double entry_timeout_); +void fuse_config_set_negative_entry_timeout(struct fuse *fuse_, + const double entry_timeout_); +void fuse_config_set_attr_timeout(struct fuse *fuse_, + const double attr_timeout_); + +int fuse_config_num_threads(const struct fuse *fuse_); +double fuse_config_get_entry_timeout(const struct fuse *fuse_); +double fuse_config_get_negative_entry_timeout(const struct fuse *fuse_); +double fuse_config_get_attr_timeout(const struct fuse *fuse_); /** * FUSE event loop with multiple threads diff --git a/libfuse/lib/fuse.c b/libfuse/lib/fuse.c index f48f5341..5e1090f4 100644 --- a/libfuse/lib/fuse.c +++ b/libfuse/lib/fuse.c @@ -4723,7 +4723,47 @@ struct fuse *fuse_new_compat25(int fd, struct fuse_args *args, FUSE_SYMVER(".symver fuse_new_compat25,fuse_new@FUSE_2.5"); -int fuse_config_num_threads(const struct fuse *f) +int +fuse_config_num_threads(const struct fuse *fuse_) { - return f->conf.threads; + return fuse_->conf.threads; +} + +void +fuse_config_set_entry_timeout(struct fuse *fuse_, + const double entry_timeout_) +{ + fuse_->conf.entry_timeout = entry_timeout_; +} + +double +fuse_config_get_entry_timeout(const struct fuse *fuse_) +{ + return fuse_->conf.entry_timeout; +} + +void +fuse_config_set_negative_entry_timeout(struct fuse *fuse_, + const double entry_timeout_) +{ + fuse_->conf.negative_timeout = entry_timeout_; +} + +double +fuse_config_get_negative_entry_timeout(const struct fuse *fuse_) +{ + return fuse_->conf.negative_timeout; +} + +void +fuse_config_set_attr_timeout(struct fuse *fuse_, + const double attr_timeout_) +{ + fuse_->conf.attr_timeout = attr_timeout_; +} + +double +fuse_config_get_attr_timeout(const struct fuse *fuse_) +{ + return fuse_->conf.attr_timeout; } diff --git a/man/mergerfs.1 b/man/mergerfs.1 index 2eaa6a76..d82f2f8e 100644 --- a/man/mergerfs.1 +++ b/man/mergerfs.1 @@ -1,7 +1,7 @@ .\"t .\" Automatically generated by Pandoc 1.19.2.4 .\" -.TH "mergerfs" "1" "2019\-02\-17" "mergerfs user manual" "" +.TH "mergerfs" "1" "2019\-02\-22" "mergerfs user manual" "" .hy .SH NAME .PP @@ -213,6 +213,16 @@ seconds. .IP \[bu] 2 \f[B]cache.statfs=\f[]: \[aq]statfs\[aq] cache timeout in seconds. (default: 0) +.IP \[bu] 2 +\f[B]cache.attr=\f[]: file attribute cache timeout in seconds. +(default: 1) +.IP \[bu] 2 +\f[B]cache.entry=\f[]: file name lookup cache timeout in seconds. +(default: 1) +.IP \[bu] 2 +\f[B]cache.negative_entry=\f[]: negative file name lookup cache +timeout in seconds. +(default: 0) .PP \f[B]NOTE:\f[] Options are evaluated in the order listed so if the options are \f[B]func.rmdir=rand,category.action=ff\f[] the diff --git a/src/fuse_getxattr.cpp b/src/fuse_getxattr.cpp index dcd3be97..19028a05 100644 --- a/src/fuse_getxattr.cpp +++ b/src/fuse_getxattr.cpp @@ -130,6 +130,18 @@ namespace l attrvalue_ = os.str(); } + static + void + getxattr_controlfile_double(const double d_, + string &attrvalue_) + { + std::ostringstream os; + + os << d_; + + attrvalue_ = os.str(); + } + static void getxattr_controlfile_time_t(const time_t time, @@ -236,7 +248,7 @@ namespace l static void - getxattr_pid(string &attrvalue) + getxattr_controlfile_pid(string &attrvalue) { int pid; char buf[32]; @@ -247,6 +259,39 @@ namespace l attrvalue = buf; } + static + void + getxattr_controlfile_cache_attr(string &attrvalue) + { + double d; + + d = fuse_config_get_attr_timeout(fuse_get_context()->fuse); + + l::getxattr_controlfile_double(d,attrvalue); + } + + static + void + getxattr_controlfile_cache_entry(string &attrvalue) + { + double d; + + d = fuse_config_get_entry_timeout(fuse_get_context()->fuse); + + l::getxattr_controlfile_double(d,attrvalue); + } + + static + void + getxattr_controlfile_cache_negative_entry(string &attrvalue) + { + double d; + + d = fuse_config_get_negative_entry_timeout(fuse_get_context()->fuse); + + l::getxattr_controlfile_double(d,attrvalue); + } + static int getxattr_controlfile(const Config &config, @@ -298,7 +343,7 @@ namespace l else if(attr[2] == "version") l::getxattr_controlfile_version(attrvalue); else if(attr[2] == "pid") - l::getxattr_pid(attrvalue); + l::getxattr_controlfile_pid(attrvalue); else if(attr[2] == "direct_io") l::getxattr_controlfile_bool(config.direct_io,attrvalue); break; @@ -312,6 +357,12 @@ namespace l l::getxattr_controlfile_uint64_t(config.open_cache.timeout,attrvalue); else if((attr[2] == "cache") && (attr[3] == "statfs")) l::getxattr_controlfile_uint64_t(fs::statvfs_cache_timeout(),attrvalue); + else if((attr[2] == "cache") && (attr[3] == "attr")) + l::getxattr_controlfile_cache_attr(attrvalue); + else if((attr[2] == "cache") && (attr[3] == "entry")) + l::getxattr_controlfile_cache_entry(attrvalue); + else if((attr[2] == "cache") && (attr[3] == "negative_entry")) + l::getxattr_controlfile_cache_negative_entry(attrvalue); break; } diff --git a/src/fuse_listxattr.cpp b/src/fuse_listxattr.cpp index 795b79ff..23c09194 100644 --- a/src/fuse_listxattr.cpp +++ b/src/fuse_listxattr.cpp @@ -45,6 +45,9 @@ namespace l const vector strs = buildvector ("user.mergerfs.branches") + ("user.mergerfs.cache.attr") + ("user.mergerfs.cache.entry") + ("user.mergerfs.cache.negative_entry") ("user.mergerfs.cache.open") ("user.mergerfs.cache.statfs") ("user.mergerfs.direct_io") diff --git a/src/fuse_setxattr.cpp b/src/fuse_setxattr.cpp index 5a6bde49..4239d02e 100644 --- a/src/fuse_setxattr.cpp +++ b/src/fuse_setxattr.cpp @@ -119,6 +119,24 @@ namespace l return 0; } + static + int + setxattr_double(const string &attrval_, + const int flags_, + double *d_) + { + int rv; + + if((flags_ & XATTR_CREATE) == XATTR_CREATE) + return -EEXIST; + + rv = num::to_double(attrval_,d_); + if(rv == -1) + return -EINVAL; + + return 0; + } + static int setxattr_time_t(const string &attrval, @@ -276,6 +294,51 @@ namespace l return rv; } + static + int + setxattr_controlfile_cache_attr(const string &attrval_, + const int flags_) + { + int rv; + double d; + + rv = l::setxattr_double(attrval_,flags_,&d); + if(rv >= 0) + fuse_config_set_attr_timeout(fuse_get_context()->fuse,d); + + return rv; + } + + static + int + setxattr_controlfile_cache_entry(const string &attrval_, + const int flags_) + { + int rv; + double d; + + rv = l::setxattr_double(attrval_,flags_,&d); + if(rv >= 0) + fuse_config_set_entry_timeout(fuse_get_context()->fuse,d); + + return rv; + } + + static + int + setxattr_controlfile_cache_negative_entry(const string &attrval_, + const int flags_) + { + int rv; + double d; + + rv = l::setxattr_double(attrval_,flags_,&d); + if(rv >= 0) + fuse_config_set_negative_entry_timeout(fuse_get_context()->fuse,d); + + return rv; + } + static int setxattr_controlfile(Config &config, @@ -367,6 +430,12 @@ namespace l config.open_cache.timeout); else if((attr[2] == "cache") && (attr[3] == "statfs")) return l::setxattr_statfs_timeout(attrval,flags); + else if((attr[2] == "cache") && (attr[3] == "attr")) + return l::setxattr_controlfile_cache_attr(attrval,flags); + else if((attr[2] == "cache") && (attr[3] == "entry")) + return l::setxattr_controlfile_cache_entry(attrval,flags); + else if((attr[2] == "cache") && (attr[3] == "negative_entry")) + return l::setxattr_controlfile_cache_negative_entry(attrval,flags); break; default: diff --git a/src/mergerfs.cpp b/src/mergerfs.cpp index 657b3610..c24ba8da 100644 --- a/src/mergerfs.cpp +++ b/src/mergerfs.cpp @@ -158,7 +158,7 @@ namespace l args.argv = argv_; args.allocated = 0; - options::parse(args,config); + options::parse(&args,&config); l::setup_resources(); l::get_fuse_operations(ops,config.nullrw); diff --git a/src/num.cpp b/src/num.cpp index 195bf55e..4483ad39 100644 --- a/src/num.cpp +++ b/src/num.cpp @@ -47,6 +47,11 @@ namespace num tmp *= (1024 * 1024 * 1024); break; + case 't': + case 'T': + tmp *= (1024ULL * 1024 * 1024 * 1024); + break; + case '\0': break; @@ -59,6 +64,22 @@ namespace num return 0; } + int + to_double(const std::string &str_, + double *d_) + { + double tmp; + char *endptr; + + tmp = strtod(str_.c_str(),&endptr); + if(*endptr != '\0') + return -1; + + *d_ = tmp; + + return 0; + } + int to_time_t(const std::string &str, time_t &value) diff --git a/src/num.hpp b/src/num.hpp index 49dcc28b..c3e3b1e7 100644 --- a/src/num.hpp +++ b/src/num.hpp @@ -22,6 +22,7 @@ namespace num { - int to_uint64_t(const std::string &str, uint64_t &value); - int to_time_t(const std::string &str, time_t &value); + int to_uint64_t(const std::string &str_, uint64_t &value_); + int to_double(const std::string &str_, double *value_); + int to_time_t(const std::string &str_, time_t &value_); } diff --git a/src/option_parser.cpp b/src/option_parser.cpp index eb2296a9..53d2a5f1 100644 --- a/src/option_parser.cpp +++ b/src/option_parser.cpp @@ -48,17 +48,17 @@ enum static void -set_option(fuse_args &args, +set_option(fuse_args *args, const std::string &option_) { - fuse_opt_add_arg(&args,"-o"); - fuse_opt_add_arg(&args,option_.c_str()); + fuse_opt_add_arg(args,"-o"); + fuse_opt_add_arg(args,option_.c_str()); } static void -set_kv_option(fuse_args &args, +set_kv_option(fuse_args *args, const std::string &key, const std::string &value) { @@ -71,7 +71,7 @@ set_kv_option(fuse_args &args, static void -set_fsname(fuse_args &args, +set_fsname(fuse_args *args, const Branches &branches_) { vector branches; @@ -90,14 +90,14 @@ set_fsname(fuse_args &args, static void -set_subtype(fuse_args &args) +set_subtype(fuse_args *args) { set_kv_option(args,"subtype","mergerfs"); } static void -set_default_options(fuse_args &args) +set_default_options(fuse_args *args) { set_option(args,"atomic_o_trunc"); set_option(args,"big_writes"); @@ -216,12 +216,19 @@ static int parse_and_process_cache(Config &config_, const string &func_, - const string &value_) + const string &value_, + fuse_args *outargs) { if(func_ == "open") return parse_and_process(value_,config_.open_cache.timeout); else if(func_ == "statfs") return parse_and_process_statfs_cache(value_); + else if(func_ == "entry") + return (set_kv_option(outargs,"entry_timeout",value_),0); + else if(func_ == "negative_entry") + return (set_kv_option(outargs,"negative_timeout",value_),0); + else if(func_ == "attr") + return (set_kv_option(outargs,"attr_timeout",value_),0); return 1; } @@ -229,8 +236,7 @@ parse_and_process_cache(Config &config_, static int parse_and_process_arg(Config &config, - const std::string &arg, - fuse_args *outargs) + const std::string &arg) { if(arg == "defaults") return 0; @@ -244,7 +250,8 @@ static int parse_and_process_kv_arg(Config &config, const std::string &key, - const std::string &value) + const std::string &value, + fuse_args *outargs) { int rv; std::vector keypart; @@ -258,7 +265,7 @@ parse_and_process_kv_arg(Config &config, else if(keypart[0] == "category") rv = config.set_category_policy(keypart[1],value); else if(keypart[0] == "cache") - rv = parse_and_process_cache(config,keypart[1],value); + rv = parse_and_process_cache(config,keypart[1],value,outargs); } else { @@ -307,11 +314,11 @@ process_opt(Config &config, switch(argvalue.size()) { case 1: - rv = parse_and_process_arg(config,argvalue[0],outargs); + rv = parse_and_process_arg(config,argvalue[0]); break; case 2: - rv = parse_and_process_kv_arg(config,argvalue[0],argvalue[1]); + rv = parse_and_process_kv_arg(config,argvalue[0],argvalue[1],outargs); break; default: @@ -362,6 +369,13 @@ usage(void) " default = 0 (disabled)\n" " -o cache.statfs= 'statfs' cache timeout in seconds. Used by\n" " policies. default = 0 (disabled)\n" + " -o cache.attr= file attribute cache timeout in seconds.\n" + " default = 1\n" + " -o cache.entry= file name lookup cache timeout in seconds.\n" + " default = 1\n" + " -o cache.negative_entry=\n" + " negative file name lookup cache timeout in\n" + " seconds. default = 0\n" " -o direct_io Bypass page caching, may increase write\n" " speeds at the cost of reads. Please read docs\n" " for more details as there are tradeoffs.\n" @@ -456,8 +470,8 @@ option_processor(void *data, namespace options { void - parse(fuse_args &args, - Config &config) + parse(fuse_args *args, + Config *config) { const struct fuse_opt opts[] = { @@ -469,13 +483,13 @@ namespace options {NULL,-1U,0} }; - fuse_opt_parse(&args, - &config, + fuse_opt_parse(args, + config, opts, ::option_processor); set_default_options(args); - set_fsname(args,config.branches); + set_fsname(args,config->branches); set_subtype(args); } } diff --git a/src/option_parser.hpp b/src/option_parser.hpp index 28c582a4..3c7702fb 100644 --- a/src/option_parser.hpp +++ b/src/option_parser.hpp @@ -23,6 +23,6 @@ namespace options { void - parse(fuse_args &args, - Config &config); + parse(fuse_args *args, + Config *config); }