From 7a057daa0c8e2d27ad0ca544710349b3d577ec3b Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Sun, 6 Jan 2019 12:52:55 -0500 Subject: [PATCH] add policy cache for 'open' A fusepath -> basepath cache for `open` to limit the overhead of FUSE in 'open, read/write, close' patterns (such as Transmission). --- README.md | 10 ++ libfuse/configure.ac | 2 +- man/mergerfs.1 | 20 +++ src/access.cpp | 2 - src/category.cpp | 69 +++++---- src/category.hpp | 105 +++++++------- src/chmod.cpp | 1 - src/chown.cpp | 1 - src/config.hpp | 4 + src/getattr.cpp | 1 - src/getxattr.cpp | 2 + src/link.cpp | 1 - src/listxattr.cpp | 22 +-- src/open.cpp | 12 +- src/option_parser.cpp | 16 ++ src/policy.cpp | 102 +++++++------ src/policy.hpp | 322 +++++++++++++++++++++-------------------- src/policy_all.cpp | 21 ++- src/policy_cache.cpp | 128 ++++++++++++++++ src/policy_cache.hpp | 61 ++++++++ src/policy_epall.cpp | 35 ++--- src/policy_epff.cpp | 36 ++--- src/policy_eplfs.cpp | 36 ++--- src/policy_eplus.cpp | 36 ++--- src/policy_epmfs.cpp | 36 ++--- src/policy_eprand.cpp | 29 ++-- src/policy_erofs.cpp | 17 +-- src/policy_ff.cpp | 21 ++- src/policy_invalid.cpp | 17 +-- src/policy_lfs.cpp | 22 ++- src/policy_lus.cpp | 22 ++- src/policy_mfs.cpp | 22 ++- src/policy_newest.cpp | 35 ++--- src/policy_rand.cpp | 29 ++-- src/readlink.cpp | 1 - src/release.cpp | 49 ++++--- src/releasedir.cpp | 29 ++-- src/removexattr.cpp | 1 - src/rename.cpp | 2 + src/rmdir.cpp | 1 - src/setxattr.cpp | 11 +- src/truncate.cpp | 1 - src/unlink.cpp | 106 +++++++------- src/utimens.cpp | 1 - 44 files changed, 853 insertions(+), 644 deletions(-) create mode 100644 src/policy_cache.cpp create mode 100644 src/policy_cache.hpp diff --git a/README.md b/README.md index bf9a588d..6f5f46d4 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ mergerfs does **not** support the copy-on-write (CoW) behavior found in **aufs** * **fsname=name**: sets the name of the filesystem as seen in **mount**, **df**, etc. Defaults to a list of the source paths concatenated together with the longest common prefix removed. * **func.<func>=<policy>**: sets the specific FUSE function's policy. See below for the list of value types. Example: **func.getattr=newest** * **category.<category>=<policy>**: Sets policy of all FUSE functions in the provided category. Example: **category.create=mfs** +* **cache.open=**: 'open' policy cache timeout in seconds. (default: 0) **NOTE:** Options are evaluated in the order listed so if the options are **func.rmdir=rand,category.action=ff** the **action** category setting will override the **rmdir** setting. @@ -491,6 +492,15 @@ It's a difficult balance between memory usage, cache bloat & duplication, and pe Given the relatively high cost of FUSE due to the kernel <-> userspace round trips there are kernel side caches for file entries and attributes. The entry cache limits the `lookup` calls to mergerfs which ask if a file exists. The attribute cache limits the need to make `getattr` calls to mergerfs which provide file attributes (mode, size, type, etc.). As with the page cache these should not be used if the underlying filesystems are being manipulated at the same time as it could lead to odd behavior or data corruption. The options for setting these are `entry_timeout` and `negative_timeout` for the entry cache and `attr_timeout` for the attributes cache. `negative_timeout` refers to the timeout for negative responses to lookups (non-existant files). +#### policy caching + +Policies are run every time a function is called. These policies can be expensive depending on the setup and usage patterns. Generally we wouldn't want to cache policy results because it may result in stale responses if the underlying drives are used directly. + +The `open` policy cache will cache the result of an `open` policy for a particular input for `cache.open` seconds or until the file is unlinked. Each file close (release) will randomly chose to clean up the cache of expired entries. + +This cache is useful in cases like that of **Transmission** which has a "open, read/write, close" pattern (which is much more costly due to the FUSE overhead than normal.) + + #### writeback caching writeback caching is a technique for improving write speeds by batching writes at a faster device and then bulk writing to the slower device. With FUSE the kernel will wait for a number of writes to be made and then send it to the filesystem as one request. mergerfs currently uses a slightly modified and vendored libfuse 2.9.7 which does not support writeback caching. However, a prototype port to libfuse 3.x has been made and the writeback cache appears to work as expected (though performance improvements greatly depend on the way the client app writes data). Once the port is complete and thoroughly tested writeback caching will be available. diff --git a/libfuse/configure.ac b/libfuse/configure.ac index 6fc80378..a669ba00 100644 --- a/libfuse/configure.ac +++ b/libfuse/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(fuse, 2.9.8-mergerfs) +AC_INIT(fuse, 2.9.7-mergerfs_2.26.0) AC_PREREQ(2.59d) AC_CONFIG_MACRO_DIR([m4]) diff --git a/man/mergerfs.1 b/man/mergerfs.1 index 6f869e60..20796fc9 100644 --- a/man/mergerfs.1 +++ b/man/mergerfs.1 @@ -211,6 +211,9 @@ Example: \f[B]func.getattr=newest\f[] \f[B]category.=\f[]: Sets policy of all FUSE functions in the provided category. Example: \f[B]category.create=mfs\f[] +.IP \[bu] 2 +\f[B]cache.open=\f[]: \[aq]open\[aq] policy cache timeout in seconds. +(default: 0) .PP \f[B]NOTE:\f[] Options are evaluated in the order listed so if the options are \f[B]func.rmdir=rand,category.action=ff\f[] the @@ -1034,6 +1037,23 @@ The options for setting these are \f[C]entry_timeout\f[] and for the attributes cache. \f[C]negative_timeout\f[] refers to the timeout for negative responses to lookups (non\-existant files). +.SS policy caching +.PP +Policies are run every time a function is called. +These policies can be expensive depending on the setup and usage +patterns. +Generally we wouldn\[aq]t want to cache policy results because it may +result in stale responses if the underlying drives are used directly. +.PP +The \f[C]open\f[] policy cache will cache the result of an \f[C]open\f[] +policy for a particular input for \f[C]cache.open\f[] seconds or until +the file is unlinked. +Each file close (release) will randomly chose to clean up the cache of +expired entries. +.PP +This cache is useful in cases like that of \f[B]Transmission\f[] which +has a "open, read/write, close" pattern (which is much more costly due +to the FUSE overhead than normal.) .SS writeback caching .PP writeback caching is a technique for improving write speeds by batching diff --git a/src/access.cpp b/src/access.cpp index bdcc8df1..96a124f4 100644 --- a/src/access.cpp +++ b/src/access.cpp @@ -26,8 +26,6 @@ using std::string; using std::vector; -using mergerfs::Policy; -using mergerfs::Category; static int diff --git a/src/category.cpp b/src/category.cpp index fc228ed2..9985b586 100644 --- a/src/category.cpp +++ b/src/category.cpp @@ -22,41 +22,38 @@ #define CATEGORY(X) Category(Category::Enum::X,#X) -namespace mergerfs +const std::vector Category::_categories_ = + buildvector + (CATEGORY(invalid)) + (CATEGORY(action)) + (CATEGORY(create)) + (CATEGORY(search)); + +const Category * const Category::categories = &_categories_[1]; + +const Category &Category::invalid = Category::categories[Category::Enum::invalid]; +const Category &Category::action = Category::categories[Category::Enum::action]; +const Category &Category::create = Category::categories[Category::Enum::create]; +const Category &Category::search = Category::categories[Category::Enum::search]; + +const Category& +Category::find(const std::string &str) { - const std::vector Category::_categories_ = - buildvector - (CATEGORY(invalid)) - (CATEGORY(action)) - (CATEGORY(create)) - (CATEGORY(search)); - - const Category * const Category::categories = &_categories_[1]; - - const Category &Category::invalid = Category::categories[Category::Enum::invalid]; - const Category &Category::action = Category::categories[Category::Enum::action]; - const Category &Category::create = Category::categories[Category::Enum::create]; - const Category &Category::search = Category::categories[Category::Enum::search]; - - const Category& - Category::find(const std::string &str) - { - for(int i = Enum::BEGIN; i != Enum::END; ++i) - { - if(categories[i] == str) - return categories[i]; - } - - return invalid; - } - - const Category& - Category::find(const Category::Enum::Type i) - { - if(i >= Category::Enum::BEGIN && - i < Category::Enum::END) - return categories[i]; - - return invalid; - } + for(int i = Enum::BEGIN; i != Enum::END; ++i) + { + if(categories[i] == str) + return categories[i]; + } + + return invalid; +} + +const Category& +Category::find(const Category::Enum::Type i) +{ + if(i >= Category::Enum::BEGIN && + i < Category::Enum::END) + return categories[i]; + + return invalid; } diff --git a/src/category.hpp b/src/category.hpp index 4c5644d1..d031f01d 100644 --- a/src/category.hpp +++ b/src/category.hpp @@ -19,69 +19,66 @@ #include #include -namespace mergerfs +class Category { - class Category +public: + struct Enum { - public: - struct Enum - { - enum Type - { - invalid = -1, - BEGIN = 0, - action = BEGIN, - create, - search, - END - }; - }; + enum Type + { + invalid = -1, + BEGIN = 0, + action = BEGIN, + create, + search, + END + }; + }; - private: - Enum::Type _enum; - std::string _str; +private: + Enum::Type _enum; + std::string _str; - public: - Category() - : _enum(invalid), - _str(invalid) - { - } +public: + Category() + : _enum(invalid), + _str(invalid) + { + } - Category(const Enum::Type enum_, - const std::string &str_) - : _enum(enum_), - _str(str_) - { - } + Category(const Enum::Type enum_, + const std::string &str_) + : _enum(enum_), + _str(str_) + { + } - public: - operator const Enum::Type() const { return _enum; } - operator const std::string&() const { return _str; } - operator const Category*() const { return this; } +public: + operator const Enum::Type() const { return _enum; } + operator const std::string&() const { return _str; } + operator const Category*() const { return this; } - bool operator==(const std::string &str_) const - { return _str == str_; } + bool operator==(const std::string &str_) const + { return _str == str_; } - bool operator==(const Enum::Type enum_) const - { return _enum == enum_; } + bool operator==(const Enum::Type enum_) const + { return _enum == enum_; } - bool operator!=(const Category &r) const - { return _enum != r._enum; } + bool operator!=(const Category &r) const + { return _enum != r._enum; } - bool operator<(const Category &r) const - { return _enum < r._enum; } + bool operator<(const Category &r) const + { return _enum < r._enum; } - public: - static const Category &find(const std::string&); - static const Category &find(const Enum::Type); +public: + static const Category &find(const std::string&); + static const Category &find(const Enum::Type); - public: - static const std::vector _categories_; - static const Category * const categories; - static const Category &invalid; - static const Category &action; - static const Category &create; - static const Category &search; - }; -} +public: + static const std::vector _categories_; + static const Category * const categories; + static const Category &invalid; + static const Category &action; + static const Category &create; + static const Category &search; +}; diff --git a/src/chmod.cpp b/src/chmod.cpp index 925a8bf5..7aa5da7b 100644 --- a/src/chmod.cpp +++ b/src/chmod.cpp @@ -29,7 +29,6 @@ using std::string; using std::vector; -using mergerfs::Policy; static int diff --git a/src/chown.cpp b/src/chown.cpp index 9784cb59..a3708f9d 100644 --- a/src/chown.cpp +++ b/src/chown.cpp @@ -29,7 +29,6 @@ using std::string; using std::vector; -using mergerfs::Policy; using mergerfs::Config; static diff --git a/src/config.hpp b/src/config.hpp index 61aa12d8..aa81c313 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -19,6 +19,7 @@ #include "branch.hpp" #include "fusefunc.hpp" #include "policy.hpp" +#include "policy_cache.hpp" #include @@ -102,6 +103,9 @@ namespace mergerfs const Policy *&unlink; const Policy *&utimens; + public: + mutable PolicyCache open_cache; + public: const std::string controlfile; diff --git a/src/getattr.cpp b/src/getattr.cpp index ba587aec..6b8c9d13 100644 --- a/src/getattr.cpp +++ b/src/getattr.cpp @@ -30,7 +30,6 @@ using std::string; using std::vector; -using mergerfs::Policy; static int diff --git a/src/getxattr.cpp b/src/getxattr.cpp index c0d0ec9c..6c9d27e6 100644 --- a/src/getxattr.cpp +++ b/src/getxattr.cpp @@ -306,6 +306,8 @@ _getxattr_controlfile(const Config &config, _getxattr_controlfile_category_policy(config,attr[3],attrvalue); else if(attr[2] == "func") _getxattr_controlfile_fusefunc_policy(config,attr[3],attrvalue); + else if((attr[2] == "cache") && (attr[3] == "open")) + _getxattr_controlfile_uint64_t(config.open_cache.timeout,attrvalue); break; } diff --git a/src/link.cpp b/src/link.cpp index 87dfc04c..3adb21fe 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -30,7 +30,6 @@ using std::string; using std::vector; -using mergerfs::Policy; using namespace mergerfs; static diff --git a/src/listxattr.cpp b/src/listxattr.cpp index 09239f0f..dfa794da 100644 --- a/src/listxattr.cpp +++ b/src/listxattr.cpp @@ -43,24 +43,26 @@ _listxattr_controlfile(char *list, string xattrs; const vector strs = buildvector - ("user.mergerfs.srcmounts") ("user.mergerfs.branches") - ("user.mergerfs.minfreespace") - ("user.mergerfs.moveonenospc") + ("user.mergerfs.cache.open") + ("user.mergerfs.direct_io") ("user.mergerfs.dropcacheonclose") - ("user.mergerfs.symlinkify") - ("user.mergerfs.symlinkify_timeout") - ("user.mergerfs.nullrw") ("user.mergerfs.ignorepponrename") ("user.mergerfs.link_cow") + ("user.mergerfs.minfreespace") + ("user.mergerfs.moveonenospc") + ("user.mergerfs.nullrw") + ("user.mergerfs.pid") + ("user.mergerfs.policies") ("user.mergerfs.security_capability") - ("user.mergerfs.xattr") + ("user.mergerfs.srcmounts") ("user.mergerfs.statfs") ("user.mergerfs.statfs_ignore") - ("user.mergerfs.direct_io") - ("user.mergerfs.policies") + ("user.mergerfs.symlinkify") + ("user.mergerfs.symlinkify_timeout") ("user.mergerfs.version") - ("user.mergerfs.pid"); + ("user.mergerfs.xattr") + ; xattrs.reserve(1024); for(size_t i = 0; i < strs.size(); i++) diff --git a/src/open.cpp b/src/open.cpp index 39b53e9b..61d5d2df 100644 --- a/src/open.cpp +++ b/src/open.cpp @@ -20,6 +20,7 @@ #include "fs_base_open.hpp" #include "fs_cow.hpp" #include "fs_path.hpp" +#include "policy_cache.hpp" #include "rwlock.hpp" #include "ugid.hpp" @@ -28,11 +29,8 @@ #include #include -#include - using std::string; using std::vector; -using mergerfs::Policy; namespace local { @@ -64,6 +62,7 @@ namespace local static int open(Policy::Func::Search searchFunc_, + PolicyCache &cache, const Branches &branches_, const uint64_t minfreespace_, const char *fusepath_, @@ -72,13 +71,13 @@ namespace local uint64_t *fh_) { int rv; - vector basepaths; + string basepath; - rv = searchFunc_(branches_,fusepath_,minfreespace_,basepaths); + rv = cache(searchFunc_,branches_,fusepath_,minfreespace_,&basepath); if(rv == -1) return -errno; - return local::open_core(*basepaths[0],fusepath_,flags_,link_cow_,fh_); + return local::open_core(basepath,fusepath_,flags_,link_cow_,fh_); } } @@ -97,6 +96,7 @@ namespace mergerfs ffi_->direct_io = config.direct_io; return local::open(config.open, + config.open_cache, config.branches, config.minfreespace, fusepath_, diff --git a/src/option_parser.cpp b/src/option_parser.cpp index 8edf6364..874089d9 100644 --- a/src/option_parser.cpp +++ b/src/option_parser.cpp @@ -202,6 +202,18 @@ parse_and_process_statfsignore(const std::string &value_, return 0; } +static +int +parse_and_process_cache(Config &config_, + const string &func_, + const string &value_) +{ + if(func_ == "open") + return parse_and_process(value_,config_.open_cache.timeout); + + return 1; +} + static int parse_and_process_arg(Config &config, @@ -233,6 +245,8 @@ parse_and_process_kv_arg(Config &config, rv = config.set_func_policy(keypart[1],value); else if(keypart[0] == "category") rv = config.set_category_policy(keypart[1],value); + else if(keypart[0] == "cache") + rv = parse_and_process_cache(config,keypart[1],value); } else { @@ -336,6 +350,8 @@ usage(void) " splice_write, splice_move\n" " -o func.=

Set function to policy

\n" " -o category.=

Set functions in category to

\n" + " -o cache.open= 'open' policy cache timeout in seconds.\n" + " default = 0 (disabled)\n" " -o direct_io Bypass page caching, may increase write\n" " speeds at the cost of reads. Please read docs\n" " for more details as there are tradeoffs.\n" diff --git a/src/policy.cpp b/src/policy.cpp index a9d48929..483e610f 100644 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -24,65 +24,63 @@ #define PRESERVES_PATH true #define DOESNT_PRESERVE_PATH false -namespace mergerfs -{ - const std::vector Policy::_policies_ = - buildvector - (POLICY(invalid,DOESNT_PRESERVE_PATH)) - (POLICY(all,DOESNT_PRESERVE_PATH)) - (POLICY(epall,PRESERVES_PATH)) - (POLICY(epff,PRESERVES_PATH)) - (POLICY(eplfs,PRESERVES_PATH)) - (POLICY(eplus,PRESERVES_PATH)) - (POLICY(epmfs,PRESERVES_PATH)) - (POLICY(eprand,PRESERVES_PATH)) - (POLICY(erofs,DOESNT_PRESERVE_PATH)) - (POLICY(ff,DOESNT_PRESERVE_PATH)) - (POLICY(lfs,DOESNT_PRESERVE_PATH)) - (POLICY(lus,DOESNT_PRESERVE_PATH)) - (POLICY(mfs,DOESNT_PRESERVE_PATH)) - (POLICY(newest,DOESNT_PRESERVE_PATH)) - (POLICY(rand,DOESNT_PRESERVE_PATH)); +const std::vector Policy::_policies_ = + buildvector + (POLICY(invalid,DOESNT_PRESERVE_PATH)) + (POLICY(all,DOESNT_PRESERVE_PATH)) + (POLICY(epall,PRESERVES_PATH)) + (POLICY(epff,PRESERVES_PATH)) + (POLICY(eplfs,PRESERVES_PATH)) + (POLICY(eplus,PRESERVES_PATH)) + (POLICY(epmfs,PRESERVES_PATH)) + (POLICY(eprand,PRESERVES_PATH)) + (POLICY(erofs,DOESNT_PRESERVE_PATH)) + (POLICY(ff,DOESNT_PRESERVE_PATH)) + (POLICY(lfs,DOESNT_PRESERVE_PATH)) + (POLICY(lus,DOESNT_PRESERVE_PATH)) + (POLICY(mfs,DOESNT_PRESERVE_PATH)) + (POLICY(newest,DOESNT_PRESERVE_PATH)) + (POLICY(rand,DOESNT_PRESERVE_PATH)); - const Policy * const Policy::policies = &_policies_[1]; +const Policy * const Policy::policies = &_policies_[1]; #define CONST_POLICY(X) const Policy &Policy::X = Policy::policies[Policy::Enum::X] - CONST_POLICY(invalid); - CONST_POLICY(all); - CONST_POLICY(epall); - CONST_POLICY(epff); - CONST_POLICY(eplfs); - CONST_POLICY(eplus); - CONST_POLICY(epmfs); - CONST_POLICY(eprand); - CONST_POLICY(erofs); - CONST_POLICY(ff); - CONST_POLICY(lfs); - CONST_POLICY(lus); - CONST_POLICY(mfs); - CONST_POLICY(newest); - CONST_POLICY(rand); +CONST_POLICY(invalid); +CONST_POLICY(all); +CONST_POLICY(epall); +CONST_POLICY(epff); +CONST_POLICY(eplfs); +CONST_POLICY(eplus); +CONST_POLICY(epmfs); +CONST_POLICY(eprand); +CONST_POLICY(erofs); +CONST_POLICY(ff); +CONST_POLICY(lfs); +CONST_POLICY(lus); +CONST_POLICY(mfs); +CONST_POLICY(newest); +CONST_POLICY(rand); - const Policy& - Policy::find(const std::string &str) +const Policy& +Policy::find(const std::string &str) +{ +for(int i = Enum::BEGIN; i != Enum::END; ++i) { - for(int i = Enum::BEGIN; i != Enum::END; ++i) - { - if(policies[i] == str) - return policies[i]; - } - - return invalid; + if(policies[i] == str) + return policies[i]; } - const Policy& - Policy::find(const Policy::Enum::Type i) - { - if(i >= Policy::Enum::BEGIN && - i < Policy::Enum::END) - return policies[i]; + return invalid; +} - return invalid; - } +const Policy& +Policy::find(const Policy::Enum::Type i) +{ + if(i >= Policy::Enum::BEGIN && + i < Policy::Enum::END) + return policies[i]; + + return invalid; } + diff --git a/src/policy.hpp b/src/policy.hpp index bd80747f..67ef6ec7 100644 --- a/src/policy.hpp +++ b/src/policy.hpp @@ -24,168 +24,178 @@ #include #include -namespace mergerfs +class Policy { - class Policy +public: + struct Enum { - public: - struct Enum - { - enum Type - { - invalid = -1, - BEGIN = 0, - all = BEGIN, - epall, - epff, - eplfs, - eplus, - epmfs, - eprand, - erofs, - ff, - lfs, - lus, - mfs, - newest, - rand, - END - }; - - static size_t begin() { return BEGIN; } - static size_t end() { return END; } - }; - - struct Func - { - typedef std::string string; - typedef std::vector strvec; - typedef std::vector cstrptrvec; - typedef const string cstring; - typedef const uint64_t cuint64_t; - typedef const strvec cstrvec; - typedef const Category::Enum::Type CType; - - typedef int (*Ptr)(CType,const Branches &,const char *,cuint64_t,cstrptrvec &); - - template - class Base + enum Type { - public: - Base(const Policy *p) - : func(p->_func) - {} - - int - operator()(const Branches &b,const char *c,cuint64_t d,cstrptrvec &e) - { - return func(T,b,c,d,e); - } - - int - operator()(const Branches &b,const string &c,cuint64_t d,cstrptrvec &e) - { - return func(T,b,c.c_str(),d,e); - } - - private: - const Ptr func; + invalid = -1, + BEGIN = 0, + all = BEGIN, + epall, + epff, + eplfs, + eplus, + epmfs, + eprand, + erofs, + ff, + lfs, + lus, + mfs, + newest, + rand, + END }; - typedef Base Action; - typedef Base Create; - typedef Base Search; - - static int invalid(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int all(CType,const Branches&,const char*,cuint64_t,cstrptrvec&); - static int epall(CType,const Branches&,const char*,cuint64_t,cstrptrvec&); - static int epff(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int eplfs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int eplus(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int epmfs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int eprand(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int erofs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int ff(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int lfs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int lus(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int mfs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int newest(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - static int rand(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); - }; + static size_t begin() { return BEGIN; } + static size_t end() { return END; } + }; - private: - Enum::Type _enum; - std::string _str; - Func::Ptr _func; - bool _path_preserving; - - public: - Policy() - : _enum(invalid), - _str(invalid), - _func(invalid), - _path_preserving(false) - { - } - - Policy(const Enum::Type enum_, - const std::string &str_, - const Func::Ptr func_, - const bool path_preserving_) - : _enum(enum_), - _str(str_), - _func(func_), - _path_preserving(path_preserving_) + struct Func + { + typedef std::string string; + typedef std::vector strvec; + typedef std::vector cstrptrvec; + typedef const string cstring; + typedef const uint64_t cuint64_t; + typedef const strvec cstrvec; + typedef const Category::Enum::Type CType; + + typedef int (*Ptr)(CType,const Branches &,const char *,cuint64_t,cstrptrvec &); + + template + class Base { - } + public: + Base(const Policy *p) + : func(p->_func) + {} - bool - path_preserving() const - { - return _path_preserving; - } - - public: - operator const Enum::Type() const { return _enum; } - operator const std::string&() const { return _str; } - operator const Func::Ptr() const { return _func; } - operator const Policy*() const { return this; } - - bool operator==(const Enum::Type enum_) const - { return _enum == enum_; } - bool operator==(const std::string &str_) const - { return _str == str_; } - bool operator==(const Func::Ptr func_) const - { return _func == func_; } - - bool operator!=(const Policy &r) const - { return _enum != r._enum; } - - bool operator<(const Policy &r) const - { return _enum < r._enum; } - - public: - static const Policy &find(const std::string&); - static const Policy &find(const Enum::Type); - - public: - static const std::vector _policies_; - static const Policy * const policies; - - static const Policy &invalid; - static const Policy &all; - static const Policy &epall; - static const Policy &epff; - static const Policy &eplfs; - static const Policy ⩱ - static const Policy &epmfs; - static const Policy &eprand; - static const Policy &erofs; - static const Policy &ff; - static const Policy &lfs; - static const Policy &lus; - static const Policy &mfs; - static const Policy &newest; - static const Policy &rand; + int + operator()(const Branches &b,const char *c,cuint64_t d,cstrptrvec &e) + { + return func(T,b,c,d,e); + } + + int + operator()(const Branches &b,const string &c,cuint64_t d,cstrptrvec &e) + { + return func(T,b,c.c_str(),d,e); + } + + int + operator()(const Branches &b,const char *c,cuint64_t d,string *e) + { + int rv; + cstrptrvec v; + + rv = func(T,b,c,d,v); + if(!v.empty()) + *e = *v[0]; + + return rv; + } + + private: + const Ptr func; + }; + + typedef Base Action; + typedef Base Create; + typedef Base Search; + + static int invalid(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int all(CType,const Branches&,const char*,cuint64_t,cstrptrvec&); + static int epall(CType,const Branches&,const char*,cuint64_t,cstrptrvec&); + static int epff(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int eplfs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int eplus(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int epmfs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int eprand(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int erofs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int ff(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int lfs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int lus(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int mfs(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int newest(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); + static int rand(CType,const Branches&,const char *,cuint64_t,cstrptrvec&); }; -} + +private: + Enum::Type _enum; + std::string _str; + Func::Ptr _func; + bool _path_preserving; + +public: + Policy() + : _enum(invalid), + _str(invalid), + _func(invalid), + _path_preserving(false) + { + } + + Policy(const Enum::Type enum_, + const std::string &str_, + const Func::Ptr func_, + const bool path_preserving_) + : _enum(enum_), + _str(str_), + _func(func_), + _path_preserving(path_preserving_) + { + } + + bool + path_preserving() const + { + return _path_preserving; + } + +public: + operator const Enum::Type() const { return _enum; } + operator const std::string&() const { return _str; } + operator const Func::Ptr() const { return _func; } + operator const Policy*() const { return this; } + + bool operator==(const Enum::Type enum_) const + { return _enum == enum_; } + bool operator==(const std::string &str_) const + { return _str == str_; } + bool operator==(const Func::Ptr func_) const + { return _func == func_; } + + bool operator!=(const Policy &r) const + { return _enum != r._enum; } + + bool operator<(const Policy &r) const + { return _enum < r._enum; } + +public: + static const Policy &find(const std::string&); + static const Policy &find(const Enum::Type); + +public: + static const std::vector _policies_; + static const Policy * const policies; + + static const Policy &invalid; + static const Policy &all; + static const Policy &epall; + static const Policy &epff; + static const Policy &eplfs; + static const Policy ⩱ + static const Policy &epmfs; + static const Policy &eprand; + static const Policy &erofs; + static const Policy &ff; + static const Policy &lfs; + static const Policy &lus; + static const Policy &mfs; + static const Policy &newest; + static const Policy &rand; +}; diff --git a/src/policy_all.cpp b/src/policy_all.cpp index f95c5ace..58007b0e 100644 --- a/src/policy_all.cpp +++ b/src/policy_all.cpp @@ -65,18 +65,15 @@ namespace all } } -namespace mergerfs +int +Policy::Func::all(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::all(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - if(type == Category::Enum::create) - return all::create(branches_,minfreespace,paths); + if(type == Category::Enum::create) + return all::create(branches_,minfreespace,paths); - return Policy::Func::epall(type,branches_,fusepath,minfreespace,paths); - } + return Policy::Func::epall(type,branches_,fusepath,minfreespace,paths); } diff --git a/src/policy_cache.cpp b/src/policy_cache.cpp new file mode 100644 index 00000000..55764275 --- /dev/null +++ b/src/policy_cache.cpp @@ -0,0 +1,128 @@ +#include "policy_cache.hpp" + +#include +#include +#include +#include + +#include + +using std::map; +using std::string; +using std::vector; + +static const uint64_t DEFAULT_TIMEOUT = 0; + +namespace local +{ + static + uint64_t + get_time(void) + { + uint64_t rv; + struct timeval now; + + ::gettimeofday(&now,NULL); + + rv = now.tv_sec; + + return rv; + } +} + + +PolicyCache::Value::Value() + : time(0), + path() +{ + +} + +PolicyCache::PolicyCache(void) + : timeout(DEFAULT_TIMEOUT) +{ +} + +void +PolicyCache::erase(const char *fusepath_) +{ + pthread_mutex_lock(&_lock); + + _cache.erase(fusepath_); + + pthread_mutex_unlock(&_lock); +} + +void +PolicyCache::cleanup(const int prob_) +{ + uint64_t now; + map::iterator i; + + if(rand() % prob_) + return; + + now = local::get_time(); + + pthread_mutex_lock(&_lock); + + i = _cache.begin(); + while(i != _cache.end()) + { + if((now - i->second.time) >= timeout) + _cache.erase(i++); + else + ++i; + } + + pthread_mutex_unlock(&_lock); +} + +void +PolicyCache::clear(void) +{ + pthread_mutex_lock(&_lock); + + _cache.clear(); + + pthread_mutex_unlock(&_lock); +} + +int +PolicyCache::operator()(Policy::Func::Search &func_, + const Branches &branches_, + const char *fusepath_, + const uint64_t minfreespace_, + std::string *branch_) +{ + int rv; + Value *v; + uint64_t now; + string branch; + + if(timeout == 0) + return func_(branches_,fusepath_,minfreespace_,branch_); + + now = local::get_time(); + + pthread_mutex_lock(&_lock); + v = &_cache[fusepath_]; + + if((now - v->time) >= timeout) + { + pthread_mutex_unlock(&_lock); + rv = func_(branches_,fusepath_,minfreespace_,&branch); + if(rv == -1) + return -1; + + pthread_mutex_lock(&_lock); + v->time = now; + v->path = branch; + } + + *branch_ = v->path; + + pthread_mutex_unlock(&_lock); + + return 0; +} diff --git a/src/policy_cache.hpp b/src/policy_cache.hpp new file mode 100644 index 00000000..d965264d --- /dev/null +++ b/src/policy_cache.hpp @@ -0,0 +1,61 @@ +/* + ISC License + + Copyright (c) 2019, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#pragma once + +#include "policy.hpp" + +#include +#include + +#include +#include + +class PolicyCache +{ +public: + struct Value + { + Value(); + + uint64_t time; + std::string path; + }; + +public: + PolicyCache(void); + +public: + void erase(const char *fusepath_); + void cleanup(const int prob_ = 1); + void clear(void); + +public: + int operator()(Policy::Func::Search &func_, + const Branches &branches_, + const char *fusepath_, + const uint64_t minfreespace_, + std::string *branch_); + +public: + uint64_t timeout; + +private: + pthread_mutex_t _lock; + std::map _cache; +}; diff --git a/src/policy_epall.cpp b/src/policy_epall.cpp index 097526dc..363c26cb 100644 --- a/src/policy_epall.cpp +++ b/src/policy_epall.cpp @@ -128,24 +128,21 @@ namespace epall } } -namespace mergerfs +int +Policy::Func::epall(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::epall(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - switch(type) - { - case Category::Enum::create: - return epall::create(branches_,fusepath,minfreespace,paths); - case Category::Enum::action: - return epall::action(branches_,fusepath,paths); - case Category::Enum::search: - default: - return epall::search(branches_,fusepath,paths); - } - } + switch(type) + { + case Category::Enum::create: + return epall::create(branches_,fusepath,minfreespace,paths); + case Category::Enum::action: + return epall::action(branches_,fusepath,paths); + case Category::Enum::search: + default: + return epall::search(branches_,fusepath,paths); + } } diff --git a/src/policy_epff.cpp b/src/policy_epff.cpp index 718b9345..e9a50135 100644 --- a/src/policy_epff.cpp +++ b/src/policy_epff.cpp @@ -27,7 +27,6 @@ using std::string; using std::vector; -using mergerfs::Category; namespace epff { @@ -126,24 +125,21 @@ namespace epff } } -namespace mergerfs +int +Policy::Func::epff(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::epff(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - switch(type) - { - case Category::Enum::create: - return epff::create(branches_,fusepath,minfreespace,paths); - case Category::Enum::action: - return epff::action(branches_,fusepath,paths); - case Category::Enum::search: - default: - return epff::search(branches_,fusepath,paths); - } - } + switch(type) + { + case Category::Enum::create: + return epff::create(branches_,fusepath,minfreespace,paths); + case Category::Enum::action: + return epff::action(branches_,fusepath,paths); + case Category::Enum::search: + default: + return epff::search(branches_,fusepath,paths); + } } diff --git a/src/policy_eplfs.cpp b/src/policy_eplfs.cpp index 9db6129a..a481b0c0 100644 --- a/src/policy_eplfs.cpp +++ b/src/policy_eplfs.cpp @@ -28,7 +28,6 @@ using std::string; using std::vector; -using mergerfs::Category; namespace eplfs { @@ -162,24 +161,21 @@ namespace eplfs } } -namespace mergerfs +int +Policy::Func::eplfs(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::eplfs(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - switch(type) - { - case Category::Enum::create: - return eplfs::create(branches_,fusepath,minfreespace,paths); - case Category::Enum::action: - return eplfs::action(branches_,fusepath,paths); - case Category::Enum::search: - default: - return eplfs::search(branches_,fusepath,paths); - } - } + switch(type) + { + case Category::Enum::create: + return eplfs::create(branches_,fusepath,minfreespace,paths); + case Category::Enum::action: + return eplfs::action(branches_,fusepath,paths); + case Category::Enum::search: + default: + return eplfs::search(branches_,fusepath,paths); + } } diff --git a/src/policy_eplus.cpp b/src/policy_eplus.cpp index a9636d9e..82accc0c 100644 --- a/src/policy_eplus.cpp +++ b/src/policy_eplus.cpp @@ -28,7 +28,6 @@ using std::string; using std::vector; -using mergerfs::Category; namespace eplus { @@ -162,24 +161,21 @@ namespace eplus } } -namespace mergerfs +int +Policy::Func::eplus(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::eplus(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - switch(type) - { - case Category::Enum::create: - return eplus::create(branches_,fusepath,minfreespace,paths); - case Category::Enum::action: - return eplus::action(branches_,fusepath,paths); - case Category::Enum::search: - default: - return eplus::search(branches_,fusepath,paths); - } - } + switch(type) + { + case Category::Enum::create: + return eplus::create(branches_,fusepath,minfreespace,paths); + case Category::Enum::action: + return eplus::action(branches_,fusepath,paths); + case Category::Enum::search: + default: + return eplus::search(branches_,fusepath,paths); + } } diff --git a/src/policy_epmfs.cpp b/src/policy_epmfs.cpp index 6450b87b..3af1a02a 100644 --- a/src/policy_epmfs.cpp +++ b/src/policy_epmfs.cpp @@ -28,7 +28,6 @@ using std::string; using std::vector; -using mergerfs::Category; namespace epmfs { @@ -162,24 +161,21 @@ namespace epmfs } } -namespace mergerfs +int +Policy::Func::epmfs(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::epmfs(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - switch(type) - { - case Category::Enum::create: - return epmfs::create(branches_,fusepath,minfreespace,paths); - case Category::Enum::action: - return epmfs::action(branches_,fusepath,paths); - case Category::Enum::search: - default: - return epmfs::search(branches_,fusepath,paths); - } - } + switch(type) + { + case Category::Enum::create: + return epmfs::create(branches_,fusepath,minfreespace,paths); + case Category::Enum::action: + return epmfs::action(branches_,fusepath,paths); + case Category::Enum::search: + default: + return epmfs::search(branches_,fusepath,paths); + } } diff --git a/src/policy_eprand.cpp b/src/policy_eprand.cpp index 4d2bf4b1..ee5eb9d3 100644 --- a/src/policy_eprand.cpp +++ b/src/policy_eprand.cpp @@ -24,21 +24,18 @@ using std::string; using std::vector; -namespace mergerfs +int +Policy::Func::eprand(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::eprand(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - int rv; - - rv = Policy::Func::epall(type,branches_,fusepath,minfreespace,paths); - if(rv == 0) - std::random_shuffle(paths.begin(),paths.end()); - - return rv; - } + int rv; + + rv = Policy::Func::epall(type,branches_,fusepath,minfreespace,paths); + if(rv == 0) + std::random_shuffle(paths.begin(),paths.end()); + + return rv; } diff --git a/src/policy_erofs.cpp b/src/policy_erofs.cpp index af6acbd0..62235678 100644 --- a/src/policy_erofs.cpp +++ b/src/policy_erofs.cpp @@ -23,15 +23,12 @@ using std::string; using std::vector; -namespace mergerfs +int +Policy::Func::erofs(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::erofs(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - return (errno=EROFS,-1); - } + return (errno=EROFS,-1); } diff --git a/src/policy_ff.cpp b/src/policy_ff.cpp index 5ca15388..282805d4 100644 --- a/src/policy_ff.cpp +++ b/src/policy_ff.cpp @@ -65,18 +65,15 @@ namespace ff } } -namespace mergerfs +int +Policy::Func::ff(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::ff(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - if(type == Category::Enum::create) - return ff::create(branches_,minfreespace,paths); + if(type == Category::Enum::create) + return ff::create(branches_,minfreespace,paths); - return Policy::Func::epff(type,branches_,fusepath,minfreespace,paths); - } + return Policy::Func::epff(type,branches_,fusepath,minfreespace,paths); } diff --git a/src/policy_invalid.cpp b/src/policy_invalid.cpp index de031ecf..5a113b49 100644 --- a/src/policy_invalid.cpp +++ b/src/policy_invalid.cpp @@ -23,15 +23,12 @@ using std::string; using std::vector; -namespace mergerfs +int +Policy::Func::invalid(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::invalid(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - return (errno=EINVAL,-1); - } + return (errno=EINVAL,-1); } diff --git a/src/policy_lfs.cpp b/src/policy_lfs.cpp index f66014c4..1ae8c873 100644 --- a/src/policy_lfs.cpp +++ b/src/policy_lfs.cpp @@ -28,7 +28,6 @@ using std::string; using std::vector; -using mergerfs::Category; namespace lfs { @@ -77,18 +76,15 @@ namespace lfs } } -namespace mergerfs +int +Policy::Func::lfs(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::lfs(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - if(type == Category::Enum::create) - return lfs::create(branches_,minfreespace,paths); + if(type == Category::Enum::create) + return lfs::create(branches_,minfreespace,paths); - return Policy::Func::eplfs(type,branches_,fusepath,minfreespace,paths); - } + return Policy::Func::eplfs(type,branches_,fusepath,minfreespace,paths); } diff --git a/src/policy_lus.cpp b/src/policy_lus.cpp index 75799066..60df2f36 100644 --- a/src/policy_lus.cpp +++ b/src/policy_lus.cpp @@ -28,7 +28,6 @@ using std::string; using std::vector; -using mergerfs::Category; namespace lus { @@ -77,18 +76,15 @@ namespace lus } } -namespace mergerfs +int +Policy::Func::lus(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::lus(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - if(type == Category::Enum::create) - return lus::create(branches_,minfreespace,paths); + if(type == Category::Enum::create) + return lus::create(branches_,minfreespace,paths); - return Policy::Func::eplus(type,branches_,fusepath,minfreespace,paths); - } + return Policy::Func::eplus(type,branches_,fusepath,minfreespace,paths); } diff --git a/src/policy_mfs.cpp b/src/policy_mfs.cpp index 471a2731..37e27b0a 100644 --- a/src/policy_mfs.cpp +++ b/src/policy_mfs.cpp @@ -27,7 +27,6 @@ using std::string; using std::vector; -using mergerfs::Category; namespace mfs { @@ -76,18 +75,15 @@ namespace mfs } } -namespace mergerfs +int +Policy::Func::mfs(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::mfs(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - if(type == Category::Enum::create) - return mfs::create(branches_,minfreespace,paths); + if(type == Category::Enum::create) + return mfs::create(branches_,minfreespace,paths); - return Policy::Func::epmfs(type,branches_,fusepath,minfreespace,paths); - } + return Policy::Func::epmfs(type,branches_,fusepath,minfreespace,paths); } diff --git a/src/policy_newest.cpp b/src/policy_newest.cpp index 80931b9d..e7331f01 100644 --- a/src/policy_newest.cpp +++ b/src/policy_newest.cpp @@ -161,24 +161,21 @@ namespace newest } } -namespace mergerfs +int +Policy::Func::newest(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::newest(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - switch(type) - { - case Category::Enum::create: - return newest::create(branches_,fusepath,minfreespace,paths); - case Category::Enum::action: - return newest::action(branches_,fusepath,paths); - case Category::Enum::search: - default: - return newest::search(branches_,fusepath,paths); - } - } + switch(type) + { + case Category::Enum::create: + return newest::create(branches_,fusepath,minfreespace,paths); + case Category::Enum::action: + return newest::action(branches_,fusepath,paths); + case Category::Enum::search: + default: + return newest::search(branches_,fusepath,paths); + } } diff --git a/src/policy_rand.cpp b/src/policy_rand.cpp index bc2a146e..a96307b8 100644 --- a/src/policy_rand.cpp +++ b/src/policy_rand.cpp @@ -24,21 +24,18 @@ using std::string; using std::vector; -namespace mergerfs +int +Policy::Func::rand(const Category::Enum::Type type, + const Branches &branches_, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) { - int - Policy::Func::rand(const Category::Enum::Type type, - const Branches &branches_, - const char *fusepath, - const uint64_t minfreespace, - vector &paths) - { - int rv; - - rv = Policy::Func::all(type,branches_,fusepath,minfreespace,paths); - if(rv == 0) - std::random_shuffle(paths.begin(),paths.end()); - - return rv; - } + int rv; + + rv = Policy::Func::all(type,branches_,fusepath,minfreespace,paths); + if(rv == 0) + std::random_shuffle(paths.begin(),paths.end()); + + return rv; } diff --git a/src/readlink.cpp b/src/readlink.cpp index 5b90e347..42ae3007 100644 --- a/src/readlink.cpp +++ b/src/readlink.cpp @@ -29,7 +29,6 @@ using std::string; using std::vector; -using mergerfs::Policy; static int diff --git a/src/release.cpp b/src/release.cpp index 196bf645..56241ab8 100644 --- a/src/release.cpp +++ b/src/release.cpp @@ -14,34 +14,37 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include - -#include - #include "config.hpp" #include "errno.hpp" #include "fileinfo.hpp" #include "fs_base_close.hpp" #include "fs_base_fadvise.hpp" -static -int -_release(FileInfo *fi, - const bool dropcacheonclose) +#include + +#include + +namespace local { - // according to Feh of nocache calling it once doesn't always work - // https://github.com/Feh/nocache - if(dropcacheonclose) - { - fs::fadvise_dontneed(fi->fd); - fs::fadvise_dontneed(fi->fd); - } + static + int + release(FileInfo *fi_, + const bool dropcacheonclose_) + { + // according to Feh of nocache calling it once doesn't always work + // https://github.com/Feh/nocache + if(dropcacheonclose_) + { + fs::fadvise_dontneed(fi_->fd); + fs::fadvise_dontneed(fi_->fd); + } - fs::close(fi->fd); + fs::close(fi_->fd); - delete fi; + delete fi_; - return 0; + return 0; + } } namespace mergerfs @@ -49,13 +52,15 @@ namespace mergerfs namespace fuse { int - release(const char *fusepath, - fuse_file_info *ffi) + release(const char *fusepath_, + fuse_file_info *ffi_) { const Config &config = Config::get(); - FileInfo *fi = reinterpret_cast(ffi->fh); + FileInfo *fi = reinterpret_cast(ffi_->fh); + + config.open_cache.cleanup(10); - return _release(fi,config.dropcacheonclose); + return local::release(fi,config.dropcacheonclose); } } } diff --git a/src/releasedir.cpp b/src/releasedir.cpp index d95486c7..4f3ee1e8 100644 --- a/src/releasedir.cpp +++ b/src/releasedir.cpp @@ -14,17 +14,21 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include - +#include "config.hpp" #include "dirinfo.hpp" -static -int -_releasedir(DirInfo *di) +#include + +namespace local { - delete di; + static + int + releasedir(DirInfo *di_) + { + delete di_; - return 0; + return 0; + } } namespace mergerfs @@ -32,12 +36,15 @@ namespace mergerfs namespace fuse { int - releasedir(const char *fusepath, - fuse_file_info *ffi) + releasedir(const char *fusepath_, + fuse_file_info *ffi_) { - DirInfo *di = reinterpret_cast(ffi->fh); + const Config &config = Config::get(); + DirInfo *di = reinterpret_cast(ffi_->fh); + + config.open_cache.cleanup(10); - return ::_releasedir(di); + return local::releasedir(di); } } } diff --git a/src/removexattr.cpp b/src/removexattr.cpp index a4227197..31561973 100644 --- a/src/removexattr.cpp +++ b/src/removexattr.cpp @@ -29,7 +29,6 @@ using std::string; using std::vector; -using mergerfs::Policy; static int diff --git a/src/rename.cpp b/src/rename.cpp index 8cfc90b5..53df9eda 100644 --- a/src/rename.cpp +++ b/src/rename.cpp @@ -283,6 +283,8 @@ namespace mergerfs const ugid::Set ugid(fc->uid,fc->gid); const rwlock::ReadGuard readlock(&config.branches_lock); + config.open_cache.erase(oldpath); + if(config.create->path_preserving() && !config.ignorepponrename) return _rename_preserve_path(config.getattr, config.rename, diff --git a/src/rmdir.cpp b/src/rmdir.cpp index 30772d7d..923e9b15 100644 --- a/src/rmdir.cpp +++ b/src/rmdir.cpp @@ -30,7 +30,6 @@ using std::string; using std::vector; -using mergerfs::Policy; static int diff --git a/src/setxattr.cpp b/src/setxattr.cpp index 2d521c4a..e39523ff 100644 --- a/src/setxattr.cpp +++ b/src/setxattr.cpp @@ -37,7 +37,6 @@ static const char SECURITY_CAPABILITY[] = "security.capability"; using std::string; using std::vector; -using mergerfs::Policy; using mergerfs::FuseFunc; using namespace mergerfs; @@ -229,6 +228,9 @@ _setxattr_controlfile_func_policy(Config &config, if((flags & XATTR_CREATE) == XATTR_CREATE) return -EEXIST; + if(funcname == "open") + config.open_cache.clear(); + rv = config.set_func_policy(funcname,attrval); if(rv == -1) return -errno; @@ -248,6 +250,9 @@ _setxattr_controlfile_category_policy(Config &config, if((flags & XATTR_CREATE) == XATTR_CREATE) return -EEXIST; + if(categoryname == "search") + config.open_cache.clear(); + rv = config.set_category_policy(categoryname,attrval); if(rv == -1) return -errno; @@ -340,6 +345,10 @@ _setxattr_controlfile(Config &config, attr[3], attrval, flags); + else if((attr[2] == "cache") && (attr[3] == "open")) + return _setxattr_uint64_t(attrval, + flags, + config.open_cache.timeout); break; default: diff --git a/src/truncate.cpp b/src/truncate.cpp index 904020d5..70e71400 100644 --- a/src/truncate.cpp +++ b/src/truncate.cpp @@ -32,7 +32,6 @@ using std::string; using std::vector; -using mergerfs::Policy; static int diff --git a/src/unlink.cpp b/src/unlink.cpp index 3bec7cc7..3b682509 100644 --- a/src/unlink.cpp +++ b/src/unlink.cpp @@ -1,5 +1,5 @@ /* - Copyright (c) 2016, Antonio SJ Musumeci + Copyright (c) 2019, Antonio SJ Musumeci Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -14,13 +14,6 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include - -#include - -#include -#include - #include "config.hpp" #include "errno.hpp" #include "fs_base_unlink.hpp" @@ -29,57 +22,66 @@ #include "rwlock.hpp" #include "ugid.hpp" +#include + +#include +#include + +#include + using std::string; using std::vector; -using mergerfs::Policy; -static -int -_unlink_loop_core(const string *basepath, - const char *fusepath, - const int error) +namespace local { - int rv; - string fullpath; + static + int + unlink_loop_core(const string *basepath_, + const char *fusepath_, + const int error_) + { + int rv; + string fullpath; - fs::path::make(basepath,fusepath,fullpath); + fs::path::make(basepath_,fusepath_,fullpath); - rv = fs::unlink(fullpath); + rv = fs::unlink(fullpath); - return error::calc(rv,error,errno); -} + return error::calc(rv,error_,errno); + } -static -int -_unlink_loop(const vector &basepaths, - const char *fusepath) -{ - int error; + static + int + unlink_loop(const vector &basepaths_, + const char *fusepath_) + { + int error; - error = -1; - for(size_t i = 0, ei = basepaths.size(); i != ei; i++) - { - error = _unlink_loop_core(basepaths[i],fusepath,error); - } + error = -1; + for(size_t i = 0, ei = basepaths_.size(); i != ei; i++) + { + error = local::unlink_loop_core(basepaths_[i],fusepath_,error); + } - return -error; -} + return -error; + } -static -int -_unlink(Policy::Func::Action actionFunc, - const Branches &branches_, - const uint64_t minfreespace, - const char *fusepath) -{ - int rv; - vector basepaths; + static + int + unlink(Policy::Func::Action actionFunc_, + const Branches &branches_, + const uint64_t minfreespace_, + const char *fusepath_) + { + int rv; + vector basepaths; - rv = actionFunc(branches_,fusepath,minfreespace,basepaths); - if(rv == -1) - return -errno; + rv = actionFunc_(branches_,fusepath_,minfreespace_,basepaths); + if(rv == -1) + return -errno; - return _unlink_loop(basepaths,fusepath); + return local::unlink_loop(basepaths,fusepath_); + } } namespace mergerfs @@ -87,17 +89,19 @@ namespace mergerfs namespace fuse { int - unlink(const char *fusepath) + unlink(const char *fusepath_) { const fuse_context *fc = fuse_get_context(); const Config &config = Config::get(fc); const ugid::Set ugid(fc->uid,fc->gid); const rwlock::ReadGuard readlock(&config.branches_lock); - return _unlink(config.unlink, - config.branches, - config.minfreespace, - fusepath); + config.open_cache.erase(fusepath_); + + return local::unlink(config.unlink, + config.branches, + config.minfreespace, + fusepath_); } } } diff --git a/src/utimens.cpp b/src/utimens.cpp index 5eb6a9b0..b678428f 100644 --- a/src/utimens.cpp +++ b/src/utimens.cpp @@ -31,7 +31,6 @@ using std::string; using std::vector; -using mergerfs::Policy; static int