From 779143f4b6dbca3772964d7a8aa92c7723655083 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Mon, 29 Feb 2016 18:02:37 -0500 Subject: [PATCH] add minfreespace checks to policy ff's create and remove fwfs fwfs is now covered by ff with the minfreespace and readonly checks --- README.md | 9 ++--- man/mergerfs.1 | 18 +++------ src/fs.cpp | 18 +++++++++ src/fs.hpp | 3 ++ src/policy.cpp | 2 - src/policy.hpp | 3 -- src/policy_ff.cpp | 61 +++++++++++++++--------------- src/policy_fwfs.cpp | 86 ------------------------------------------- src/policy_newest.cpp | 52 +++++++++++++++++++++----- 9 files changed, 106 insertions(+), 146 deletions(-) delete mode 100644 src/policy_fwfs.cpp diff --git a/README.md b/README.md index 85f03e0e..a0cec89f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ mergerfs -o<options> <srcmounts> <mountpoint> * **defaults**: a shortcut for FUSE's **atomic_o_trunc**, **auto_cache**, **big_writes**, **default_permissions**, **splice_move**, **splice_read**, and **splice_write**. These options seem to provide the best performance. * **direct_io**: causes FUSE to bypass an addition caching step which can increase write speeds at the detriment of read speed. -* **minfreespace**: the minimum space value used for the **lfs**, **fwfs**, **eplfs**, & **epmfs** policies. Understands 'K', 'M', and 'G' to represent kilobyte, megabyte, and gigabyte respectively. (default: 4G) +* **minfreespace**: the minimum space value used for creation policies. Understands 'K', 'M', and 'G' to represent kilobyte, megabyte, and gigabyte respectively. (default: 4G) * **moveonenospc**: when enabled (set to **true**) if a **write** fails with **ENOSPC** a scan of all drives will be done looking for the drive with most free space which is at least the size of the file plus the amount which failed to write. An attempt to move the file to that drive will occur (keeping all metadata possible) and if successful the original is unlinked and the write retried. (default: false) * **func.<func>=<policy>**: sets the specific FUSE function's policy. See below for the list of value types. Example: **func.getattr=newest** * **category.<category>=<policy>**: Sets policy of all FUSE functions in the provided category. Example: **category.create=mfs** @@ -78,7 +78,7 @@ Due to FUSE limitations **ioctl** behaves differently if its acting on a directo #### Policy descriptions #### -Generally speaking most policies when called to create will filter out drives which are readonly or have less than `minfreespace`. +Most policies when called to create will filter out drives which are readonly or have less than **minfreespace**. | Policy | Description | |--------------|-------------| @@ -87,7 +87,6 @@ Generally speaking most policies when called to create will filter out drives wh | epmfs (existing path, most free space) | If the path exists on multiple drives use the one with the most free space. Falls back to **mfs**. | | erofs | Exclusively return **-1** with **errno** set to **EROFS**. By setting **create** functions to this you can in effect turn the filesystem readonly. | | ff (first found) | Given the order of the drives, as defined at mount time or when configured via xattr interface, act on the first one found. | -| fwfs (first with free space) | Pick the first drive which has at least **minfreespace**. Falls back to **mfs**. | | lfs (least free space) | Pick the drive with the least available free space but more than **minfreespace**. Falls back to **mfs**. | | mfs (most free space) | Use the drive with the most available free space. Falls back to **ff**. | | newest (newest file) | Pick the file / directory with the largest mtime. | @@ -245,9 +244,9 @@ user.mergerfs.moveonenospc: false [trapexit:/tmp/mount] $ xattr -p user.mergerfs.category.search .mergerfs ff -[trapexit:/tmp/mount] $ xattr -w user.mergerfs.category.search fwfs .mergerfs +[trapexit:/tmp/mount] $ xattr -w user.mergerfs.category.search newest .mergerfs [trapexit:/tmp/mount] $ xattr -p user.mergerfs.category.search .mergerfs -fwfs +newest [trapexit:/tmp/mount] $ xattr -w user.mergerfs.srcmounts +/tmp/c .mergerfs [trapexit:/tmp/mount] $ xattr -p user.mergerfs.srcmounts .mergerfs diff --git a/man/mergerfs.1 b/man/mergerfs.1 index 5773c842..52b3a6a0 100644 --- a/man/mergerfs.1 +++ b/man/mergerfs.1 @@ -44,8 +44,8 @@ These options seem to provide the best performance. \f[B]direct_io\f[]: causes FUSE to bypass an addition caching step which can increase write speeds at the detriment of read speed. .IP \[bu] 2 -\f[B]minfreespace\f[]: the minimum space value used for the -\f[B]lfs\f[], \f[B]fwfs\f[], \f[B]eplfs\f[], & \f[B]epmfs\f[] policies. +\f[B]minfreespace\f[]: the minimum space value used for creation +policies. Understands \[aq]K\[aq], \[aq]M\[aq], and \[aq]G\[aq] to represent kilobyte, megabyte, and gigabyte respectively. (default: 4G) @@ -161,8 +161,8 @@ In other cases where something may be searched (to confirm a directory exists across all source mounts) then \f[B]getattr\f[] will be used. .SS Policy descriptions .PP -Generally speaking most policies when called to create will filter out -drives which are readonly or have less than \f[C]minfreespace\f[]. +Most policies when called to create will filter out drives which are +readonly or have less than \f[B]minfreespace\f[]. .PP .TS tab(@); @@ -212,12 +212,6 @@ Given the order of the drives, as defined at mount time or when configured via xattr interface, act on the first one found. T} T{ -fwfs (first with free space) -T}@T{ -Pick the first drive which has at least \f[B]minfreespace\f[]. -Falls back to \f[B]mfs\f[]. -T} -T{ lfs (least free space) T}@T{ Pick the drive with the least available free space but more than @@ -533,9 +527,9 @@ user.mergerfs.moveonenospc:\ false [trapexit:/tmp/mount]\ $\ xattr\ \-p\ user.mergerfs.category.search\ .mergerfs ff -[trapexit:/tmp/mount]\ $\ xattr\ \-w\ user.mergerfs.category.search\ fwfs\ .mergerfs +[trapexit:/tmp/mount]\ $\ xattr\ \-w\ user.mergerfs.category.search\ newest\ .mergerfs [trapexit:/tmp/mount]\ $\ xattr\ \-p\ user.mergerfs.category.search\ .mergerfs -fwfs +newest [trapexit:/tmp/mount]\ $\ xattr\ \-w\ user.mergerfs.srcmounts\ +/tmp/c\ .mergerfs [trapexit:/tmp/mount]\ $\ xattr\ \-p\ user.mergerfs.srcmounts\ .mergerfs diff --git a/src/fs.cpp b/src/fs.cpp index 4dc42cee..4e651edd 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -70,6 +70,24 @@ namespace fs return exists(path,st); } + bool + exists(const string &path, + bool &readonly, + size_t &spaceavail) + { + bool rv; + struct statvfs st; + + rv = exists(path,st); + if(rv) + { + readonly = StatVFS::readonly(st); + spaceavail = StatVFS::spaceavail(st); + } + + return rv; + } + bool exists_on_rw_fs(const string &path, struct statvfs &st) diff --git a/src/fs.hpp b/src/fs.hpp index 364fa3ed..2889daac 100644 --- a/src/fs.hpp +++ b/src/fs.hpp @@ -31,6 +31,9 @@ namespace fs bool exists(const string &path, struct statvfs &st); bool exists(const string &path); + bool exists(const string &path, + bool &readonly, + size_t &spaceavail); bool exists_on_rw_fs(const string &path, struct statvfs &st); diff --git a/src/policy.cpp b/src/policy.cpp index 684369eb..d779c118 100644 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -34,7 +34,6 @@ namespace mergerfs (POLICY(epmfs,PRESERVES_PATH)) (POLICY(erofs,DOESNT_PRESERVE_PATH)) (POLICY(ff,DOESNT_PRESERVE_PATH)) - (POLICY(fwfs,DOESNT_PRESERVE_PATH)) (POLICY(lfs,DOESNT_PRESERVE_PATH)) (POLICY(mfs,DOESNT_PRESERVE_PATH)) (POLICY(newest,DOESNT_PRESERVE_PATH)) @@ -50,7 +49,6 @@ namespace mergerfs CONST_POLICY(epmfs); CONST_POLICY(erofs); CONST_POLICY(ff); - CONST_POLICY(fwfs); CONST_POLICY(lfs); CONST_POLICY(mfs); CONST_POLICY(newest); diff --git a/src/policy.hpp b/src/policy.hpp index 7cfe3ba3..dcaf220c 100644 --- a/src/policy.hpp +++ b/src/policy.hpp @@ -47,7 +47,6 @@ namespace mergerfs epmfs, erofs, ff, - fwfs, lfs, mfs, newest, @@ -100,7 +99,6 @@ namespace mergerfs static int epmfs(CType,cstrvec&,const char *,csize_t,cstrptrvec&); static int erofs(CType,cstrvec&,const char *,csize_t,cstrptrvec&); static int ff(CType,cstrvec&,const char *,csize_t,cstrptrvec&); - static int fwfs(CType,cstrvec&,const char *,csize_t,cstrptrvec&); static int lfs(CType,cstrvec&,const char *,csize_t,cstrptrvec&); static int mfs(CType,cstrvec&,const char *,csize_t,cstrptrvec&); static int newest(CType,cstrvec&,const char *,csize_t,cstrptrvec&); @@ -172,7 +170,6 @@ namespace mergerfs static const Policy &epmfs; static const Policy &erofs; static const Policy &ff; - static const Policy &fwfs; static const Policy &lfs; static const Policy &mfs; static const Policy &newest; diff --git a/src/policy_ff.cpp b/src/policy_ff.cpp index 0a2d2ee8..099e1a5c 100644 --- a/src/policy_ff.cpp +++ b/src/policy_ff.cpp @@ -15,7 +15,6 @@ */ #include -#include #include #include @@ -23,7 +22,6 @@ #include "fs.hpp" #include "fs_path.hpp" #include "policy.hpp" -#include "statvfs_util.hpp" using std::string; using std::vector; @@ -31,54 +29,54 @@ using std::size_t; static int -_ff_rw(const vector &basepaths, - vector &paths) +_ff(const vector &basepaths, + const char *fusepath, + vector &paths) { - struct statvfs st; - const string *fallback = NULL; + string fullpath; for(size_t i = 0, ei = basepaths.size(); i != ei; i++) { const string *basepath = &basepaths[i]; - if(!fs::exists(basepath->c_str(),st)) - continue; + fs::path::make(basepath,fusepath,fullpath); - if(StatVFS::readonly(st)) - { - if(fallback == NULL) - fallback = basepath; - continue; - } + if(!fs::exists(fullpath)) + continue; paths.push_back(basepath); return POLICY_SUCCESS; } - if(fallback == NULL) - return POLICY_FAIL_ENOENT; - - paths.push_back(fallback); - - return POLICY_SUCCESS; + return POLICY_FAIL_ENOENT; } static int -_ff_ro(const vector &basepaths, - const char *fusepath, - vector &paths) +_ff_create(const vector &basepaths, + const size_t minfreespace, + vector &paths) { - string fullpath; + bool readonly; + size_t spaceavail; + const string *fallback; + fallback = NULL; for(size_t i = 0, ei = basepaths.size(); i != ei; i++) { const string *basepath = &basepaths[i]; - fs::path::make(basepath,fusepath,fullpath); + if(!fs::exists(*basepath,readonly,spaceavail)) + continue; - if(!fs::exists(fullpath)) + if(readonly) + continue; + + if(fallback == NULL) + fallback = basepath; + + if(spaceavail < minfreespace) continue; paths.push_back(basepath); @@ -86,7 +84,12 @@ _ff_ro(const vector &basepaths, return POLICY_SUCCESS; } - return POLICY_FAIL_ENOENT; + if(fallback == NULL) + return POLICY_FAIL_ENOENT; + + paths.push_back(fallback); + + return POLICY_SUCCESS; } namespace mergerfs @@ -99,8 +102,8 @@ namespace mergerfs vector &paths) { if(type == Category::Enum::create) - return _ff_rw(basepaths,paths); + return _ff_create(basepaths,minfreespace,paths); - return _ff_ro(basepaths,fusepath,paths); + return _ff(basepaths,fusepath,paths); } } diff --git a/src/policy_fwfs.cpp b/src/policy_fwfs.cpp deleted file mode 100644 index 3c52c305..00000000 --- a/src/policy_fwfs.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright (c) 2016, Antonio SJ Musumeci - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ - -#include -#include - -#include -#include - -#include "fs.hpp" -#include "fs_path.hpp" -#include "policy.hpp" -#include "statvfs_util.hpp" - -using std::string; -using std::vector; -using std::size_t; -using mergerfs::Policy; -using mergerfs::Category; - -static -int -_fwfs(const vector &basepaths, - const char *fusepath, - const size_t minfreespace, - const bool needswritablefs, - vector &paths) -{ - string fullpath; - struct statvfs st; - - for(size_t i = 0, size = basepaths.size(); i != size; i++) - { - const string *basepath = &basepaths[i]; - - fs::path::make(basepath,fusepath,fullpath); - - if(!fs::available(fullpath,needswritablefs,st)) - continue; - - if(StatVFS::spaceavail(st) < minfreespace) - continue; - - paths.push_back(basepath); - - return POLICY_SUCCESS; - } - - return (errno=ENOENT,POLICY_FAIL); -} - -namespace mergerfs -{ - int - Policy::Func::fwfs(const Category::Enum::Type type, - const vector &basepaths, - const char *fusepath, - const size_t minfreespace, - vector &paths) - { - int rv; - const char *fp = - ((type == Category::Enum::create) ? "" : fusepath); - const bool needswritablefs = - (type == Category::Enum::create); - - rv = _fwfs(basepaths,fp,minfreespace,needswritablefs,paths); - if(POLICY_FAILED(rv)) - rv = Policy::Func::mfs(type,basepaths,fusepath,minfreespace,paths); - - return rv; - } -} diff --git a/src/policy_newest.cpp b/src/policy_newest.cpp index c0e83cda..c824c305 100644 --- a/src/policy_newest.cpp +++ b/src/policy_newest.cpp @@ -16,26 +16,62 @@ #include #include -#include #include #include #include +#include "fs.hpp" #include "fs_path.hpp" #include "policy.hpp" #include "success_fail.hpp" -#include "statvfs_util.hpp" using std::string; using std::vector; using std::size_t; +static +int +_newest_create(const vector &basepaths, + const char *fusepath, + vector &paths) +{ + time_t newest; + string fullpath; + struct stat st; + const string *newestbasepath; + + newest = std::numeric_limits::min(); + newestbasepath = NULL; + for(size_t i = 0, ei = basepaths.size(); i != ei; i++) + { + const string *basepath = &basepaths[i]; + + fs::path::make(basepath,fusepath,fullpath); + + if(!fs::exists(fullpath,st)) + continue; + if(st.st_mtime < newest) + continue; + if(!fs::exists_on_rw_fs(fullpath)) + continue; + + newest = st.st_mtime; + newestbasepath = basepath; + } + + if(newestbasepath == NULL) + return POLICY_FAIL_ENOENT; + + paths.push_back(newestbasepath); + + return POLICY_SUCCESS; +} + static int _newest(const vector &basepaths, const char *fusepath, - const bool needswritablefs, vector &paths) { time_t newest; @@ -55,15 +91,13 @@ _newest(const vector &basepaths, continue; if(st.st_mtime < newest) continue; - if(needswritablefs && !fs::exists_on_rw_fs(fullpath)) - continue; newest = st.st_mtime; newestbasepath = basepath; } if(newestbasepath == NULL) - return (errno=ENOENT,POLICY_FAIL); + return POLICY_FAIL_ENOENT; paths.push_back(newestbasepath); @@ -79,9 +113,9 @@ namespace mergerfs const size_t minfreespace, vector &paths) { - const bool needswritablefs = - (type == Category::Enum::create); + if(type == Category::Enum::create) + return _newest_create(basepaths,fusepath,paths); - return _newest(basepaths,fusepath,needswritablefs,paths); + return _newest(basepaths,fusepath,paths); } }