From a93ab6c2f597258ff6bf82eca3038a39955d1ce6 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Mon, 11 Jul 2016 10:08:25 -0400 Subject: [PATCH] add existing path first found policy. closes #289 --- README.md | 5 +- man/mergerfs.1 | 30 +++++++++- src/policy.cpp | 2 + src/policy.hpp | 3 + src/policy_epff.cpp | 131 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 src/policy_epff.cpp diff --git a/README.md b/README.md index 41ee0809..2e6c702e 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Due to FUSE limitations **ioctl** behaves differently if its acting on a directo | Policy | Description | |--------------|-------------| | all | Search category: acts like **ff**. Action category: apply to all found. Create category: for **mkdir**, **mknod**, and **symlink** it will apply to all found. **create** works like **ff**. It will exclude readonly drives and those with free space less than **minfreespace**. | +| epff | Given the order of the drives, as defined at mount time or when configured via the xattr interface, act on the first one found where the path already exists. For **create** cateogry it will exclude readonly drives and those with free space less than **minfreespace** (unless there is no other option). Falls back to **ff**. | | eplfs (existing path, least free space) | If the path exists on multiple drives use the one with the least free space. For **create** category it will exclude readonly drives and those with free space less than **minfreespace**. Falls back to **lfs**. | | eplus (existing path, least used space) | If the path exists on multiple drives the the one with the least used space. For **create** category it will exclude readonly drives and those with free space less than **minfreespace**. Falls back to **lus**. | | epmfs (existing path, most free space) | If the path exists on multiple drives use the one with the most free space. For **create** category it will exclude readonly drives and those with free space less than **minfreespace**. Falls back to **mfs**. | @@ -97,7 +98,7 @@ Due to FUSE limitations **ioctl** behaves differently if its acting on a directo | newest (newest file) | Pick the file / directory with the largest mtime. For **create** category it will exclude readonly drives and those with free space less than **minfreespace** (unless there is no other option). | | rand (random) | Calls **all** and then randomizes. | -**eplfs**, **eplus**, and **epmf** are path preserving policies. As the descriptions above explain they will only consider drives where the path being accessed exists. Non-path preserving policies will clone paths as necessary. +**epff**, **eplfs**, **eplus**, and **epmf** are path preserving policies. As the descriptions above explain they will only consider drives where the path being accessed exists. Non-path preserving policies will clone paths as necessary. #### Defaults #### @@ -113,7 +114,7 @@ Due to FUSE limitations **ioctl** behaves differently if its acting on a directo Originally mergerfs would return EXDEV whenever a rename was requested which was cross directory in any way. This made the code simple and was technically complient with POSIX requirements. However, many applications fail to handle EXDEV at all and treat it as a normal error or they only partially support EXDEV (don't respond the same as `mv` would). Such apps include: gvfsd-fuse v1.20.3 and prior, Finder / CIFS/SMB client in Apple OSX 10.9+, NZBGet, Samba's recycling bin feature. -* If using a **create** policy which tries to preserve directory paths (epmfs,eplfs) +* If using a **create** policy which tries to preserve directory paths (epff,eplfs,eplus,epmfs) * Using the **rename** policy get the list of files to rename * For each file attempt rename: * If failure with ENOENT run **create** policy diff --git a/man/mergerfs.1 b/man/mergerfs.1 index 496ff573..ba521f9d 100644 --- a/man/mergerfs.1 +++ b/man/mergerfs.1 @@ -203,6 +203,17 @@ It will exclude readonly drives and those with free space less than \f[B]minfreespace\f[]. T} T{ +epff +T}@T{ +Given the order of the drives, as defined at mount time or when +configured via the xattr interface, act on the first one found where the +path already exists. +For \f[B]create\f[] cateogry it will exclude readonly drives and those +with free space less than \f[B]minfreespace\f[] (unless there is no +other option). +Falls back to \f[B]ff\f[]. +T} +T{ eplfs (existing path, least free space) T}@T{ If the path exists on multiple drives use the one with the least free @@ -284,6 +295,12 @@ T}@T{ Calls \f[B]all\f[] and then randomizes. T} .TE +.PP +\f[B]epff\f[], \f[B]eplfs\f[], \f[B]eplus\f[], and \f[B]epmf\f[] are +path preserving policies. +As the descriptions above explain they will only consider drives where +the path being accessed exists. +Non\-path preserving policies will clone paths as necessary. .SS Defaults .PP .TS @@ -337,7 +354,7 @@ Such apps include: gvfsd\-fuse v1.20.3 and prior, Finder / CIFS/SMB client in Apple OSX 10.9+, NZBGet, Samba\[aq]s recycling bin feature. .IP \[bu] 2 If using a \f[B]create\f[] policy which tries to preserve directory -paths (epmfs,eplfs) +paths (epff,eplfs,eplus,epmfs) .IP \[bu] 2 Using the \f[B]rename\f[] policy get the list of files to rename .IP \[bu] 2 @@ -850,6 +867,17 @@ Drives can fail and all other data will continue to be accessable. Yes. It will be represented immediately in the pool as the policies would describe. +.SS Why do I get an "out of space" error even though the system says +there\[aq]s lots of space left? +.PP +Please reread the sections above about policies, path preserving, and +the \f[B]moveonenospc\f[] option. +If the policy is path preserving and a drive is almost full and the +drive the policy would pick then the writing of the file may fill the +drive and receive ENOSPC errors. +That is expected with those settings. +If you don\[aq]t want that: enable \f[B]moveonenospc\f[] and don\[aq]t +use a path preserving policy. .SS It\[aq]s mentioned that there are some security issues with mhddfs. What are they? How does mergerfs address them? .PP diff --git a/src/policy.cpp b/src/policy.cpp index f8d40f22..bd07c0de 100644 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -30,6 +30,7 @@ namespace mergerfs buildvector (POLICY(invalid,DOESNT_PRESERVE_PATH)) (POLICY(all,DOESNT_PRESERVE_PATH)) + (POLICY(epff,PRESERVES_PATH)) (POLICY(eplfs,PRESERVES_PATH)) (POLICY(eplus,PRESERVES_PATH)) (POLICY(epmfs,PRESERVES_PATH)) @@ -47,6 +48,7 @@ namespace mergerfs CONST_POLICY(invalid); CONST_POLICY(all); + CONST_POLICY(epff); CONST_POLICY(eplfs); CONST_POLICY(eplus); CONST_POLICY(epmfs); diff --git a/src/policy.hpp b/src/policy.hpp index 0119fe4c..09a29145 100644 --- a/src/policy.hpp +++ b/src/policy.hpp @@ -43,6 +43,7 @@ namespace mergerfs invalid = -1, BEGIN = 0, all = BEGIN, + epff, eplfs, eplus, epmfs, @@ -96,6 +97,7 @@ namespace mergerfs static int invalid(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int all(CType,cstrvec&,const char*,cuint64_t,cstrptrvec&); + static int epff(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int eplfs(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int eplus(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int epmfs(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); @@ -169,6 +171,7 @@ namespace mergerfs static const Policy &invalid; static const Policy &all; + static const Policy &epff; static const Policy &eplfs; static const Policy ⩱ static const Policy &epmfs; diff --git a/src/policy_epff.cpp b/src/policy_epff.cpp new file mode 100644 index 00000000..b8981185 --- /dev/null +++ b/src/policy_epff.cpp @@ -0,0 +1,131 @@ +/* + Copyright (c) 2016, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include + +#include +#include + +#include "fs.hpp" +#include "fs_path.hpp" +#include "policy.hpp" + +using std::string; +using std::vector; +using mergerfs::Category; + +static +int +_epff_create(const vector &basepaths, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) +{ + string fullpath; + const string *fallback; + + fallback = NULL; + for(size_t i = 0, ei = basepaths.size(); i != ei; i++) + { + bool readonly; + uint64_t spaceavail; + uint64_t _spaceused; + const string *basepath = &basepaths[i]; + + fs::path::make(basepath,fusepath,fullpath); + + if(!fs::exists(fullpath)) + continue; + if(!fs::info(*basepath,readonly,spaceavail,_spaceused)) + continue; + if(readonly) + continue; + if(fallback == NULL) + fallback = basepath; + if(spaceavail < minfreespace) + continue; + + paths.push_back(basepath); + + return POLICY_SUCCESS; + } + + if(fallback == NULL) + return POLICY_FAIL_ENOENT; + + paths.push_back(fallback); + + return POLICY_SUCCESS; +} + +static +int +_epff_other(const vector &basepaths, + const char *fusepath, + vector &paths) +{ + string fullpath; + + for(size_t i = 0, ei = basepaths.size(); i != ei; i++) + { + const string *basepath = &basepaths[i]; + + fs::path::make(basepath,fusepath,fullpath); + + if(!fs::exists(fullpath)) + continue; + + paths.push_back(basepath); + + return POLICY_SUCCESS; + } + + return POLICY_FAIL_ENOENT; +} + +static +int +_epff(const Category::Enum::Type type, + const vector &basepaths, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) +{ + if(type == Category::Enum::create) + return _epff_create(basepaths,fusepath,minfreespace,paths); + + return _epff_other(basepaths,fusepath,paths); +} + + +namespace mergerfs +{ + int + Policy::Func::epff(const Category::Enum::Type type, + const vector &basepaths, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) + { + int rv; + + rv = _epff(type,basepaths,fusepath,minfreespace,paths); + if(POLICY_FAILED(rv)) + rv = Policy::Func::ff(type,basepaths,fusepath,minfreespace,paths); + + return rv; + } +}