From 0fceb8e0a32eac72198644fb5c899880f73e66c0 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Mon, 1 Aug 2016 15:51:28 -0400 Subject: [PATCH] add epall and eprand policies --- README.md | 4 +- man/mergerfs.1 | 23 +++++++-- src/policy.cpp | 4 ++ src/policy.hpp | 6 +++ src/policy_epall.cpp | 105 ++++++++++++++++++++++++++++++++++++++++++ src/policy_eprand.cpp | 46 ++++++++++++++++++ 6 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 src/policy_epall.cpp create mode 100644 src/policy_eprand.cpp diff --git a/README.md b/README.md index ef7ecab7..6666f6ab 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ % mergerfs(1) mergerfs user manual % Antonio SJ Musumeci -% 2016-05-17 +% 2016-08-01 # NAME @@ -86,10 +86,12 @@ Due to FUSE limitations **ioctl** behaves differently if its acting on a directo | Policy | Description | |--------------|-------------| | all | Search category: acts like **ff**. Action category: apply to all found. Create category: for **mkdir**, **mknod**, and **symlink** it will apply to all found. **create** works like **ff**. It will exclude readonly drives and those with free space less than **minfreespace**. | +| epall (existing path, all) | Search category: acts like **epff**. Action category: apply to all found. Create category: for **mkdir**, **mknod**, and **symlink** it will apply to all existing paths found. **create** works like **epff**. It will exclude readonly drives and those with free space less than **minfreespace**. | | epff | Given the order of the drives, as defined at mount time or when configured via the xattr interface, act on the first one found where the path already exists. For **create** cateogry it will exclude readonly drives and those with free space less than **minfreespace** (unless there is no other option). Falls back to **ff**. | | eplfs (existing path, least free space) | If the path exists on multiple drives use the one with the least free space. For **create** category it will exclude readonly drives and those with free space less than **minfreespace**. Falls back to **lfs**. | | eplus (existing path, least used space) | If the path exists on multiple drives the the one with the least used space. For **create** category it will exclude readonly drives and those with free space less than **minfreespace**. Falls back to **lus**. | | epmfs (existing path, most free space) | If the path exists on multiple drives use the one with the most free space. For **create** category it will exclude readonly drives and those with free space less than **minfreespace**. Falls back to **mfs**. | +| eprand (existing path, random) | Calls **epall** and then randomizes. | | erofs | Exclusively return **-1** with **errno** set to **EROFS**. By setting **create** functions to this you can in effect turn the filesystem readonly. | | ff (first found) | Given the order of the drives, as defined at mount time or when configured via xattr interface, act on the first one found. For **create** category it will exclude readonly drives and those with free space less than **minfreespace** (unless there is no other option). | | lfs (least free space) | Pick the drive with the least available free space. For **create** category it will exclude readonly drives and those with free space less than **minfreespace**. Falls back to **mfs**. | diff --git a/man/mergerfs.1 b/man/mergerfs.1 index ba521f9d..69afb380 100644 --- a/man/mergerfs.1 +++ b/man/mergerfs.1 @@ -1,5 +1,5 @@ .\"t -.TH "mergerfs" "1" "2016\-05\-17" "mergerfs user manual" "" +.TH "mergerfs" "1" "2016\-08\-01" "mergerfs user manual" "" .SH NAME .PP mergerfs \- another (FUSE based) union filesystem @@ -8,7 +8,7 @@ mergerfs \- another (FUSE based) union filesystem mergerfs \-o .SH DESCRIPTION .PP -\f[B]mergerfs\f[] is a union filesystem geared towards simplifing +\f[B]mergerfs\f[] is a union filesystem geared towards simplifying storage and management of files across numerous commodity storage devices. It is similar to \f[B]mhddfs\f[], \f[B]unionfs\f[], and \f[B]aufs\f[]. @@ -30,7 +30,8 @@ Opportunistic credential caching .IP \[bu] 2 Works with heterogeneous filesystem types .IP \[bu] 2 -Handling of writes to full drives +Handling of writes to full drives (transparently move file to drive with +capacity) .IP \[bu] 2 Handles pool of readonly and read/write drives .SH OPTIONS @@ -203,6 +204,17 @@ It will exclude readonly drives and those with free space less than \f[B]minfreespace\f[]. T} T{ +epall (existing path, all) +T}@T{ +Search category: acts like \f[B]epff\f[]. +Action category: apply to all found. +Create category: for \f[B]mkdir\f[], \f[B]mknod\f[], and +\f[B]symlink\f[] it will apply to all existing paths found. +\f[B]create\f[] works like \f[B]epff\f[]. +It will exclude readonly drives and those with free space less than +\f[B]minfreespace\f[]. +T} +T{ epff T}@T{ Given the order of the drives, as defined at mount time or when @@ -241,6 +253,11 @@ with free space less than \f[B]minfreespace\f[]. Falls back to \f[B]mfs\f[]. T} T{ +eprand (existing path, random) +T}@T{ +Calls \f[B]epall\f[] and then randomizes. +T} +T{ erofs T}@T{ Exclusively return \f[B]\-1\f[] with \f[B]errno\f[] set to diff --git a/src/policy.cpp b/src/policy.cpp index bd07c0de..f443045f 100644 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -30,10 +30,12 @@ namespace mergerfs buildvector (POLICY(invalid,DOESNT_PRESERVE_PATH)) (POLICY(all,DOESNT_PRESERVE_PATH)) + (POLICY(epall,PRESERVES_PATH)) (POLICY(epff,PRESERVES_PATH)) (POLICY(eplfs,PRESERVES_PATH)) (POLICY(eplus,PRESERVES_PATH)) (POLICY(epmfs,PRESERVES_PATH)) + (POLICY(eprand,PRESERVES_PATH)) (POLICY(erofs,DOESNT_PRESERVE_PATH)) (POLICY(ff,DOESNT_PRESERVE_PATH)) (POLICY(lfs,DOESNT_PRESERVE_PATH)) @@ -48,10 +50,12 @@ namespace mergerfs CONST_POLICY(invalid); CONST_POLICY(all); + CONST_POLICY(epall); CONST_POLICY(epff); CONST_POLICY(eplfs); CONST_POLICY(eplus); CONST_POLICY(epmfs); + CONST_POLICY(eprand); CONST_POLICY(erofs); CONST_POLICY(ff); CONST_POLICY(lfs); diff --git a/src/policy.hpp b/src/policy.hpp index 09a29145..1fda5d9a 100644 --- a/src/policy.hpp +++ b/src/policy.hpp @@ -43,10 +43,12 @@ namespace mergerfs invalid = -1, BEGIN = 0, all = BEGIN, + epall, epff, eplfs, eplus, epmfs, + eprand, erofs, ff, lfs, @@ -97,10 +99,12 @@ namespace mergerfs static int invalid(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int all(CType,cstrvec&,const char*,cuint64_t,cstrptrvec&); + static int epall(CType,cstrvec&,const char*,cuint64_t,cstrptrvec&); static int epff(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int eplfs(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int eplus(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int epmfs(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); + static int eprand(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int erofs(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int ff(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); static int lfs(CType,cstrvec&,const char *,cuint64_t,cstrptrvec&); @@ -171,10 +175,12 @@ namespace mergerfs static const Policy &invalid; static const Policy &all; + static const Policy &epall; static const Policy &epff; static const Policy &eplfs; static const Policy ⩱ static const Policy &epmfs; + static const Policy &eprand; static const Policy &erofs; static const Policy &ff; static const Policy &lfs; diff --git a/src/policy_epall.cpp b/src/policy_epall.cpp new file mode 100644 index 00000000..a4fd5720 --- /dev/null +++ b/src/policy_epall.cpp @@ -0,0 +1,105 @@ +/* + Copyright (c) 2016, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include + +#include +#include + +#include "fs.hpp" +#include "fs_path.hpp" +#include "policy.hpp" + +using std::string; +using std::vector; + +static +int +_epall_create(const vector &basepaths, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) +{ + string fullpath; + + for(size_t i = 0, ei = basepaths.size(); i != ei; i++) + { + bool readonly; + uint64_t spaceavail; + uint64_t _spaceused; + const string *basepath = &basepaths[i]; + + fs::path::make(basepath,fusepath,fullpath); + + if(!fs::exists(fullpath)) + continue; + if(!fs::info(*basepath,readonly,spaceavail,_spaceused)) + continue; + if(readonly) + continue; + if(spaceavail < minfreespace) + continue; + + paths.push_back(basepath); + } + + if(paths.empty()) + return POLICY_FAIL_ENOENT; + + return POLICY_SUCCESS; +} + +static +int +_epall_other(const vector &basepaths, + const char *fusepath, + vector &paths) +{ + string fullpath; + + for(size_t i = 0, ei = basepaths.size(); i != ei; i++) + { + const string *basepath = &basepaths[i]; + + fs::path::make(basepath,fusepath,fullpath); + + if(!fs::exists(fullpath)) + continue; + + paths.push_back(basepath); + } + + if(paths.empty()) + return POLICY_FAIL_ENOENT; + + return POLICY_SUCCESS; +} + +namespace mergerfs +{ + int + Policy::Func::epall(const Category::Enum::Type type, + const vector &basepaths, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) + { + if(type == Category::Enum::create) + return _epall_create(basepaths,fusepath,minfreespace,paths); + + return _epall_other(basepaths,fusepath,paths); + } +} diff --git a/src/policy_eprand.cpp b/src/policy_eprand.cpp new file mode 100644 index 00000000..4e8ffb84 --- /dev/null +++ b/src/policy_eprand.cpp @@ -0,0 +1,46 @@ +/* + Copyright (c) 2016, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include + +#include +#include +#include + +#include "policy.hpp" +#include "success_fail.hpp" + +using std::string; +using std::vector; + +namespace mergerfs +{ + int + Policy::Func::eprand(const Category::Enum::Type type, + const vector &basepaths, + const char *fusepath, + const uint64_t minfreespace, + vector &paths) + { + int rv; + + rv = Policy::Func::epall(type,basepaths,fusepath,minfreespace,paths); + if(POLICY_SUCCEEDED(rv)) + std::random_shuffle(paths.begin(),paths.end()); + + return rv; + } +}