From a2bddec8c738a08b4f70f189b90543cb2effdfa9 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Thu, 29 Jun 2017 23:57:15 -0400 Subject: [PATCH] add 'ignore path preserving on rename' feature --- README.md | 9 ++++++++- man/mergerfs.1 | 23 ++++++++++++++++++++--- src/config.cpp | 1 + src/config.hpp | 1 + src/getxattr.cpp | 4 ++++ src/link.cpp | 2 +- src/listxattr.cpp | 2 ++ src/option_parser.cpp | 5 +++++ src/rename.cpp | 2 +- src/setxattr.cpp | 4 ++++ 10 files changed, 47 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index eb2c7782..6d957ac1 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ mergerfs -o<options> <srcmounts> <mountpoint> * **symlinkify**: when enabled (set to **true**) and a file is not writable and its mtime or ctime is older than **symlinkify_timeout** files will be reported as symlinks to the original files. Please read more below before using. (default: false) * **symlinkify_timeout**: time to wait, in seconds, to activate the **symlinkify** behavior. (default: 3600) * **nullrw**: turns reads and writes into no-ops. The request will succeed but do nothing. Useful for benchmarking mergerfs. (default: false) +* **ignorepponrename**: ignore path preserving on rename. Typically rename and link act differently depending on the policy of `create` (read below). Enabling this will cause rename and link to always use the non-path preserving behavior. This means files, when renamed or linked, will stay on the same drive. * **fsname**: sets the name of the filesystem as seen in **mount**, **df**, etc. Defaults to a list of the source paths concatenated together with the longest common prefix removed. * **func.<func>=<policy>**: sets the specific FUSE function's policy. See below for the list of value types. Example: **func.getattr=newest** * **category.<category>=<policy>**: Sets policy of all FUSE functions in the provided category. Example: **category.create=mfs** @@ -165,7 +166,7 @@ When using non-path preserving policies where something is created paths will be #### rename & link #### -**NOTE:** If you're receiving errors from software when files are moved / renamed then you should consider changing the create policy to one which is **not** path preserving or contacting the author of the offending software and requesting that `EXDEV` be properly handled. +**NOTE:** If you're receiving errors from software when files are moved / renamed then you should consider changing the create policy to one which is **not** path preserving, enabling `ignorepponrename`, or contacting the author of the offending software and requesting that `EXDEV` be properly handled. [rename](http://man7.org/linux/man-pages/man2/rename.2.html) is a tricky function in a merged system. Under normal situations rename only works within a single filesystem or device. If a rename can't be done atomically due to the source and destination paths existing on different mount points it will return **-1** with **errno = EXDEV** (cross device). @@ -407,6 +408,12 @@ Try enabling the `use_ino` option. Some have reported that it fixes the issue. Be sure to turn off `direct_io`. rtorrent and some other applications use [mmap](http://linux.die.net/man/2/mmap) to read and write to files and offer no failback to traditional methods. FUSE does not currently support mmap while using `direct_io`. There will be a performance penalty on writes with `direct_io` off as well as the problem of double caching but it's the only way to get such applications to work. If the performance loss is too high for other apps you can mount mergerfs twice. Once with `direct_io` enabled and one without it. +#### Plex doesn't work with mergerfs + +It does. If you're trying to put Plex's config / metadata on mergerfs you have to leave `direct_io` off because Plex is using sqlite which apparently needs mmap. mmap doesn't work with `direct_io`. + +If the issue is that scanning doesn't seem to pick up media then be sure to set `func.getattr=newest` as mentioned above. + #### mmap performance is really bad There [is a bug](https://lkml.org/lkml/2016/3/16/260) in caching which affects overall performance of mmap through FUSE in Linux 4.x kernels. It is fixed in [4.4.10 and 4.5.4](https://lkml.org/lkml/2016/5/11/59). diff --git a/man/mergerfs.1 b/man/mergerfs.1 index ffda9cd5..bdf62cfb 100644 --- a/man/mergerfs.1 +++ b/man/mergerfs.1 @@ -96,6 +96,13 @@ The request will succeed but do nothing. Useful for benchmarking mergerfs. (default: false) .IP \[bu] 2 +\f[B]ignorepponrename\f[]: ignore path preserving on rename. +Typically rename and link act differently depending on the policy of +\f[C]create\f[] (read below). +Enabling this will cause rename and link to always use the non\-path +preserving behavior. +This means files, when renamed or linked, will stay on the same drive. +.IP \[bu] 2 \f[B]fsname\f[]: sets the name of the filesystem as seen in \f[B]mount\f[], \f[B]df\f[], etc. Defaults to a list of the source paths concatenated together with the @@ -463,9 +470,9 @@ T} .PP \f[B]NOTE:\f[] If you\[aq]re receiving errors from software when files are moved / renamed then you should consider changing the create policy -to one which is \f[B]not\f[] path preserving or contacting the author of -the offending software and requesting that \f[C]EXDEV\f[] be properly -handled. +to one which is \f[B]not\f[] path preserving, enabling +\f[C]ignorepponrename\f[], or contacting the author of the offending +software and requesting that \f[C]EXDEV\f[] be properly handled. .PP rename (http://man7.org/linux/man-pages/man2/rename.2.html) is a tricky function in a merged system. @@ -920,6 +927,16 @@ to get such applications to work. If the performance loss is too high for other apps you can mount mergerfs twice. Once with \f[C]direct_io\f[] enabled and one without it. +.SS Plex doesn\[aq]t work with mergerfs +.PP +It does. +If you\[aq]re trying to put Plex\[aq]s config / metadata on mergerfs you +have to leave \f[C]direct_io\f[] off because Plex is using sqlite which +apparently needs mmap. +mmap doesn\[aq]t work with \f[C]direct_io\f[]. +.PP +If the issue is that scanning doesn\[aq]t seem to pick up media then be +sure to set \f[C]func.getattr=newest\f[] as mentioned above. .SS mmap performance is really bad .PP There is a bug (https://lkml.org/lkml/2016/3/16/260) in caching which diff --git a/src/config.cpp b/src/config.cpp index 0fc81e39..5bf1d733 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -44,6 +44,7 @@ namespace mergerfs symlinkify(false), symlinkify_timeout(3600), nullrw(false), + ignorepponrename(false), POLICYINIT(access), POLICYINIT(chmod), POLICYINIT(chown), diff --git a/src/config.hpp b/src/config.hpp index f829c9e3..ef194f08 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -52,6 +52,7 @@ namespace mergerfs bool symlinkify; time_t symlinkify_timeout; bool nullrw; + bool ignorepponrename; public: const Policy *policies[FuseFunc::Enum::END]; diff --git a/src/getxattr.cpp b/src/getxattr.cpp index 7f4005d3..e86d424b 100644 --- a/src/getxattr.cpp +++ b/src/getxattr.cpp @@ -193,6 +193,10 @@ _getxattr_controlfile(const Config &config, _getxattr_controlfile_bool(config.symlinkify,attrvalue); else if(attr[2] == "symlinkify_timeout") _getxattr_controlfile_time_t(config.symlinkify_timeout,attrvalue); + else if(attr[2] == "nullrw") + _getxattr_controlfile_bool(config.nullrw,attrvalue); + else if(attr[2] == "ignorepponrename") + _getxattr_controlfile_bool(config.ignorepponrename,attrvalue); else if(attr[2] == "policies") _getxattr_controlfile_policies(config,attrvalue); else if(attr[2] == "version") diff --git a/src/link.cpp b/src/link.cpp index c48a25b1..dd3ab49e 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -240,7 +240,7 @@ namespace mergerfs const ugid::Set ugid(fc->uid,fc->gid); const rwlock::ReadGuard readlock(&config.srcmountslock); - if(config.create->path_preserving()) + if(config.create->path_preserving() && !config.ignorepponrename) return _link_preserve_path(config.getattr, config.link, config.create, diff --git a/src/listxattr.cpp b/src/listxattr.cpp index 7dbee28b..61d593de 100644 --- a/src/listxattr.cpp +++ b/src/listxattr.cpp @@ -49,6 +49,8 @@ _listxattr_controlfile(char *list, ("user.mergerfs.dropcacheonclose") ("user.mergerfs.symlinkify") ("user.mergerfs.symlinkify_timeout") + ("user.mergerfs.nullrw") + ("user.mergerfs.ignorepponrename") ("user.mergerfs.policies") ("user.mergerfs.version") ("user.mergerfs.pid"); diff --git a/src/option_parser.cpp b/src/option_parser.cpp index 9aae0c32..f721f680 100644 --- a/src/option_parser.cpp +++ b/src/option_parser.cpp @@ -192,6 +192,8 @@ parse_and_process_kv_arg(Config &config, rv = parse_and_process(value,config.symlinkify_timeout); else if(key == "nullrw") rv = parse_and_process(value,config.nullrw); + else if(key == "ignorepponrename") + rv = parse_and_process(value,config.ignorepponrename); } if(rv == -1) @@ -294,6 +296,9 @@ usage(void) " timeout in seconds before will turn to symlinks.\n" " default=3600\n" " -o nullrw= Disables reads and writes. For benchmarking.\n" + " -o ignorepponrename=\n" + " Ignore path preserving when performing renames\n" + " and links. default = false" << std::endl; } diff --git a/src/rename.cpp b/src/rename.cpp index 2d4811c5..b65d464e 100644 --- a/src/rename.cpp +++ b/src/rename.cpp @@ -301,7 +301,7 @@ namespace mergerfs const ugid::Set ugid(fc->uid,fc->gid); const rwlock::ReadGuard readlock(&config.srcmountslock); - if(config.create->path_preserving()) + if(config.create->path_preserving() && !config.ignorepponrename) return _rename_preserve_path(config.getattr, config.rename, config.create, diff --git a/src/setxattr.cpp b/src/setxattr.cpp index f54668b9..4c46371b 100644 --- a/src/setxattr.cpp +++ b/src/setxattr.cpp @@ -293,6 +293,10 @@ _setxattr_controlfile(Config &config, return _setxattr_time_t(attrval, flags, config.symlinkify_timeout); + else if(attr[2] == "ignorepponrename") + return _setxattr_bool(attrval, + flags, + config.ignorepponrename); break; case 4: