diff --git a/.cirrus.yml b/.cirrus.yml index ba419f2e..11b9ec5b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,6 +19,18 @@ linux_task: - tools/install-build-pkgs - make STATIC=1 LTO=1 +linux_task: + name: "centos:6" + container: + image: centos:6 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - make + - make rpm + linux_task: name: "centos:7" container: @@ -42,3 +54,138 @@ linux_task: - tools/install-build-pkgs - make - make rpm + +linux_task: + name: "ubuntu:20.04" + container: + image: ubuntu:20.04 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true + +linux_task: + name: "ubuntu:19.10" + container: + image: ubuntu:19.10 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true + +linux_task: + name: "ubuntu:18.04" + container: + image: ubuntu:18.04 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true + +linux_task: + name: "ubuntu:16.04" + container: + image: ubuntu:16.04 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true + +linux_task: + name: "ubuntu:14.04" + container: + image: ubuntu:14.04 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true + +linux_task: + name: "debian:10" + container: + image: debian:10 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true + +linux_task: + name: "debian:9" + container: + image: debian:9 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true + +linux_task: + name: "debian:8" + container: + image: debian:8 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true + +linux_task: + name: "debian:7" + container: + image: debian:8 + cpu: 4 + memory: 2G + timeout_in: 10m + script: + - tools/install-build-pkgs + - git fetch + - make deb + - apt-get -y install fuse + - dpkg -i ../*.deb + - mergerfs -v || true diff --git a/README.md b/README.md index 040eabd7..382c6423 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ % mergerfs(1) mergerfs user manual % Antonio SJ Musumeci -% 2020-07-14 +% 2020-07-21 # NAME @@ -241,13 +241,14 @@ Runtime extended attribute support can be managed via the `xattr` option. By def # FUNCTIONS / POLICIES / CATEGORIES -The POSIX filesystem API is made up of a number of functions. **creat**, **stat**, **chown**, etc. In mergerfs most of the core functions are grouped into 3 categories: **action**, **create**, and **search**. These functions and categories can be assigned a policy which dictates what file or directory is chosen when performing that behavior. Any policy can be assigned to a function or category though some may not be very useful in practice. For instance: **rand** (random) may be useful for file creation (create) but could lead to very odd behavior if used for `chmod` if there were more than one copy of the file. +The POSIX filesystem API is made up of a number of functions. **creat**, **stat**, **chown**, etc. For ease of configuration in mergerfs most of the core functions are grouped into 3 categories: **action**, **create**, and **search**. These functions and categories can be assigned a policy which dictates which underlying branch/file/directory is chosen when performing that behavior. Any policy can be assigned to a function or category though some may not be very useful in practice. For instance: **rand** (random) may be useful for file creation (create) but could lead to very odd behavior if used for `chmod` if there were more than one copy of the file. Some functions, listed in the category `N/A` below, can not be assigned the normal policies. All functions which work on file handles use the handle which was acquired by `open` or `create`. `readdir` has no real need for a policy given the purpose is merely to return a list of entries in a directory. `statfs`'s behavior can be modified via other options. That said many times the current FUSE kernel driver will not always provide the file handle when a client calls `fgetattr`, `fchown`, `fchmod`, `futimens`, `ftruncate`, etc. This means it will call the regular, path based, versions. When using policies which are based on a branch's available space the base path provided is used. Not the full path to the file in question. Meaning that sub mounts won't be considered in the space calculations. The reason is that it doesn't really work for non-path preserving policies and can lead to non-obvious behaviors. -#### Function / Category classifications + +#### Functions and their Category classifications | Category | FUSE Functions | |----------|-------------------------------------------------------------------------------------| @@ -269,34 +270,42 @@ A path preserving policy will only consider drives where the relative path being When using non-path preserving policies paths will be cloned to target drives as necessary. +With the `msp` or `most shared path` policies they are defined as `path preserving` for the purpose of controlling `link` and `rename`'s behaviors since `ignorepponrename` is available to disable that behavior. In mergerfs v3.0 the path preserving behavior of rename and link will likely be separated from the policy all together. + #### Filters -Policies basically search branches and create a list of files / paths for functions to work on. The policy is responsible for filtering and sorting. The policy type defines the sorting but filtering is mostly uniform as described below. +Policies basically search branches and create a list of files / paths for functions to work on. The policy is responsible for filtering and sorting. Filters include **minfreespace**, whether or not a branch is mounted read only, and the branch tagging (RO,NC,RW). The policy defines the sorting but filtering is mostly uniform as described below. * No **search** policies filter. * All **action** policies will filter out branches which are mounted **read-only** or tagged as **RO (read-only)**. * All **create** policies will filter out branches which are mounted **read-only**, tagged **RO (read-only)** or **NC (no create)**, or has available space less than `minfreespace`. -If all branches are filtered an error will be returned. Typically **EROFS** (read-only filesystem) or **ENOSPC** (no space left on device) depending on the reasons. +If all branches are filtered an error will be returned. Typically **EROFS** (read-only filesystem) or **ENOSPC** (no space left on device) depending on the most recent reason for filtering a branch. #### Policy descriptions +Because of the nature of the behavior the policies act diffierently depending on the function it is used with (based on the category). + + | Policy | Description | |------------------|------------------------------------------------------------| -| all | Search category: same as **epall**. Action category: same as **epall**. Create category: for **mkdir**, **mknod**, and **symlink** it will apply to all branches. **create** works like **ff**. | -| epall (existing path, all) | Search category: same as **epff** (but more expensive because it doesn't stop after finding a valid branch). Action category: apply to all found. Create category: for **mkdir**, **mknod**, and **symlink** it will apply to all found. **create** works like **epff** (but more expensive because it doesn't stop after finding a valid branch). | +| all | Search: same as **epall**. Action: same as **epall**. Create: for **mkdir**, **mknod**, and **symlink** it will apply to all branches. **create** works like **ff**. | +| epall (existing path, all) | Search: same as **epff** (but more expensive because it doesn't stop after finding a valid branch). Action: apply to all found. Create: for **mkdir**, **mknod**, and **symlink** it will apply to all found. **create** works like **epff** (but more expensive because it doesn't stop after finding a valid branch). | | epff (existing path, first found) | Given the order of the branches, as defined at mount time or configured at runtime, act on the first one found where the relative path exists. | | eplfs (existing path, least free space) | Of all the branches on which the relative path exists choose the drive with the least free space. | | eplus (existing path, least used space) | Of all the branches on which the relative path exists choose the drive with the least used space. | | epmfs (existing path, most free space) | Of all the branches on which the relative path exists choose the drive with the most free space. | | eprand (existing path, random) | Calls **epall** and then randomizes. Returns 1. | | erofs | Exclusively return **-1** with **errno** set to **EROFS** (read-only filesystem). | -| ff (first found) | Search category: same as **epff**. Action category: same as **epff**. Create category: Given the order of the drives, as defined at mount time or configured at runtime, act on the first one found. | -| lfs (least free space) | Search category: same as **eplfs**. Action category: same as **eplfs**. Create category: Pick the drive with the least available free space. | -| lus (least used space) | Search category: same as **eplus**. Action category: same as **eplus**. Create category: Pick the drive with the least used space. | -| mfs (most free space) | Search category: same as **epmfs**. Action category: same as **epmfs**. Create category: Pick the drive with the most available free space. | +| ff (first found) | Search: same as **epff**. Action: same as **epff**. Create: Given the order of the drives, as defined at mount time or configured at runtime, act on the first one found. | +| lfs (least free space) | Search: same as **eplfs**. Action: same as **eplfs**. Create: Pick the drive with the least available free space. | +| lus (least used space) | Search: same as **eplus**. Action: same as **eplus**. Create: Pick the drive with the least used space. | +| mfs (most free space) | Search: same as **epmfs**. Action: same as **epmfs**. Create: Pick the drive with the most available free space. | +| msplfs (most shared path, least free space) | Search: same as **eplfs**. Action: same as **eplfs**. Create: like **eplfs** but walk back the path if it fails to find a branch at that level. | +| msplus (most shared path, least used space) | Search: same as **eplus**. Action: same as **eplus**. Create: like **eplus** but walk back the path if it fails to find a branch at that level. | +| mspmfs (most shared path, most free space) | Search: same as **epmfss**. Action: same as **epmfs**. Create: like **eplmfs** but walk back the path if it fails to find a branch at that level. | | newest | Pick the file / directory with the largest mtime. | | rand (random) | Calls **all** and then randomizes. Returns 1. | @@ -990,7 +999,9 @@ Please update. This is only happened to mergerfs versions at or below v2.25.x an #### How well does mergerfs scale? Is it "production ready?" -Users have reported running mergerfs on everything from a Raspberry Pi to dual socket Xeon systems with >20 cores. I'm aware of at least a few companies which use mergerfs in production. [Open Media Vault](https://www.openmediavault.org) includes mergerfs as its sole solution for pooling drives. +Users have reported running mergerfs on everything from a Raspberry Pi to dual socket Xeon systems with >20 cores. I'm aware of at least a few companies which use mergerfs in production. [Open Media Vault](https://www.openmediavault.org) includes mergerfs as its sole solution for pooling drives. The author of mergerfs had it running for over 300 days managing 16+ drives with reasonably heavy 24/7 read and write usage. Stopping only after the machine's power supply died. + +Most serious issues (crashes or data corruption) have been due to kernel bugs. All of which are fixed in stable releases. #### Can mergerfs be used with drives which already have data / are in use? @@ -1007,9 +1018,9 @@ See the previous question's answer. #### What policies should I use? -Unless you're doing something more niche the average user is probably best off using `mfs` for `category.create`. It will spread files out across your branches based on available space. You may want to use `lus` if you prefer a slightly different distribution of data if you have a mix of smaller and larger drives. Generally though `mfs`, `lus`, or even `rand` are good for the general use case. If you are starting with an imbalanced pool you can use the tool **mergerfs.balance** to redistribute files across the pool. +Unless you're doing something more niche the average user is probably best off using `mfs` for `category.create`. It will spread files out across your branches based on available space. Use `mspmfs` if you want to try to colocate the data a bit more. You may want to use `lus` if you prefer a slightly different distribution of data if you have a mix of smaller and larger drives. Generally though `mfs`, `lus`, or even `rand` are good for the general use case. If you are starting with an imbalanced pool you can use the tool **mergerfs.balance** to redistribute files across the pool. -If you really wish to try to colocate files based on directory you can set `func.create` to `epmfs` or similar and `func.mkdir` to `rand` or `eprand` depending on if you just want to colocate generally or on specific branches. Either way the *need* to colocate is rare. For instance: if you wish to remove the drive regularly and want the data to predictably be on that drive or if you don't use backup at all and don't wish to replace that data piecemeal. In which case using path preservation can help but will require some manual attention. Colocating after the fact can be accomplished using the **mergerfs.consolidate** tool. +If you really wish to try to colocate files based on directory you can set `func.create` to `epmfs` or similar and `func.mkdir` to `rand` or `eprand` depending on if you just want to colocate generally or on specific branches. Either way the *need* to colocate is rare. For instance: if you wish to remove the drive regularly and want the data to predictably be on that drive or if you don't use backup at all and don't wish to replace that data piecemeal. In which case using path preservation can help but will require some manual attention. Colocating after the fact can be accomplished using the **mergerfs.consolidate** tool. If you don't need strict colocation which the `ep` policies provide then you can use the `msp` based policies which will walk back the path till finding a branch that works. Ultimately there is no correct answer. It is a preference or based on some particular need. mergerfs is very easy to test and experiment with. I suggest creating a test setup and experimenting to get a sense of what you want. @@ -1027,7 +1038,7 @@ That said, for the average person, the following should be fine: #### Why are all my files ending up on 1 drive?! -Did you start with empty drives? Did you explicitly configure a `category.create` policy? Are you using a path preserving policy? +Did you start with empty drives? Did you explicitly configure a `category.create` policy? Are you using an `existing path` / `path preserving` policy? The default create policy is `epmfs`. That is a path preserving algorithm. With such a policy for `mkdir` and `create` with a set of empty drives it will select only 1 drive when the first directory is created. Anything, files or directories, created in that first directory will be placed on the same branch because it is preserving paths. @@ -1057,7 +1068,9 @@ If using a network filesystem such as NFS, SMB, CIFS (Samba) be sure to pay clos #### Is my OS's libfuse needed for mergerfs to work? -No. Normally `mount.fuse` is needed to get mergerfs (or any FUSE filesystem to mount using the `mount` command but in vendoring the libfuse library the `mount.fuse` app has been renamed to `mount.mergerfs` meaning the filesystem type in `fstab` can simply be `mergerfs`. +No. Normally `mount.fuse` is needed to get mergerfs (or any FUSE filesystem to mount using the `mount` command but in vendoring the libfuse library the `mount.fuse` app has been renamed to `mount.mergerfs` meaning the filesystem type in `fstab` can simply be `mergerfs`. That said there should be no harm in having it installed and continuing to using `fuse.mergerfs` as the type in `/etc/fstab`. + +If `mergerfs` doesn't work as a type it could be due to how the `mount.mergerfs` tool was installed. Must be in `/sbin/` with proper permissions. #### Why was libfuse embedded into mergerfs? @@ -1149,12 +1162,14 @@ If you don't care about path preservation then simply change the `create` policy #### Why does the total available space in mergerfs not equal outside? -Are you using ext4? With reserve for root? mergerfs uses available space for statfs calculations. If you've reserved space for root then it won't show up. +Are you using ext2/3/4? With reserve for root? mergerfs uses available space for statfs calculations. If you've reserved space for root then it won't show up. + +You can remove the reserve by running: `tune2fs -m 0 ` #### Can mergerfs mounts be exported over NFS? -Yes, however if you do anything which may changes files out of band (including for example using the `newest` policy) it will result in "stale file handle" errors. +Yes, however if you do anything which may changes files out of band (including for example using the `newest` policy) it will result in "stale file handle" errors unless properly setup. Be sure to use the following options: @@ -1168,6 +1183,11 @@ Be sure to use the following options: Yes. While some users have reported problems it appears to always be related to how Samba is setup in relation to permissions. +#### Can mergerfs mounts be used over SSHFS? + +Yes. + + #### I notice massive slowdowns of writes when enabling cache.files. When file caching is enabled in any form (`cache.files!=off` or `direct_io=false`) it will issue `getxattr` requests for `security.capability` prior to *every single write*. This will usually result in a performance degregation, especially when using a network filesystem (such as NFS or CIFS/SMB/Samba.) Unfortunately at this moment the kernel is not caching the response. @@ -1248,6 +1268,7 @@ This software is free to use and released under a very liberal license. That sai * https://spawn.link * https://github.com/trapexit/mergerfs +* https://github.com/trapexit/mergerfs/wiki * https://github.com/trapexit/mergerfs-tools * https://github.com/trapexit/scorch * https://github.com/trapexit/bbf diff --git a/man/mergerfs.1 b/man/mergerfs.1 index 1ab48833..0aa05af1 100644 --- a/man/mergerfs.1 +++ b/man/mergerfs.1 @@ -1,7 +1,7 @@ .\"t .\" Automatically generated by Pandoc 1.19.2.4 .\" -.TH "mergerfs" "1" "2020\-07\-14" "mergerfs user manual" "" +.TH "mergerfs" "1" "2020\-07\-21" "mergerfs user manual" "" .hy .SH NAME .PP @@ -568,10 +568,12 @@ runtime control via the hidden file to stop working. .PP The POSIX filesystem API is made up of a number of functions. \f[B]creat\f[], \f[B]stat\f[], \f[B]chown\f[], etc. -In mergerfs most of the core functions are grouped into 3 categories: -\f[B]action\f[], \f[B]create\f[], and \f[B]search\f[]. +For ease of configuration in mergerfs most of the core functions are +grouped into 3 categories: \f[B]action\f[], \f[B]create\f[], and +\f[B]search\f[]. These functions and categories can be assigned a policy which dictates -what file or directory is chosen when performing that behavior. +which underlying branch/file/directory is chosen when performing that +behavior. Any policy can be assigned to a function or category though some may not be very useful in practice. For instance: \f[B]rand\f[] (random) may be useful for file creation @@ -598,7 +600,7 @@ Meaning that sub mounts won\[aq]t be considered in the space calculations. The reason is that it doesn\[aq]t really work for non\-path preserving policies and can lead to non\-obvious behaviors. -.SS Function / Category classifications +.SS Functions and their Category classifications .PP .TS tab(@); @@ -651,12 +653,21 @@ path being accessed already exists. .PP When using non\-path preserving policies paths will be cloned to target drives as necessary. +.PP +With the \f[C]msp\f[] or \f[C]most\ shared\ path\f[] policies they are +defined as \f[C]path\ preserving\f[] for the purpose of controlling +\f[C]link\f[] and \f[C]rename\f[]\[aq]s behaviors since +\f[C]ignorepponrename\f[] is available to disable that behavior. +In mergerfs v3.0 the path preserving behavior of rename and link will +likely be separated from the policy all together. .SS Filters .PP Policies basically search branches and create a list of files / paths for functions to work on. The policy is responsible for filtering and sorting. -The policy type defines the sorting but filtering is mostly uniform as +Filters include \f[B]minfreespace\f[], whether or not a branch is +mounted read only, and the branch tagging (RO,NC,RW). +The policy defines the sorting but filtering is mostly uniform as described below. .IP \[bu] 2 No \f[B]search\f[] policies filter. @@ -670,9 +681,13 @@ create)\f[], or has available space less than \f[C]minfreespace\f[]. .PP If all branches are filtered an error will be returned. Typically \f[B]EROFS\f[] (read\-only filesystem) or \f[B]ENOSPC\f[] (no -space left on device) depending on the reasons. +space left on device) depending on the most recent reason for filtering +a branch. .SS Policy descriptions .PP +Because of the nature of the behavior the policies act diffierently +depending on the function it is used with (based on the category). +.PP .TS tab(@); lw(16.6n) lw(53.4n). @@ -685,20 +700,20 @@ _ T{ all T}@T{ -Search category: same as \f[B]epall\f[]. -Action category: same as \f[B]epall\f[]. -Create category: for \f[B]mkdir\f[], \f[B]mknod\f[], and -\f[B]symlink\f[] it will apply to all branches. +Search: same as \f[B]epall\f[]. +Action: same as \f[B]epall\f[]. +Create: for \f[B]mkdir\f[], \f[B]mknod\f[], and \f[B]symlink\f[] it will +apply to all branches. \f[B]create\f[] works like \f[B]ff\f[]. T} T{ epall (existing path, all) T}@T{ -Search category: same as \f[B]epff\f[] (but more expensive because it -doesn\[aq]t stop after finding a valid branch). -Action category: apply to all found. -Create category: for \f[B]mkdir\f[], \f[B]mknod\f[], and -\f[B]symlink\f[] it will apply to all found. +Search: same as \f[B]epff\f[] (but more expensive because it doesn\[aq]t +stop after finding a valid branch). +Action: apply to all found. +Create: for \f[B]mkdir\f[], \f[B]mknod\f[], and \f[B]symlink\f[] it will +apply to all found. \f[B]create\f[] works like \f[B]epff\f[] (but more expensive because it doesn\[aq]t stop after finding a valid branch). T} @@ -741,31 +756,55 @@ T} T{ ff (first found) T}@T{ -Search category: same as \f[B]epff\f[]. -Action category: same as \f[B]epff\f[]. -Create category: Given the order of the drives, as defined at mount time -or configured at runtime, act on the first one found. +Search: same as \f[B]epff\f[]. +Action: same as \f[B]epff\f[]. +Create: Given the order of the drives, as defined at mount time or +configured at runtime, act on the first one found. T} T{ lfs (least free space) T}@T{ -Search category: same as \f[B]eplfs\f[]. -Action category: same as \f[B]eplfs\f[]. -Create category: Pick the drive with the least available free space. +Search: same as \f[B]eplfs\f[]. +Action: same as \f[B]eplfs\f[]. +Create: Pick the drive with the least available free space. T} T{ lus (least used space) T}@T{ -Search category: same as \f[B]eplus\f[]. -Action category: same as \f[B]eplus\f[]. -Create category: Pick the drive with the least used space. +Search: same as \f[B]eplus\f[]. +Action: same as \f[B]eplus\f[]. +Create: Pick the drive with the least used space. T} T{ mfs (most free space) T}@T{ -Search category: same as \f[B]epmfs\f[]. -Action category: same as \f[B]epmfs\f[]. -Create category: Pick the drive with the most available free space. +Search: same as \f[B]epmfs\f[]. +Action: same as \f[B]epmfs\f[]. +Create: Pick the drive with the most available free space. +T} +T{ +msplfs (most shared path, least free space) +T}@T{ +Search: same as \f[B]eplfs\f[]. +Action: same as \f[B]eplfs\f[]. +Create: like \f[B]eplfs\f[] but walk back the path if it fails to find a +branch at that level. +T} +T{ +msplus (most shared path, least used space) +T}@T{ +Search: same as \f[B]eplus\f[]. +Action: same as \f[B]eplus\f[]. +Create: like \f[B]eplus\f[] but walk back the path if it fails to find a +branch at that level. +T} +T{ +mspmfs (most shared path, most free space) +T}@T{ +Search: same as \f[B]epmfss\f[]. +Action: same as \f[B]epmfs\f[]. +Create: like \f[B]eplmfs\f[] but walk back the path if it fails to find +a branch at that level. T} T{ newest @@ -946,7 +985,7 @@ Remove the target from all drives with no source file Remove the source from all drives which failed to rename .RE .PP -The removals are subject to normal entitlement checks. +The the removals are subject to normal entitlement checks. .PP The above behavior will help minimize the likelihood of EXDEV being returned but it will still be possible. @@ -1774,10 +1813,11 @@ Subsonic (http://subsonic.org), etc. can use directory mtime (http://linux.die.net/man/2/stat) to more efficiently determine whether to scan for new content rather than simply performing a full scan. -If using the default \f[B]getattr\f[] policy of \f[B]ff\f[] its possible -those programs will miss an update on account of it returning the first -directory found\[aq]s \f[B]stat\f[] info and its a later directory on -another mount which had the \f[B]mtime\f[] recently updated. +If using the default \f[B]getattr\f[] policy of \f[B]ff\f[] it\[aq]s +possible those programs will miss an update on account of it returning +the first directory found\[aq]s \f[B]stat\f[] info and its a later +directory on another mount which had the \f[B]mtime\f[] recently +updated. To fix this you will want to set \f[B]func.getattr=newest\f[]. Remember though that this is just \f[B]stat\f[]. If the file is later \f[B]open\f[]\[aq]ed or \f[B]unlink\f[]\[aq]ed and @@ -2046,7 +2086,7 @@ trapexit. There seems to be an issue with Linux version \f[C]4.9.0\f[] and above in which an invalid message appears to be transmitted to libfuse (used by mergerfs) causing it to exit. -No messages will be printed in any logs as it's not a proper crash. +No messages will be printed in any logs as it\[aq]s not a proper crash. Debugging of the issue is still ongoing and can be followed via the fuse\-devel thread (https://sourceforge.net/p/fuse/mailman/message/35662577). @@ -2111,6 +2151,13 @@ I\[aq]m aware of at least a few companies which use mergerfs in production. Open Media Vault (https://www.openmediavault.org) includes mergerfs as its sole solution for pooling drives. +The author of mergerfs had it running for over 300 days managing 16+ +drives with reasonably heavy 24/7 read and write usage. +Stopping only after the machine\[aq]s power supply died. +.PP +Most serious issues (crashes or data corruption) have been due to kernel +bugs. +All of which are fixed in stable releases. .SS Can mergerfs be used with drives which already have data / are in use? .PP @@ -2131,6 +2178,7 @@ See the previous question\[aq]s answer. Unless you\[aq]re doing something more niche the average user is probably best off using \f[C]mfs\f[] for \f[C]category.create\f[]. It will spread files out across your branches based on available space. +Use \f[C]mspmfs\f[] if you want to try to colocate the data a bit more. You may want to use \f[C]lus\f[] if you prefer a slightly different distribution of data if you have a mix of smaller and larger drives. Generally though \f[C]mfs\f[], \f[C]lus\f[], or even \f[C]rand\f[] are @@ -2150,6 +2198,9 @@ In which case using path preservation can help but will require some manual attention. Colocating after the fact can be accomplished using the \f[B]mergerfs.consolidate\f[] tool. +If you don\[aq]t need strict colocation which the \f[C]ep\f[] policies +provide then you can use the \f[C]msp\f[] based policies which will walk +back the path till finding a branch that works. .PP Ultimately there is no correct answer. It is a preference or based on some particular need. @@ -2178,7 +2229,8 @@ That said, for the average person, the following should be fine: .PP Did you start with empty drives? Did you explicitly configure a \f[C]category.create\f[] policy? -Are you using a path preserving policy? +Are you using an \f[C]existing\ path\f[] / \f[C]path\ preserving\f[] +policy? .PP The default create policy is \f[C]epmfs\f[]. That is a path preserving algorithm. @@ -2198,8 +2250,8 @@ act of creating paths on the drives you want the data to land on before transferring your data. Setting \f[C]func.mkdir=epall\f[] can simplify managing path preservation for \f[C]create\f[]. -Or use \f[C]func.mkdir=rand\f[] if you\[aq]re interested in just grouping -together directory content by drive. +Or use \f[C]func.mkdir=rand\f[] if you\[aq]re interested in just +grouping together directory content by drive. .SS Do hard links work? .PP Yes. @@ -2256,6 +2308,12 @@ filesystem to mount using the \f[C]mount\f[] command but in vendoring the libfuse library the \f[C]mount.fuse\f[] app has been renamed to \f[C]mount.mergerfs\f[] meaning the filesystem type in \f[C]fstab\f[] can simply be \f[C]mergerfs\f[]. +That said there should be no harm in having it installed and continuing +to using \f[C]fuse.mergerfs\f[] as the type in \f[C]/etc/fstab\f[]. +.PP +If \f[C]mergerfs\f[] doesn\[aq]t work as a type it could be due to how +the \f[C]mount.mergerfs\f[] tool was installed. +Must be in \f[C]/sbin/\f[] with proper permissions. .SS Why was libfuse embedded into mergerfs? .IP "1." 3 A significant number of users use mergerfs on distros with old versions @@ -2281,7 +2339,7 @@ Longer term the plan is to rewrite mergerfs to use the low level API. See above first. .PP If/when mergerfs is rewritten to use the low\-level API then it\[aq]ll -be plausible to support system libfuse but till then it's simply too much +be plausible to support system libfuse but till then its simply too much work to manage the differences across the versions. .SS Why use mergerfs over mhddfs? .PP @@ -2342,9 +2400,9 @@ here (https://utcc.utoronto.ca/~cks/space/blog/solaris/ZFSWhyNoRealReshaping). .PP UnRAID is a full OS and its storage layer, as I understand, is proprietary and closed source. -Users who have experience with both have said they prefer the flexibility -offered by mergerfs and for some the fact it is free and open source is -important. +Users who have experience with both have said they prefer the +flexibility offered by mergerfs and for some the fact it is free and +open source is important. .PP There are a number of UnRAID users who use mergerfs as well though I\[aq]m not entirely familiar with the use case. @@ -2365,8 +2423,8 @@ If you need that kind of device performance aggregation or high availability you should stick with RAID. .SS Can drives be written to directly? Outside of mergerfs while pooled? .PP -Yes, however it's not recommended to use the same file from within the -pool and from without at the same time (particularly writing). +Yes, however it\[aq]s not recommended to use the same file from within +the pool and from without at the same time (particularly writing). Especially if using caching of any kind (cache.files, cache.entry, cache.attr, cache.negative_entry, cache.symlinks, cache.readdir, etc.) as there could be a conflict between cached data and not. @@ -2403,20 +2461,23 @@ filesystem. If you don\[aq]t care about path preservation then simply change the \f[C]create\f[] policy to one which isn\[aq]t. \f[C]mfs\f[] is probably what most are looking for. -The reason it's not default is because it was originally set to +The reason it\[aq]s not default is because it was originally set to \f[C]epmfs\f[] and changing it now would change people\[aq]s setup. Such a setting change will likely occur in mergerfs 3. .SS Why does the total available space in mergerfs not equal outside? .PP -Are you using ext4? +Are you using ext2/3/4? With reserve for root? mergerfs uses available space for statfs calculations. If you\[aq]ve reserved space for root then it won\[aq]t show up. +.PP +You can remove the reserve by running: +\f[C]tune2fs\ \-m\ 0\ \f[] .SS Can mergerfs mounts be exported over NFS? .PP Yes, however if you do anything which may changes files out of band (including for example using the \f[C]newest\f[] policy) it will result -in "stale file handle" errors. +in "stale file handle" errors unless properly setup. .PP Be sure to use the following options: .IP \[bu] 2 @@ -2430,6 +2491,9 @@ inodecalc=path\-hash Yes. While some users have reported problems it appears to always be related to how Samba is setup in relation to permissions. +.SS Can mergerfs mounts be used over SSHFS? +.PP +Yes. .SS I notice massive slowdowns of writes when enabling cache.files. .PP When file caching is enabled in any form (\f[C]cache.files!=off\f[] or @@ -2598,6 +2662,8 @@ https://spawn.link .IP \[bu] 2 https://github.com/trapexit/mergerfs .IP \[bu] 2 +https://github.com/trapexit/mergerfs/wiki +.IP \[bu] 2 https://github.com/trapexit/mergerfs\-tools .IP \[bu] 2 https://github.com/trapexit/scorch diff --git a/src/fs_exists.hpp b/src/fs_exists.hpp index 7895f390..0c62846e 100644 --- a/src/fs_exists.hpp +++ b/src/fs_exists.hpp @@ -48,6 +48,18 @@ namespace fs return fs::exists(path_,&st); } + static + inline + bool + exists(const std::string &basepath_, + const std::string &relpath_) + { + std::string fullpath; + + fullpath = fs::path::make(basepath_,relpath_); + + return fs::exists(fullpath); + } static inline diff --git a/src/fs_path.cpp b/src/fs_path.cpp index e14522d0..ba0919d8 100644 --- a/src/fs_path.cpp +++ b/src/fs_path.cpp @@ -29,7 +29,7 @@ namespace fs { namespace path { - string + string dirname(const char *path_) { string path(path_); diff --git a/src/policy.cpp b/src/policy.cpp index 483e610f..2059cbf9 100644 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -39,6 +39,9 @@ const std::vector Policy::_policies_ = (POLICY(lfs,DOESNT_PRESERVE_PATH)) (POLICY(lus,DOESNT_PRESERVE_PATH)) (POLICY(mfs,DOESNT_PRESERVE_PATH)) + (POLICY(msplfs,PRESERVES_PATH)) + (POLICY(msplus,PRESERVES_PATH)) + (POLICY(mspmfs,PRESERVES_PATH)) (POLICY(newest,DOESNT_PRESERVE_PATH)) (POLICY(rand,DOESNT_PRESERVE_PATH)); @@ -59,6 +62,9 @@ CONST_POLICY(ff); CONST_POLICY(lfs); CONST_POLICY(lus); CONST_POLICY(mfs); +CONST_POLICY(msplfs); +CONST_POLICY(msplus); +CONST_POLICY(mspmfs); CONST_POLICY(newest); CONST_POLICY(rand); @@ -83,4 +89,3 @@ Policy::find(const Policy::Enum::Type i) return invalid; } - diff --git a/src/policy.hpp b/src/policy.hpp index 763861da..39a51d35 100644 --- a/src/policy.hpp +++ b/src/policy.hpp @@ -20,7 +20,6 @@ #include "category.hpp" #include "fs.hpp" -#include #include #include @@ -45,6 +44,9 @@ public: lfs, lus, mfs, + msplfs, + msplus, + mspmfs, newest, rand, END @@ -123,6 +125,9 @@ public: static int lfs(CType,const Branches&,const char *,cuint64_t,strvec*); static int lus(CType,const Branches&,const char *,cuint64_t,strvec*); static int mfs(CType,const Branches&,const char *,cuint64_t,strvec*); + static int msplfs(CType,const Branches&,const char *,cuint64_t,strvec*); + static int msplus(CType,const Branches&,const char *,cuint64_t,strvec*); + static int mspmfs(CType,const Branches&,const char *,cuint64_t,strvec*); static int newest(CType,const Branches&,const char *,cuint64_t,strvec*); static int rand(CType,const Branches&,const char *,cuint64_t,strvec*); }; @@ -201,6 +206,9 @@ public: static const Policy &lfs; static const Policy &lus; static const Policy &mfs; + static const Policy &msplfs; + static const Policy &msplus; + static const Policy &mspmfs; static const Policy &newest; static const Policy &rand; }; diff --git a/src/policy_msplfs.cpp b/src/policy_msplfs.cpp new file mode 100644 index 00000000..ea9e2a40 --- /dev/null +++ b/src/policy_msplfs.cpp @@ -0,0 +1,123 @@ +/* + Copyright (c) 2016, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include "errno.hpp" +#include "fs.hpp" +#include "fs_exists.hpp" +#include "fs_info.hpp" +#include "fs_path.hpp" +#include "fs_statvfs_cache.hpp" +#include "policy.hpp" +#include "policy_error.hpp" +#include "rwlock.hpp" + +#include +#include +#include + +using std::string; +using std::vector; + + +namespace msplfs +{ + static + const + string* + create_1(const Branches &branches_, + const string &fusepath_, + const uint64_t minfreespace_, + int *err_) + { + int rv; + uint64_t lfs; + string fusepath; + fs::info_t info; + const Branch *branch; + const string *basepath; + + basepath = NULL; + lfs = std::numeric_limits::max(); + for(size_t i = 0, ei = branches_.size(); i != ei; i++) + { + branch = &branches_[i]; + + if(!fs::exists(branch->path,fusepath)) + error_and_continue(*err_,ENOENT); + if(branch->ro_or_nc()) + error_and_continue(*err_,EROFS); + rv = fs::info(&branch->path,&info); + if(rv == -1) + error_and_continue(*err_,ENOENT); + if(info.readonly) + error_and_continue(*err_,EROFS); + if(info.spaceavail < minfreespace_) + error_and_continue(*err_,ENOSPC); + if(info.spaceavail > lfs) + continue; + + lfs = info.spaceavail; + basepath = &branch->path; + } + + return basepath; + } + + static + int + create(const Branches &branches_, + const char *fusepath_, + const uint64_t minfreespace_, + vector *paths_) + { + int error; + string fusepath; + const string *basepath; + rwlock::ReadGuard guard(&branches_.lock); + + error = ENOENT; + fusepath = fusepath_; + do + { + basepath = create_1(branches_,fusepath,minfreespace_,&error); + if(basepath) + break; + + fusepath = fs::path::dirname(fusepath); + } + while(!fusepath.empty()); + + if(basepath == NULL) + return (errno=error,-1); + + paths_->push_back(*basepath); + + return 0; + } +} + +int +Policy::Func::msplfs(const Category::Enum::Type type_, + const Branches &branches_, + const char *fusepath_, + const uint64_t minfreespace_, + vector *paths_) +{ + if(type_ == Category::Enum::create) + return msplfs::create(branches_,fusepath_,minfreespace_,paths_); + + return Policy::Func::eplfs(type_,branches_,fusepath_,minfreespace_,paths_); +} diff --git a/src/policy_msplus.cpp b/src/policy_msplus.cpp new file mode 100644 index 00000000..7405eb55 --- /dev/null +++ b/src/policy_msplus.cpp @@ -0,0 +1,123 @@ +/* + Copyright (c) 2016, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include "errno.hpp" +#include "fs.hpp" +#include "fs_exists.hpp" +#include "fs_info.hpp" +#include "fs_path.hpp" +#include "fs_statvfs_cache.hpp" +#include "policy.hpp" +#include "policy_error.hpp" +#include "rwlock.hpp" + +#include +#include +#include + +using std::string; +using std::vector; + + +namespace msplus +{ + static + const + string* + create_1(const Branches &branches_, + const string &fusepath_, + const uint64_t minfreespace_, + int *err_) + { + int rv; + uint64_t lus; + string fusepath; + fs::info_t info; + const Branch *branch; + const string *basepath; + + basepath = NULL; + lus = std::numeric_limits::max(); + for(size_t i = 0, ei = branches_.size(); i != ei; i++) + { + branch = &branches_[i]; + + if(!fs::exists(branch->path,fusepath)) + error_and_continue(*err_,ENOENT); + if(branch->ro_or_nc()) + error_and_continue(*err_,EROFS); + rv = fs::info(&branch->path,&info); + if(rv == -1) + error_and_continue(*err_,ENOENT); + if(info.readonly) + error_and_continue(*err_,EROFS); + if(info.spaceavail < minfreespace_) + error_and_continue(*err_,ENOSPC); + if(info.spaceused >= lus) + continue; + + lus = info.spaceused;; + basepath = &branch->path; + } + + return basepath; + } + + static + int + create(const Branches &branches_, + const char *fusepath_, + const uint64_t minfreespace_, + vector *paths_) + { + int error; + string fusepath; + const string *basepath; + rwlock::ReadGuard guard(&branches_.lock); + + error = ENOENT; + fusepath = fusepath_; + do + { + basepath = create_1(branches_,fusepath,minfreespace_,&error); + if(basepath) + break; + + fusepath = fs::path::dirname(fusepath); + } + while(!fusepath.empty()); + + if(basepath == NULL) + return (errno=error,-1); + + paths_->push_back(*basepath); + + return 0; + } +} + +int +Policy::Func::msplus(const Category::Enum::Type type_, + const Branches &branches_, + const char *fusepath_, + const uint64_t minfreespace_, + vector *paths_) +{ + if(type_ == Category::Enum::create) + return msplus::create(branches_,fusepath_,minfreespace_,paths_); + + return Policy::Func::eplus(type_,branches_,fusepath_,minfreespace_,paths_); +} diff --git a/src/policy_mspmfs.cpp b/src/policy_mspmfs.cpp new file mode 100644 index 00000000..685980d8 --- /dev/null +++ b/src/policy_mspmfs.cpp @@ -0,0 +1,123 @@ +/* + Copyright (c) 2016, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include "errno.hpp" +#include "fs.hpp" +#include "fs_exists.hpp" +#include "fs_info.hpp" +#include "fs_path.hpp" +#include "fs_statvfs_cache.hpp" +#include "policy.hpp" +#include "policy_error.hpp" +#include "rwlock.hpp" + +#include +#include +#include + +using std::string; +using std::vector; + + +namespace mspmfs +{ + static + const + string* + create_1(const Branches &branches_, + const string &fusepath_, + const uint64_t minfreespace_, + int *err_) + { + int rv; + uint64_t mfs; + string fusepath; + fs::info_t info; + const Branch *branch; + const string *basepath; + + basepath = NULL; + mfs = std::numeric_limits::min(); + for(size_t i = 0, ei = branches_.size(); i != ei; i++) + { + branch = &branches_[i]; + + if(!fs::exists(branch->path,fusepath)) + error_and_continue(*err_,ENOENT); + if(branch->ro_or_nc()) + error_and_continue(*err_,EROFS); + rv = fs::info(&branch->path,&info); + if(rv == -1) + error_and_continue(*err_,ENOENT); + if(info.readonly) + error_and_continue(*err_,EROFS); + if(info.spaceavail < minfreespace_) + error_and_continue(*err_,ENOSPC); + if(info.spaceavail < mfs) + continue; + + mfs = info.spaceavail; + basepath = &branch->path; + } + + return basepath; + } + + static + int + create(const Branches &branches_, + const char *fusepath_, + const uint64_t minfreespace_, + vector *paths_) + { + int error; + string fusepath; + const string *basepath; + rwlock::ReadGuard guard(&branches_.lock); + + error = ENOENT; + fusepath = fusepath_; + do + { + basepath = create_1(branches_,fusepath,minfreespace_,&error); + if(basepath) + break; + + fusepath = fs::path::dirname(fusepath); + } + while(!fusepath.empty()); + + if(basepath == NULL) + return (errno=error,-1); + + paths_->push_back(*basepath); + + return 0; + } +} + +int +Policy::Func::mspmfs(const Category::Enum::Type type_, + const Branches &branches_, + const char *fusepath_, + const uint64_t minfreespace_, + vector *paths_) +{ + if(type_ == Category::Enum::create) + return mspmfs::create(branches_,fusepath_,minfreespace_,paths_); + + return Policy::Func::epmfs(type_,branches_,fusepath_,minfreespace_,paths_); +} diff --git a/tools/git2debcl b/tools/git2debcl index 47ebe4ec..8d0423ce 100755 --- a/tools/git2debcl +++ b/tools/git2debcl @@ -80,7 +80,6 @@ def main(): args.version = git_version() tags = git_tags() - if args.version in tags: idx = tags.index(args.version) tags = tags[idx:]