Browse Source

Create ld_preload to reopen files on underlying filesystem (#1294)

pull/1295/head
trapexit 11 months ago
committed by GitHub
parent
commit
d4dc0701d6
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 97
      .cirrus.yml
  2. 2
      LICENSE
  3. 29
      Makefile
  4. 48
      README.md
  5. 0
      buildtools/create-branches
  6. 0
      buildtools/git2debcl
  7. 0
      buildtools/install-build-pkgs
  8. 0
      buildtools/update-version
  9. 52
      man/mergerfs.1
  10. 1
      mergerfs.spec
  11. 427
      tools/preload.c

97
.cirrus.yml

@ -5,7 +5,7 @@ freebsd_task:
env: env:
ASSUME_ALWAYS_YES: yes ASSUME_ALWAYS_YES: yes
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- gmake -j4 - gmake -j4
freebsd_task: freebsd_task:
@ -15,7 +15,7 @@ freebsd_task:
env: env:
ASSUME_ALWAYS_YES: yes ASSUME_ALWAYS_YES: yes
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- gmake -j4 - gmake -j4
freebsd_task: freebsd_task:
@ -25,7 +25,7 @@ freebsd_task:
env: env:
ASSUME_ALWAYS_YES: yes ASSUME_ALWAYS_YES: yes
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- gmake -j4 - gmake -j4
freebsd_task: freebsd_task:
@ -35,7 +35,7 @@ freebsd_task:
env: env:
ASSUME_ALWAYS_YES: yes ASSUME_ALWAYS_YES: yes
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- gmake -j4 - gmake -j4
@ -43,7 +43,7 @@ freebsd_task:
# osx_instance: # osx_instance:
# image: catalina-base # image: catalina-base
# script: # script:
# - tools/install-build-pkgs
# - buildtools/install-build-pkgs
# - gmake -j4 # - gmake -j4
linux_task: linux_task:
@ -54,7 +54,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -65,7 +65,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -76,7 +76,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -87,7 +87,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -98,7 +98,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -109,7 +109,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -120,7 +120,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -131,7 +131,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -142,7 +142,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make STATIC=1 LTO=1 - make STATIC=1 LTO=1
linux_task: linux_task:
@ -153,7 +153,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -165,7 +165,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -177,7 +177,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -189,7 +189,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -201,7 +201,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -213,7 +213,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -225,7 +225,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -237,7 +237,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -249,7 +249,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -261,7 +261,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -273,7 +273,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -285,7 +285,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -297,7 +297,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- make - make
- make rpm - make rpm
@ -309,7 +309,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -324,7 +324,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -339,7 +339,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -354,7 +354,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -369,7 +369,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -384,7 +384,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -399,7 +399,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -414,7 +414,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -429,7 +429,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -444,7 +444,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -459,7 +459,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -474,7 +474,22 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch
- make deb
- apt-get -y install fuse
- dpkg -i ../*.deb
- mergerfs -v || true
linux_task:
name: "debian:12"
container:
image: debian:12
cpu: 4
memory: 4G
timeout_in: 15m
script:
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -489,7 +504,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse
@ -504,7 +519,7 @@ linux_task:
memory: 4G memory: 4G
timeout_in: 15m timeout_in: 15m
script: script:
- tools/install-build-pkgs
- buildtools/install-build-pkgs
- git fetch - git fetch
- make deb - make deb
- apt-get -y install fuse - apt-get -y install fuse

2
LICENSE

@ -1,7 +1,7 @@
/* /*
ISC License ISC License
Copyright (c) 2023, Antonio SJ Musumeci <trapexit@spawn.link>
Copyright (c) 2024, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above purpose with or without fee is hereby granted, provided that the above

29
Makefile

@ -1,4 +1,4 @@
# Copyright (c) 2016, Antonio SJ Musumeci <trapexit@spawn.link>
# Copyright (c) 2024, Antonio SJ Musumeci <trapexit@spawn.link>
# #
# Permission to use, copy, modify, and/or distribute this software for any # Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above # purpose with or without fee is hereby granted, provided that the above
@ -26,7 +26,7 @@ STRIP = strip
PANDOC = pandoc PANDOC = pandoc
SED = sed SED = sed
RPMBUILD = rpmbuild RPMBUILD = rpmbuild
GIT2DEBCL = ./tools/git2debcl
GIT2DEBCL = ./buildtools/git2debcl
PKGCONFIG = pkg-config PKGCONFIG = pkg-config
GIT_REPO = 0 GIT_REPO = 0
@ -68,6 +68,10 @@ TESTS_DEPS = $(TESTS:tests/%.cpp=build/.tests/%.d)
TESTS_DEPS += $(DEPS) TESTS_DEPS += $(DEPS)
MANPAGE = mergerfs.1 MANPAGE = mergerfs.1
CFLAGS ?= ${OPT_FLAGS}
CFLAGS := ${CFLAGS} \
-Wall \
-Wno-unused-result
CXXFLAGS ?= ${OPT_FLAGS} CXXFLAGS ?= ${OPT_FLAGS}
CXXFLAGS := \ CXXFLAGS := \
${CXXFLAGS} \ ${CXXFLAGS} \
@ -92,6 +96,7 @@ LDFLAGS := \
-pthread \ -pthread \
-lrt -lrt
# https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
DESTDIR = DESTDIR =
PREFIX = /usr/local PREFIX = /usr/local
EXEC_PREFIX = $(PREFIX) EXEC_PREFIX = $(PREFIX)
@ -99,11 +104,13 @@ DATAROOTDIR = $(PREFIX)/share
DATADIR = $(DATAROOTDIR) DATADIR = $(DATAROOTDIR)
BINDIR = $(EXEC_PREFIX)/bin BINDIR = $(EXEC_PREFIX)/bin
SBINDIR = $(EXEC_PREFIX)/sbin SBINDIR = $(EXEC_PREFIX)/sbin
LIBDIR = $(EXEC_PREFIX)/lib
MANDIR = $(DATAROOTDIR)/man MANDIR = $(DATAROOTDIR)/man
MAN1DIR = $(MANDIR)/man1 MAN1DIR = $(MANDIR)/man1
INSTALLBINDIR = $(DESTDIR)$(BINDIR) INSTALLBINDIR = $(DESTDIR)$(BINDIR)
INSTALLSBINDIR = $(DESTDIR)$(SBINDIR) INSTALLSBINDIR = $(DESTDIR)$(SBINDIR)
INSTALLLIBDIR = $(DESTDIR)$(LIBDIR)/mergerfs
INSTALLMAN1DIR = $(DESTDIR)$(MAN1DIR) INSTALLMAN1DIR = $(DESTDIR)$(MAN1DIR)
.PHONY: all .PHONY: all
@ -141,7 +148,7 @@ endif
.PHONY: version .PHONY: version
version: version:
tools/update-version
./buildtools/update-version
build/stamp: build/stamp:
$(MKDIR) -p build/.src build/.tests $(MKDIR) -p build/.src build/.tests
@ -153,6 +160,10 @@ build/.src/%.o: src/%.cpp
build/.tests/%.o: tests/%.cpp build/.tests/%.o: tests/%.cpp
$(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(FUSE_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) -c $< -o $@ $(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(FUSE_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) -c $< -o $@
build/preload.so: build/stamp tools/preload.c
$(CC) -shared -fPIC $(CFLAGS) $(CPPFLAGS) -o $@ tools/preload.c
preload: build/preload.so
.PHONY: clean .PHONY: clean
clean: rpm-clean clean: rpm-clean
@ -166,7 +177,7 @@ ifeq ($(GIT_REPO),1)
endif endif
.PHONY: install .PHONY: install
install: install-base install-mount-tools install-man
install: install-base install-mount-tools install-preload install-man
install-base: build/mergerfs install-base: build/mergerfs
$(MKDIR) -p "$(INSTALLBINDIR)" $(MKDIR) -p "$(INSTALLBINDIR)"
@ -180,6 +191,10 @@ install-man: $(MANPAGE)
$(MKDIR) -p "$(INSTALLMAN1DIR)" $(MKDIR) -p "$(INSTALLMAN1DIR)"
$(INSTALL) -v -m 0644 "man/$(MANPAGE)" "$(INSTALLMAN1DIR)/$(MANPAGE)" $(INSTALL) -v -m 0644 "man/$(MANPAGE)" "$(INSTALLMAN1DIR)/$(MANPAGE)"
install-preload: preload
$(MKDIR) -p "$(INSTALLLIBDIR)"
$(INSTALL) -v -m 444 "build/preload.so" "$(INSTALLLIBDIR)/preload.so"
install-strip: install-base install-strip: install-base
$(STRIP) "$(INSTALLBINDIR)/mergerfs" $(STRIP) "$(INSTALLBINDIR)/mergerfs"
@ -225,13 +240,13 @@ endif
signed-deb: signed-deb:
$(MAKE) distclean $(MAKE) distclean
$(MAKE) debian-changelog $(MAKE) debian-changelog
dpkg-source -b .
# dpkg-source -b .
dpkg-buildpackage -nc dpkg-buildpackage -nc
deb: deb:
$(MAKE) distclean $(MAKE) distclean
$(MAKE) debian-changelog $(MAKE) debian-changelog
dpkg-source -b .
# dpkg-source -b .
dpkg-buildpackage -nc -uc -us dpkg-buildpackage -nc -uc -us
.PHONY: rpm-clean .PHONY: rpm-clean
@ -250,7 +265,7 @@ rpm: tarball
.PHONY: install-build-pkgs .PHONY: install-build-pkgs
install-build-pkgs: install-build-pkgs:
tools/install-build-pkgs
./buildtools/install-build-pkgs
.PHONY: libfuse .PHONY: libfuse
libfuse: libfuse:

48
README.md

@ -937,7 +937,7 @@ By default FUSE would issue a flush before the release of a file
descriptor. This was considered a bit aggressive and a feature added descriptor. This was considered a bit aggressive and a feature added
to give the FUSE server the ability to choose when that happens. to give the FUSE server the ability to choose when that happens.
Options:
Options:
* always * always
* never * never
* opened-for-write * opened-for-write
@ -1288,6 +1288,52 @@ typedef char IOCTL_BUF[4096];
# TOOLING # TOOLING
## preload.so
EXPERIMENTAL
This preloadable library overrides the creation and opening of files
in order to simulate passthrough file IO. It catches the
open/creat/fopen calls, lets mergerfs do the call, queries mergerfs
for the branch the file exists on, and reopens the file on the underlying
filesystem. Meaning that you will get native read/write performance.
This will only work on dynamically linked software. Anything
statically compiled will not work. Many GoLang and Rust apps are
statically compiled.
The library will not interfere with non-mergerfs filesystems.
While the library was written to account for a number of edgecases
there could be some yet accounted for so please report any oddities.
### general usage
```
LD_PRELOAD=/usr/lib/mergerfs/preload.so touch /mnt/mergerfs/filename
```
### Docker usage
Assume `/mnt/fs0` and `/mnt/fs1` are pooled with mergerfs at
`/mnt/mergerfs`.
Remember that you must bind into the container the original host paths
to the same locations otherwise the preload module will not be able to
find the files.
```
docker run \
-e LD_PRELOAD=/usr/lib/mergerfs/preload.so \
-v /usr/lib/mergerfs/preload.so:/usr/lib/mergerfs/preload.so:ro \
-v /mnt:/mnt \
ubuntu:latest \
bash
```
## Misc
* https://github.com/trapexit/mergerfs-tools * https://github.com/trapexit/mergerfs-tools
* mergerfs.ctl: A tool to make it easier to query and configure mergerfs at runtime * mergerfs.ctl: A tool to make it easier to query and configure mergerfs at runtime
* mergerfs.fsck: Provides permissions and ownership auditing and the ability to fix them * mergerfs.fsck: Provides permissions and ownership auditing and the ability to fix them

0
tools/create-branches → buildtools/create-branches

0
tools/git2debcl → buildtools/git2debcl

0
tools/install-build-pkgs → buildtools/install-build-pkgs

0
tools/update-version → buildtools/update-version

52
man/mergerfs.1

@ -325,14 +325,12 @@ by the number of process threads plus read thread count.
on file close. on file close.
Mostly for when writeback is enabled or merging network filesystems. Mostly for when writeback is enabled or merging network filesystems.
(default: opened-for-write) (default: opened-for-write)
.RS 2
.IP \[bu] 2 .IP \[bu] 2
\f[B]scheduling-priority=INT\f[R]: Set mergerfs\[cq] scheduling \f[B]scheduling-priority=INT\f[R]: Set mergerfs\[cq] scheduling
priority. priority.
Valid values range from -20 to 19. Valid values range from -20 to 19.
See \f[C]setpriority\f[R] man page for more details. See \f[C]setpriority\f[R] man page for more details.
(default: -10) (default: -10)
.RE
.IP \[bu] 2 .IP \[bu] 2
\f[B]fsname=STR\f[R]: Sets the name of the filesystem as seen in \f[B]fsname=STR\f[R]: Sets the name of the filesystem as seen in
\f[B]mount\f[R], \f[B]df\f[R], etc. \f[B]mount\f[R], \f[B]df\f[R], etc.
@ -428,8 +426,8 @@ Use \f[C]async_read=false\f[R] instead.
.IP \[bu] 2 .IP \[bu] 2
\f[B]splice_move\f[R]: deprecated - Does nothing. \f[B]splice_move\f[R]: deprecated - Does nothing.
.IP \[bu] 2 .IP \[bu] 2
\f[B]allow_other\f[R]: deprecated - mergerfs always sets this FUSE
option as normal permissions can be used to limit access.
\f[B]allow_other\f[R]: deprecated - mergerfs v2.35.0 and newer sets this
FUSE option automatically if running as root.
.IP \[bu] 2 .IP \[bu] 2
\f[B]use_ino\f[R]: deprecated - mergerfs should always control inode \f[B]use_ino\f[R]: deprecated - mergerfs should always control inode
calculation so this is enabled all the time. calculation so this is enabled all the time.
@ -1724,6 +1722,52 @@ IOCTL_INVALIDATE_ALL_NODES: Same as SIGUSR1.
Send invalidation notifications to the kernel for all files causing Send invalidation notifications to the kernel for all files causing
unused files to be released from memory. unused files to be released from memory.
.SH TOOLING .SH TOOLING
.SS preload.so
.PP
EXPERIMENTAL
.PP
This preloadable library overrides the creation and opening of files in
order to simulate passthrough file IO.
It catches the open/creat/fopen calls, lets mergerfs do the call,
queries mergerfs for the branch the file exists on, and reopens the file
on the underlying filesystem.
Meaning that you will get native read/write performance.
.PP
This will only work on dynamically linked software.
Anything statically compiled will not work.
Many GoLang and Rust apps are statically compiled.
.PP
The library will not interfere with non-mergerfs filesystems.
.PP
While the library was written to account for a number of edgecases there
could be some yet accounted for so please report any oddities.
.SS general usage
.IP
.nf
\f[C]
LD_PRELOAD=/usr/lib/mergerfs/preload.so touch /mnt/mergerfs/filename
\f[R]
.fi
.SS Docker usage
.PP
Assume \f[C]/mnt/fs0\f[R] and \f[C]/mnt/fs1\f[R] are pooled with
mergerfs at \f[C]/mnt/mergerfs\f[R].
.PP
Remember that you must bind into the container the original host paths
to the same locations otherwise the preload module will not be able to
find the files.
.IP
.nf
\f[C]
docker run \[rs]
-e LD_PRELOAD=/usr/lib/mergerfs/preload.so \[rs]
-v /usr/lib/mergerfs/preload.so:/usr/lib/mergerfs/preload.so:ro \[rs]
-v /mnt:/mnt \[rs]
ubuntu:latest \[rs]
bash
\f[R]
.fi
.SS Misc
.IP \[bu] 2 .IP \[bu] 2
https://github.com/trapexit/mergerfs-tools https://github.com/trapexit/mergerfs-tools
.RS 2 .RS 2

1
mergerfs.spec

@ -35,6 +35,7 @@ make install PREFIX=%{_prefix} DESTDIR=%{buildroot}
/usr/bin/mergerfs /usr/bin/mergerfs
/usr/bin/mergerfs-fusermount /usr/bin/mergerfs-fusermount
/sbin/mount.mergerfs /sbin/mount.mergerfs
/usr/lib/mergerfs/preload.so
%doc %{_mandir}/* %doc %{_mandir}/*
%changelog %changelog

427
tools/preload.c

@ -0,0 +1,427 @@
/*
ISC License
Copyright (c) 2024, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#define _GNU_SOURCE
#define _POSIX_C_SOURCE 200809L
#include <assert.h>
#include <dlfcn.h>
#include <stdio.h>
#include <string.h>
#include <sys/ioctl.h>
#include <stdarg.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <fcntl.h>
typedef char IOCTL_BUF[4096];
#define IOCTL_APP_TYPE 0xDF
#define IOCTL_FILE_INFO _IOWR(IOCTL_APP_TYPE,0,IOCTL_BUF)
#define LOAD_FUNC(func) \
do \
{ \
if(!_libc_##func) \
_libc_##func = (func##_func_t)dlsym(RTLD_NEXT,#func); \
assert(_libc_##func != NULL); \
} \
while(0)
typedef int (*open_func_t)(const char*, int, ...);
typedef int (*open64_func_t)(const char*, int, ...);
typedef int (*openat_func_t)(int, const char*, int, ...);
typedef int (*openat64_func_t)(int, const char*, int, ...);
typedef int (*creat_func_t)(const char*, mode_t);
typedef int (*creat64_func_t)(const char*, mode_t);
typedef FILE* (*fopen_func_t)(const char*, const char*);
typedef FILE* (*fopen64_func_t)(const char*, const char*);
static open_func_t _libc_open = NULL;
static open64_func_t _libc_open64 = NULL;
static openat_func_t _libc_openat = NULL;
static openat64_func_t _libc_openat64 = NULL;
static fopen_func_t _libc_fopen = NULL;
static fopen64_func_t _libc_fopen64 = NULL;
static creat_func_t _libc_creat = NULL;
static creat64_func_t _libc_creat64 = NULL;
static
int
get_underlying_filepath(int fd_,
char *filepath_)
{
int rv;
strcpy(filepath_,"fullpath");
rv = ioctl(fd_,IOCTL_FILE_INFO,filepath_);
if(rv == -1)
return -1;
return rv;
}
static
void
strip_exec(const char *orig_mode_,
char *new_mode_)
{
size_t i;
size_t j;
for(i = j = 0; orig_mode_[i]; i++)
{
if(orig_mode_[i] == 'x')
continue;
new_mode_[j++] = orig_mode_[i];
}
new_mode_[j] = '\0';
}
int
open(const char *pathname_,
int flags_,
...)
{
int rv;
int fd;
mode_t mode;
struct stat st;
LOAD_FUNC(open);
mode = 0;
if(flags_ & O_CREAT)
{
va_list args;
va_start(args,flags_);
mode = va_arg(args,mode_t);
va_end(args);
}
fd = _libc_open(pathname_,flags_,mode);
if(fd == -1)
return -1;
if(flags_ & (O_DIRECTORY|O_PATH))
return fd;
rv = fstat(fd,&st);
if(rv == -1)
return fd;
if((st.st_mode & S_IFMT) != S_IFREG)
return fd;
IOCTL_BUF real_pathname;
rv = get_underlying_filepath(fd,real_pathname);
if(rv == -1)
return fd;
flags_ &= ~(O_EXCL|O_CREAT);
rv = _libc_open(real_pathname,flags_,mode);
if(rv == -1)
return fd;
close(fd);
return rv;
}
int
open64(const char *pathname_,
int flags_,
...)
{
int rv;
int fd;
mode_t mode;
struct stat st;
LOAD_FUNC(open64);
mode = 0;
if(flags_ & O_CREAT)
{
va_list args;
va_start(args,flags_);
mode = va_arg(args,mode_t);
va_end(args);
}
fd = _libc_open64(pathname_,flags_,mode);
if(fd == -1)
return -1;
if(flags_ & (O_DIRECTORY|O_PATH))
return fd;
rv = fstat(fd,&st);
if(rv == -1)
return fd;
if((st.st_mode & S_IFMT) != S_IFREG)
return fd;
IOCTL_BUF real_pathname;
rv = get_underlying_filepath(fd,real_pathname);
if(rv == -1)
return fd;
flags_ &= ~(O_EXCL|O_CREAT);
rv = _libc_open64(real_pathname,flags_,mode);
if(rv == -1)
return fd;
close(fd);
return rv;
}
int
openat(int dirfd_,
const char *pathname_,
int flags_,
...)
{
int rv;
int fd;
mode_t mode;
struct stat st;
LOAD_FUNC(openat);
mode = 0;
if(flags_ & O_CREAT)
{
va_list args;
va_start(args,flags_);
mode = va_arg(args,mode_t);
va_end(args);
}
fd = _libc_openat(dirfd_,pathname_,flags_,mode);
if(fd == -1)
return -1;
if(flags_ & (O_DIRECTORY|O_PATH))
return fd;
rv = fstat(fd,&st);
if(rv == -1)
return fd;
if((st.st_mode & S_IFMT) != S_IFREG)
return fd;
IOCTL_BUF real_pathname;
rv = get_underlying_filepath(fd,real_pathname);
if(rv == -1)
return fd;
flags_ &= ~(O_EXCL|O_CREAT);
rv = _libc_openat(dirfd_,real_pathname,flags_,mode);
if(rv == -1)
return fd;
close(fd);
return rv;
}
int
openat64(int dirfd_,
const char *pathname_,
int flags_,
...)
{
int rv;
int fd;
mode_t mode;
struct stat st;
LOAD_FUNC(openat64);
mode = 0;
if(flags_ & O_CREAT)
{
va_list args;
va_start(args,flags_);
mode = va_arg(args,mode_t);
va_end(args);
}
fd = _libc_openat64(dirfd_,pathname_,flags_,mode);
if(fd == -1)
return -1;
if(flags_ & (O_DIRECTORY|O_PATH))
return fd;
rv = fstat(fd,&st);
if(rv == -1)
return fd;
if((st.st_mode & S_IFMT) != S_IFREG)
return fd;
IOCTL_BUF real_pathname;
rv = get_underlying_filepath(fd,real_pathname);
if(rv == -1)
return fd;
flags_ &= ~(O_EXCL|O_CREAT);
rv = _libc_openat64(dirfd_,real_pathname,flags_,mode);
if(rv == -1)
return fd;
close(fd);
return rv;
}
FILE*
fopen(const char *pathname_,
const char *mode_)
{
int fd;
int rv;
FILE *f;
FILE *f2;
struct stat st;
LOAD_FUNC(fopen);
f = _libc_fopen(pathname_,mode_);
if(f == NULL)
return NULL;
fd = fileno(f);
if(fd == -1)
return f;
rv = fstat(fd,&st);
if(rv == -1)
return f;
if((st.st_mode & S_IFMT) != S_IFREG)
return f;
IOCTL_BUF real_pathname;
rv = get_underlying_filepath(fd,real_pathname);
if(rv == -1)
return f;
char new_mode[64];
strip_exec(mode_,new_mode);
f2 = _libc_fopen(real_pathname,new_mode);
if(f2 == NULL)
return f;
fclose(f);
return f2;
}
FILE*
fopen64(const char *pathname_,
const char *mode_)
{
int fd;
int rv;
FILE *f;
FILE *f2;
struct stat st;
LOAD_FUNC(fopen64);
f = _libc_fopen64(pathname_,mode_);
if(f == NULL)
return NULL;
fd = fileno(f);
if(fd == -1)
return f;
rv = fstat(fd,&st);
if(rv == -1)
return f;
if((st.st_mode & S_IFMT) != S_IFREG)
return f;
IOCTL_BUF real_pathname;
rv = get_underlying_filepath(fd,real_pathname);
if(rv == -1)
return f;
char new_mode[64];
strip_exec(mode_,new_mode);
f2 = _libc_fopen64(real_pathname,new_mode);
if(f2 == NULL)
return f;
fclose(f);
return f2;
}
int
creat(const char *pathname_,
mode_t mode_)
{
int fd;
int rv;
LOAD_FUNC(creat);
fd = _libc_creat(pathname_,mode_);
if(fd == -1)
return -1;
IOCTL_BUF real_pathname;
rv = get_underlying_filepath(fd,real_pathname);
if(rv == -1)
return fd;
rv = _libc_creat(real_pathname,mode_);
if(rv == -1)
return fd;
close(fd);
return rv;
}
int
creat64(const char *pathname_,
mode_t mode_)
{
int fd;
int rv;
LOAD_FUNC(creat64);
fd = _libc_creat64(pathname_,mode_);
if(fd == -1)
return -1;
IOCTL_BUF real_pathname;
rv = get_underlying_filepath(fd,real_pathname);
if(rv == -1)
return fd;
rv = _libc_creat64(real_pathname,mode_);
if(rv == -1)
return fd;
close(fd);
return rv;
}
Loading…
Cancel
Save