mirror of
https://github.com/trapexit/mergerfs.git
synced 2025-02-19 03:12:47 +08:00
Create ld_preload to reopen files on underlying filesystem (#1294)
This commit is contained in:
parent
c1c2f075ee
commit
d4dc0701d6
97
.cirrus.yml
97
.cirrus.yml
|
@ -5,7 +5,7 @@ freebsd_task:
|
|||
env:
|
||||
ASSUME_ALWAYS_YES: yes
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- gmake -j4
|
||||
|
||||
freebsd_task:
|
||||
|
@ -15,7 +15,7 @@ freebsd_task:
|
|||
env:
|
||||
ASSUME_ALWAYS_YES: yes
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- gmake -j4
|
||||
|
||||
freebsd_task:
|
||||
|
@ -25,7 +25,7 @@ freebsd_task:
|
|||
env:
|
||||
ASSUME_ALWAYS_YES: yes
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- gmake -j4
|
||||
|
||||
freebsd_task:
|
||||
|
@ -35,7 +35,7 @@ freebsd_task:
|
|||
env:
|
||||
ASSUME_ALWAYS_YES: yes
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- gmake -j4
|
||||
|
||||
|
||||
|
@ -43,7 +43,7 @@ freebsd_task:
|
|||
# osx_instance:
|
||||
# image: catalina-base
|
||||
# script:
|
||||
# - tools/install-build-pkgs
|
||||
# - buildtools/install-build-pkgs
|
||||
# - gmake -j4
|
||||
|
||||
linux_task:
|
||||
|
@ -54,7 +54,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -65,7 +65,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -76,7 +76,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -87,7 +87,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -98,7 +98,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -109,7 +109,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -120,7 +120,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -131,7 +131,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -142,7 +142,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make STATIC=1 LTO=1
|
||||
|
||||
linux_task:
|
||||
|
@ -153,7 +153,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -165,7 +165,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -177,7 +177,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -189,7 +189,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -201,7 +201,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -213,7 +213,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -225,7 +225,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -237,7 +237,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -249,7 +249,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -261,7 +261,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -273,7 +273,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -285,7 +285,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -297,7 +297,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- make
|
||||
- make rpm
|
||||
|
||||
|
@ -309,7 +309,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -324,7 +324,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -339,7 +339,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -354,7 +354,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -369,7 +369,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -384,7 +384,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -399,7 +399,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -414,7 +414,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -429,7 +429,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -444,7 +444,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -459,7 +459,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -474,7 +474,22 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
- dpkg -i ../*.deb
|
||||
- mergerfs -v || true
|
||||
|
||||
linux_task:
|
||||
name: "debian:12"
|
||||
container:
|
||||
image: debian:12
|
||||
cpu: 4
|
||||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -489,7 +504,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
@ -504,7 +519,7 @@ linux_task:
|
|||
memory: 4G
|
||||
timeout_in: 15m
|
||||
script:
|
||||
- tools/install-build-pkgs
|
||||
- buildtools/install-build-pkgs
|
||||
- git fetch
|
||||
- make deb
|
||||
- apt-get -y install fuse
|
||||
|
|
2
LICENSE
2
LICENSE
|
@ -1,7 +1,7 @@
|
|||
/*
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2023, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
Copyright (c) 2024, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
|
|
29
Makefile
29
Makefile
|
@ -1,4 +1,4 @@
|
|||
# Copyright (c) 2016, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
# Copyright (c) 2024, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
#
|
||||
# Permission to use, copy, modify, and/or distribute this software for any
|
||||
# purpose with or without fee is hereby granted, provided that the above
|
||||
|
@ -26,7 +26,7 @@ STRIP = strip
|
|||
PANDOC = pandoc
|
||||
SED = sed
|
||||
RPMBUILD = rpmbuild
|
||||
GIT2DEBCL = ./tools/git2debcl
|
||||
GIT2DEBCL = ./buildtools/git2debcl
|
||||
PKGCONFIG = pkg-config
|
||||
|
||||
GIT_REPO = 0
|
||||
|
@ -68,6 +68,10 @@ TESTS_DEPS = $(TESTS:tests/%.cpp=build/.tests/%.d)
|
|||
TESTS_DEPS += $(DEPS)
|
||||
|
||||
MANPAGE = mergerfs.1
|
||||
CFLAGS ?= ${OPT_FLAGS}
|
||||
CFLAGS := ${CFLAGS} \
|
||||
-Wall \
|
||||
-Wno-unused-result
|
||||
CXXFLAGS ?= ${OPT_FLAGS}
|
||||
CXXFLAGS := \
|
||||
${CXXFLAGS} \
|
||||
|
@ -92,6 +96,7 @@ LDFLAGS := \
|
|||
-pthread \
|
||||
-lrt
|
||||
|
||||
# https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
|
||||
DESTDIR =
|
||||
PREFIX = /usr/local
|
||||
EXEC_PREFIX = $(PREFIX)
|
||||
|
@ -99,11 +104,13 @@ DATAROOTDIR = $(PREFIX)/share
|
|||
DATADIR = $(DATAROOTDIR)
|
||||
BINDIR = $(EXEC_PREFIX)/bin
|
||||
SBINDIR = $(EXEC_PREFIX)/sbin
|
||||
LIBDIR = $(EXEC_PREFIX)/lib
|
||||
MANDIR = $(DATAROOTDIR)/man
|
||||
MAN1DIR = $(MANDIR)/man1
|
||||
|
||||
INSTALLBINDIR = $(DESTDIR)$(BINDIR)
|
||||
INSTALLSBINDIR = $(DESTDIR)$(SBINDIR)
|
||||
INSTALLLIBDIR = $(DESTDIR)$(LIBDIR)/mergerfs
|
||||
INSTALLMAN1DIR = $(DESTDIR)$(MAN1DIR)
|
||||
|
||||
.PHONY: all
|
||||
|
@ -141,7 +148,7 @@ endif
|
|||
|
||||
.PHONY: version
|
||||
version:
|
||||
tools/update-version
|
||||
./buildtools/update-version
|
||||
|
||||
build/stamp:
|
||||
$(MKDIR) -p build/.src build/.tests
|
||||
|
@ -153,6 +160,10 @@ build/.src/%.o: src/%.cpp
|
|||
build/.tests/%.o: tests/%.cpp
|
||||
$(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(FUSE_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) -c $< -o $@
|
||||
|
||||
build/preload.so: build/stamp tools/preload.c
|
||||
$(CC) -shared -fPIC $(CFLAGS) $(CPPFLAGS) -o $@ tools/preload.c
|
||||
|
||||
preload: build/preload.so
|
||||
|
||||
.PHONY: clean
|
||||
clean: rpm-clean
|
||||
|
@ -166,7 +177,7 @@ ifeq ($(GIT_REPO),1)
|
|||
endif
|
||||
|
||||
.PHONY: install
|
||||
install: install-base install-mount-tools install-man
|
||||
install: install-base install-mount-tools install-preload install-man
|
||||
|
||||
install-base: build/mergerfs
|
||||
$(MKDIR) -p "$(INSTALLBINDIR)"
|
||||
|
@ -180,6 +191,10 @@ install-man: $(MANPAGE)
|
|||
$(MKDIR) -p "$(INSTALLMAN1DIR)"
|
||||
$(INSTALL) -v -m 0644 "man/$(MANPAGE)" "$(INSTALLMAN1DIR)/$(MANPAGE)"
|
||||
|
||||
install-preload: preload
|
||||
$(MKDIR) -p "$(INSTALLLIBDIR)"
|
||||
$(INSTALL) -v -m 444 "build/preload.so" "$(INSTALLLIBDIR)/preload.so"
|
||||
|
||||
install-strip: install-base
|
||||
$(STRIP) "$(INSTALLBINDIR)/mergerfs"
|
||||
|
||||
|
@ -225,13 +240,13 @@ endif
|
|||
signed-deb:
|
||||
$(MAKE) distclean
|
||||
$(MAKE) debian-changelog
|
||||
dpkg-source -b .
|
||||
# dpkg-source -b .
|
||||
dpkg-buildpackage -nc
|
||||
|
||||
deb:
|
||||
$(MAKE) distclean
|
||||
$(MAKE) debian-changelog
|
||||
dpkg-source -b .
|
||||
# dpkg-source -b .
|
||||
dpkg-buildpackage -nc -uc -us
|
||||
|
||||
.PHONY: rpm-clean
|
||||
|
@ -250,7 +265,7 @@ rpm: tarball
|
|||
|
||||
.PHONY: install-build-pkgs
|
||||
install-build-pkgs:
|
||||
tools/install-build-pkgs
|
||||
./buildtools/install-build-pkgs
|
||||
|
||||
.PHONY: libfuse
|
||||
libfuse:
|
||||
|
|
48
README.md
48
README.md
|
@ -937,7 +937,7 @@ By default FUSE would issue a flush before the release of a file
|
|||
descriptor. This was considered a bit aggressive and a feature added
|
||||
to give the FUSE server the ability to choose when that happens.
|
||||
|
||||
Options:
|
||||
Options:
|
||||
* always
|
||||
* never
|
||||
* opened-for-write
|
||||
|
@ -1288,6 +1288,52 @@ typedef char IOCTL_BUF[4096];
|
|||
|
||||
# TOOLING
|
||||
|
||||
## preload.so
|
||||
|
||||
EXPERIMENTAL
|
||||
|
||||
This preloadable library overrides the creation and opening of files
|
||||
in order to simulate passthrough file IO. It catches the
|
||||
open/creat/fopen calls, lets mergerfs do the call, queries mergerfs
|
||||
for the branch the file exists on, and reopens the file on the underlying
|
||||
filesystem. Meaning that you will get native read/write performance.
|
||||
|
||||
This will only work on dynamically linked software. Anything
|
||||
statically compiled will not work. Many GoLang and Rust apps are
|
||||
statically compiled.
|
||||
|
||||
The library will not interfere with non-mergerfs filesystems.
|
||||
|
||||
While the library was written to account for a number of edgecases
|
||||
there could be some yet accounted for so please report any oddities.
|
||||
|
||||
|
||||
### general usage
|
||||
|
||||
```
|
||||
LD_PRELOAD=/usr/lib/mergerfs/preload.so touch /mnt/mergerfs/filename
|
||||
```
|
||||
|
||||
### Docker usage
|
||||
|
||||
Assume `/mnt/fs0` and `/mnt/fs1` are pooled with mergerfs at
|
||||
`/mnt/mergerfs`.
|
||||
|
||||
Remember that you must bind into the container the original host paths
|
||||
to the same locations otherwise the preload module will not be able to
|
||||
find the files.
|
||||
|
||||
```
|
||||
docker run \
|
||||
-e LD_PRELOAD=/usr/lib/mergerfs/preload.so \
|
||||
-v /usr/lib/mergerfs/preload.so:/usr/lib/mergerfs/preload.so:ro \
|
||||
-v /mnt:/mnt \
|
||||
ubuntu:latest \
|
||||
bash
|
||||
```
|
||||
|
||||
## Misc
|
||||
|
||||
* https://github.com/trapexit/mergerfs-tools
|
||||
* mergerfs.ctl: A tool to make it easier to query and configure mergerfs at runtime
|
||||
* mergerfs.fsck: Provides permissions and ownership auditing and the ability to fix them
|
||||
|
|
|
@ -325,14 +325,12 @@ by the number of process threads plus read thread count.
|
|||
on file close.
|
||||
Mostly for when writeback is enabled or merging network filesystems.
|
||||
(default: opened-for-write)
|
||||
.RS 2
|
||||
.IP \[bu] 2
|
||||
\f[B]scheduling-priority=INT\f[R]: Set mergerfs\[cq] scheduling
|
||||
priority.
|
||||
Valid values range from -20 to 19.
|
||||
See \f[C]setpriority\f[R] man page for more details.
|
||||
(default: -10)
|
||||
.RE
|
||||
.IP \[bu] 2
|
||||
\f[B]fsname=STR\f[R]: Sets the name of the filesystem as seen in
|
||||
\f[B]mount\f[R], \f[B]df\f[R], etc.
|
||||
|
@ -428,8 +426,8 @@ Use \f[C]async_read=false\f[R] instead.
|
|||
.IP \[bu] 2
|
||||
\f[B]splice_move\f[R]: deprecated - Does nothing.
|
||||
.IP \[bu] 2
|
||||
\f[B]allow_other\f[R]: deprecated - mergerfs always sets this FUSE
|
||||
option as normal permissions can be used to limit access.
|
||||
\f[B]allow_other\f[R]: deprecated - mergerfs v2.35.0 and newer sets this
|
||||
FUSE option automatically if running as root.
|
||||
.IP \[bu] 2
|
||||
\f[B]use_ino\f[R]: deprecated - mergerfs should always control inode
|
||||
calculation so this is enabled all the time.
|
||||
|
@ -1724,6 +1722,52 @@ IOCTL_INVALIDATE_ALL_NODES: Same as SIGUSR1.
|
|||
Send invalidation notifications to the kernel for all files causing
|
||||
unused files to be released from memory.
|
||||
.SH TOOLING
|
||||
.SS preload.so
|
||||
.PP
|
||||
EXPERIMENTAL
|
||||
.PP
|
||||
This preloadable library overrides the creation and opening of files in
|
||||
order to simulate passthrough file IO.
|
||||
It catches the open/creat/fopen calls, lets mergerfs do the call,
|
||||
queries mergerfs for the branch the file exists on, and reopens the file
|
||||
on the underlying filesystem.
|
||||
Meaning that you will get native read/write performance.
|
||||
.PP
|
||||
This will only work on dynamically linked software.
|
||||
Anything statically compiled will not work.
|
||||
Many GoLang and Rust apps are statically compiled.
|
||||
.PP
|
||||
The library will not interfere with non-mergerfs filesystems.
|
||||
.PP
|
||||
While the library was written to account for a number of edgecases there
|
||||
could be some yet accounted for so please report any oddities.
|
||||
.SS general usage
|
||||
.IP
|
||||
.nf
|
||||
\f[C]
|
||||
LD_PRELOAD=/usr/lib/mergerfs/preload.so touch /mnt/mergerfs/filename
|
||||
\f[R]
|
||||
.fi
|
||||
.SS Docker usage
|
||||
.PP
|
||||
Assume \f[C]/mnt/fs0\f[R] and \f[C]/mnt/fs1\f[R] are pooled with
|
||||
mergerfs at \f[C]/mnt/mergerfs\f[R].
|
||||
.PP
|
||||
Remember that you must bind into the container the original host paths
|
||||
to the same locations otherwise the preload module will not be able to
|
||||
find the files.
|
||||
.IP
|
||||
.nf
|
||||
\f[C]
|
||||
docker run \[rs]
|
||||
-e LD_PRELOAD=/usr/lib/mergerfs/preload.so \[rs]
|
||||
-v /usr/lib/mergerfs/preload.so:/usr/lib/mergerfs/preload.so:ro \[rs]
|
||||
-v /mnt:/mnt \[rs]
|
||||
ubuntu:latest \[rs]
|
||||
bash
|
||||
\f[R]
|
||||
.fi
|
||||
.SS Misc
|
||||
.IP \[bu] 2
|
||||
https://github.com/trapexit/mergerfs-tools
|
||||
.RS 2
|
||||
|
|
|
@ -35,6 +35,7 @@ make install PREFIX=%{_prefix} DESTDIR=%{buildroot}
|
|||
/usr/bin/mergerfs
|
||||
/usr/bin/mergerfs-fusermount
|
||||
/sbin/mount.mergerfs
|
||||
/usr/lib/mergerfs/preload.so
|
||||
%doc %{_mandir}/*
|
||||
|
||||
%changelog
|
||||
|
|
427
tools/preload.c
Normal file
427
tools/preload.c
Normal file
|
@ -0,0 +1,427 @@
|
|||
/*
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2024, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
|
||||
#define _GNU_SOURCE
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
|
||||
#include <assert.h>
|
||||
#include <dlfcn.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <stdarg.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
typedef char IOCTL_BUF[4096];
|
||||
#define IOCTL_APP_TYPE 0xDF
|
||||
#define IOCTL_FILE_INFO _IOWR(IOCTL_APP_TYPE,0,IOCTL_BUF)
|
||||
|
||||
#define LOAD_FUNC(func) \
|
||||
do \
|
||||
{ \
|
||||
if(!_libc_##func) \
|
||||
_libc_##func = (func##_func_t)dlsym(RTLD_NEXT,#func); \
|
||||
assert(_libc_##func != NULL); \
|
||||
} \
|
||||
while(0)
|
||||
|
||||
typedef int (*open_func_t)(const char*, int, ...);
|
||||
typedef int (*open64_func_t)(const char*, int, ...);
|
||||
typedef int (*openat_func_t)(int, const char*, int, ...);
|
||||
typedef int (*openat64_func_t)(int, const char*, int, ...);
|
||||
typedef int (*creat_func_t)(const char*, mode_t);
|
||||
typedef int (*creat64_func_t)(const char*, mode_t);
|
||||
typedef FILE* (*fopen_func_t)(const char*, const char*);
|
||||
typedef FILE* (*fopen64_func_t)(const char*, const char*);
|
||||
|
||||
static open_func_t _libc_open = NULL;
|
||||
static open64_func_t _libc_open64 = NULL;
|
||||
static openat_func_t _libc_openat = NULL;
|
||||
static openat64_func_t _libc_openat64 = NULL;
|
||||
static fopen_func_t _libc_fopen = NULL;
|
||||
static fopen64_func_t _libc_fopen64 = NULL;
|
||||
static creat_func_t _libc_creat = NULL;
|
||||
static creat64_func_t _libc_creat64 = NULL;
|
||||
|
||||
static
|
||||
int
|
||||
get_underlying_filepath(int fd_,
|
||||
char *filepath_)
|
||||
{
|
||||
int rv;
|
||||
|
||||
strcpy(filepath_,"fullpath");
|
||||
rv = ioctl(fd_,IOCTL_FILE_INFO,filepath_);
|
||||
if(rv == -1)
|
||||
return -1;
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
strip_exec(const char *orig_mode_,
|
||||
char *new_mode_)
|
||||
{
|
||||
size_t i;
|
||||
size_t j;
|
||||
|
||||
for(i = j = 0; orig_mode_[i]; i++)
|
||||
{
|
||||
if(orig_mode_[i] == 'x')
|
||||
continue;
|
||||
new_mode_[j++] = orig_mode_[i];
|
||||
}
|
||||
|
||||
new_mode_[j] = '\0';
|
||||
}
|
||||
|
||||
int
|
||||
open(const char *pathname_,
|
||||
int flags_,
|
||||
...)
|
||||
{
|
||||
int rv;
|
||||
int fd;
|
||||
mode_t mode;
|
||||
struct stat st;
|
||||
|
||||
LOAD_FUNC(open);
|
||||
|
||||
mode = 0;
|
||||
if(flags_ & O_CREAT)
|
||||
{
|
||||
va_list args;
|
||||
va_start(args,flags_);
|
||||
mode = va_arg(args,mode_t);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
fd = _libc_open(pathname_,flags_,mode);
|
||||
if(fd == -1)
|
||||
return -1;
|
||||
|
||||
if(flags_ & (O_DIRECTORY|O_PATH))
|
||||
return fd;
|
||||
rv = fstat(fd,&st);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
if((st.st_mode & S_IFMT) != S_IFREG)
|
||||
return fd;
|
||||
|
||||
IOCTL_BUF real_pathname;
|
||||
rv = get_underlying_filepath(fd,real_pathname);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
flags_ &= ~(O_EXCL|O_CREAT);
|
||||
rv = _libc_open(real_pathname,flags_,mode);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
close(fd);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
int
|
||||
open64(const char *pathname_,
|
||||
int flags_,
|
||||
...)
|
||||
{
|
||||
int rv;
|
||||
int fd;
|
||||
mode_t mode;
|
||||
struct stat st;
|
||||
|
||||
LOAD_FUNC(open64);
|
||||
|
||||
mode = 0;
|
||||
if(flags_ & O_CREAT)
|
||||
{
|
||||
va_list args;
|
||||
va_start(args,flags_);
|
||||
mode = va_arg(args,mode_t);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
fd = _libc_open64(pathname_,flags_,mode);
|
||||
if(fd == -1)
|
||||
return -1;
|
||||
|
||||
if(flags_ & (O_DIRECTORY|O_PATH))
|
||||
return fd;
|
||||
rv = fstat(fd,&st);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
if((st.st_mode & S_IFMT) != S_IFREG)
|
||||
return fd;
|
||||
|
||||
IOCTL_BUF real_pathname;
|
||||
rv = get_underlying_filepath(fd,real_pathname);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
flags_ &= ~(O_EXCL|O_CREAT);
|
||||
rv = _libc_open64(real_pathname,flags_,mode);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
close(fd);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
int
|
||||
openat(int dirfd_,
|
||||
const char *pathname_,
|
||||
int flags_,
|
||||
...)
|
||||
{
|
||||
int rv;
|
||||
int fd;
|
||||
mode_t mode;
|
||||
struct stat st;
|
||||
|
||||
LOAD_FUNC(openat);
|
||||
|
||||
mode = 0;
|
||||
if(flags_ & O_CREAT)
|
||||
{
|
||||
va_list args;
|
||||
va_start(args,flags_);
|
||||
mode = va_arg(args,mode_t);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
fd = _libc_openat(dirfd_,pathname_,flags_,mode);
|
||||
if(fd == -1)
|
||||
return -1;
|
||||
|
||||
if(flags_ & (O_DIRECTORY|O_PATH))
|
||||
return fd;
|
||||
rv = fstat(fd,&st);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
if((st.st_mode & S_IFMT) != S_IFREG)
|
||||
return fd;
|
||||
|
||||
IOCTL_BUF real_pathname;
|
||||
rv = get_underlying_filepath(fd,real_pathname);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
flags_ &= ~(O_EXCL|O_CREAT);
|
||||
rv = _libc_openat(dirfd_,real_pathname,flags_,mode);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
close(fd);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
int
|
||||
openat64(int dirfd_,
|
||||
const char *pathname_,
|
||||
int flags_,
|
||||
...)
|
||||
{
|
||||
int rv;
|
||||
int fd;
|
||||
mode_t mode;
|
||||
struct stat st;
|
||||
|
||||
LOAD_FUNC(openat64);
|
||||
|
||||
mode = 0;
|
||||
if(flags_ & O_CREAT)
|
||||
{
|
||||
va_list args;
|
||||
va_start(args,flags_);
|
||||
mode = va_arg(args,mode_t);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
fd = _libc_openat64(dirfd_,pathname_,flags_,mode);
|
||||
if(fd == -1)
|
||||
return -1;
|
||||
|
||||
if(flags_ & (O_DIRECTORY|O_PATH))
|
||||
return fd;
|
||||
rv = fstat(fd,&st);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
if((st.st_mode & S_IFMT) != S_IFREG)
|
||||
return fd;
|
||||
|
||||
IOCTL_BUF real_pathname;
|
||||
rv = get_underlying_filepath(fd,real_pathname);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
flags_ &= ~(O_EXCL|O_CREAT);
|
||||
rv = _libc_openat64(dirfd_,real_pathname,flags_,mode);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
close(fd);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
FILE*
|
||||
fopen(const char *pathname_,
|
||||
const char *mode_)
|
||||
{
|
||||
int fd;
|
||||
int rv;
|
||||
FILE *f;
|
||||
FILE *f2;
|
||||
struct stat st;
|
||||
|
||||
LOAD_FUNC(fopen);
|
||||
|
||||
f = _libc_fopen(pathname_,mode_);
|
||||
if(f == NULL)
|
||||
return NULL;
|
||||
|
||||
fd = fileno(f);
|
||||
if(fd == -1)
|
||||
return f;
|
||||
|
||||
rv = fstat(fd,&st);
|
||||
if(rv == -1)
|
||||
return f;
|
||||
if((st.st_mode & S_IFMT) != S_IFREG)
|
||||
return f;
|
||||
|
||||
IOCTL_BUF real_pathname;
|
||||
rv = get_underlying_filepath(fd,real_pathname);
|
||||
if(rv == -1)
|
||||
return f;
|
||||
|
||||
char new_mode[64];
|
||||
strip_exec(mode_,new_mode);
|
||||
f2 = _libc_fopen(real_pathname,new_mode);
|
||||
if(f2 == NULL)
|
||||
return f;
|
||||
|
||||
fclose(f);
|
||||
|
||||
return f2;
|
||||
}
|
||||
|
||||
FILE*
|
||||
fopen64(const char *pathname_,
|
||||
const char *mode_)
|
||||
{
|
||||
int fd;
|
||||
int rv;
|
||||
FILE *f;
|
||||
FILE *f2;
|
||||
struct stat st;
|
||||
|
||||
LOAD_FUNC(fopen64);
|
||||
|
||||
f = _libc_fopen64(pathname_,mode_);
|
||||
if(f == NULL)
|
||||
return NULL;
|
||||
|
||||
fd = fileno(f);
|
||||
if(fd == -1)
|
||||
return f;
|
||||
|
||||
rv = fstat(fd,&st);
|
||||
if(rv == -1)
|
||||
return f;
|
||||
if((st.st_mode & S_IFMT) != S_IFREG)
|
||||
return f;
|
||||
|
||||
IOCTL_BUF real_pathname;
|
||||
rv = get_underlying_filepath(fd,real_pathname);
|
||||
if(rv == -1)
|
||||
return f;
|
||||
|
||||
char new_mode[64];
|
||||
strip_exec(mode_,new_mode);
|
||||
f2 = _libc_fopen64(real_pathname,new_mode);
|
||||
if(f2 == NULL)
|
||||
return f;
|
||||
|
||||
fclose(f);
|
||||
|
||||
return f2;
|
||||
}
|
||||
|
||||
int
|
||||
creat(const char *pathname_,
|
||||
mode_t mode_)
|
||||
{
|
||||
int fd;
|
||||
int rv;
|
||||
|
||||
LOAD_FUNC(creat);
|
||||
|
||||
fd = _libc_creat(pathname_,mode_);
|
||||
if(fd == -1)
|
||||
return -1;
|
||||
|
||||
IOCTL_BUF real_pathname;
|
||||
rv = get_underlying_filepath(fd,real_pathname);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
rv = _libc_creat(real_pathname,mode_);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
close(fd);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
int
|
||||
creat64(const char *pathname_,
|
||||
mode_t mode_)
|
||||
{
|
||||
int fd;
|
||||
int rv;
|
||||
|
||||
LOAD_FUNC(creat64);
|
||||
|
||||
fd = _libc_creat64(pathname_,mode_);
|
||||
if(fd == -1)
|
||||
return -1;
|
||||
|
||||
IOCTL_BUF real_pathname;
|
||||
rv = get_underlying_filepath(fd,real_pathname);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
rv = _libc_creat64(real_pathname,mode_);
|
||||
if(rv == -1)
|
||||
return fd;
|
||||
|
||||
close(fd);
|
||||
|
||||
return rv;
|
||||
}
|
Loading…
Reference in New Issue
Block a user