github-mirror/mergerfs
2
0
Fork 0
mirror of https://github.com/trapexit/mergerfs.git synced 2025-01-22 15:57:16 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #556 from trapexit/direct

Browse Source 
set direct_io per open/create, now runtime configurable
This commit is contained in:
trapexit 2019-01-11 15:36:39 -05:00 committed by GitHub
commit 56f7df028b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 336 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
libfuse/lib/fuse.c
View File

@ -70,7 +70,6 @@ struct fuse_config {
int set_mode;
int set_uid;
int set_gid;
int direct_io;
int kernel_cache;
int auto_cache;
int intr;
@ -3138,8 +3137,6 @@ static void fuse_lib_create(fuse_req_t req, fuse_ino_t parent,
fuse_fs_release(f->fs, path, fi);
forget_node(f, e.ino, 1);
} else {
if (f->conf.direct_io)
fi->direct_io = 1;
if (f->conf.kernel_cache)
fi->keep_cache = 1;
@ -3215,8 +3212,6 @@ static void fuse_lib_open(fuse_req_t req, fuse_ino_t ino,
fuse_prepare_interrupt(f, req, &d);
err = fuse_fs_open(f->fs, path, fi);
if (!err) {
if (f->conf.direct_io)
fi->direct_io = 1;
if (f->conf.kernel_cache)
fi->keep_cache = 1;
@ -4385,7 +4380,6 @@ static const struct fuse_opt fuse_lib_opts[] = {
FUSE_LIB_OPT("hard_remove", hard_remove, 1),
FUSE_LIB_OPT("use_ino", use_ino, 1),
FUSE_LIB_OPT("readdir_ino", readdir_ino, 1),
FUSE_LIB_OPT("direct_io", direct_io, 1),
FUSE_LIB_OPT("kernel_cache", kernel_cache, 1),
FUSE_LIB_OPT("auto_cache", auto_cache, 1),
FUSE_LIB_OPT("noauto_cache", auto_cache, 0),
@ -4416,7 +4410,6 @@ static void fuse_lib_help(void)
" -o hard_remove immediate removal (don't hide files)\n"
" -o use_ino let filesystem set inode numbers\n"
" -o readdir_ino try to fill in d_ino in readdir\n"
" -o direct_io use direct I/O\n"
" -o kernel_cache cache files in kernel\n"
" -o [no]auto_cache enable caching based on modification times (off)\n"
" -o umask=M set file permissions (octal)\n"

157
src/create.cpp
View File

@ -14,11 +14,6 @@
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <fuse.h>
#include <string>
#include <vector>
#include "config.hpp"
#include "errno.hpp"
#include "fileinfo.hpp"
@ -29,81 +24,92 @@
#include "rwlock.hpp"
#include "ugid.hpp"
#include <fuse.h>
#include <string>
#include <vector>
using std::string;
using std::vector;
using namespace mergerfs;
static
int
_create_core(const string &fullpath,
mode_t mode,
const mode_t umask,
const int flags)
namespace local
{
if(!fs::acl::dir_has_defaults(fullpath))
mode &= ~umask;
static
int
create_core(const string &fullpath_,
mode_t mode_,
const mode_t umask_,
const int flags_)
{
if(!fs::acl::dir_has_defaults(fullpath_))
mode_ &= ~umask_;
return fs::open(fullpath,flags,mode);
}
return fs::open(fullpath_,flags_,mode_);
}
static
int
_create_core(const string &createpath,
const char *fusepath,
const mode_t mode,
const mode_t umask,
const int flags,
uint64_t &fh)
{
int rv;
string fullpath;
static
int
create_core(const string &createpath_,
const char *fusepath_,
const mode_t mode_,
const mode_t umask_,
const int flags_,
uint64_t *fh_)
{
int rv;
string fullpath;
fs::path::make(&createpath,fusepath,fullpath);
fs::path::make(&createpath_,fusepath_,fullpath);
rv = _create_core(fullpath,mode,umask,flags);
if(rv == -1)
return -errno;
rv = local::create_core(fullpath,mode_,umask_,flags_);
if(rv == -1)
return -errno;
fh = reinterpret_cast<uint64_t>(new FileInfo(rv,fusepath));
*fh_ = reinterpret_cast<uint64_t>(new FileInfo(rv,fusepath_));
return 0;
}
return 0;
}
static
int
_create(Policy::Func::Search searchFunc,
Policy::Func::Create createFunc,
const Branches &branches_,
const uint64_t minfreespace,
const char *fusepath,
const mode_t mode,
const mode_t umask,
const int flags,
uint64_t &fh)
{
int rv;
string fullpath;
string fusedirpath;
vector<const string*> createpaths;
vector<const string*> existingpaths;
static
int
create(Policy::Func::Search searchFunc_,
Policy::Func::Create createFunc_,
const Branches &branches_,
const uint64_t minfreespace_,
const char *fusepath_,
const mode_t mode_,
const mode_t umask_,
const int flags_,
uint64_t *fh_)
{
int rv;
string fullpath;
string fusedirpath;
vector<const string*> createpaths;
vector<const string*> existingpaths;
fusedirpath = fs::path::dirname(fusepath);
fusedirpath = fs::path::dirname(fusepath_);
rv = searchFunc(branches_,fusedirpath,minfreespace,existingpaths);
if(rv == -1)
return -errno;
rv = searchFunc_(branches_,fusedirpath,minfreespace_,existingpaths);
if(rv == -1)
return -errno;
rv = createFunc(branches_,fusedirpath,minfreespace,createpaths);
if(rv == -1)
return -errno;
rv = createFunc_(branches_,fusedirpath,minfreespace_,createpaths);
if(rv == -1)
return -errno;
rv = fs::clonepath_as_root(*existingpaths[0],*createpaths[0],fusedirpath);
if(rv == -1)
return -errno;
rv = fs::clonepath_as_root(*existingpaths[0],*createpaths[0],fusedirpath);
if(rv == -1)
return -errno;
return _create_core(*createpaths[0],
fusepath,
mode,umask,flags,fh);
return local::create_core(*createpaths[0],
fusepath_,
mode_,
umask_,
flags_,
fh_);
}
}
namespace mergerfs
@ -111,24 +117,25 @@ namespace mergerfs
namespace fuse
{
int
create(const char *fusepath,
mode_t mode,
fuse_file_info *ffi)
create(const char *fusepath_,
mode_t mode_,
fuse_file_info *ffi_)
{
const fuse_context *fc = fuse_get_context();
const Config &config = Config::get(fc);
const ugid::Set ugid(fc->uid,fc->gid);
const rwlock::ReadGuard readlock(&config.branches_lock);
return _create(config.getattr,
config.create,
config.branches,
config.minfreespace,
fusepath,
mode,
fc->umask,
ffi->flags,
ffi->fh);
ffi_->direct_io = config.direct_io;
return local::create(config.getattr,
config.create,
config.branches,
config.minfreespace,
fusepath_,
mode_,
fc->umask,
ffi_->flags,
&ffi_->fh);
}
}
}

10
src/fs_path.hpp
View File

@ -46,6 +46,16 @@ namespace fs
base += suffix;
}
inline
void
make(const string &base_,
const char *suffix_,
string *output_)
{
*output_ = base_;
*output_ += suffix_;
}
inline
void
make(const string *base,

24
src/getxattr.cpp
View File

@ -14,17 +14,6 @@
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <fuse.h>
#include <stdio.h>
#include <string.h>
#include <algorithm>
#include <set>
#include <sstream>
#include <string>
#include <vector>
#include "config.hpp"
#include "errno.hpp"
#include "fs_base_getxattr.hpp"
@ -34,6 +23,17 @@
#include "ugid.hpp"
#include "version.hpp"
#include <fuse.h>
#include <algorithm>
#include <set>
#include <sstream>
#include <string>
#include <vector>
#include <stdio.h>
#include <string.h>
static const char SECURITY_CAPABILITY[] = "security.capability";
using std::string;
@ -297,6 +297,8 @@ _getxattr_controlfile(const Config &config,
_getxattr_controlfile_version(attrvalue);
else if(attr[2] == "pid")
_getxattr_pid(attrvalue);
else if(attr[2] == "direct_io")
_getxattr_controlfile_bool(config.direct_io,attrvalue);
break;
case 4:

1
src/listxattr.cpp
View File

@ -57,6 +57,7 @@ _listxattr_controlfile(char *list,
("user.mergerfs.xattr")
("user.mergerfs.statfs")
("user.mergerfs.statfs_ignore")
("user.mergerfs.direct_io")
("user.mergerfs.policies")
("user.mergerfs.version")
("user.mergerfs.pid");

13
src/mergerfs.cpp
View File

@ -70,7 +70,6 @@ namespace local
static
void
get_fuse_operations(struct fuse_operations &ops,
const bool direct_io,
const bool nullrw)
{
ops.flag_nullpath_ok = true;
@ -105,9 +104,7 @@ namespace local
ops.poll = NULL;
ops.read = (nullrw ?
mergerfs::fuse::read_null :
(direct_io ?
mergerfs::fuse::read_direct_io :
mergerfs::fuse::read));
mergerfs::fuse::read);
ops.read_buf = (nullrw ?
NULL :
mergerfs::fuse::read_buf);
@ -127,9 +124,7 @@ namespace local
ops.utimens = mergerfs::fuse::utimens;
ops.write = (nullrw ?
mergerfs::fuse::write_null :
(direct_io ?
mergerfs::fuse::write_direct_io :
mergerfs::fuse::write));
mergerfs::fuse::write);
ops.write_buf = (nullrw ?
mergerfs::fuse::write_buf_null :
mergerfs::fuse::write_buf);
@ -168,9 +163,7 @@ namespace mergerfs
mergerfs::options::parse(args,config);
local::setup_resources();
local::get_fuse_operations(ops,
config.direct_io,
config.nullrw);
local::get_fuse_operations(ops,config.nullrw);
return fuse_main(args.argc,
args.argv,

92
src/open.cpp
View File

@ -14,8 +14,6 @@
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <fuse.h>
#include "config.hpp"
#include "errno.hpp"
#include "fileinfo.hpp"
@ -25,58 +23,63 @@
#include "rwlock.hpp"
#include "ugid.hpp"
#include <fcntl.h>
#include <fuse.h>
#include <string>
#include <vector>
#include <fcntl.h>
using std::string;
using std::vector;
using mergerfs::Policy;
static
int
_open_core(const string *basepath_,
const char *fusepath_,
const int flags_,
const bool link_cow_,
uint64_t &fh_)
namespace local
{
int fd;
string fullpath;
static
int
open_core(const string &basepath_,
const char *fusepath_,
const int flags_,
const bool link_cow_,
uint64_t *fh_)
{
int fd;
string fullpath;
fs::path::make(basepath_,fusepath_,fullpath);
fs::path::make(basepath_,fusepath_,&fullpath);
if(link_cow_ && fs::cow::is_eligible(fullpath.c_str(),flags_))
fs::cow::break_link(fullpath.c_str());
if(link_cow_ && fs::cow::is_eligible(fullpath.c_str(),flags_))
fs::cow::break_link(fullpath.c_str());
fd = fs::open(fullpath,flags_);
if(fd == -1)
return -errno;
fd = fs::open(fullpath,flags_);
if(fd == -1)
return -errno;
fh_ = reinterpret_cast<uint64_t>(new FileInfo(fd,fusepath_));
*fh_ = reinterpret_cast<uint64_t>(new FileInfo(fd,fusepath_));
return 0;
}
return 0;
}
static
int
_open(Policy::Func::Search searchFunc_,
const Branches &branches_,
const uint64_t minfreespace_,
const char *fusepath_,
const int flags_,
const bool link_cow_,
uint64_t &fh_)
{
int rv;
vector<const string*> basepaths;
static
int
open(Policy::Func::Search searchFunc_,
const Branches &branches_,
const uint64_t minfreespace_,
const char *fusepath_,
const int flags_,
const bool link_cow_,
uint64_t *fh_)
{
int rv;
vector<const string*> basepaths;
rv = searchFunc_(branches_,fusepath_,minfreespace_,basepaths);
if(rv == -1)
return -errno;
rv = searchFunc_(branches_,fusepath_,minfreespace_,basepaths);
if(rv == -1)
return -errno;
return _open_core(basepaths[0],fusepath_,flags_,link_cow_,fh_);
return local::open_core(*basepaths[0],fusepath_,flags_,link_cow_,fh_);
}
}
namespace mergerfs
@ -92,13 +95,14 @@ namespace mergerfs
const ugid::Set ugid(fc->uid,fc->gid);
const rwlock::ReadGuard readlock(&config.branches_lock);
return _open(config.open,
config.branches,
config.minfreespace,
fusepath_,
ffi_->flags,
config.link_cow,
ffi_->fh);
ffi_->direct_io = config.direct_io;
return local::open(config.open,
config.branches,
config.minfreespace,
fusepath_,
ffi_->flags,
config.link_cow,
&ffi_->fh);
}
}
}

10
src/opendir.cpp
View File

@ -14,19 +14,19 @@
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <fuse.h>
#include "dirinfo.hpp"
#include <fuse.h>
namespace mergerfs
{
namespace fuse
{
int
opendir(const char *fusepath,
fuse_file_info *ffi)
opendir(const char *fusepath_,
fuse_file_info *ffi_)
{
ffi->fh = reinterpret_cast<uint64_t>(new DirInfo(fusepath));
ffi_->fh = reinterpret_cast<uint64_t>(new DirInfo(fusepath_));
return 0;
}

32
src/option_parser.cpp
View File

@ -14,20 +14,6 @@
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <fuse.h>
#include <stddef.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string>
#include <vector>
#include <sstream>
#include <iostream>
#include <iomanip>
#include "config.hpp"
#include "errno.hpp"
#include "fs_glob.hpp"
@ -36,6 +22,20 @@
#include "str.hpp"
#include "version.hpp"
#include <fuse.h>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
using std::string;
using std::vector;
using namespace mergerfs;
@ -211,7 +211,7 @@ parse_and_process_arg(Config &config,
if(arg == "defaults")
return (set_default_options(*outargs),0);
else if(arg == "direct_io")
return (config.direct_io=true,1);
return (config.direct_io=true,0);
return 1;
}
@ -336,7 +336,7 @@ usage(void)
" splice_write, splice_move\n"
" -o func.<f>=<p> Set function <f> to policy <p>\n"
" -o category.<c>=<p> Set functions in category <c> to <p>\n"
" -o direct_io Bypass additional caching, increases write\n"
" -o direct_io Bypass page caching, may increase write\n"
" speeds at the cost of reads. Please read docs\n"
" for more details as there are tradeoffs.\n"
" -o use_ino Have mergerfs generate inode values rather than\n"

109
src/read.cpp
View File

@ -14,50 +14,49 @@
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <fuse.h>
#include <string.h>
#include <string>
#include "errno.hpp"
#include "fileinfo.hpp"
#include "fs_base_read.hpp"
static
inline
int
_read(const int fd,
void *buf,
const size_t count,
const off_t offset)
#include <fuse.h>
namespace local
{
int rv;
static
inline
int
read_regular(const int fd_,
void *buf_,
const size_t count_,
const off_t offset_)
{
int rv;
rv = fs::pread(fd,buf,count,offset);
if(rv == -1)
return -errno;
if(rv == 0)
return 0;
rv = fs::pread(fd_,buf_,count_,offset_);
if(rv == -1)
return -errno;
if(rv == 0)
return 0;
return count;
}
return count_;
}
static
inline
int
_read_direct_io(const int fd,
void *buf,
const size_t count,
const off_t offset)
{
int rv;
static
inline
int
read_direct_io(const int fd_,
void *buf_,
const size_t count_,
const off_t offset_)
{
int rv;
rv = fs::pread(fd,buf,count,offset);
if(rv == -1)
return -errno;
rv = fs::pread(fd_,buf_,count_,offset_);
if(rv == -1)
return -errno;
return rv;
return rv;
}
}
namespace mergerfs
@ -65,38 +64,30 @@ namespace mergerfs
namespace fuse
{
int
read(const char *fusepath,
char *buf,
size_t count,
off_t offset,
fuse_file_info *ffi)
read(const char *fusepath_,
char *buf_,
size_t count_,
off_t offset_,
fuse_file_info *ffi_)
{
FileInfo *fi = reinterpret_cast<FileInfo*>(ffi->fh);
FileInfo *fi;
return ::_read(fi->fd,buf,count,offset);
fi = reinterpret_cast<FileInfo*>(ffi_->fh);
if(ffi_->direct_io)
return local::read_direct_io(fi->fd,buf_,count_,offset_);
return local::read_regular(fi->fd,buf_,count_,offset_);
}
int
read_direct_io(const char *fusepath,
char *buf,
size_t count,
off_t offset,
fuse_file_info *ffi)
{
FileInfo *fi = reinterpret_cast<FileInfo*>(ffi->fh);
return ::_read_direct_io(fi->fd,buf,count,offset);
}
int
read_null(const char *fusepath,
char *buf,
size_t count,
off_t offset,
fuse_file_info *ffi)
read_null(const char *fusepath_,
char *buf_,
size_t count_,
off_t offset_,
fuse_file_info *ffi_)
{
return count;
return count_;
}
}
}

20
src/setxattr.cpp
View File

@ -14,14 +14,6 @@
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <fuse.h>
#include <string.h>
#include <sstream>
#include <string>
#include <vector>
#include "config.hpp"
#include "errno.hpp"
#include "fs_base_setxattr.hpp"
@ -33,6 +25,14 @@
#include "str.hpp"
#include "ugid.hpp"
#include <fuse.h>
#include <sstream>
#include <string>
#include <vector>
#include <string.h>
static const char SECURITY_CAPABILITY[] = "security.capability";
using std::string;
@ -323,6 +323,10 @@ _setxattr_controlfile(Config &config,
return _setxattr_statfsignore(attrval,
flags,
config.statfs_ignore);
else if(attr[2] == "direct_io")
return _setxattr_bool(attrval,
flags,
config.direct_io);
break;
case 4:

195
src/write.cpp
View File

@ -33,119 +33,116 @@ using std::vector;
typedef int (*WriteFunc)(const int,const void*,const size_t,const off_t);
static
bool
_out_of_space(const int error)
namespace local
{
return ((error == ENOSPC) ||
(error == EDQUOT));
static
bool
out_of_space(const int error_)
{
return ((error_ == ENOSPC) ||
(error_ == EDQUOT));
}
static
int
write_regular(const int fd_,
const void *buf_,
const size_t count_,
const off_t offset_)
{
int rv;
rv = fs::pwrite(fd_,buf_,count_,offset_);
if(rv == -1)
return -errno;
if(rv == 0)
return 0;
return count_;
}
static
int
write_direct_io(const int fd_,
const void *buf_,
const size_t count_,
const off_t offset_)
{
int rv;
rv = fs::pwrite(fd_,buf_,count_,offset_);
if(rv == -1)
return -errno;
return rv;
}
static
int
write(WriteFunc func_,
const char *buf_,
const size_t count_,
const off_t offset_,
fuse_file_info *ffi_)
{
int rv;
FileInfo* fi;
fi = reinterpret_cast<FileInfo*>(ffi_->fh);
rv = func_(fi->fd,buf_,count_,offset_);
if(local::out_of_space(-rv))
{
const fuse_context *fc = fuse_get_context();
const Config &config = Config::get(fc);
if(config.moveonenospc)
{
vector<string> paths;
const ugid::Set ugid(0,0);
const rwlock::ReadGuard readlock(&config.branches_lock);
config.branches.to_paths(paths);
rv = fs::movefile(paths,fi->fusepath,count_,fi->fd);
if(rv == -1)
return -ENOSPC;
rv = func_(fi->fd,buf_,count_,offset_);
}
}
return rv;
}
}
static
inline
int
_write(const int fd,
const void *buf,
const size_t count,
const off_t offset)
{
int rv;
rv = fs::pwrite(fd,buf,count,offset);
if(rv == -1)
return -errno;
if(rv == 0)
return 0;
return count;
}
static
inline
int
_write_direct_io(const int fd,
const void *buf,
const size_t count,
const off_t offset)
{
int rv;
rv = fs::pwrite(fd,buf,count,offset);
if(rv == -1)
return -errno;
return rv;
}
namespace mergerfs
{
namespace fuse
{
static
inline
int
write(WriteFunc func,
const char *buf,
const size_t count,
const off_t offset,
fuse_file_info *ffi)
write(const char *fusepath_,
const char *buf_,
size_t count_,
off_t offset_,
fuse_file_info *ffi_)
{
int rv;
FileInfo* fi = reinterpret_cast<FileInfo*>(ffi->fh);
WriteFunc wf;
rv = func(fi->fd,buf,count,offset);
if(_out_of_space(-rv))
{
const fuse_context *fc = fuse_get_context();
const Config &config = Config::get(fc);
wf = ((ffi_->direct_io) ?
local::write_direct_io :
local::write_regular);
if(config.moveonenospc)
{
vector<string> paths;
const ugid::Set ugid(0,0);
const rwlock::ReadGuard readlock(&config.branches_lock);
config.branches.to_paths(paths);
rv = fs::movefile(paths,fi->fusepath,count,fi->fd);
if(rv == -1)
return -ENOSPC;
rv = func(fi->fd,buf,count,offset);
}
}
return rv;
return local::write(wf,buf_,count_,offset_,ffi_);
}
int
write(const char *fusepath,
const char *buf,
size_t count,
off_t offset,
fuse_file_info *ffi)
write_null(const char *fusepath_,
const char *buf_,
size_t count_,
off_t offset_,
fuse_file_info *ffi_)
{
return write(_write,buf,count,offset,ffi);
}
int
write_direct_io(const char *fusepath,
const char *buf,
size_t count,
off_t offset,
fuse_file_info *ffi)
{
return write(_write_direct_io,buf,count,offset,ffi);
}
int
write_null(const char *fusepath,
const char *buf,
size_t count,
off_t offset,
fuse_file_info *ffi)
{
return count;
return count_;
}
}
}