Mirror
/
mergerfs
mirror of https://github.com/trapexit/mergerfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Change the "devino" inode calculation 

Use hashing of branch path rather than use st_dev.
pull/1443/head
Antonio SJ Musumeci 3 weeks ago
parent
ad54c8be19
commit
77bf16e9ae
15 changed files with 1891 additions and 246 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      libfuse/include/fuse.h
  2. 6
      src/fh.hpp
  3. 9
      src/fileinfo.hpp
  4. 173
      src/fs_inode.cpp
  5. 40
      src/fs_inode.hpp
  6. 2
      src/fuse_create.cpp
  7. 28
      src/fuse_fgetattr.cpp
  8. 4
      src/fuse_getattr.cpp
  9. 2
      src/fuse_open.cpp
  10. 80
      src/fuse_readdir_cor.cpp
  11. 53
      src/fuse_readdir_cosr.cpp
  12. 24
      src/fuse_readdir_seq.cpp
  13. 4
      src/fuse_statx.cpp
  14. 2
      src/fuse_symlink.cpp
  15. 1709
      src/nonstd/string_view.hpp

1
libfuse/include/fuse.h
View File

@ -11,6 +11,7 @@
#include "extern_c.h"
#include "fuse_common.h"
#include "fuse_kernel.h"
#include <fcntl.h>
#include <time.h>

6
src/fh.hpp
View File

@ -22,6 +22,12 @@
class FH
{
public:
FH(const std::string fusepath_)
: fusepath(std::move(fusepath_))
{
}
FH(const char *fusepath_)
: fusepath(fusepath_)
{

9
src/fileinfo.hpp
View File

@ -26,17 +26,20 @@
class FileInfo : public FH
{
public:
FileInfo(int const fd_,
char const *fusepath_,
bool const direct_io_)
FileInfo(const int fd_,
const std::string branchpath_,
const char *fusepath_,
const bool direct_io_)
: FH(fusepath_),
fd(fd_),
branchpath(std::move(branchpath_)),
direct_io(direct_io_)
{
}
public:
int fd;
std::string branchpath;
uint32_t direct_io:1;
std::mutex mutex;
};

173
src/fs_inode.cpp
View File

@ -29,9 +29,18 @@
#include <string.h>
#include <sys/stat.h>
typedef uint64_t (*inodefunc_t)(const char*,const uint64_t,const mode_t,const dev_t,const ino_t);
using namespace nonstd;
typedef uint64_t (*inodefunc_t)(const string_view,
const string_view,
const mode_t,
const ino_t);
static uint64_t hybrid_hash(const string_view,
const string_view,
const mode_t,
const ino_t);
static uint64_t hybrid_hash(const char*,const uint64_t,const mode_t,const dev_t,const ino_t);
static inodefunc_t g_func = hybrid_hash;
@ -47,40 +56,40 @@ h64_to_h32(uint64_t h_)
static
uint64_t
passthrough(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
passthrough(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return ino_;
}
static
uint64_t
path_hash(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
path_hash(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return rapidhash(fusepath_,fusepath_len_);
uint64_t seed;
seed = rapidhash(&fusepath_[0],fusepath_.size());
return seed;
}
static
uint64_t
path_hash32(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
path_hash32(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
uint64_t h;
h = path_hash(fusepath_,
fusepath_len_,
h = path_hash(branch_path_,
fusepath_,
mode_,
dev_,
ino_);
return h64_to_h32(h);
@ -88,34 +97,31 @@ path_hash32(const char *fusepath_,
static
uint64_t
devino_hash(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
devino_hash(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
uint64_t buf[2];
uint64_t seed;
buf[0] = dev_;
buf[1] = ino_;
seed = rapidhash(&branch_path_[0],branch_path_.size());
seed = rapidhash_withSeed(&ino_,sizeof(ino_),seed);
return rapidhash((void*)&buf[0],sizeof(buf));
return seed;
}
static
uint64_t
devino_hash32(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
devino_hash32(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
uint64_t h;
h = devino_hash(fusepath_,
fusepath_len_,
h = devino_hash(branch_path_,
fusepath_,
mode_,
dev_,
ino_);
return h64_to_h32(h);
@ -123,28 +129,26 @@ devino_hash32(const char *fusepath_,
static
uint64_t
hybrid_hash(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
hybrid_hash(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return (S_ISDIR(mode_) ?
path_hash(fusepath_,fusepath_len_,mode_,dev_,ino_) :
devino_hash(fusepath_,fusepath_len_,mode_,dev_,ino_));
path_hash(branch_path_,fusepath_,mode_,ino_) :
devino_hash(branch_path_,fusepath_,mode_,ino_));
}
static
uint64_t
hybrid_hash32(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
hybrid_hash32(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return (S_ISDIR(mode_) ?
path_hash32(fusepath_,fusepath_len_,mode_,dev_,ino_) :
devino_hash32(fusepath_,fusepath_len_,mode_,dev_,ino_));
path_hash32(branch_path_,fusepath_,mode_,ino_) :
devino_hash32(branch_path_,fusepath_,mode_,ino_));
}
namespace fs
@ -196,71 +200,34 @@ namespace fs
}
uint64_t
calc(const char *fusepath_,
const uint64_t fusepath_len_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
calc(const string_view branch_path_,
const string_view fusepath_,
const mode_t mode_,
const ino_t ino_)
{
return g_func(fusepath_,fusepath_len_,mode_,dev_,ino_);
}
uint64_t
calc(std::string const &fusepath_,
const mode_t mode_,
const dev_t dev_,
const ino_t ino_)
{
return calc(fusepath_.c_str(),
fusepath_.size(),
mode_,
dev_,
ino_);
return g_func(branch_path_,fusepath_,mode_,ino_);
}
void
calc(const char *fusepath_,
const uint64_t fusepath_len_,
struct stat *st_)
calc(const string_view branch_path_,
const string_view fusepath_,
struct stat *st_)
{
st_->st_ino = calc(fusepath_,
fusepath_len_,
st_->st_ino = calc(branch_path_,
fusepath_,
st_->st_mode,
st_->st_dev,
st_->st_ino);
}
void
calc(const char *fusepath_,
const uint64_t fusepath_len_,
calc(const string_view branch_path_,
const string_view fusepath_,
struct fuse_statx *st_)
{
st_->ino = calc(fusepath_,
fusepath_len_,
st_->ino = calc(branch_path_,
fusepath_,
st_->mode,
st_->dev_major ^ st_->dev_minor,
st_->ino);
}
void
calc(const char *fusepath_,
struct stat *st_)
{
calc(fusepath_,strlen(fusepath_),st_);
}
void
calc(const char *fusepath_,
struct fuse_statx *st_)
{
calc(fusepath_,strlen(fusepath_),st_);
}
void
calc(const std::string &fusepath_,
struct stat *st_)
{
calc(fusepath_.c_str(),fusepath_.size(),st_);
}
}
}

40
src/fs_inode.hpp
View File

@ -20,6 +20,9 @@
#include "fuse_kernel.h"
#include "nonstd/string_view.hpp"
#include "ghc/filesystem.hpp"
#include <cstdint>
#include <string>
@ -30,34 +33,19 @@ namespace fs
{
namespace inode
{
static const uint64_t MAGIC = 0x7472617065786974;
int set_algo(const std::string &s);
std::string get_algo(void);
uint64_t calc(const char *fusepath,
const uint64_t fusepath_len,
const mode_t mode,
const dev_t dev,
const ino_t ino);
uint64_t calc(std::string const &fusepath,
mode_t const mode,
dev_t const dev,
ino_t ino);
void calc(const char *fusepath,
const uint64_t fusepath_len,
struct stat *st);
void calc(const char *fusepath,
const uint64_t fusepath_len,
struct fuse_statx *st);
void calc(const char *fusepath,
struct stat *st);
void calc(const char *fusepath,
struct fuse_statx *st);
void calc(const std::string &fusepath,
struct stat *st);
uint64_t calc(const nonstd::string_view basepath,
const nonstd::string_view fusepath,
const mode_t mode,
const ino_t ino);
void calc(const nonstd::string_view basepath,
const nonstd::string_view fusepath,
struct stat *st);
void calc(const nonstd::string_view basepath,
const nonstd::string_view fusepath,
struct fuse_statx *st);
}
}

2
src/fuse_create.cpp
View File

@ -163,7 +163,7 @@ namespace l
if(rv == -1)
return -errno;
fi = new FileInfo(rv,fusepath_,ffi_->direct_io);
fi = new FileInfo(rv,createpath_,fusepath_,ffi_->direct_io);
ffi_->fh = reinterpret_cast<uint64_t>(fi);

28
src/fuse_fgetattr.cpp
View File

@ -23,26 +23,6 @@
#include "fuse.h"
namespace l
{
static
int
fgetattr(const int fd_,
const std::string &fusepath_,
struct stat *st_)
{
int rv;
rv = fs::fstat(fd_,st_);
if(rv == -1)
return -errno;
fs::inode::calc(fusepath_,st_);
return 0;
}
}
namespace FUSE
{
int
@ -54,7 +34,13 @@ namespace FUSE
Config::Read cfg;
FileInfo *fi = reinterpret_cast<FileInfo*>(ffi_->fh);
rv = l::fgetattr(fi->fd,fi->fusepath,st_);
rv = fs::fstat(fi->fd,st_);
if(rv == -1)
return -errno;
fs::inode::calc(fi->branchpath,
fi->fusepath,
st_);
timeout_->entry = ((rv >= 0) ?
cfg->cache_entry :

4
src/fuse_getattr.cpp
View File

@ -77,7 +77,7 @@ namespace l
static const time_t now = ::time(NULL);
st_->st_dev = 0;
st_->st_ino = fs::inode::MAGIC;
st_->st_ino = 0;
st_->st_mode = (S_IFREG|S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH);
st_->st_nlink = 1;
st_->st_uid = uid;
@ -141,7 +141,7 @@ namespace l
if(symlinkify_ && symlinkify::can_be_symlink(*st_,symlinkify_timeout_))
symlinkify::convert(fullpath,st_);
fs::inode::calc(fusepath_,st_);
fs::inode::calc(basepaths[0],fusepath_,st_);
return 0;
}

2
src/fuse_open.cpp
View File

@ -211,7 +211,7 @@ namespace l
if(fd == -1)
return -errno;
fi = new FileInfo(fd,fusepath_,ffi_->direct_io);
fi = new FileInfo(fd,basepath_,fusepath_,ffi_->direct_io);
ffi_->fh = reinterpret_cast<uint64_t>(fi);

80
src/fuse_readdir_cor.cpp
View File

@ -20,7 +20,6 @@
#include "dirinfo.hpp"
#include "errno.hpp"
#include "fs_close.hpp"
#include "fs_devid.hpp"
#include "fs_getdents64.hpp"
#include "fs_inode.hpp"
#include "fs_open.hpp"
@ -77,24 +76,25 @@ namespace l
static
inline
int
readdir(std::string basepath_,
HashSet &names_,
fuse_dirents_t *buf_,
std::mutex &mutex_)
readdir(const std::string &branch_path_,
const std::string &rel_dirpath_,
HashSet &names_,
fuse_dirents_t *buf_,
std::mutex &mutex_)
{
int rv;
int dfd;
dev_t dev;
std::string filepath;
std::string rel_filepath;
std::string abs_dirpath;
dfd = fs::open_dir_ro(basepath_);
abs_dirpath = fs::path::make(branch_path_,rel_dirpath_);
dfd = fs::open_dir_ro(abs_dirpath);
if(dfd == -1)
return errno;
DEFER{ fs::close(dfd); };
dev = fs::devid(dfd);
rv = 0;
for(;;)
{
@ -121,10 +121,10 @@ namespace l
if(rv == 0)
continue;
filepath = fs::path::make(basepath_,d->name);
d->ino = fs::inode::calc(filepath,
rel_filepath = fs::path::make(rel_dirpath_,d->name);
d->ino = fs::inode::calc(branch_path_,
rel_filepath,
DTTOIF(d->type),
dev,
d->ino);
rv = fuse_dirents_add_linux(buf_,d,namelen);
@ -142,27 +142,31 @@ namespace l
int
concurrent_readdir(ThreadPool &tp_,
const Branches::CPtr &branches_,
const char *dirname_,
const std::string &rel_dirpath_,
fuse_dirents_t *buf_,
uid_t const uid_,
gid_t const gid_)
const uid_t uid_,
const gid_t gid_)
{
HashSet names;
std::mutex mutex;
std::vector<std::future<int>> futures;
fuse_dirents_reset(buf_);
futures.reserve(branches_->size());
for(auto const &branch : *branches_)
{
auto func = [&,dirname_,buf_,uid_,gid_]()
{
std::string basepath;
ugid::Set const ugid(uid_,gid_);
basepath = fs::path::make(branch.path,dirname_);
for(const auto &branch : *branches_)
{
auto func =
[&,buf_,uid_,gid_]()
{
const ugid::Set ugid(uid_,gid_);
return l::readdir(basepath,names,buf_,mutex);
};
return l::readdir(branch.path,
rel_dirpath_,
names,
buf_,
mutex);
};
auto rv = tp_.enqueue_task(func);
@ -175,20 +179,6 @@ namespace l
return -error;
}
static
int
readdir(ThreadPool &tp_,
const Branches::CPtr &branches_,
const char *dirname_,
fuse_dirents_t *buf_,
uid_t const uid_,
gid_t const gid_)
{
fuse_dirents_reset(buf_);
return l::concurrent_readdir(tp_,branches_,dirname_,buf_,uid_,gid_);
}
}
int
@ -199,10 +189,10 @@ FUSE::ReadDirCOR::operator()(fuse_file_info_t const *ffi_,
DirInfo *di = reinterpret_cast<DirInfo*>(ffi_->fh);
const fuse_context *fc = fuse_get_context();
return l::readdir(_tp,
cfg->branches,
di->fusepath.c_str(),
buf_,
fc->uid,
fc->gid);
return l::concurrent_readdir(_tp,
cfg->branches,
di->fusepath,
buf_,
fc->uid,
fc->gid);
}

53
src/fuse_readdir_cosr.cpp
View File

@ -20,7 +20,6 @@
#include "dirinfo.hpp"
#include "errno.hpp"
#include "fs_closedir.hpp"
#include "fs_devid.hpp"
#include "fs_dirfd.hpp"
#include "fs_inode.hpp"
#include "fs_opendir.hpp"
@ -50,6 +49,7 @@ namespace l
{
struct DirRV
{
const std::string *branch_path;
DIR *dir;
int err;
};
@ -99,29 +99,30 @@ namespace l
std::vector<std::future<DirRV>>
opendir(ThreadPool &tp_,
const Branches::CPtr &branches_,
char const *dirname_,
const std::string &rel_dirpath_,
uid_t const uid_,
gid_t const gid_)
{
std::vector<std::future<DirRV>> futures;
futures.reserve(branches_->size());
for(auto const &branch : *branches_)
{
auto func = [&branch,dirname_,uid_,gid_]()
{
DirRV rv;
std::string basepath;
ugid::Set const ugid(uid_,gid_);
auto func =
[&branch,&rel_dirpath_,uid_,gid_]()
{
DIR *dir;
std::string abs_dirpath;
const ugid::Set ugid(uid_,gid_);
basepath = fs::path::make(branch.path,dirname_);
abs_dirpath = fs::path::make(branch.path,rel_dirpath_);
errno = 0;
rv.dir = fs::opendir(basepath);
rv.err = errno;
errno = 0;
dir = fs::opendir(abs_dirpath);
return rv;
};
return DirRV{&branch.path,dir,errno};
};
auto rv = tp_.enqueue_task(func);
@ -135,28 +136,26 @@ namespace l
inline
int
readdir(std::vector<std::future<DirRV>> &dh_futures_,
char const *dirname_,
const std::string &rel_dirpath_,
fuse_dirents_t *buf_)
{
Error error;
HashSet names;
std::string fullpath;
std::string rel_filepath;
for(auto &dh_future : dh_futures_)
{
int rv;
dev_t dev;
DirRV dirrv;
dirrv = dh_future.get();
error = dirrv.err;
if(dirrv.dir == NULL)
continue;
DEFER { fs::closedir(dirrv.dir); };
dev = fs::devid(dirrv.dir);
rv = 0;
for(dirent *de = fs::readdir(dirrv.dir); de && !rv; de = fs::readdir(dirrv.dir))
{
@ -168,11 +167,11 @@ namespace l
if(rv == 0)
continue;
fullpath = fs::path::make(dirname_,de->d_name);
de->d_ino = fs::inode::calc(fullpath,
DTTOIF(de->d_type),
dev,
de->d_ino);
rel_filepath = fs::path::make(rel_dirpath_,de->d_name);
de->d_ino = fs::inode::calc(*dirrv.branch_path,
rel_filepath,
DTTOIF(de->d_type),
de->d_ino);
rv = fuse_dirents_add(buf_,de,namelen);
if(rv == 0)
@ -190,7 +189,7 @@ namespace l
int
readdir(ThreadPool &tp_,
const Branches::CPtr &branches_,
const char *dirname_,
const std::string &rel_dirpath_,
fuse_dirents_t *buf_,
uid_t const uid_,
gid_t const gid_)
@ -200,8 +199,8 @@ namespace l
fuse_dirents_reset(buf_);
futures = l::opendir(tp_,branches_,dirname_,uid_,gid_);
rv = l::readdir(futures,dirname_,buf_);
futures = l::opendir(tp_,branches_,rel_dirpath_,uid_,gid_);
rv = l::readdir(futures,rel_dirpath_,buf_);
return rv;
}
@ -217,7 +216,7 @@ FUSE::ReadDirCOSR::operator()(fuse_file_info_t const *ffi_,
return l::readdir(_tp,
cfg->branches,
di->fusepath.c_str(),
di->fusepath,
buf_,
fc->uid,
fc->gid);

24
src/fuse_readdir_seq.cpp
View File

@ -23,7 +23,6 @@
#include "dirinfo.hpp"
#include "errno.hpp"
#include "fs_closedir.hpp"
#include "fs_devid.hpp"
#include "fs_dirfd.hpp"
#include "fs_inode.hpp"
#include "fs_opendir.hpp"
@ -85,34 +84,31 @@ namespace l
static
int
readdir(const Branches::CPtr &branches_,
const char *dirname_,
const std::string &rel_dirpath_,
fuse_dirents_t *buf_)
{
Error error;
HashSet names;
std::string basepath;
std::string fullpath;
std::string rel_filepath;
std::string abs_dirpath;
fuse_dirents_reset(buf_);
for(auto const &branch : *branches_)
for(const auto &branch : *branches_)
{
int rv;
DIR *dh;
dev_t dev;
basepath = fs::path::make(branch.path,dirname_);
abs_dirpath = fs::path::make(branch.path,rel_dirpath_);
errno = 0;
dh = fs::opendir(basepath);
dh = fs::opendir(abs_dirpath);
error = errno;
if(!dh)
continue;
DEFER{ fs::closedir(dh); };
dev = fs::devid(dh);
rv = 0;
for(dirent *de = fs::readdir(dh); de && !rv; de = fs::readdir(dh))
{
@ -124,10 +120,10 @@ namespace l
if(rv == 0)
continue;
fullpath = fs::path::make(dirname_,de->d_name);
de->d_ino = fs::inode::calc(fullpath,
rel_filepath = fs::path::make(rel_dirpath_,de->d_name);
de->d_ino = fs::inode::calc(branch.path,
rel_filepath,
DTTOIF(de->d_type),
dev,
de->d_ino);
rv = fuse_dirents_add(buf_,de,namelen);
@ -150,6 +146,6 @@ FUSE::ReadDirSeq::operator()(fuse_file_info_t const *ffi_,
const ugid::Set ugid(fc->uid,fc->gid);
return l::readdir(cfg->branches,
di->fusepath.c_str(),
di->fusepath,
buf_);
}

4
src/fuse_statx.cpp
View File

@ -60,7 +60,7 @@ _statx_controlfile(struct fuse_statx *st_)
static const gid_t gid = ::getgid();
static const time_t now = ::time(NULL);
st_->ino = fs::inode::MAGIC;
st_->ino = 0;
st_->mode = (S_IFREG|S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH);
st_->nlink = 1;
st_->uid = uid;
@ -124,7 +124,7 @@ _statx(const Policy::Search &searchFunc_,
if(symlinkify_ && symlinkify::can_be_symlink(*st_,symlinkify_timeout_))
symlinkify::convert(fullpath,st_);
fs::inode::calc(fusepath_,st_);
fs::inode::calc(basepaths[0],fusepath_,st_);
return 0;
}

2
src/fuse_symlink.cpp
View File

@ -74,7 +74,7 @@ namespace l
{
fs::lstat(fullnewpath,st_);
if(st_->st_ino != 0)
fs::inode::calc(linkpath_,st_);
fs::inode::calc(newbasepath_,linkpath_,st_);
}
return error::calc(rv,error_,errno);

1709
src/nonstd/string_view.hpp
File diff suppressed because it is too large
View File

Write Preview
Loading…
Cancel
Save