Mirror
/
mergerfs
mirror of https://github.com/trapexit/mergerfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #677 from trapexit/moveonenospc 

fix short writes on >2GB files when cloning file
pull/685/head 2.28.3
trapexit 5 years ago
committed by GitHub
parent
026dc5f750 4d82ed9324
commit
6058ac0716
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 286 additions and 212 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 38
      src/fs_clonefile.cpp
  2. 20
      src/fs_copy_file_range.hpp
  3. 82
      src/fs_copy_file_range_linux.icpp
  4. 67
      src/fs_copydata_copy_file_range.cpp
  5. 24
      src/fs_copydata_copy_file_range.hpp
  6. 113
      src/fs_copydata_readwrite.cpp
  7. 4
      src/fs_copydata_readwrite.hpp
  8. 110
      src/fs_copyfile.cpp

38
src/fs_clonefile.cpp
View File

@ -26,17 +26,20 @@
#include "fs_base_stat.hpp"
#include "fs_base_utime.hpp"
#include "fs_copy_file_range.hpp"
#include "fs_copyfile.hpp"
#include "fs_copydata_copy_file_range.hpp"
#include "fs_copydata_readwrite.hpp"
#include "fs_ficlone.hpp"
#include "fs_sendfile.hpp"
#include "fs_xattr.hpp"
static
int
copydata(const int src_fd_,
namespace l
{
static
int
copydata(const int src_fd_,
const int dst_fd_,
const size_t count_)
{
{
int rv;
rv = fs::ftruncate(dst_fd_,count_);
@ -50,21 +53,17 @@ copydata(const int src_fd_,
fs::fadvise_willneed(src_fd_,0,count_);
fs::fadvise_sequential(src_fd_,0,count_);
rv = fs::copy_file_range(src_fd_,dst_fd_,count_);
if(rv != -1)
return rv;
rv = fs::sendfile(src_fd_,dst_fd_,count_);
rv = fs::copydata_copy_file_range(src_fd_,dst_fd_);
if(rv != -1)
return rv;
return fs::copyfile(src_fd_,dst_fd_);
}
return fs::copydata_readwrite(src_fd_,dst_fd_);
}
static
bool
ignorable_error(const int err_)
{
static
bool
ignorable_error(const int err_)
{
switch(err_)
{
case ENOTTY:
@ -76,6 +75,7 @@ ignorable_error(const int err_)
}
return false;
}
}
namespace fs
@ -91,16 +91,16 @@ namespace fs
if(rv == -1)
return -1;
rv = ::copydata(src_fd_,dst_fd_,src_st.st_size);
rv = l::copydata(src_fd_,dst_fd_,src_st.st_size);
if(rv == -1)
return -1;
rv = fs::attr::copy(src_fd_,dst_fd_);
if((rv == -1) && !ignorable_error(errno))
if((rv == -1) && !l::ignorable_error(errno))
return -1;
rv = fs::xattr::copy(src_fd_,dst_fd_);
if((rv == -1) && !ignorable_error(errno))
if((rv == -1) && !l::ignorable_error(errno))
return -1;
rv = fs::fchown_check_on_error(dst_fd_,src_st);

20
src/fs_copy_file_range.hpp
View File

@ -24,17 +24,11 @@
namespace fs
{
ssize_t
copy_file_range(const int fd_in_,
int64_t *off_in_,
const int fd_out_,
int64_t *off_out_,
const size_t len_,
const unsigned int flags_);
ssize_t
copy_file_range(const int fd_in_,
const int fd_out_,
const size_t len_,
const unsigned int flags_ = 0);
int64_t
copy_file_range(const int fd_in,
int64_t *off_in,
const int fd_out,
int64_t *off_out,
const uint64_t len,
const unsigned int flags);
}

82
src/fs_copy_file_range_linux.icpp
View File

@ -32,33 +32,39 @@
namespace l
{
static
ssize_t
copy_file_range_(int fd_in_,
int64_t *off_in_,
int fd_out_,
int64_t *off_out_,
size_t len_,
unsigned int flags_)
int64_t
copy_file_range_(int src_fd_,
int64_t *src_off_,
int tgt_fd_,
int64_t *tgt_off_,
const uint64_t len_,
const unsigned int flags_)
{
#ifdef SYS_copy_file_range
ssize_t rv;
loff_t off_in;
loff_t off_out;
int64_t rv;
loff_t src_off;
loff_t tgt_off;
loff_t *src_off_ptr;
loff_t *tgt_off_ptr;
off_in = *off_in_;
off_out = *off_out_;
src_off = ((src_off_ == NULL) ? 0 : *src_off_);
tgt_off = ((tgt_off_ == NULL) ? 0 : *tgt_off_);
src_off_ptr = ((src_off_ == NULL) ? NULL : &src_off);
tgt_off_ptr = ((tgt_off_ == NULL) ? NULL : &tgt_off);
rv = ::syscall(SYS_copy_file_range,
fd_in_,
&off_in,
fd_out_,
&off_out,
src_fd_,
src_off_ptr,
tgt_fd_,
tgt_off_ptr,
len_,
flags_);
if(rv != -1)
{
*off_in_ = off_in;
*off_out_ = off_out;
if(src_off_ != NULL)
*src_off_ = src_off;
if(tgt_off_ != NULL)
*tgt_off_ = tgt_off;
}
return rv;
@ -70,38 +76,18 @@ namespace l
namespace fs
{
ssize_t
copy_file_range(const int fd_in_,
int64_t *off_in_,
const int fd_out_,
int64_t *off_out_,
const size_t len_,
int64_t
copy_file_range(const int src_fd_,
int64_t *src_off_,
const int tgt_fd_,
int64_t *tgt_off_,
const uint64_t len_,
const unsigned int flags_)
{
return l::copy_file_range_(fd_in_,
off_in_,
fd_out_,
off_out_,
len_,
flags_);
}
ssize_t
copy_file_range(const int fd_in_,
const int fd_out_,
const size_t len_,
const unsigned int flags_)
{
int64_t off_in;
int64_t off_out;
off_in = 0;
off_out = 0;
return fs::copy_file_range(fd_in_,
&off_in,
fd_out_,
&off_out,
return l::copy_file_range_(src_fd_,
src_off_,
tgt_fd_,
tgt_off_,
len_,
flags_);
}

67
src/fs_copydata_copy_file_range.cpp
View File

@ -0,0 +1,67 @@
/*
Copyright (c) 2019, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include "errno.hpp"
#include "fs_base_stat.hpp"
#include "fs_copy_file_range.hpp"
namespace l
{
int64_t
copydata_copy_file_range(const int src_fd_,
const int dst_fd_,
uint64_t size_)
{
int64_t rv;
uint64_t len;
int64_t src_off;
int64_t dst_off;
src_off = 0;
dst_off = 0;
len = size_;
do
{
rv = fs::copy_file_range(src_fd_,&src_off,dst_fd_,&dst_off,len,0);
if(rv == -1)
return -1;
len -= rv;
}
while((len > 0) && (rv > 0));
return size_;
}
}
namespace fs
{
int64_t
copydata_copy_file_range(const int src_fd_,
const int dst_fd_)
{
int rv;
struct stat st;
rv = fs::fstat(src_fd_,&st);
if(rv == -1)
return -1;
return l::copydata_copy_file_range(src_fd_,
dst_fd_,
st.st_size);
}
}

24
src/fs_copydata_copy_file_range.hpp
View File

@ -0,0 +1,24 @@
/*
Copyright (c) 2016, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#pragma once
namespace fs
{
int
copydata_copy_file_range(const int src_fd,
const int dst_fd);
}

113
src/fs_copydata_readwrite.cpp
View File

@ -0,0 +1,113 @@
/*
Copyright (c) 2018, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include "errno.hpp"
#include "fs_base_stat.hpp"
#include "fs_base_lseek.hpp"
#include "fs_base_read.hpp"
#include "fs_base_write.hpp"
#include <vector>
using std::vector;
namespace l
{
static
int
writen(const int fd_,
const char *buf_,
const size_t count_)
{
size_t nleft;
ssize_t nwritten;
nleft = count_;
do
{
nwritten = fs::write(fd_,buf_,nleft);
if((nwritten == -1) && (errno == EINTR))
continue;
if(nwritten == -1)
return -1;
nleft -= nwritten;
buf_ += nwritten;
}
while(nleft > 0);
return count_;
}
static
int
copydata_readwrite(const int src_fd_,
const int dst_fd_,
const size_t count_,
const size_t blocksize_)
{
ssize_t nr;
ssize_t nw;
ssize_t bufsize;
size_t totalwritten;
vector<char> buf;
bufsize = (blocksize_ * 16);
buf.resize(bufsize);
fs::lseek(src_fd_,0,SEEK_SET);
totalwritten = 0;
while(totalwritten < count_)
{
nr = fs::read(src_fd_,&buf[0],bufsize);
if(nr == 0)
return totalwritten;
if((nr == -1) && (errno == EINTR))
continue;
if(nr == -1)
return -1;
nw = l::writen(dst_fd_,&buf[0],nr);
if(nw == -1)
return -1;
totalwritten += nw;
}
return totalwritten;
}
}
namespace fs
{
int
copydata_readwrite(const int src_fd_,
const int dst_fd_)
{
int rv;
struct stat st;
rv = fs::fstat(src_fd_,&st);
if(rv == -1)
return -1;
return l::copydata_readwrite(src_fd_,
dst_fd_,
st.st_size,
st.st_blksize);
}
}

4
src/fs_copyfile.hpp → src/fs_copydata_readwrite.hpp
View File

@ -19,6 +19,6 @@
namespace fs
{
int
copyfile(const int src_fd_,
const int dst_fd_);
copydata_readwrite(const int src_fd,
const int dst_fd);
}

110
src/fs_copyfile.cpp
View File

@ -1,110 +0,0 @@
/*
Copyright (c) 2018, Antonio SJ Musumeci <trapexit@spawn.link>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include "errno.hpp"
#include "fs_base_stat.hpp"
#include "fs_base_lseek.hpp"
#include "fs_base_read.hpp"
#include "fs_base_write.hpp"
#include <vector>
using std::vector;
static
int
writen(const int fd_,
const char *buf_,
const size_t count_)
{
size_t nleft;
ssize_t nwritten;
nleft = count_;
do
{
nwritten = fs::write(fd_,buf_,nleft);
if((nwritten == -1) && (errno == EINTR))
continue;
if(nwritten == -1)
return -1;
nleft -= nwritten;
buf_ += nwritten;
}
while(nleft > 0);
return count_;
}
static
int
copyfile(const int src_fd_,
const int dst_fd_,
const size_t count_,
const size_t blocksize_)
{
ssize_t nr;
ssize_t nw;
ssize_t bufsize;
size_t totalwritten;
vector<char> buf;
bufsize = (blocksize_ * 16);
buf.resize(bufsize);
fs::lseek(src_fd_,0,SEEK_SET);
totalwritten = 0;
while(totalwritten < count_)
{
nr = fs::read(src_fd_,&buf[0],bufsize);
if(nr == 0)
return totalwritten;
if((nr == -1) && (errno == EINTR))
continue;
if(nr == -1)
return -1;
nw = writen(dst_fd_,&buf[0],nr);
if(nw == -1)
return -1;
totalwritten += nw;
}
return totalwritten;
}
namespace fs
{
int
copyfile(const int src_fd_,
const int dst_fd_)
{
int rv;
struct stat st;
rv = fs::fstat(src_fd_,&st);
if(rv == -1)
return rv;
return ::copyfile(src_fd_,
dst_fd_,
st.st_size,
st.st_blksize);
}
}
Write Preview
Loading…
Cancel
Save