From 4d82ed9324d66f62973f93b3eb8dd5e5378f4f7d Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Mon, 14 Oct 2019 10:14:02 -0400 Subject: [PATCH] fix short writes on >2GB files when cloning file --- src/fs_clonefile.cpp | 78 ++++++------ src/fs_copy_file_range.hpp | 20 ++-- src/fs_copy_file_range_linux.icpp | 82 ++++++------- src/fs_copydata_copy_file_range.cpp | 67 +++++++++++ src/fs_copydata_copy_file_range.hpp | 24 ++++ src/fs_copydata_readwrite.cpp | 113 ++++++++++++++++++ ...copyfile.hpp => fs_copydata_readwrite.hpp} | 4 +- src/fs_copyfile.cpp | 110 ----------------- 8 files changed, 286 insertions(+), 212 deletions(-) create mode 100644 src/fs_copydata_copy_file_range.cpp create mode 100644 src/fs_copydata_copy_file_range.hpp create mode 100644 src/fs_copydata_readwrite.cpp rename src/{fs_copyfile.hpp => fs_copydata_readwrite.hpp} (91%) delete mode 100644 src/fs_copyfile.cpp diff --git a/src/fs_clonefile.cpp b/src/fs_clonefile.cpp index fef0dc3d..1064d9fa 100644 --- a/src/fs_clonefile.cpp +++ b/src/fs_clonefile.cpp @@ -26,56 +26,56 @@ #include "fs_base_stat.hpp" #include "fs_base_utime.hpp" #include "fs_copy_file_range.hpp" -#include "fs_copyfile.hpp" +#include "fs_copydata_copy_file_range.hpp" +#include "fs_copydata_readwrite.hpp" #include "fs_ficlone.hpp" #include "fs_sendfile.hpp" #include "fs_xattr.hpp" -static -int -copydata(const int src_fd_, - const int dst_fd_, - const size_t count_) +namespace l { - int rv; - - rv = fs::ftruncate(dst_fd_,count_); - if(rv == -1) - return -1; + static + int + copydata(const int src_fd_, + const int dst_fd_, + const size_t count_) + { + int rv; - rv = fs::ficlone(src_fd_,dst_fd_); - if(rv != -1) - return rv; + rv = fs::ftruncate(dst_fd_,count_); + if(rv == -1) + return -1; - fs::fadvise_willneed(src_fd_,0,count_); - fs::fadvise_sequential(src_fd_,0,count_); + rv = fs::ficlone(src_fd_,dst_fd_); + if(rv != -1) + return rv; - rv = fs::copy_file_range(src_fd_,dst_fd_,count_); - if(rv != -1) - return rv; + fs::fadvise_willneed(src_fd_,0,count_); + fs::fadvise_sequential(src_fd_,0,count_); - rv = fs::sendfile(src_fd_,dst_fd_,count_); - if(rv != -1) - return rv; + rv = fs::copydata_copy_file_range(src_fd_,dst_fd_); + if(rv != -1) + return rv; - return fs::copyfile(src_fd_,dst_fd_); -} + return fs::copydata_readwrite(src_fd_,dst_fd_); + } -static -bool -ignorable_error(const int err_) -{ - switch(err_) - { - case ENOTTY: - case ENOTSUP: + static + bool + ignorable_error(const int err_) + { + switch(err_) + { + case ENOTTY: + case ENOTSUP: #if ENOTSUP != EOPNOTSUPP - case EOPNOTSUPP: + case EOPNOTSUPP: #endif - return true; - } + return true; + } - return false; + return false; + } } namespace fs @@ -91,16 +91,16 @@ namespace fs if(rv == -1) return -1; - rv = ::copydata(src_fd_,dst_fd_,src_st.st_size); + rv = l::copydata(src_fd_,dst_fd_,src_st.st_size); if(rv == -1) return -1; rv = fs::attr::copy(src_fd_,dst_fd_); - if((rv == -1) && !ignorable_error(errno)) + if((rv == -1) && !l::ignorable_error(errno)) return -1; rv = fs::xattr::copy(src_fd_,dst_fd_); - if((rv == -1) && !ignorable_error(errno)) + if((rv == -1) && !l::ignorable_error(errno)) return -1; rv = fs::fchown_check_on_error(dst_fd_,src_st); diff --git a/src/fs_copy_file_range.hpp b/src/fs_copy_file_range.hpp index 140092df..aab5fc9e 100644 --- a/src/fs_copy_file_range.hpp +++ b/src/fs_copy_file_range.hpp @@ -24,17 +24,11 @@ namespace fs { - ssize_t - copy_file_range(const int fd_in_, - int64_t *off_in_, - const int fd_out_, - int64_t *off_out_, - const size_t len_, - const unsigned int flags_); - - ssize_t - copy_file_range(const int fd_in_, - const int fd_out_, - const size_t len_, - const unsigned int flags_ = 0); + int64_t + copy_file_range(const int fd_in, + int64_t *off_in, + const int fd_out, + int64_t *off_out, + const uint64_t len, + const unsigned int flags); } diff --git a/src/fs_copy_file_range_linux.icpp b/src/fs_copy_file_range_linux.icpp index d640f32f..7df4f845 100644 --- a/src/fs_copy_file_range_linux.icpp +++ b/src/fs_copy_file_range_linux.icpp @@ -32,33 +32,39 @@ namespace l { static - ssize_t - copy_file_range_(int fd_in_, - int64_t *off_in_, - int fd_out_, - int64_t *off_out_, - size_t len_, - unsigned int flags_) + int64_t + copy_file_range_(int src_fd_, + int64_t *src_off_, + int tgt_fd_, + int64_t *tgt_off_, + const uint64_t len_, + const unsigned int flags_) { #ifdef SYS_copy_file_range - ssize_t rv; - loff_t off_in; - loff_t off_out; + int64_t rv; + loff_t src_off; + loff_t tgt_off; + loff_t *src_off_ptr; + loff_t *tgt_off_ptr; - off_in = *off_in_; - off_out = *off_out_; + src_off = ((src_off_ == NULL) ? 0 : *src_off_); + tgt_off = ((tgt_off_ == NULL) ? 0 : *tgt_off_); + src_off_ptr = ((src_off_ == NULL) ? NULL : &src_off); + tgt_off_ptr = ((tgt_off_ == NULL) ? NULL : &tgt_off); rv = ::syscall(SYS_copy_file_range, - fd_in_, - &off_in, - fd_out_, - &off_out, + src_fd_, + src_off_ptr, + tgt_fd_, + tgt_off_ptr, len_, flags_); if(rv != -1) { - *off_in_ = off_in; - *off_out_ = off_out; + if(src_off_ != NULL) + *src_off_ = src_off; + if(tgt_off_ != NULL) + *tgt_off_ = tgt_off; } return rv; @@ -70,38 +76,18 @@ namespace l namespace fs { - ssize_t - copy_file_range(const int fd_in_, - int64_t *off_in_, - const int fd_out_, - int64_t *off_out_, - const size_t len_, + int64_t + copy_file_range(const int src_fd_, + int64_t *src_off_, + const int tgt_fd_, + int64_t *tgt_off_, + const uint64_t len_, const unsigned int flags_) { - return l::copy_file_range_(fd_in_, - off_in_, - fd_out_, - off_out_, - len_, - flags_); - } - - ssize_t - copy_file_range(const int fd_in_, - const int fd_out_, - const size_t len_, - const unsigned int flags_) - { - int64_t off_in; - int64_t off_out; - - off_in = 0; - off_out = 0; - - return fs::copy_file_range(fd_in_, - &off_in, - fd_out_, - &off_out, + return l::copy_file_range_(src_fd_, + src_off_, + tgt_fd_, + tgt_off_, len_, flags_); } diff --git a/src/fs_copydata_copy_file_range.cpp b/src/fs_copydata_copy_file_range.cpp new file mode 100644 index 00000000..a3bff9d9 --- /dev/null +++ b/src/fs_copydata_copy_file_range.cpp @@ -0,0 +1,67 @@ +/* + Copyright (c) 2019, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include "errno.hpp" +#include "fs_base_stat.hpp" +#include "fs_copy_file_range.hpp" + +namespace l +{ + int64_t + copydata_copy_file_range(const int src_fd_, + const int dst_fd_, + uint64_t size_) + { + int64_t rv; + uint64_t len; + int64_t src_off; + int64_t dst_off; + + src_off = 0; + dst_off = 0; + len = size_; + do + { + rv = fs::copy_file_range(src_fd_,&src_off,dst_fd_,&dst_off,len,0); + if(rv == -1) + return -1; + + len -= rv; + } + while((len > 0) && (rv > 0)); + + return size_; + } +} + +namespace fs +{ + int64_t + copydata_copy_file_range(const int src_fd_, + const int dst_fd_) + { + int rv; + struct stat st; + + rv = fs::fstat(src_fd_,&st); + if(rv == -1) + return -1; + + return l::copydata_copy_file_range(src_fd_, + dst_fd_, + st.st_size); + } +} diff --git a/src/fs_copydata_copy_file_range.hpp b/src/fs_copydata_copy_file_range.hpp new file mode 100644 index 00000000..f23299c4 --- /dev/null +++ b/src/fs_copydata_copy_file_range.hpp @@ -0,0 +1,24 @@ +/* + Copyright (c) 2016, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#pragma once + +namespace fs +{ + int + copydata_copy_file_range(const int src_fd, + const int dst_fd); +} diff --git a/src/fs_copydata_readwrite.cpp b/src/fs_copydata_readwrite.cpp new file mode 100644 index 00000000..ccf2fc13 --- /dev/null +++ b/src/fs_copydata_readwrite.cpp @@ -0,0 +1,113 @@ +/* + Copyright (c) 2018, Antonio SJ Musumeci + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#include "errno.hpp" +#include "fs_base_stat.hpp" +#include "fs_base_lseek.hpp" +#include "fs_base_read.hpp" +#include "fs_base_write.hpp" + +#include + +using std::vector; + +namespace l +{ + static + int + writen(const int fd_, + const char *buf_, + const size_t count_) + { + size_t nleft; + ssize_t nwritten; + + nleft = count_; + do + { + nwritten = fs::write(fd_,buf_,nleft); + if((nwritten == -1) && (errno == EINTR)) + continue; + if(nwritten == -1) + return -1; + + nleft -= nwritten; + buf_ += nwritten; + } + while(nleft > 0); + + return count_; + } + + static + int + copydata_readwrite(const int src_fd_, + const int dst_fd_, + const size_t count_, + const size_t blocksize_) + { + ssize_t nr; + ssize_t nw; + ssize_t bufsize; + size_t totalwritten; + vector buf; + + bufsize = (blocksize_ * 16); + buf.resize(bufsize); + + fs::lseek(src_fd_,0,SEEK_SET); + + totalwritten = 0; + while(totalwritten < count_) + { + nr = fs::read(src_fd_,&buf[0],bufsize); + if(nr == 0) + return totalwritten; + if((nr == -1) && (errno == EINTR)) + continue; + if(nr == -1) + return -1; + + nw = l::writen(dst_fd_,&buf[0],nr); + if(nw == -1) + return -1; + + totalwritten += nw; + } + + return totalwritten; + } +} + +namespace fs +{ + int + copydata_readwrite(const int src_fd_, + const int dst_fd_) + { + int rv; + struct stat st; + + rv = fs::fstat(src_fd_,&st); + if(rv == -1) + return -1; + + return l::copydata_readwrite(src_fd_, + dst_fd_, + st.st_size, + st.st_blksize); + } +} diff --git a/src/fs_copyfile.hpp b/src/fs_copydata_readwrite.hpp similarity index 91% rename from src/fs_copyfile.hpp rename to src/fs_copydata_readwrite.hpp index 86118fd7..9e900d03 100644 --- a/src/fs_copyfile.hpp +++ b/src/fs_copydata_readwrite.hpp @@ -19,6 +19,6 @@ namespace fs { int - copyfile(const int src_fd_, - const int dst_fd_); + copydata_readwrite(const int src_fd, + const int dst_fd); } diff --git a/src/fs_copyfile.cpp b/src/fs_copyfile.cpp deleted file mode 100644 index dd0fbf68..00000000 --- a/src/fs_copyfile.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - Copyright (c) 2018, Antonio SJ Musumeci - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ - -#include "errno.hpp" -#include "fs_base_stat.hpp" -#include "fs_base_lseek.hpp" -#include "fs_base_read.hpp" -#include "fs_base_write.hpp" - -#include - -using std::vector; - -static -int -writen(const int fd_, - const char *buf_, - const size_t count_) -{ - size_t nleft; - ssize_t nwritten; - - nleft = count_; - do - { - nwritten = fs::write(fd_,buf_,nleft); - if((nwritten == -1) && (errno == EINTR)) - continue; - if(nwritten == -1) - return -1; - - nleft -= nwritten; - buf_ += nwritten; - } - while(nleft > 0); - - return count_; -} - -static -int -copyfile(const int src_fd_, - const int dst_fd_, - const size_t count_, - const size_t blocksize_) -{ - ssize_t nr; - ssize_t nw; - ssize_t bufsize; - size_t totalwritten; - vector buf; - - bufsize = (blocksize_ * 16); - buf.resize(bufsize); - - fs::lseek(src_fd_,0,SEEK_SET); - - totalwritten = 0; - while(totalwritten < count_) - { - nr = fs::read(src_fd_,&buf[0],bufsize); - if(nr == 0) - return totalwritten; - if((nr == -1) && (errno == EINTR)) - continue; - if(nr == -1) - return -1; - - nw = writen(dst_fd_,&buf[0],nr); - if(nw == -1) - return -1; - - totalwritten += nw; - } - - return totalwritten; -} - -namespace fs -{ - int - copyfile(const int src_fd_, - const int dst_fd_) - { - int rv; - struct stat st; - - rv = fs::fstat(src_fd_,&st); - if(rv == -1) - return rv; - - return ::copyfile(src_fd_, - dst_fd_, - st.st_size, - st.st_blksize); - } -}