Browse Source
Use filer-side copy for mounted whole-file copy_file_range (#8747)
Use filer-side copy for mounted whole-file copy_file_range (#8747)
* Optimize mounted whole-file copy_file_range * Address mounted copy review feedback * Harden mounted copy fast path --------- Co-authored-by: Copilot <copilot@github.com>pull/8761/head
Chris Lu
1 week ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 1292 additions and 60 deletions
-
60weed/command/mount_std.go
-
17weed/filer/copy_params.go
-
55weed/mount/weedfs.go
-
355weed/mount/weedfs_file_copy_range.go
-
355weed/mount/weedfs_file_copy_range_test.go
-
4weed/server/filer_server.go
-
283weed/server/filer_server_handlers_copy.go
-
223weed/server/filer_server_handlers_copy_test.go
@ -0,0 +1,17 @@ |
|||||
|
package filer |
||||
|
|
||||
|
const ( |
||||
|
CopyQueryParamFrom = "cp.from" |
||||
|
CopyQueryParamOverwrite = "overwrite" |
||||
|
CopyQueryParamDataOnly = "dataOnly" |
||||
|
CopyQueryParamRequestID = "copy.requestId" |
||||
|
CopyQueryParamSourceInode = "copy.srcInode" |
||||
|
CopyQueryParamSourceMtime = "copy.srcMtime" |
||||
|
CopyQueryParamSourceSize = "copy.srcSize" |
||||
|
CopyQueryParamDestinationInode = "copy.dstInode" |
||||
|
CopyQueryParamDestinationMtime = "copy.dstMtime" |
||||
|
CopyQueryParamDestinationSize = "copy.dstSize" |
||||
|
|
||||
|
CopyResponseHeaderCommitted = "X-SeaweedFS-Copy-Committed" |
||||
|
CopyResponseHeaderRequestID = "X-SeaweedFS-Copy-Request-ID" |
||||
|
) |
||||
@ -0,0 +1,355 @@ |
|||||
|
package mount |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"errors" |
||||
|
"path/filepath" |
||||
|
"testing" |
||||
|
|
||||
|
"github.com/seaweedfs/go-fuse/v2/fuse" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
func TestWholeFileServerCopyCandidate(t *testing.T) { |
||||
|
wfs := newCopyRangeTestWFS() |
||||
|
|
||||
|
srcPath := util.FullPath("/src.txt") |
||||
|
dstPath := util.FullPath("/dst.txt") |
||||
|
srcInode := wfs.inodeToPath.Lookup(srcPath, 1, false, false, 0, true) |
||||
|
dstInode := wfs.inodeToPath.Lookup(dstPath, 1, false, false, 0, true) |
||||
|
|
||||
|
srcHandle := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ |
||||
|
Name: "src.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100644, |
||||
|
FileSize: 5, |
||||
|
Inode: srcInode, |
||||
|
}, |
||||
|
Content: []byte("hello"), |
||||
|
}) |
||||
|
dstHandle := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ |
||||
|
Name: "dst.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100644, |
||||
|
Inode: dstInode, |
||||
|
}, |
||||
|
}) |
||||
|
|
||||
|
srcHandle.RememberPath(srcPath) |
||||
|
dstHandle.RememberPath(dstPath) |
||||
|
|
||||
|
in := &fuse.CopyFileRangeIn{ |
||||
|
FhIn: uint64(srcHandle.fh), |
||||
|
FhOut: uint64(dstHandle.fh), |
||||
|
OffIn: 0, |
||||
|
OffOut: 0, |
||||
|
Len: 8, |
||||
|
} |
||||
|
|
||||
|
copyRequest, ok := wholeFileServerCopyCandidate(srcHandle, dstHandle, in) |
||||
|
if !ok { |
||||
|
t.Fatal("expected whole-file server copy candidate") |
||||
|
} |
||||
|
if copyRequest.srcPath != srcPath { |
||||
|
t.Fatalf("source path = %q, want %q", copyRequest.srcPath, srcPath) |
||||
|
} |
||||
|
if copyRequest.dstPath != dstPath { |
||||
|
t.Fatalf("destination path = %q, want %q", copyRequest.dstPath, dstPath) |
||||
|
} |
||||
|
if copyRequest.sourceSize != 5 { |
||||
|
t.Fatalf("source size = %d, want 5", copyRequest.sourceSize) |
||||
|
} |
||||
|
if copyRequest.srcInode == 0 || copyRequest.dstInode == 0 { |
||||
|
t.Fatalf("expected inode preconditions, got src=%d dst=%d", copyRequest.srcInode, copyRequest.dstInode) |
||||
|
} |
||||
|
|
||||
|
srcHandle.dirtyMetadata = true |
||||
|
if _, ok := wholeFileServerCopyCandidate(srcHandle, dstHandle, in); ok { |
||||
|
t.Fatal("dirty source handle should disable whole-file server copy") |
||||
|
} |
||||
|
srcHandle.dirtyMetadata = false |
||||
|
|
||||
|
in.Len = 4 |
||||
|
if _, ok := wholeFileServerCopyCandidate(srcHandle, dstHandle, in); ok { |
||||
|
t.Fatal("short copy request should disable whole-file server copy") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestCopyFileRangeUsesServerSideWholeFileCopy(t *testing.T) { |
||||
|
wfs := newCopyRangeTestWFS() |
||||
|
|
||||
|
srcPath := util.FullPath("/src.txt") |
||||
|
dstPath := util.FullPath("/dst.txt") |
||||
|
srcInode := wfs.inodeToPath.Lookup(srcPath, 1, false, false, 0, true) |
||||
|
dstInode := wfs.inodeToPath.Lookup(dstPath, 1, false, false, 0, true) |
||||
|
|
||||
|
srcHandle := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ |
||||
|
Name: "src.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100644, |
||||
|
FileSize: 5, |
||||
|
Inode: srcInode, |
||||
|
}, |
||||
|
Content: []byte("hello"), |
||||
|
}) |
||||
|
dstHandle := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ |
||||
|
Name: "dst.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100644, |
||||
|
Inode: dstInode, |
||||
|
}, |
||||
|
}) |
||||
|
|
||||
|
srcHandle.RememberPath(srcPath) |
||||
|
dstHandle.RememberPath(dstPath) |
||||
|
|
||||
|
originalCopy := performServerSideWholeFileCopy |
||||
|
defer func() { |
||||
|
performServerSideWholeFileCopy = originalCopy |
||||
|
}() |
||||
|
|
||||
|
var called bool |
||||
|
performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { |
||||
|
called = true |
||||
|
if gotWFS != wfs { |
||||
|
t.Fatalf("wfs = %p, want %p", gotWFS, wfs) |
||||
|
} |
||||
|
if copyRequest.srcPath != srcPath { |
||||
|
t.Fatalf("source path = %q, want %q", copyRequest.srcPath, srcPath) |
||||
|
} |
||||
|
if copyRequest.dstPath != dstPath { |
||||
|
t.Fatalf("destination path = %q, want %q", copyRequest.dstPath, dstPath) |
||||
|
} |
||||
|
return &filer_pb.Entry{ |
||||
|
Name: "dst.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100644, |
||||
|
FileSize: 5, |
||||
|
Mime: "text/plain; charset=utf-8", |
||||
|
}, |
||||
|
Content: []byte("hello"), |
||||
|
}, serverSideWholeFileCopyCommitted, nil |
||||
|
} |
||||
|
|
||||
|
written, status := wfs.CopyFileRange(make(chan struct{}), &fuse.CopyFileRangeIn{ |
||||
|
FhIn: uint64(srcHandle.fh), |
||||
|
FhOut: uint64(dstHandle.fh), |
||||
|
OffIn: 0, |
||||
|
OffOut: 0, |
||||
|
Len: 8, |
||||
|
}) |
||||
|
if status != fuse.OK { |
||||
|
t.Fatalf("CopyFileRange status = %v, want OK", status) |
||||
|
} |
||||
|
if written != 5 { |
||||
|
t.Fatalf("CopyFileRange wrote %d bytes, want 5", written) |
||||
|
} |
||||
|
if !called { |
||||
|
t.Fatal("expected server-side whole-file copy path to be used") |
||||
|
} |
||||
|
|
||||
|
gotEntry := dstHandle.GetEntry().GetEntry() |
||||
|
if gotEntry.Attributes == nil || gotEntry.Attributes.FileSize != 5 { |
||||
|
t.Fatalf("destination size = %v, want 5", gotEntry.GetAttributes().GetFileSize()) |
||||
|
} |
||||
|
if string(gotEntry.Content) != "hello" { |
||||
|
t.Fatalf("destination content = %q, want %q", string(gotEntry.Content), "hello") |
||||
|
} |
||||
|
if dstHandle.dirtyMetadata { |
||||
|
t.Fatal("server-side whole-file copy should leave destination handle clean") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestCopyFileRangeDoesNotFallbackAfterCommittedServerCopyRefreshFailure(t *testing.T) { |
||||
|
wfs := newCopyRangeTestWFSWithMetaCache(t) |
||||
|
|
||||
|
srcPath := util.FullPath("/src.txt") |
||||
|
dstPath := util.FullPath("/dst.txt") |
||||
|
srcInode := wfs.inodeToPath.Lookup(srcPath, 1, false, false, 0, true) |
||||
|
dstInode := wfs.inodeToPath.Lookup(dstPath, 1, false, false, 0, true) |
||||
|
|
||||
|
srcHandle := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ |
||||
|
Name: "src.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100644, |
||||
|
FileSize: 5, |
||||
|
Mime: "text/plain; charset=utf-8", |
||||
|
Md5: []byte("abcde"), |
||||
|
Inode: srcInode, |
||||
|
}, |
||||
|
Content: []byte("hello"), |
||||
|
}) |
||||
|
dstHandle := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ |
||||
|
Name: "dst.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100600, |
||||
|
Inode: dstInode, |
||||
|
}, |
||||
|
}) |
||||
|
|
||||
|
srcHandle.RememberPath(srcPath) |
||||
|
dstHandle.RememberPath(dstPath) |
||||
|
|
||||
|
originalCopy := performServerSideWholeFileCopy |
||||
|
defer func() { |
||||
|
performServerSideWholeFileCopy = originalCopy |
||||
|
}() |
||||
|
|
||||
|
performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { |
||||
|
if gotWFS != wfs || copyRequest.srcPath != srcPath || copyRequest.dstPath != dstPath { |
||||
|
t.Fatalf("unexpected server-side copy call: wfs=%p src=%q dst=%q", gotWFS, copyRequest.srcPath, copyRequest.dstPath) |
||||
|
} |
||||
|
return nil, serverSideWholeFileCopyCommitted, errors.New("reload copied entry: transient filer read failure") |
||||
|
} |
||||
|
|
||||
|
written, status := wfs.CopyFileRange(make(chan struct{}), &fuse.CopyFileRangeIn{ |
||||
|
FhIn: uint64(srcHandle.fh), |
||||
|
FhOut: uint64(dstHandle.fh), |
||||
|
OffIn: 0, |
||||
|
OffOut: 0, |
||||
|
Len: 8, |
||||
|
}) |
||||
|
if status != fuse.OK { |
||||
|
t.Fatalf("CopyFileRange status = %v, want OK", status) |
||||
|
} |
||||
|
if written != 5 { |
||||
|
t.Fatalf("CopyFileRange wrote %d bytes, want 5", written) |
||||
|
} |
||||
|
if dstHandle.dirtyMetadata { |
||||
|
t.Fatal("committed server-side copy should not fall back to dirty-page copy") |
||||
|
} |
||||
|
|
||||
|
gotEntry := dstHandle.GetEntry().GetEntry() |
||||
|
if gotEntry.GetAttributes().GetFileSize() != 5 { |
||||
|
t.Fatalf("destination size = %d, want 5", gotEntry.GetAttributes().GetFileSize()) |
||||
|
} |
||||
|
if gotEntry.GetAttributes().GetFileMode() != 0100600 { |
||||
|
t.Fatalf("destination mode = %#o, want %#o", gotEntry.GetAttributes().GetFileMode(), uint32(0100600)) |
||||
|
} |
||||
|
if string(gotEntry.GetContent()) != "hello" { |
||||
|
t.Fatalf("destination content = %q, want %q", string(gotEntry.GetContent()), "hello") |
||||
|
} |
||||
|
|
||||
|
cachedEntry, err := wfs.metaCache.FindEntry(context.Background(), dstPath) |
||||
|
if err != nil { |
||||
|
t.Fatalf("metaCache find entry: %v", err) |
||||
|
} |
||||
|
if cachedEntry.FileSize != 5 { |
||||
|
t.Fatalf("metaCache destination size = %d, want 5", cachedEntry.FileSize) |
||||
|
} |
||||
|
if cachedEntry.Mime != "text/plain; charset=utf-8" { |
||||
|
t.Fatalf("metaCache destination mime = %q, want %q", cachedEntry.Mime, "text/plain; charset=utf-8") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestCopyFileRangeReturnsEIOForAmbiguousServerSideCopy(t *testing.T) { |
||||
|
wfs := newCopyRangeTestWFS() |
||||
|
|
||||
|
srcPath := util.FullPath("/src.txt") |
||||
|
dstPath := util.FullPath("/dst.txt") |
||||
|
srcInode := wfs.inodeToPath.Lookup(srcPath, 1, false, false, 0, true) |
||||
|
dstInode := wfs.inodeToPath.Lookup(dstPath, 1, false, false, 0, true) |
||||
|
|
||||
|
srcHandle := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{ |
||||
|
Name: "src.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100644, |
||||
|
FileSize: 5, |
||||
|
Inode: srcInode, |
||||
|
}, |
||||
|
Content: []byte("hello"), |
||||
|
}) |
||||
|
dstHandle := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{ |
||||
|
Name: "dst.txt", |
||||
|
Attributes: &filer_pb.FuseAttributes{ |
||||
|
FileMode: 0100600, |
||||
|
Inode: dstInode, |
||||
|
}, |
||||
|
}) |
||||
|
|
||||
|
srcHandle.RememberPath(srcPath) |
||||
|
dstHandle.RememberPath(dstPath) |
||||
|
|
||||
|
originalCopy := performServerSideWholeFileCopy |
||||
|
defer func() { |
||||
|
performServerSideWholeFileCopy = originalCopy |
||||
|
}() |
||||
|
|
||||
|
performServerSideWholeFileCopy = func(cancel <-chan struct{}, gotWFS *WFS, copyRequest wholeFileServerCopyRequest) (*filer_pb.Entry, serverSideWholeFileCopyOutcome, error) { |
||||
|
if gotWFS != wfs || copyRequest.srcPath != srcPath || copyRequest.dstPath != dstPath { |
||||
|
t.Fatalf("unexpected server-side copy call: wfs=%p src=%q dst=%q", gotWFS, copyRequest.srcPath, copyRequest.dstPath) |
||||
|
} |
||||
|
return nil, serverSideWholeFileCopyAmbiguous, errors.New("transport timeout after request dispatch") |
||||
|
} |
||||
|
|
||||
|
written, status := wfs.CopyFileRange(make(chan struct{}), &fuse.CopyFileRangeIn{ |
||||
|
FhIn: uint64(srcHandle.fh), |
||||
|
FhOut: uint64(dstHandle.fh), |
||||
|
OffIn: 0, |
||||
|
OffOut: 0, |
||||
|
Len: 8, |
||||
|
}) |
||||
|
if status != fuse.EIO { |
||||
|
t.Fatalf("CopyFileRange status = %v, want EIO", status) |
||||
|
} |
||||
|
if written != 0 { |
||||
|
t.Fatalf("CopyFileRange wrote %d bytes, want 0", written) |
||||
|
} |
||||
|
if dstHandle.dirtyMetadata { |
||||
|
t.Fatal("ambiguous server-side copy should not fall back to dirty-page copy") |
||||
|
} |
||||
|
if dstHandle.GetEntry().GetEntry().GetAttributes().GetFileSize() != 0 { |
||||
|
t.Fatalf("destination size = %d, want 0", dstHandle.GetEntry().GetEntry().GetAttributes().GetFileSize()) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func newCopyRangeTestWFS() *WFS { |
||||
|
wfs := &WFS{ |
||||
|
option: &Option{ |
||||
|
ChunkSizeLimit: 1024, |
||||
|
ConcurrentReaders: 1, |
||||
|
VolumeServerAccess: "filerProxy", |
||||
|
FilerAddresses: []pb.ServerAddress{"127.0.0.1:8888"}, |
||||
|
}, |
||||
|
inodeToPath: NewInodeToPath(util.FullPath("/"), 0), |
||||
|
fhMap: NewFileHandleToInode(), |
||||
|
fhLockTable: util.NewLockTable[FileHandleId](), |
||||
|
} |
||||
|
wfs.copyBufferPool.New = func() any { |
||||
|
return make([]byte, 1024) |
||||
|
} |
||||
|
return wfs |
||||
|
} |
||||
|
|
||||
|
func newCopyRangeTestWFSWithMetaCache(t *testing.T) *WFS { |
||||
|
t.Helper() |
||||
|
|
||||
|
wfs := newCopyRangeTestWFS() |
||||
|
root := util.FullPath("/") |
||||
|
wfs.inodeToPath.MarkChildrenCached(root) |
||||
|
uidGidMapper, err := meta_cache.NewUidGidMapper("", "") |
||||
|
if err != nil { |
||||
|
t.Fatalf("create uid/gid mapper: %v", err) |
||||
|
} |
||||
|
wfs.metaCache = meta_cache.NewMetaCache( |
||||
|
filepath.Join(t.TempDir(), "meta"), |
||||
|
uidGidMapper, |
||||
|
root, |
||||
|
func(path util.FullPath) { |
||||
|
wfs.inodeToPath.MarkChildrenCached(path) |
||||
|
}, |
||||
|
func(path util.FullPath) bool { |
||||
|
return wfs.inodeToPath.IsChildrenCached(path) |
||||
|
}, |
||||
|
func(util.FullPath, *filer_pb.Entry) {}, |
||||
|
nil, |
||||
|
) |
||||
|
t.Cleanup(func() { |
||||
|
wfs.metaCache.Shutdown() |
||||
|
}) |
||||
|
|
||||
|
return wfs |
||||
|
} |
||||
@ -0,0 +1,223 @@ |
|||||
|
package weed_server |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"testing" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/filer" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
func TestCopyEntryRefreshesDestinationTimestamps(t *testing.T) { |
||||
|
fs := &FilerServer{} |
||||
|
|
||||
|
oldTime := time.Unix(123, 0) |
||||
|
srcEntry := &filer.Entry{ |
||||
|
FullPath: util.FullPath("/src.txt"), |
||||
|
Attr: filer.Attr{ |
||||
|
Mtime: oldTime, |
||||
|
Crtime: oldTime, |
||||
|
}, |
||||
|
Content: []byte("hello"), |
||||
|
} |
||||
|
|
||||
|
before := time.Now().Add(-time.Second) |
||||
|
copied, err := fs.copyEntry(context.Background(), srcEntry, util.FullPath("/dst.txt"), nil) |
||||
|
after := time.Now().Add(time.Second) |
||||
|
if err != nil { |
||||
|
t.Fatalf("copyEntry: %v", err) |
||||
|
} |
||||
|
|
||||
|
if copied.Crtime.Before(before) || copied.Crtime.After(after) { |
||||
|
t.Fatalf("copied Crtime = %v, want between %v and %v", copied.Crtime, before, after) |
||||
|
} |
||||
|
if copied.Mtime.Before(before) || copied.Mtime.After(after) { |
||||
|
t.Fatalf("copied Mtime = %v, want between %v and %v", copied.Mtime, before, after) |
||||
|
} |
||||
|
if copied.Crtime.Equal(oldTime) || copied.Mtime.Equal(oldTime) { |
||||
|
t.Fatalf("destination timestamps should differ from source timestamps: src=%v copied=(%v,%v)", oldTime, copied.Crtime, copied.Mtime) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestPreserveDestinationMetadataForDataCopy(t *testing.T) { |
||||
|
dstTime := time.Unix(100, 0) |
||||
|
srcTime := time.Unix(200, 0) |
||||
|
|
||||
|
dstEntry := &filer.Entry{ |
||||
|
FullPath: util.FullPath("/dst.txt"), |
||||
|
Attr: filer.Attr{ |
||||
|
Mtime: dstTime, |
||||
|
Crtime: dstTime, |
||||
|
Mode: 0100600, |
||||
|
Uid: 101, |
||||
|
Gid: 202, |
||||
|
TtlSec: 17, |
||||
|
UserName: "dst-user", |
||||
|
GroupNames: []string{"dst-group"}, |
||||
|
SymlinkTarget: "", |
||||
|
Rdev: 9, |
||||
|
Inode: 1234, |
||||
|
}, |
||||
|
Extended: map[string][]byte{ |
||||
|
"user.color": []byte("blue"), |
||||
|
}, |
||||
|
Remote: &filer_pb.RemoteEntry{ |
||||
|
StorageName: "remote-store", |
||||
|
RemoteETag: "remote-etag", |
||||
|
RemoteSize: 7, |
||||
|
}, |
||||
|
Quota: 77, |
||||
|
WORMEnforcedAtTsNs: 88, |
||||
|
HardLinkId: filer.HardLinkId([]byte("hard-link")), |
||||
|
HardLinkCounter: 3, |
||||
|
} |
||||
|
copiedEntry := &filer.Entry{ |
||||
|
FullPath: util.FullPath("/dst.txt"), |
||||
|
Attr: filer.Attr{ |
||||
|
Mtime: srcTime, |
||||
|
Crtime: srcTime, |
||||
|
Mode: 0100644, |
||||
|
Uid: 11, |
||||
|
Gid: 22, |
||||
|
Mime: "text/plain", |
||||
|
Md5: []byte("source-md5"), |
||||
|
FileSize: 5, |
||||
|
UserName: "src-user", |
||||
|
GroupNames: []string{"src-group"}, |
||||
|
}, |
||||
|
Content: []byte("hello"), |
||||
|
Extended: map[string][]byte{ |
||||
|
"user.color": []byte("red"), |
||||
|
}, |
||||
|
Quota: 5, |
||||
|
} |
||||
|
|
||||
|
before := time.Now().Add(-time.Second) |
||||
|
preserveDestinationMetadataForDataCopy(dstEntry, copiedEntry) |
||||
|
after := time.Now().Add(time.Second) |
||||
|
|
||||
|
if copiedEntry.Mode != dstEntry.Mode || copiedEntry.Uid != dstEntry.Uid || copiedEntry.Gid != dstEntry.Gid { |
||||
|
t.Fatalf("destination ownership/mode not preserved: got mode=%#o uid=%d gid=%d", copiedEntry.Mode, copiedEntry.Uid, copiedEntry.Gid) |
||||
|
} |
||||
|
if copiedEntry.Crtime != dstEntry.Crtime || copiedEntry.Inode != dstEntry.Inode { |
||||
|
t.Fatalf("destination identity not preserved: got crtime=%v inode=%d", copiedEntry.Crtime, copiedEntry.Inode) |
||||
|
} |
||||
|
if copiedEntry.Mtime.Before(before) || copiedEntry.Mtime.After(after) { |
||||
|
t.Fatalf("destination mtime = %v, want between %v and %v", copiedEntry.Mtime, before, after) |
||||
|
} |
||||
|
if copiedEntry.FileSize != 5 || copiedEntry.Mime != "text/plain" || string(copiedEntry.Md5) != "source-md5" { |
||||
|
t.Fatalf("copied data attributes changed unexpectedly: size=%d mime=%q md5=%q", copiedEntry.FileSize, copiedEntry.Mime, string(copiedEntry.Md5)) |
||||
|
} |
||||
|
if string(copiedEntry.Extended["user.color"]) != "blue" { |
||||
|
t.Fatalf("destination xattrs not preserved: got %q", string(copiedEntry.Extended["user.color"])) |
||||
|
} |
||||
|
if copiedEntry.Remote == nil || copiedEntry.Remote.StorageName != "remote-store" { |
||||
|
t.Fatalf("destination remote metadata not preserved: %+v", copiedEntry.Remote) |
||||
|
} |
||||
|
if copiedEntry.Quota != 77 || copiedEntry.WORMEnforcedAtTsNs != 88 { |
||||
|
t.Fatalf("destination quota/WORM not preserved: quota=%d worm=%d", copiedEntry.Quota, copiedEntry.WORMEnforcedAtTsNs) |
||||
|
} |
||||
|
if string(copiedEntry.HardLinkId) != "hard-link" || copiedEntry.HardLinkCounter != 3 { |
||||
|
t.Fatalf("destination hard-link metadata not preserved: id=%q count=%d", string(copiedEntry.HardLinkId), copiedEntry.HardLinkCounter) |
||||
|
} |
||||
|
|
||||
|
dstEntry.GroupNames[0] = "mutated" |
||||
|
dstEntry.Extended["user.color"][0] = 'g' |
||||
|
dstEntry.Remote.StorageName = "mutated-remote" |
||||
|
if copiedEntry.GroupNames[0] != "dst-group" { |
||||
|
t.Fatalf("group names should be cloned, got %q", copiedEntry.GroupNames[0]) |
||||
|
} |
||||
|
if string(copiedEntry.Extended["user.color"]) != "blue" { |
||||
|
t.Fatalf("extended metadata should be cloned, got %q", string(copiedEntry.Extended["user.color"])) |
||||
|
} |
||||
|
if copiedEntry.Remote.StorageName != "remote-store" { |
||||
|
t.Fatalf("remote metadata should be cloned, got %q", copiedEntry.Remote.StorageName) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestValidateCopySourcePreconditions(t *testing.T) { |
||||
|
srcInode := uint64(101) |
||||
|
srcMtime := int64(200) |
||||
|
srcSize := int64(5) |
||||
|
preconditions := copyRequestPreconditions{ |
||||
|
srcInode: &srcInode, |
||||
|
srcMtime: &srcMtime, |
||||
|
srcSize: &srcSize, |
||||
|
} |
||||
|
|
||||
|
srcEntry := &filer.Entry{ |
||||
|
FullPath: util.FullPath("/src.txt"), |
||||
|
Attr: filer.Attr{ |
||||
|
Inode: srcInode, |
||||
|
Mtime: time.Unix(srcMtime, 0), |
||||
|
FileSize: uint64(srcSize), |
||||
|
}, |
||||
|
Content: []byte("hello"), |
||||
|
} |
||||
|
|
||||
|
if err := validateCopySourcePreconditions(preconditions, srcEntry); err != nil { |
||||
|
t.Fatalf("validate source preconditions: %v", err) |
||||
|
} |
||||
|
|
||||
|
changedSize := int64(6) |
||||
|
preconditions.srcSize = &changedSize |
||||
|
if err := validateCopySourcePreconditions(preconditions, srcEntry); err == nil { |
||||
|
t.Fatal("expected source size mismatch to fail") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestValidateCopyDestinationPreconditions(t *testing.T) { |
||||
|
dstInode := uint64(202) |
||||
|
dstMtime := int64(300) |
||||
|
dstSize := int64(0) |
||||
|
preconditions := copyRequestPreconditions{ |
||||
|
dstInode: &dstInode, |
||||
|
dstMtime: &dstMtime, |
||||
|
dstSize: &dstSize, |
||||
|
} |
||||
|
|
||||
|
dstEntry := &filer.Entry{ |
||||
|
FullPath: util.FullPath("/dst.txt"), |
||||
|
Attr: filer.Attr{ |
||||
|
Inode: dstInode, |
||||
|
Mtime: time.Unix(dstMtime, 0), |
||||
|
FileSize: uint64(dstSize), |
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
if err := validateCopyDestinationPreconditions(preconditions, dstEntry); err != nil { |
||||
|
t.Fatalf("validate destination preconditions: %v", err) |
||||
|
} |
||||
|
|
||||
|
if err := validateCopyDestinationPreconditions(preconditions, nil); err == nil { |
||||
|
t.Fatal("expected missing destination to fail") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func TestRecentCopyRequestDeduplicatesByRequestID(t *testing.T) { |
||||
|
fs := &FilerServer{ |
||||
|
recentCopyRequests: make(map[string]recentCopyRequest), |
||||
|
} |
||||
|
|
||||
|
requestID := "copy-req" |
||||
|
fingerprint := "src|dst|1" |
||||
|
fs.rememberRecentCopyRequest(requestID, fingerprint) |
||||
|
|
||||
|
handled, err := fs.handleRecentCopyRequest(requestID, fingerprint) |
||||
|
if err != nil { |
||||
|
t.Fatalf("handle recent copy request: %v", err) |
||||
|
} |
||||
|
if !handled { |
||||
|
t.Fatal("expected recent copy request to be recognized") |
||||
|
} |
||||
|
|
||||
|
handled, err = fs.handleRecentCopyRequest(requestID, "different") |
||||
|
if err == nil { |
||||
|
t.Fatal("expected reused request id with different fingerprint to fail") |
||||
|
} |
||||
|
if handled { |
||||
|
t.Fatal("reused request id with different fingerprint should not be treated as handled") |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue