Browse Source
Fix S3 delete for non-empty directory markers (#8740)
Fix S3 delete for non-empty directory markers (#8740)
* Fix S3 delete for non-empty directory markers * Address review feedback on directory marker deletes * Stabilize FUSE concurrent directory operationsfix/subscribe-metadata-slow-consumer-blocked
committed by
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 385 additions and 31 deletions
-
25test/fuse_integration/framework_test.go
-
45test/s3/normal/s3_integration_test.go
-
21weed/mount/filehandle.go
-
52weed/mount/filehandle_test.go
-
6weed/mount/weedfs_async_flush.go
-
3weed/mount/weedfs_file_sync.go
-
1weed/mount/weedfs_filehandle.go
-
7weed/mount/weedfs_rename.go
-
93weed/s3api/filer_util.go
-
152weed/s3api/filer_util_delete_test.go
-
2weed/s3api/s3api_object_handlers_copy.go
-
7weed/s3api/s3api_object_handlers_delete.go
-
2weed/s3api/s3api_object_versioning.go
@ -0,0 +1,52 @@ |
|||
package mount |
|||
|
|||
import ( |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/util" |
|||
) |
|||
|
|||
func TestFileHandleFullPathFallsBackAfterForget(t *testing.T) { |
|||
wfs := &WFS{ |
|||
inodeToPath: NewInodeToPath(util.FullPath("/"), 0), |
|||
} |
|||
|
|||
fullPath := util.FullPath("/worker_0/subdir_0/test.txt") |
|||
inode := wfs.inodeToPath.Lookup(fullPath, 1, false, false, 0, true) |
|||
|
|||
fh := &FileHandle{ |
|||
inode: inode, |
|||
wfs: wfs, |
|||
} |
|||
fh.RememberPath(fullPath) |
|||
|
|||
wfs.inodeToPath.Forget(inode, 1, nil) |
|||
|
|||
if got := fh.FullPath(); got != fullPath { |
|||
t.Fatalf("FullPath() after forget = %q, want %q", got, fullPath) |
|||
} |
|||
} |
|||
|
|||
func TestFileHandleFullPathUsesSavedRenamePathAfterForget(t *testing.T) { |
|||
wfs := &WFS{ |
|||
inodeToPath: NewInodeToPath(util.FullPath("/"), 0), |
|||
} |
|||
|
|||
oldPath := util.FullPath("/worker_0/subdir_0/test.txt") |
|||
newPath := util.FullPath("/worker_0/subdir_1/test.txt") |
|||
inode := wfs.inodeToPath.Lookup(oldPath, 1, false, false, 0, true) |
|||
|
|||
fh := &FileHandle{ |
|||
inode: inode, |
|||
wfs: wfs, |
|||
} |
|||
fh.RememberPath(oldPath) |
|||
|
|||
wfs.inodeToPath.MovePath(oldPath, newPath) |
|||
fh.RememberPath(newPath) |
|||
wfs.inodeToPath.Forget(inode, 1, nil) |
|||
|
|||
if got := fh.FullPath(); got != newPath { |
|||
t.Fatalf("FullPath() after rename+forget = %q, want %q", got, newPath) |
|||
} |
|||
} |
|||
@ -0,0 +1,152 @@ |
|||
package s3api |
|||
|
|||
import ( |
|||
"context" |
|||
"errors" |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/filer" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" |
|||
"github.com/stretchr/testify/assert" |
|||
"github.com/stretchr/testify/require" |
|||
grpc "google.golang.org/grpc" |
|||
"google.golang.org/grpc/codes" |
|||
"google.golang.org/grpc/status" |
|||
) |
|||
|
|||
type deleteObjectEntryTestClient struct { |
|||
filer_pb.SeaweedFilerClient |
|||
|
|||
deleteResp *filer_pb.DeleteEntryResponse |
|||
deleteErr error |
|||
lookupResp *filer_pb.LookupDirectoryEntryResponse |
|||
lookupErr error |
|||
updateErr error |
|||
|
|||
deleteReq *filer_pb.DeleteEntryRequest |
|||
lookupReq *filer_pb.LookupDirectoryEntryRequest |
|||
updateReq *filer_pb.UpdateEntryRequest |
|||
} |
|||
|
|||
func (c *deleteObjectEntryTestClient) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest, _ ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) { |
|||
c.deleteReq = req |
|||
if c.deleteResp == nil { |
|||
return &filer_pb.DeleteEntryResponse{}, c.deleteErr |
|||
} |
|||
return c.deleteResp, c.deleteErr |
|||
} |
|||
|
|||
func (c *deleteObjectEntryTestClient) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { |
|||
c.lookupReq = req |
|||
if c.lookupResp == nil { |
|||
return &filer_pb.LookupDirectoryEntryResponse{}, c.lookupErr |
|||
} |
|||
return c.lookupResp, c.lookupErr |
|||
} |
|||
|
|||
func (c *deleteObjectEntryTestClient) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) { |
|||
c.updateReq = req |
|||
return &filer_pb.UpdateEntryResponse{}, c.updateErr |
|||
} |
|||
|
|||
func TestDeleteObjectEntryDemotesNonEmptyDirectoryMarker(t *testing.T) { |
|||
client := &deleteObjectEntryTestClient{ |
|||
deleteResp: &filer_pb.DeleteEntryResponse{ |
|||
Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos", |
|||
}, |
|||
lookupResp: &filer_pb.LookupDirectoryEntryResponse{ |
|||
Entry: &filer_pb.Entry{ |
|||
Name: "photos", |
|||
IsDirectory: true, |
|||
Attributes: &filer_pb.FuseAttributes{ |
|||
Mime: "application/octet-stream", |
|||
Md5: []byte{1, 2, 3, 4}, |
|||
FileSize: 4, |
|||
}, |
|||
Content: []byte("test"), |
|||
Extended: map[string][]byte{ |
|||
s3_constants.ExtETagKey: []byte("etag"), |
|||
s3_constants.ExtAmzOwnerKey: []byte("owner"), |
|||
s3_constants.AmzUserMetaPrefix + "Color": []byte("blue"), |
|||
s3_constants.AmzObjectTaggingPrefix + "k": []byte("v"), |
|||
"xattr-keep": []byte("keep-me"), |
|||
"x-seaweedfs-internal": []byte("keep-me-too"), |
|||
}, |
|||
}, |
|||
}, |
|||
} |
|||
|
|||
err := deleteObjectEntry(client, "/buckets/test", "photos", true, false) |
|||
require.NoError(t, err) |
|||
require.NotNil(t, client.lookupReq) |
|||
require.NotNil(t, client.updateReq) |
|||
|
|||
updated := client.updateReq.Entry |
|||
require.NotNil(t, updated) |
|||
assert.False(t, updated.IsDirectoryKeyObject()) |
|||
assert.Equal(t, "", updated.Attributes.Mime) |
|||
assert.Empty(t, updated.Attributes.Md5) |
|||
assert.Zero(t, updated.Attributes.FileSize) |
|||
assert.Nil(t, updated.Content) |
|||
assert.Nil(t, updated.Chunks) |
|||
assert.Equal(t, map[string][]byte{ |
|||
"xattr-keep": []byte("keep-me"), |
|||
"x-seaweedfs-internal": []byte("keep-me-too"), |
|||
}, updated.Extended) |
|||
} |
|||
|
|||
func TestDeleteObjectEntryTreatsImplicitDirectoryAsSuccessfulNoop(t *testing.T) { |
|||
client := &deleteObjectEntryTestClient{ |
|||
deleteResp: &filer_pb.DeleteEntryResponse{ |
|||
Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos", |
|||
}, |
|||
lookupResp: &filer_pb.LookupDirectoryEntryResponse{ |
|||
Entry: &filer_pb.Entry{ |
|||
Name: "photos", |
|||
IsDirectory: true, |
|||
Attributes: &filer_pb.FuseAttributes{}, |
|||
}, |
|||
}, |
|||
} |
|||
|
|||
err := deleteObjectEntry(client, "/buckets/test", "photos", true, false) |
|||
require.NoError(t, err) |
|||
require.NotNil(t, client.lookupReq) |
|||
assert.Nil(t, client.updateReq) |
|||
} |
|||
|
|||
func TestDeleteObjectEntryIgnoresConcurrentUpdateNotFound(t *testing.T) { |
|||
client := &deleteObjectEntryTestClient{ |
|||
deleteResp: &filer_pb.DeleteEntryResponse{ |
|||
Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos", |
|||
}, |
|||
lookupResp: &filer_pb.LookupDirectoryEntryResponse{ |
|||
Entry: &filer_pb.Entry{ |
|||
Name: "photos", |
|||
IsDirectory: true, |
|||
Attributes: &filer_pb.FuseAttributes{ |
|||
Mime: "application/octet-stream", |
|||
}, |
|||
}, |
|||
}, |
|||
updateErr: status.Error(codes.NotFound, "already removed"), |
|||
} |
|||
|
|||
err := deleteObjectEntry(client, "/buckets/test", "photos", true, false) |
|||
require.NoError(t, err) |
|||
require.NotNil(t, client.lookupReq) |
|||
require.NotNil(t, client.updateReq) |
|||
} |
|||
|
|||
func TestDeleteObjectEntryPropagatesNonDirectoryDeleteErrors(t *testing.T) { |
|||
client := &deleteObjectEntryTestClient{ |
|||
deleteErr: errors.New("boom"), |
|||
} |
|||
|
|||
err := deleteObjectEntry(client, "/buckets/test", "photos", true, false) |
|||
require.Error(t, err) |
|||
assert.Contains(t, err.Error(), "boom") |
|||
assert.Nil(t, client.lookupReq) |
|||
assert.Nil(t, client.updateReq) |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue