Browse Source
Merge pull request #2083 from kmlebedev/s3BucketNotEmpty
s3 test bucket delete nonempty
pull/2856/head
Chris Lu
3 years ago
committed by
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with
54 additions and
29 deletions
docker/compose/local-s3tests-compose.yml
weed/command/filer.go
weed/command/s3.go
weed/command/server.go
weed/s3api/s3api_bucket_handlers.go
weed/s3api/s3api_server.go
@ -24,7 +24,7 @@ services:
- 8888 : 8888
- 18888 : 18888
- 8000 : 8000
command : 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000'
command : 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false '
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
depends_on:
@ -38,7 +38,7 @@ services:
S3TEST_CONF : "s3tests.conf"
NOSETESTS_OPTIONS : "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
NOSETESTS_ATTR : "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
NOSETESTS_EXCLUDE : "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_ concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket|list_multipart_upload_owner)"
NOSETESTS_EXCLUDE : "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket|list_multipart_upload_owner|multipart_upload_small )"
depends_on:
- master
- volume
@ -89,6 +89,7 @@ func init() {
filerS3Options . config = cmdFiler . Flag . String ( "s3.config" , "" , "path to the config file" )
filerS3Options . auditLogConfig = cmdFiler . Flag . String ( "s3.auditLogConfig" , "" , "path to the audit log config file" )
filerS3Options . allowEmptyFolder = cmdFiler . Flag . Bool ( "s3.allowEmptyFolder" , true , "allow empty folders" )
filerS3Options . allowDeleteBucketNotEmpty = cmdFiler . Flag . Bool ( "s3.allowDeleteBucketNotEmpty" , true , "allow recursive deleting all entries along with bucket" )
// start webdav on filer
filerStartWebDav = cmdFiler . Flag . Bool ( "webdav" , false , "whether to start webdav gateway" )
@ -33,6 +33,7 @@ type S3Options struct {
tlsCertificate * string
metricsHttpPort * int
allowEmptyFolder * bool
allowDeleteBucketNotEmpty * bool
auditLogConfig * string
localFilerSocket * string
}
@ -49,6 +50,7 @@ func init() {
s3StandaloneOptions . tlsCertificate = cmdS3 . Flag . String ( "cert.file" , "" , "path to the TLS certificate file" )
s3StandaloneOptions . metricsHttpPort = cmdS3 . Flag . Int ( "metricsPort" , 0 , "Prometheus metrics listen port" )
s3StandaloneOptions . allowEmptyFolder = cmdS3 . Flag . Bool ( "allowEmptyFolder" , true , "allow empty folders" )
s3StandaloneOptions . allowDeleteBucketNotEmpty = cmdS3 . Flag . Bool ( "allowDeleteBucketNotEmpty" , true , "allow recursive deleting all entries along with bucket" )
}
var cmdS3 = & Command {
@ -185,6 +187,7 @@ func (s3opt *S3Options) startS3Server() bool {
BucketsPath : filerBucketsPath ,
GrpcDialOption : grpcDialOption ,
AllowEmptyFolder : * s3opt . allowEmptyFolder ,
AllowDeleteBucketNotEmpty : * s3opt . allowDeleteBucketNotEmpty ,
LocalFilerSocket : s3opt . localFilerSocket ,
} )
if s3ApiServer_err != nil {
@ -138,6 +138,7 @@ func init() {
s3Options . config = cmdServer . Flag . String ( "s3.config" , "" , "path to the config file" )
s3Options . auditLogConfig = cmdServer . Flag . String ( "s3.auditLogConfig" , "" , "path to the audit log config file" )
s3Options . allowEmptyFolder = cmdServer . Flag . Bool ( "s3.allowEmptyFolder" , true , "allow empty folders" )
s3Options . allowDeleteBucketNotEmpty = cmdServer . Flag . Bool ( "s3.allowDeleteBucketNotEmpty" , true , "allow recursive deleting all entries along with bucket" )
iamOptions . port = cmdServer . Flag . Int ( "iam.port" , 8111 , "iam server http listen port" )
@ -3,6 +3,7 @@ package s3api
import (
"context"
"encoding/xml"
"errors"
"fmt"
"math"
"net/http"
@ -148,6 +149,15 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
}
err := s3a . WithFilerClient ( false , func ( client filer_pb . SeaweedFilerClient ) error {
if ! s3a . option . AllowDeleteBucketNotEmpty {
entries , _ , err := s3a . list ( s3a . option . BucketsPath + "/" + bucket , "" , "" , false , 1 )
if err != nil {
return fmt . Errorf ( "failed to list bucket %s: %v" , bucket , err )
}
if len ( entries ) > 0 {
return errors . New ( s3err . GetAPIError ( s3err . ErrBucketNotEmpty ) . Code )
}
}
// delete collection
deleteCollectionRequest := & filer_pb . DeleteCollectionRequest {
@ -162,6 +172,15 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
return nil
} )
if err != nil {
s3ErrorCode := s3err . ErrInternalError
if err . Error ( ) == s3err . GetAPIError ( s3err . ErrBucketNotEmpty ) . Code {
s3ErrorCode = s3err . ErrBucketNotEmpty
}
s3err . WriteErrorResponse ( w , r , s3ErrorCode )
return
}
err = s3a . rm ( s3a . option . BucketsPath , bucket , false , true )
if err != nil {
@ -26,6 +26,7 @@ type S3ApiServerOption struct {
BucketsPath string
GrpcDialOption grpc . DialOption
AllowEmptyFolder bool
AllowDeleteBucketNotEmpty bool
LocalFilerSocket * string
}