diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index f7dba0c39..fe6ce4a80 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -169,6 +169,7 @@ jobs: s3tests_boto3/functional/test_s3.py::test_multipart_upload \ s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \ s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \ + s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small \ s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \ s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \ s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \ @@ -181,7 +182,6 @@ jobs: s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \ s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \ s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \ - s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last \ s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \ s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \ s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \ diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index b2da8193f..e9cd6a0c4 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -25,7 +25,10 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) -const multipartExt = ".part" +const ( + multipartExt = ".part" + multiPartMinSize = 5 * 1024 * 1024 +) type InitiateMultipartUploadResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"` @@ -75,6 +78,10 @@ type CompleteMultipartUploadResult struct { func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) { glog.V(2).Infof("completeMultipartUpload input %v", input) + if len(parts.Parts) == 0 { + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompletedNoSuchUpload).Inc() + return nil, s3err.ErrNoSuchUpload + } completedPartNumbers := []int{} completedPartMap := make(map[int][]string) for _, part := range parts.Parts { @@ -83,8 +90,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa } completedPartMap[part.PartNumber] = append(completedPartMap[part.PartNumber], part.ETag) } - uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId + sort.Ints(completedPartNumbers) + uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList) if err != nil { glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries)) @@ -118,6 +126,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa } deleteEntries := []*filer_pb.Entry{} partEntries := make(map[int][]*filer_pb.Entry, len(entries)) + entityTooSmall := false for _, entry := range entries { foundEntry := false glog.V(4).Infof("completeMultipartUpload part entries %s", entry.Name) @@ -156,16 +165,23 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa partEntries[partNumber] = append(partEntries[partNumber], entry) foundEntry = true } - if !foundEntry { + if foundEntry { + if len(completedPartNumbers) > 1 && partNumber != completedPartNumbers[len(completedPartNumbers)-1] && + entry.Attributes.FileSize < multiPartMinSize { + glog.Warningf("completeMultipartUpload %s part file size less 5mb", entry.Name) + entityTooSmall = true + } + } else { deleteEntries = append(deleteEntries, entry) } } - + if entityTooSmall { + stats.S3HandlerCounter.WithLabelValues(stats.ErrorCompleteEntityTooSmall).Inc() + return nil, s3err.ErrEntityTooSmall + } mime := pentry.Attributes.Mime - var finalParts []*filer_pb.FileChunk var offset int64 - sort.Ints(completedPartNumbers) for _, partNumber := range completedPartNumbers { partEntriesByNumber, ok := partEntries[partNumber] if !ok { diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index cfc0fbeb0..c0a6e99be 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -46,8 +46,9 @@ const ( // s3 handler ErrorCompletedNoSuchUpload = "errorCompletedNoSuchUpload" - ErrorCompletedPartEmpty = "ErrorCompletedPartEmpty" - ErrorCompletedPartNumber = "ErrorCompletedPartNumber" + ErrorCompleteEntityTooSmall = "errorCompleteEntityTooSmall" + ErrorCompletedPartEmpty = "errorCompletedPartEmpty" + ErrorCompletedPartNumber = "errorCompletedPartNumber" ErrorCompletedPartNotFound = "errorCompletedPartNotFound" ErrorCompletedEtagInvalid = "errorCompletedEtagInvalid" ErrorCompletedEtagMismatch = "errorCompletedEtagMismatch"