Browse Source

fix s3test test_bucket_listv2_delimiter_prefix_ends_with_delimiter (#4399)

* fix s3test test_bucket_listv2_delimiter_prefix_ends_with_delimiter

* fix list with delimiter and start token

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
pull/4404/head
Konstantin Lebedev 2 years ago
committed by GitHub
parent
commit
5614ad0000
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 5
      weed/s3api/s3api_object_handlers.go
  2. 25
      weed/s3api/s3api_objects_list_handlers.go

5
weed/s3api/s3api_object_handlers.go

@ -94,13 +94,16 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
defer dataReader.Close()
objectContentType := r.Header.Get("Content-Type")
if strings.HasSuffix(object, "/") && r.ContentLength == 0 {
if strings.HasSuffix(object, "/") && r.ContentLength <= 1024 {
if err := s3a.mkdir(
s3a.option.BucketsPath, bucket+strings.TrimSuffix(object, "/"),
func(entry *filer_pb.Entry) {
if objectContentType == "" {
objectContentType = s3_constants.FolderMimeType
}
if r.ContentLength > 0 {
entry.Content, _ = io.ReadAll(r.Body)
}
entry.Attributes.Mime = objectContentType
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)

25
weed/s3api/s3api_objects_list_handlers.go

@ -142,7 +142,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
var nextMarker string
cursor := &ListingCursor{
maxKeys: maxKeys,
prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/"),
prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/") && len(originalMarker) == 0,
}
// check filer
@ -152,14 +152,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
empty = false
if entry.IsDirectory {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
})
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
} else if entry.IsDirectoryKeyObject() {
if entry.IsDirectoryKeyObject() {
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
@ -171,6 +164,13 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
StorageClass: "STANDARD",
})
cursor.maxKeys--
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
} else if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
})
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
}
} else {
storageClass := "STANDARD"
@ -238,7 +238,11 @@ type ListingCursor struct {
func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, alignedMarker string) {
// alignedDir should not end with "/"
// alignedDir, alignedPrefix, alignedMarker should only have "/" in middle
if len(marker) == 0 {
prefix = strings.Trim(prefix, "/")
} else {
prefix = strings.TrimLeft(prefix, "/")
}
marker = strings.TrimLeft(marker, "/")
if prefix == "" {
return "", "", marker
@ -368,6 +372,9 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
if delimiter != "/" || cursor.prefixEndsOnDelimiter {
if cursor.prefixEndsOnDelimiter {
cursor.prefixEndsOnDelimiter = false
if entry.IsDirectoryKeyObject() {
eachEntryFn(dir, entry)
}
} else {
eachEntryFn(dir, entry)
}

Loading…
Cancel
Save