Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fix s3 tests 

bucket_list_delimiter_prefix
bucket_list_delimiter_prefix_underscore
bucket_list_delimiter_prefix_ends_with_delimiter
pull/3130/head
Konstantin Lebedev 3 years ago
parent
b86628f85d
commit
c07820178f
3 changed files with 52 additions and 30 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      weed/pb/filer_pb/filer_pb_helper.go
  2. 8
      weed/s3api/s3api_object_handlers.go
  3. 56
      weed/s3api/s3api_objects_list_handlers.go

4
weed/pb/filer_pb/filer_pb_helper.go
View File

@ -17,6 +17,10 @@ func (entry *Entry) IsInRemoteOnly() bool {
return len(entry.Chunks) == 0 && entry.RemoteEntry != nil && entry.RemoteEntry.RemoteSize > 0
}
func (entry *Entry) IsDirectoryKeyObject() bool {
return entry.IsDirectory && entry.Attributes != nil && entry.Attributes.Mime != ""
}
func (entry *Entry) FileMode() (fileMode os.FileMode) {
if entry != nil && entry.Attributes != nil {
fileMode = os.FileMode(entry.Attributes.FileMode)

8
weed/s3api/s3api_object_handlers.go
View File

@ -92,16 +92,20 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
}
defer dataReader.Close()
objectContentType := r.Header.Get("Content-Type")
if strings.HasSuffix(object, "/") {
if err := s3a.mkdir(s3a.option.BucketsPath, bucket+strings.TrimSuffix(object, "/"), func(entry *filer_pb.Entry) {
entry.Attributes.Mime = r.Header.Get("Content-Type")
if objectContentType == "" {
objectContentType = "httpd/unix-directory"
}
entry.Attributes.Mime = objectContentType
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
} else {
uploadUrl := s3a.toFilerUrl(bucket, object)
if r.Header.Get("Content-Type") == "" {
if objectContentType == "" {
dataReader = mimeDetect(r, dataReader)
}

56
weed/s3api/s3api_objects_list_handlers.go
View File

@ -133,6 +133,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
reqDir = reqDir[1:]
}
bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
bucketPrefixLen := len(bucketPrefix)
reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
if strings.HasSuffix(reqDir, "/") {
reqDir = strings.TrimSuffix(reqDir, "/")
@ -147,14 +148,14 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
// check filer
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, false, bucketPrefixLen, func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectory {
if delimiter == "/" {
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[bucketPrefixLen:],
})
}
if entry.Attributes.Mime == "" || !strings.HasSuffix(entry.Name, "/") {
if !(entry.IsDirectoryKeyObject() && strings.HasSuffix(entry.Name, "/")) {
return
}
}
@ -163,7 +164,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
storageClass = string(v)
}
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[bucketPrefixLen:],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)),
@ -174,6 +175,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
StorageClass: StorageClass(storageClass),
})
})
glog.V(4).Infof("end doListFilerEntries isTruncated:%v nextMarker:%v reqDir: %v prefix: %v", isTruncated, nextMarker, reqDir, prefix)
if doErr != nil {
return doErr
}
@ -187,14 +189,14 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
reqDir, prefix = filepath.Split(strings.TrimSuffix(reqDir, "/"))
reqDir = strings.TrimSuffix(reqDir, "/")
}
_, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectory && entry.Attributes.Mime != "" && entry.Name == prefix {
_, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, false, bucketPrefixLen, func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectoryKeyObject() && entry.Name == prefix {
storageClass := "STANDARD"
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
storageClass = string(v)
}
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[bucketPrefixLen:],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + fmt.Sprintf("%x", entry.Attributes.Md5) + "\"",
Size: int64(filer.FileSize(entry)),
@ -211,6 +213,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
}
}
if len(nextMarker) > 0 {
nextMarker = nextMarker[bucketPrefixLen:]
}
response = ListBucketResult{
Name: bucket,
Prefix: originalPrefix,
@ -229,7 +235,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
return
}
func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, subEntries bool, bucketPrefixLen int, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
// invariants
// prefix and marker should be under dir, marker may contain "/"
// maxKeys should be updated for each recursion
@ -242,10 +248,15 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
}
if strings.Contains(marker, "/") {
if strings.HasSuffix(marker, "/") {
marker = strings.TrimSuffix(marker, "/")
}
sepIndex := strings.Index(marker, "/")
subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
glog.V(4).Infoln("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, false, eachEntryFn)
if sepIndex != -1 {
subPrefix, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
subDir := fmt.Sprintf("%s/%s", dir[0:bucketPrefixLen-1], subPrefix)
if strings.HasPrefix(subDir, dir) {
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, subDir, "", maxKeys, subMarker, delimiter, false, false, bucketPrefixLen, eachEntryFn)
if subErr != nil {
err = subErr
return
@ -253,9 +264,11 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
counter += subCounter
isTruncated = isTruncated || subIsTruncated
maxKeys -= subCounter
nextMarker = subDir + "/" + subNextMarker
nextMarker = subNextMarker
// finished processing this sub directory
marker = subDir
marker = subPrefix
}
}
}
if maxKeys <= 0 {
return
@ -293,45 +306,46 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
return
}
entry := resp.Entry
nextMarker = entry.Name
nextMarker = dir + "/" + entry.Name
if entry.IsDirectory {
// println("ListEntries", dir, "dir:", entry.Name)
if entry.Name == ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
continue
}
if delimiter != "/" {
if delimiter == "" {
eachEntryFn(dir, entry)
// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, eachEntryFn)
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, true, bucketPrefixLen, eachEntryFn)
if subErr != nil {
err = fmt.Errorf("doListFilerEntries2: %v", subErr)
return
}
// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
if subCounter == 0 && entry.Attributes.Mime != "" {
if subCounter == 0 && entry.IsDirectoryKeyObject() {
entry.Name += "/"
eachEntryFn(dir, entry)
counter++
}
counter += subCounter
nextMarker = entry.Name + "/" + subNextMarker
nextMarker = subNextMarker
if subIsTruncated {
isTruncated = true
return
}
} else {
} else if delimiter == "/" {
var isEmpty bool
if !s3a.option.AllowEmptyFolder && entry.Attributes.Mime == "" {
if !s3a.option.AllowEmptyFolder && !entry.IsDirectoryKeyObject() {
if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
glog.Errorf("check empty folder %s: %v", dir, err)
}
}
if !isEmpty {
nextMarker += "/"
eachEntryFn(dir, entry)
counter++
}
}
} else {
} else if !(delimiter == "/" && subEntries) {
// println("ListEntries", dir, "file:", entry.Name)
eachEntryFn(dir, entry)
counter++

Write Preview
Loading…
Cancel
Save