You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

190 lines
4.8 KiB

7 years ago
7 years ago
7 years ago
6 years ago
7 years ago
  1. package s3api
  2. import (
  3. "context"
  4. "fmt"
  5. "net/http"
  6. "net/url"
  7. "path/filepath"
  8. "strconv"
  9. "strings"
  10. "time"
  11. "github.com/chrislusf/seaweedfs/weed/filer2"
  12. "github.com/chrislusf/seaweedfs/weed/glog"
  13. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  14. "github.com/gorilla/mux"
  15. )
  16. const (
  17. maxObjectListSizeLimit = 1000 // Limit number of objects in a listObjectsResponse.
  18. )
  19. func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
  20. // https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
  21. // collect parameters
  22. vars := mux.Vars(r)
  23. bucket := vars["bucket"]
  24. glog.V(4).Infof("read v2: %v", vars)
  25. originalPrefix, marker, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
  26. if maxKeys < 0 {
  27. writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
  28. return
  29. }
  30. if delimiter != "" && delimiter != "/" {
  31. writeErrorResponse(w, ErrNotImplemented, r.URL)
  32. return
  33. }
  34. if marker == "" {
  35. marker = startAfter
  36. }
  37. ctx := context.Background()
  38. response, err := s3a.listFilerEntries(ctx, bucket, originalPrefix, maxKeys, marker)
  39. if err != nil {
  40. writeErrorResponse(w, ErrInternalError, r.URL)
  41. return
  42. }
  43. writeSuccessResponseXML(w, encodeResponse(response))
  44. }
  45. func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
  46. // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
  47. // collect parameters
  48. vars := mux.Vars(r)
  49. bucket := vars["bucket"]
  50. ctx := context.Background()
  51. originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
  52. if maxKeys < 0 {
  53. writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
  54. return
  55. }
  56. if delimiter != "" && delimiter != "/" {
  57. writeErrorResponse(w, ErrNotImplemented, r.URL)
  58. return
  59. }
  60. response, err := s3a.listFilerEntries(ctx, bucket, originalPrefix, maxKeys, marker)
  61. if err != nil {
  62. writeErrorResponse(w, ErrInternalError, r.URL)
  63. return
  64. }
  65. writeSuccessResponseXML(w, encodeResponse(response))
  66. }
  67. func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPrefix string, maxKeys int, marker string) (response ListBucketResult, err error) {
  68. // convert full path prefix into directory name and prefix for entry name
  69. dir, prefix := filepath.Split(originalPrefix)
  70. if strings.HasPrefix(dir, "/") {
  71. dir = dir[1:]
  72. }
  73. // check filer
  74. err = s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
  75. request := &filer_pb.ListEntriesRequest{
  76. Directory: fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, dir),
  77. Prefix: prefix,
  78. Limit: uint32(maxKeys + 1),
  79. StartFromFileName: marker,
  80. InclusiveStartFrom: false,
  81. }
  82. resp, err := client.ListEntries(ctx, request)
  83. if err != nil {
  84. return fmt.Errorf("list buckets: %v", err)
  85. }
  86. var contents []ListEntry
  87. var commonPrefixes []PrefixEntry
  88. var counter int
  89. var lastEntryName string
  90. var isTruncated bool
  91. for _, entry := range resp.Entries {
  92. counter++
  93. if counter > maxKeys {
  94. isTruncated = true
  95. break
  96. }
  97. lastEntryName = entry.Name
  98. if entry.IsDirectory {
  99. commonPrefixes = append(commonPrefixes, PrefixEntry{
  100. Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
  101. })
  102. } else {
  103. contents = append(contents, ListEntry{
  104. Key: fmt.Sprintf("%s%s", dir, entry.Name),
  105. LastModified: time.Unix(entry.Attributes.Mtime, 0),
  106. ETag: "\"" + filer2.ETag(entry.Chunks) + "\"",
  107. Size: int64(filer2.TotalSize(entry.Chunks)),
  108. Owner: CanonicalUser{
  109. ID: fmt.Sprintf("%x", entry.Attributes.Uid),
  110. DisplayName: entry.Attributes.UserName,
  111. },
  112. StorageClass: "STANDARD",
  113. })
  114. }
  115. }
  116. response = ListBucketResult{
  117. Name: bucket,
  118. Prefix: originalPrefix,
  119. Marker: marker,
  120. NextMarker: lastEntryName,
  121. MaxKeys: maxKeys,
  122. Delimiter: "/",
  123. IsTruncated: isTruncated,
  124. Contents: contents,
  125. CommonPrefixes: commonPrefixes,
  126. }
  127. glog.V(4).Infof("read directory: %v, found: %v, %+v", request, counter, response)
  128. return nil
  129. })
  130. return
  131. }
  132. func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
  133. prefix = values.Get("prefix")
  134. token = values.Get("continuation-token")
  135. startAfter = values.Get("start-after")
  136. delimiter = values.Get("delimiter")
  137. if values.Get("max-keys") != "" {
  138. maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
  139. } else {
  140. maxkeys = maxObjectListSizeLimit
  141. }
  142. fetchOwner = values.Get("fetch-owner") == "true"
  143. return
  144. }
  145. func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
  146. prefix = values.Get("prefix")
  147. marker = values.Get("marker")
  148. delimiter = values.Get("delimiter")
  149. if values.Get("max-keys") != "" {
  150. maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
  151. } else {
  152. maxkeys = maxObjectListSizeLimit
  153. }
  154. return
  155. }