You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

184 lines
4.9 KiB

7 years ago
7 years ago
7 years ago
7 years ago
  1. package s3api
  2. import (
  3. "context"
  4. "fmt"
  5. "net/http"
  6. "net/url"
  7. "path/filepath"
  8. "strconv"
  9. "time"
  10. "github.com/aws/aws-sdk-go/aws"
  11. "github.com/aws/aws-sdk-go/service/s3"
  12. "github.com/chrislusf/seaweedfs/weed/filer2"
  13. "github.com/chrislusf/seaweedfs/weed/glog"
  14. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  15. "github.com/gorilla/mux"
  16. )
  17. const (
  18. maxObjectListSizeLimit = 1000 // Limit number of objects in a listObjectsResponse.
  19. )
  20. func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
  21. // https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
  22. // collect parameters
  23. vars := mux.Vars(r)
  24. bucket := vars["bucket"]
  25. glog.V(4).Infof("read v2: %v", vars)
  26. originalPrefix, marker, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
  27. if maxKeys < 0 {
  28. writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
  29. return
  30. }
  31. if delimiter != "" && delimiter != "/" {
  32. writeErrorResponse(w, ErrNotImplemented, r.URL)
  33. return
  34. }
  35. if marker == "" {
  36. marker = startAfter
  37. }
  38. response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker)
  39. if err != nil {
  40. writeErrorResponse(w, ErrInternalError, r.URL)
  41. return
  42. }
  43. writeSuccessResponseXML(w, encodeResponse(response))
  44. }
  45. func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
  46. // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
  47. // collect parameters
  48. vars := mux.Vars(r)
  49. bucket := vars["bucket"]
  50. originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
  51. if maxKeys < 0 {
  52. writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
  53. return
  54. }
  55. if delimiter != "" && delimiter != "/" {
  56. writeErrorResponse(w, ErrNotImplemented, r.URL)
  57. return
  58. }
  59. response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker)
  60. if err != nil {
  61. writeErrorResponse(w, ErrInternalError, r.URL)
  62. return
  63. }
  64. writeSuccessResponseXML(w, encodeResponse(response))
  65. }
  66. func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response *s3.ListObjectsOutput, err error) {
  67. // convert full path prefix into directory name and prefix for entry name
  68. dir, prefix := filepath.Split(originalPrefix)
  69. // check filer
  70. err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  71. request := &filer_pb.ListEntriesRequest{
  72. Directory: fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, dir),
  73. Prefix: prefix,
  74. Limit: uint32(maxKeys + 1),
  75. StartFromFileName: marker,
  76. InclusiveStartFrom: false,
  77. }
  78. resp, err := client.ListEntries(context.Background(), request)
  79. if err != nil {
  80. return fmt.Errorf("list buckets: %v", err)
  81. }
  82. var contents []*s3.Object
  83. var commonPrefixes []*s3.CommonPrefix
  84. var counter int
  85. var lastEntryName string
  86. var isTruncated bool
  87. for _, entry := range resp.Entries {
  88. counter++
  89. if counter > maxKeys {
  90. isTruncated = true
  91. break
  92. }
  93. lastEntryName = entry.Name
  94. if entry.IsDirectory {
  95. commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{
  96. Prefix: aws.String(fmt.Sprintf("%s%s/", dir, entry.Name)),
  97. })
  98. } else {
  99. contents = append(contents, &s3.Object{
  100. Key: aws.String(fmt.Sprintf("%s%s", dir, entry.Name)),
  101. LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)),
  102. ETag: aws.String("\"2345sgfwetrewrt\""), // TODO add etag
  103. Size: aws.Int64(int64(filer2.TotalSize(entry.Chunks))),
  104. Owner: &s3.Owner{
  105. ID: aws.String("bcaf161ca5fb16fd081034f"),
  106. DisplayName: aws.String("webfile"),
  107. },
  108. StorageClass: aws.String("STANDARD"),
  109. })
  110. }
  111. }
  112. response = &s3.ListObjectsOutput{
  113. Name: aws.String(bucket),
  114. Prefix: aws.String(originalPrefix),
  115. Marker: aws.String(marker),
  116. NextMarker: aws.String(lastEntryName),
  117. MaxKeys: aws.Int64(int64(maxKeys)),
  118. Delimiter: aws.String("/"),
  119. IsTruncated: aws.Bool(isTruncated),
  120. Contents: contents,
  121. CommonPrefixes: commonPrefixes,
  122. }
  123. glog.V(4).Infof("read directory: %v, found: %v", request, counter)
  124. return nil
  125. })
  126. return
  127. }
  128. func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
  129. prefix = values.Get("prefix")
  130. token = values.Get("continuation-token")
  131. startAfter = values.Get("start-after")
  132. delimiter = values.Get("delimiter")
  133. if values.Get("max-keys") != "" {
  134. maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
  135. } else {
  136. maxkeys = maxObjectListSizeLimit
  137. }
  138. fetchOwner = values.Get("fetch-owner") == "true"
  139. return
  140. }
  141. func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
  142. prefix = values.Get("prefix")
  143. marker = values.Get("marker")
  144. delimiter = values.Get("delimiter")
  145. if values.Get("max-keys") != "" {
  146. maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
  147. } else {
  148. maxkeys = maxObjectListSizeLimit
  149. }
  150. return
  151. }