You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

218 lines
6.8 KiB

6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
  1. package s3api
  2. import (
  3. "encoding/xml"
  4. "fmt"
  5. "path/filepath"
  6. "strconv"
  7. "strings"
  8. "time"
  9. "github.com/aws/aws-sdk-go/aws"
  10. "github.com/aws/aws-sdk-go/service/s3"
  11. "github.com/google/uuid"
  12. "github.com/chrislusf/seaweedfs/weed/filer2"
  13. "github.com/chrislusf/seaweedfs/weed/glog"
  14. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  15. )
  16. type InitiateMultipartUploadResult struct {
  17. XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
  18. s3.CreateMultipartUploadOutput
  19. }
  20. func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code ErrorCode) {
  21. uploadId, _ := uuid.NewRandom()
  22. uploadIdString := uploadId.String()
  23. if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
  24. if entry.Extended == nil {
  25. entry.Extended = make(map[string][]byte)
  26. }
  27. entry.Extended["key"] = []byte(*input.Key)
  28. }); err != nil {
  29. glog.Errorf("NewMultipartUpload error: %v", err)
  30. return nil, ErrInternalError
  31. }
  32. output = &InitiateMultipartUploadResult{
  33. CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
  34. Bucket: input.Bucket,
  35. Key: objectKey(input.Key),
  36. UploadId: aws.String(uploadIdString),
  37. },
  38. }
  39. return
  40. }
  41. type CompleteMultipartUploadResult struct {
  42. XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
  43. s3.CompleteMultipartUploadOutput
  44. }
  45. func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code ErrorCode) {
  46. uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
  47. entries, err := s3a.list(uploadDirectory, "", "", false, 0)
  48. if err != nil || len(entries) == 0 {
  49. glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries))
  50. return nil, ErrNoSuchUpload
  51. }
  52. var finalParts []*filer_pb.FileChunk
  53. var offset int64
  54. for _, entry := range entries {
  55. if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
  56. for _, chunk := range entry.Chunks {
  57. p := &filer_pb.FileChunk{
  58. FileId: chunk.GetFileIdString(),
  59. Offset: offset,
  60. Size: chunk.Size,
  61. Mtime: chunk.Mtime,
  62. CipherKey: chunk.CipherKey,
  63. ETag: chunk.ETag,
  64. }
  65. finalParts = append(finalParts, p)
  66. offset += int64(chunk.Size)
  67. }
  68. }
  69. }
  70. entryName := filepath.Base(*input.Key)
  71. dirName := filepath.Dir(*input.Key)
  72. if dirName == "." {
  73. dirName = ""
  74. }
  75. if strings.HasPrefix(dirName, "/") {
  76. dirName = dirName[1:]
  77. }
  78. dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName)
  79. // remove suffix '/'
  80. if strings.HasSuffix(dirName, "/") {
  81. dirName = dirName[:len(dirName)-1]
  82. }
  83. err = s3a.mkFile(dirName, entryName, finalParts)
  84. if err != nil {
  85. glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
  86. return nil, ErrInternalError
  87. }
  88. output = &CompleteMultipartUploadResult{
  89. CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
  90. Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer, dirName, entryName)),
  91. Bucket: input.Bucket,
  92. ETag: aws.String("\"" + filer2.ETagChunks(finalParts) + "\""),
  93. Key: objectKey(input.Key),
  94. },
  95. }
  96. if err = s3a.rm(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil {
  97. glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err)
  98. }
  99. return
  100. }
  101. func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code ErrorCode) {
  102. exists, err := s3a.exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true)
  103. if err != nil {
  104. glog.V(1).Infof("bucket %s abort upload %s: %v", *input.Bucket, *input.UploadId, err)
  105. return nil, ErrNoSuchUpload
  106. }
  107. if exists {
  108. err = s3a.rm(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true, true)
  109. }
  110. if err != nil {
  111. glog.V(1).Infof("bucket %s remove upload %s: %v", *input.Bucket, *input.UploadId, err)
  112. return nil, ErrInternalError
  113. }
  114. return &s3.AbortMultipartUploadOutput{}, ErrNone
  115. }
  116. type ListMultipartUploadsResult struct {
  117. XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
  118. s3.ListMultipartUploadsOutput
  119. }
  120. func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
  121. output = &ListMultipartUploadsResult{
  122. ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
  123. Bucket: input.Bucket,
  124. Delimiter: input.Delimiter,
  125. EncodingType: input.EncodingType,
  126. KeyMarker: input.KeyMarker,
  127. MaxUploads: input.MaxUploads,
  128. Prefix: input.Prefix,
  129. },
  130. }
  131. entries, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), *input.Prefix, *input.KeyMarker, true, uint32(*input.MaxUploads))
  132. if err != nil {
  133. glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err)
  134. return
  135. }
  136. for _, entry := range entries {
  137. if entry.Extended != nil {
  138. key := entry.Extended["key"]
  139. output.Uploads = append(output.Uploads, &s3.MultipartUpload{
  140. Key: objectKey(aws.String(string(key))),
  141. UploadId: aws.String(entry.Name),
  142. })
  143. }
  144. }
  145. return
  146. }
  147. type ListPartsResult struct {
  148. XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
  149. s3.ListPartsOutput
  150. }
  151. func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
  152. output = &ListPartsResult{
  153. ListPartsOutput: s3.ListPartsOutput{
  154. Bucket: input.Bucket,
  155. Key: objectKey(input.Key),
  156. UploadId: input.UploadId,
  157. MaxParts: input.MaxParts, // the maximum number of parts to return.
  158. PartNumberMarker: input.PartNumberMarker, // the part number starts after this, exclusive
  159. },
  160. }
  161. entries, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d.part", *input.PartNumberMarker), false, uint32(*input.MaxParts))
  162. if err != nil {
  163. glog.Errorf("listObjectParts %s %s error: %v", *input.Bucket, *input.UploadId, err)
  164. return nil, ErrNoSuchUpload
  165. }
  166. for _, entry := range entries {
  167. if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
  168. partNumberString := entry.Name[:len(entry.Name)-len(".part")]
  169. partNumber, err := strconv.Atoi(partNumberString)
  170. if err != nil {
  171. glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err)
  172. continue
  173. }
  174. output.Parts = append(output.Parts, &s3.Part{
  175. PartNumber: aws.Int64(int64(partNumber)),
  176. LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0).UTC()),
  177. Size: aws.Int64(int64(filer2.FileSize(entry))),
  178. ETag: aws.String("\"" + filer2.ETag(entry) + "\""),
  179. })
  180. }
  181. }
  182. return
  183. }