You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

269 lines
8.0 KiB

4 years ago
4 years ago
6 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package s3api
  2. import (
  3. "fmt"
  4. "github.com/chrislusf/seaweedfs/weed/glog"
  5. "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
  6. weed_server "github.com/chrislusf/seaweedfs/weed/server"
  7. "net/http"
  8. "net/url"
  9. "strconv"
  10. "strings"
  11. "github.com/aws/aws-sdk-go/aws"
  12. "github.com/aws/aws-sdk-go/service/s3"
  13. )
  14. const (
  15. maxObjectListSizeLimit = 10000 // Limit number of objects in a listObjectsResponse.
  16. maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
  17. maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
  18. globalMaxPartID = 100000
  19. )
  20. // NewMultipartUploadHandler - New multipart upload.
  21. func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
  22. bucket, object := getBucketAndObject(r)
  23. createMultipartUploadInput := &s3.CreateMultipartUploadInput{
  24. Bucket: aws.String(bucket),
  25. Key: objectKey(aws.String(object)),
  26. Metadata: make(map[string]*string),
  27. }
  28. metadata := weed_server.SaveAmzMetaData(r, nil, false)
  29. for k, v := range metadata {
  30. createMultipartUploadInput.Metadata[k] = aws.String(string(v))
  31. }
  32. contentType := r.Header.Get("Content-Type")
  33. if contentType != "" {
  34. createMultipartUploadInput.ContentType = &contentType
  35. }
  36. response, errCode := s3a.createMultipartUpload(createMultipartUploadInput)
  37. glog.V(2).Info("NewMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)
  38. if errCode != s3err.ErrNone {
  39. s3err.WriteErrorResponse(w, errCode, r)
  40. return
  41. }
  42. writeSuccessResponseXML(w, response)
  43. }
  44. // CompleteMultipartUploadHandler - Completes multipart upload.
  45. func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
  46. bucket, object := getBucketAndObject(r)
  47. // Get upload id.
  48. uploadID, _, _, _ := getObjectResources(r.URL.Query())
  49. response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
  50. Bucket: aws.String(bucket),
  51. Key: objectKey(aws.String(object)),
  52. UploadId: aws.String(uploadID),
  53. })
  54. glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)
  55. if errCode != s3err.ErrNone {
  56. s3err.WriteErrorResponse(w, errCode, r)
  57. return
  58. }
  59. writeSuccessResponseXML(w, response)
  60. }
  61. // AbortMultipartUploadHandler - Aborts multipart upload.
  62. func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
  63. bucket, object := getBucketAndObject(r)
  64. // Get upload id.
  65. uploadID, _, _, _ := getObjectResources(r.URL.Query())
  66. response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
  67. Bucket: aws.String(bucket),
  68. Key: objectKey(aws.String(object)),
  69. UploadId: aws.String(uploadID),
  70. })
  71. if errCode != s3err.ErrNone {
  72. s3err.WriteErrorResponse(w, errCode, r)
  73. return
  74. }
  75. glog.V(2).Info("AbortMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)))
  76. writeSuccessResponseXML(w, response)
  77. }
  78. // ListMultipartUploadsHandler - Lists multipart uploads.
  79. func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
  80. bucket, _ := getBucketAndObject(r)
  81. prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query())
  82. if maxUploads < 0 {
  83. s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxUploads, r)
  84. return
  85. }
  86. if keyMarker != "" {
  87. // Marker not common with prefix is not implemented.
  88. if !strings.HasPrefix(keyMarker, prefix) {
  89. s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
  90. return
  91. }
  92. }
  93. response, errCode := s3a.listMultipartUploads(&s3.ListMultipartUploadsInput{
  94. Bucket: aws.String(bucket),
  95. Delimiter: aws.String(delimiter),
  96. EncodingType: aws.String(encodingType),
  97. KeyMarker: aws.String(keyMarker),
  98. MaxUploads: aws.Int64(int64(maxUploads)),
  99. Prefix: aws.String(prefix),
  100. UploadIdMarker: aws.String(uploadIDMarker),
  101. })
  102. glog.V(2).Info("ListMultipartUploadsHandler", string(s3err.EncodeXMLResponse(response)), errCode)
  103. if errCode != s3err.ErrNone {
  104. s3err.WriteErrorResponse(w, errCode, r)
  105. return
  106. }
  107. // TODO handle encodingType
  108. writeSuccessResponseXML(w, response)
  109. }
  110. // ListObjectPartsHandler - Lists object parts in a multipart upload.
  111. func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
  112. bucket, object := getBucketAndObject(r)
  113. uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
  114. if partNumberMarker < 0 {
  115. s3err.WriteErrorResponse(w, s3err.ErrInvalidPartNumberMarker, r)
  116. return
  117. }
  118. if maxParts < 0 {
  119. s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
  120. return
  121. }
  122. response, errCode := s3a.listObjectParts(&s3.ListPartsInput{
  123. Bucket: aws.String(bucket),
  124. Key: objectKey(aws.String(object)),
  125. MaxParts: aws.Int64(int64(maxParts)),
  126. PartNumberMarker: aws.Int64(int64(partNumberMarker)),
  127. UploadId: aws.String(uploadID),
  128. })
  129. glog.V(2).Info("ListObjectPartsHandler", string(s3err.EncodeXMLResponse(response)), errCode)
  130. if errCode != s3err.ErrNone {
  131. s3err.WriteErrorResponse(w, errCode, r)
  132. return
  133. }
  134. writeSuccessResponseXML(w, response)
  135. }
  136. // PutObjectPartHandler - Put an object part in a multipart upload.
  137. func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
  138. bucket, _ := getBucketAndObject(r)
  139. uploadID := r.URL.Query().Get("uploadId")
  140. exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
  141. if !exists {
  142. s3err.WriteErrorResponse(w, s3err.ErrNoSuchUpload, r)
  143. return
  144. }
  145. partIDString := r.URL.Query().Get("partNumber")
  146. partID, err := strconv.Atoi(partIDString)
  147. if err != nil {
  148. s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r)
  149. return
  150. }
  151. if partID > globalMaxPartID {
  152. s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
  153. return
  154. }
  155. dataReader := r.Body
  156. if s3a.iam.isEnabled() {
  157. rAuthType := getRequestAuthType(r)
  158. var s3ErrCode s3err.ErrorCode
  159. switch rAuthType {
  160. case authTypeStreamingSigned:
  161. dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
  162. case authTypeSignedV2, authTypePresignedV2:
  163. _, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
  164. case authTypePresigned, authTypeSigned:
  165. _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
  166. }
  167. if s3ErrCode != s3err.ErrNone {
  168. s3err.WriteErrorResponse(w, s3ErrCode, r)
  169. return
  170. }
  171. }
  172. defer dataReader.Close()
  173. uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
  174. s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, bucket)
  175. etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
  176. if errCode != s3err.ErrNone {
  177. s3err.WriteErrorResponse(w, errCode, r)
  178. return
  179. }
  180. setEtag(w, etag)
  181. writeSuccessResponseEmpty(w)
  182. }
  183. func (s3a *S3ApiServer) genUploadsFolder(bucket string) string {
  184. return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket)
  185. }
  186. // Parse bucket url queries for ?uploads
  187. func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
  188. prefix = values.Get("prefix")
  189. keyMarker = values.Get("key-marker")
  190. uploadIDMarker = values.Get("upload-id-marker")
  191. delimiter = values.Get("delimiter")
  192. if values.Get("max-uploads") != "" {
  193. maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
  194. } else {
  195. maxUploads = maxUploadsList
  196. }
  197. encodingType = values.Get("encoding-type")
  198. return
  199. }
  200. // Parse object url queries
  201. func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
  202. uploadID = values.Get("uploadId")
  203. partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
  204. if values.Get("max-parts") != "" {
  205. maxParts, _ = strconv.Atoi(values.Get("max-parts"))
  206. } else {
  207. maxParts = maxPartsList
  208. }
  209. encodingType = values.Get("encoding-type")
  210. return
  211. }
  212. type byCompletedPartNumber []*s3.CompletedPart
  213. func (a byCompletedPartNumber) Len() int { return len(a) }
  214. func (a byCompletedPartNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  215. func (a byCompletedPartNumber) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber }