You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

189 lines
5.5 KiB

6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "io"
  5. "net/http"
  6. "path"
  7. "strconv"
  8. "strings"
  9. "time"
  10. "github.com/chrislusf/seaweedfs/weed/filer2"
  11. "github.com/chrislusf/seaweedfs/weed/glog"
  12. "github.com/chrislusf/seaweedfs/weed/operation"
  13. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  14. "github.com/chrislusf/seaweedfs/weed/security"
  15. "github.com/chrislusf/seaweedfs/weed/stats"
  16. "github.com/chrislusf/seaweedfs/weed/util"
  17. )
  18. func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
  19. replication string, collection string, dataCenter string) bool {
  20. if r.Method != "POST" {
  21. glog.V(4).Infoln("AutoChunking not supported for method", r.Method)
  22. return false
  23. }
  24. // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
  25. query := r.URL.Query()
  26. parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
  27. maxMB := int32(parsedMaxMB)
  28. if maxMB <= 0 && fs.option.MaxMB > 0 {
  29. maxMB = int32(fs.option.MaxMB)
  30. }
  31. if maxMB <= 0 {
  32. glog.V(4).Infoln("AutoChunking not enabled")
  33. return false
  34. }
  35. glog.V(4).Infoln("AutoChunking level set to", maxMB, "(MB)")
  36. chunkSize := 1024 * 1024 * maxMB
  37. contentLength := int64(0)
  38. if contentLengthHeader := r.Header["Content-Length"]; len(contentLengthHeader) == 1 {
  39. contentLength, _ = strconv.ParseInt(contentLengthHeader[0], 10, 64)
  40. if contentLength <= int64(chunkSize) {
  41. glog.V(4).Infoln("Content-Length of", contentLength, "is less than the chunk size of", chunkSize, "so autoChunking will be skipped.")
  42. return false
  43. }
  44. }
  45. if contentLength <= 0 {
  46. glog.V(4).Infoln("Content-Length value is missing or unexpected so autoChunking will be skipped.")
  47. return false
  48. }
  49. reply, err := fs.doAutoChunk(ctx, w, r, contentLength, chunkSize, replication, collection, dataCenter)
  50. if err != nil {
  51. writeJsonError(w, r, http.StatusInternalServerError, err)
  52. } else if reply != nil {
  53. writeJsonQuiet(w, r, http.StatusCreated, reply)
  54. }
  55. return true
  56. }
  57. func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
  58. contentLength int64, chunkSize int32, replication string, collection string, dataCenter string) (filerResult *FilerPostResult, replyerr error) {
  59. stats.FilerRequestCounter.WithLabelValues("postAutoChunk").Inc()
  60. start := time.Now()
  61. defer func() {
  62. stats.FilerRequestHistogram.WithLabelValues("postAutoChunk").Observe(time.Since(start).Seconds())
  63. }()
  64. multipartReader, multipartReaderErr := r.MultipartReader()
  65. if multipartReaderErr != nil {
  66. return nil, multipartReaderErr
  67. }
  68. part1, part1Err := multipartReader.NextPart()
  69. if part1Err != nil {
  70. return nil, part1Err
  71. }
  72. fileName := part1.FileName()
  73. if fileName != "" {
  74. fileName = path.Base(fileName)
  75. }
  76. var fileChunks []*filer_pb.FileChunk
  77. chunkOffset := int64(0)
  78. for chunkOffset < contentLength {
  79. limitedReader := io.LimitReader(part1, int64(chunkSize))
  80. // assign one file id for one chunk
  81. fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(w, r, replication, collection, dataCenter)
  82. if assignErr != nil {
  83. return nil, assignErr
  84. }
  85. // upload the chunk to the volume server
  86. chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(len(fileChunks)+1), 10)
  87. uploadedSize, uploadErr := fs.doUpload(urlLocation, w, r, limitedReader, chunkName, "", fileId, auth)
  88. if uploadErr != nil {
  89. return nil, uploadErr
  90. }
  91. // if last chunk exhausted the reader exactly at the border
  92. if uploadedSize == 0 {
  93. break
  94. }
  95. // Save to chunk manifest structure
  96. fileChunks = append(fileChunks,
  97. &filer_pb.FileChunk{
  98. FileId: fileId,
  99. Offset: chunkOffset,
  100. Size: uint64(uploadedSize),
  101. Mtime: time.Now().UnixNano(),
  102. },
  103. )
  104. glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d) of %d", fileName, len(fileChunks), fileId, chunkOffset, chunkOffset+int64(uploadedSize), contentLength)
  105. // if last chunk was not at full chunk size, but already exhausted the reader
  106. if uploadedSize < int64(chunkSize) {
  107. break
  108. }
  109. // reset variables for the next chunk
  110. chunkOffset = chunkOffset + int64(uploadedSize)
  111. }
  112. path := r.URL.Path
  113. if strings.HasSuffix(path, "/") {
  114. if fileName != "" {
  115. path += fileName
  116. }
  117. }
  118. glog.V(4).Infoln("saving", path)
  119. entry := &filer2.Entry{
  120. FullPath: filer2.FullPath(path),
  121. Attr: filer2.Attr{
  122. Mtime: time.Now(),
  123. Crtime: time.Now(),
  124. Mode: 0660,
  125. Uid: OS_UID,
  126. Gid: OS_GID,
  127. Replication: replication,
  128. Collection: collection,
  129. TtlSec: int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)),
  130. },
  131. Chunks: fileChunks,
  132. }
  133. filerResult = &FilerPostResult{
  134. Name: fileName,
  135. Size: chunkOffset,
  136. }
  137. if dbErr := fs.filer.CreateEntry(ctx, entry, false); dbErr != nil {
  138. fs.filer.DeleteChunks(entry.Chunks)
  139. replyerr = dbErr
  140. filerResult.Error = dbErr.Error()
  141. glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr)
  142. return
  143. }
  144. return
  145. }
  146. func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request,
  147. limitedReader io.Reader, fileName string, contentType string, fileId string, auth security.EncodedJwt) (size int64, err error) {
  148. stats.FilerRequestCounter.WithLabelValues("postAutoChunkUpload").Inc()
  149. start := time.Now()
  150. defer func() {
  151. stats.FilerRequestHistogram.WithLabelValues("postAutoChunkUpload").Observe(time.Since(start).Seconds())
  152. }()
  153. uploadResult, uploadError := operation.Upload(urlLocation, fileName, limitedReader, false, contentType, nil, auth)
  154. if uploadError != nil {
  155. return 0, uploadError
  156. }
  157. return int64(uploadResult.Size), nil
  158. }