You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

348 lines
10 KiB

5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
6 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "crypto/md5"
  5. "fmt"
  6. "hash"
  7. "io"
  8. "io/ioutil"
  9. "net/http"
  10. "os"
  11. "path"
  12. "strconv"
  13. "strings"
  14. "time"
  15. "github.com/chrislusf/seaweedfs/weed/filer"
  16. "github.com/chrislusf/seaweedfs/weed/util/log"
  17. "github.com/chrislusf/seaweedfs/weed/operation"
  18. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  19. xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
  20. "github.com/chrislusf/seaweedfs/weed/security"
  21. "github.com/chrislusf/seaweedfs/weed/stats"
  22. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  23. "github.com/chrislusf/seaweedfs/weed/util"
  24. )
  25. func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
  26. // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
  27. query := r.URL.Query()
  28. parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
  29. maxMB := int32(parsedMaxMB)
  30. if maxMB <= 0 && fs.option.MaxMB > 0 {
  31. maxMB = int32(fs.option.MaxMB)
  32. }
  33. chunkSize := 1024 * 1024 * maxMB
  34. stats.FilerRequestCounter.WithLabelValues("postAutoChunk").Inc()
  35. start := time.Now()
  36. defer func() {
  37. stats.FilerRequestHistogram.WithLabelValues("postAutoChunk").Observe(time.Since(start).Seconds())
  38. }()
  39. var reply *FilerPostResult
  40. var err error
  41. var md5bytes []byte
  42. if r.Method == "POST" {
  43. if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
  44. reply, err = fs.mkdir(ctx, w, r)
  45. } else {
  46. reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, so)
  47. }
  48. } else {
  49. reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, so)
  50. }
  51. if err != nil {
  52. writeJsonError(w, r, http.StatusInternalServerError, err)
  53. } else if reply != nil {
  54. if len(md5bytes) > 0 {
  55. w.Header().Set("Content-MD5", util.Base64Encode(md5bytes))
  56. }
  57. writeJsonQuiet(w, r, http.StatusCreated, reply)
  58. }
  59. }
  60. func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  61. multipartReader, multipartReaderErr := r.MultipartReader()
  62. if multipartReaderErr != nil {
  63. return nil, nil, multipartReaderErr
  64. }
  65. part1, part1Err := multipartReader.NextPart()
  66. if part1Err != nil {
  67. return nil, nil, part1Err
  68. }
  69. fileName := part1.FileName()
  70. if fileName != "" {
  71. fileName = path.Base(fileName)
  72. }
  73. contentType := part1.Header.Get("Content-Type")
  74. if contentType == "application/octet-stream" {
  75. contentType = ""
  76. }
  77. fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, so)
  78. if err != nil {
  79. return nil, nil, err
  80. }
  81. fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks)
  82. if replyerr != nil {
  83. log.Infof("manifestize %s: %v", r.RequestURI, replyerr)
  84. return
  85. }
  86. md5bytes = md5Hash.Sum(nil)
  87. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset)
  88. return
  89. }
  90. func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  91. fileName := ""
  92. contentType := ""
  93. fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, so)
  94. if err != nil {
  95. return nil, nil, err
  96. }
  97. fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks)
  98. if replyerr != nil {
  99. log.Infof("manifestize %s: %v", r.RequestURI, replyerr)
  100. return
  101. }
  102. md5bytes = md5Hash.Sum(nil)
  103. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset)
  104. return
  105. }
  106. func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64) (filerResult *FilerPostResult, replyerr error) {
  107. // detect file mode
  108. modeStr := r.URL.Query().Get("mode")
  109. if modeStr == "" {
  110. modeStr = "0660"
  111. }
  112. mode, err := strconv.ParseUint(modeStr, 8, 32)
  113. if err != nil {
  114. log.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  115. mode = 0660
  116. }
  117. // fix the path
  118. path := r.URL.Path
  119. if strings.HasSuffix(path, "/") {
  120. if fileName != "" {
  121. path += fileName
  122. }
  123. }
  124. // fix the crTime
  125. existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
  126. crTime := time.Now()
  127. if err == nil && existingEntry != nil {
  128. crTime = existingEntry.Crtime
  129. }
  130. log.Trace("saving", path)
  131. entry := &filer.Entry{
  132. FullPath: util.FullPath(path),
  133. Attr: filer.Attr{
  134. Mtime: time.Now(),
  135. Crtime: crTime,
  136. Mode: os.FileMode(mode),
  137. Uid: OS_UID,
  138. Gid: OS_GID,
  139. Replication: so.Replication,
  140. Collection: so.Collection,
  141. TtlSec: so.TtlSeconds,
  142. Mime: contentType,
  143. Md5: md5bytes,
  144. FileSize: uint64(chunkOffset),
  145. },
  146. Chunks: fileChunks,
  147. }
  148. filerResult = &FilerPostResult{
  149. Name: fileName,
  150. Size: chunkOffset,
  151. }
  152. if entry.Extended == nil {
  153. entry.Extended = make(map[string][]byte)
  154. }
  155. fs.saveAmzMetaData(r, entry)
  156. for k, v := range r.Header {
  157. if len(v) > 0 && strings.HasPrefix(k, needle.PairNamePrefix) {
  158. entry.Extended[k] = []byte(v[0])
  159. }
  160. }
  161. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil {
  162. fs.filer.DeleteChunks(entry.Chunks)
  163. replyerr = dbErr
  164. filerResult.Error = dbErr.Error()
  165. log.Infof("failing to write %s to filer server : %v", path, dbErr)
  166. }
  167. return filerResult, replyerr
  168. }
  169. func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error) {
  170. var fileChunks []*filer_pb.FileChunk
  171. md5Hash := md5.New()
  172. var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))
  173. chunkOffset := int64(0)
  174. for {
  175. limitedReader := io.LimitReader(partReader, int64(chunkSize))
  176. // assign one file id for one chunk
  177. fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
  178. if assignErr != nil {
  179. return nil, nil, 0, assignErr
  180. }
  181. // upload the chunk to the volume server
  182. uploadResult, uploadErr := fs.doUpload(urlLocation, w, r, limitedReader, fileName, contentType, nil, auth)
  183. if uploadErr != nil {
  184. return nil, nil, 0, uploadErr
  185. }
  186. // if last chunk exhausted the reader exactly at the border
  187. if uploadResult.Size == 0 {
  188. break
  189. }
  190. // Save to chunk manifest structure
  191. fileChunks = append(fileChunks, uploadResult.ToPbFileChunk(fileId, chunkOffset))
  192. log.Tracef("uploaded %s chunk %d to %s [%d,%d)", fileName, len(fileChunks), fileId, chunkOffset, chunkOffset+int64(uploadResult.Size))
  193. // reset variables for the next chunk
  194. chunkOffset = chunkOffset + int64(uploadResult.Size)
  195. // if last chunk was not at full chunk size, but already exhausted the reader
  196. if int64(uploadResult.Size) < int64(chunkSize) {
  197. break
  198. }
  199. }
  200. return fileChunks, md5Hash, chunkOffset, nil
  201. }
  202. func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error) {
  203. stats.FilerRequestCounter.WithLabelValues("postAutoChunkUpload").Inc()
  204. start := time.Now()
  205. defer func() {
  206. stats.FilerRequestHistogram.WithLabelValues("postAutoChunkUpload").Observe(time.Since(start).Seconds())
  207. }()
  208. uploadResult, err, _ := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
  209. return uploadResult, err
  210. }
  211. func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
  212. return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
  213. // assign one file id for one chunk
  214. fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
  215. if assignErr != nil {
  216. return nil, "", "", assignErr
  217. }
  218. // upload the chunk to the volume server
  219. uploadResult, uploadErr, _ := operation.Upload(urlLocation, name, fs.option.Cipher, reader, false, "", nil, auth)
  220. if uploadErr != nil {
  221. return nil, "", "", uploadErr
  222. }
  223. return uploadResult.ToPbFileChunk(fileId, offset), so.Collection, so.Replication, nil
  224. }
  225. }
  226. func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http.Request) (filerResult *FilerPostResult, replyerr error) {
  227. // detect file mode
  228. modeStr := r.URL.Query().Get("mode")
  229. if modeStr == "" {
  230. modeStr = "0660"
  231. }
  232. mode, err := strconv.ParseUint(modeStr, 8, 32)
  233. if err != nil {
  234. log.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  235. mode = 0660
  236. }
  237. // fix the path
  238. path := r.URL.Path
  239. if strings.HasSuffix(path, "/") {
  240. path = path[:len(path)-1]
  241. }
  242. existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
  243. if err == nil && existingEntry != nil {
  244. replyerr = fmt.Errorf("dir %s already exists", path)
  245. return
  246. }
  247. log.Trace("mkdir", path)
  248. entry := &filer.Entry{
  249. FullPath: util.FullPath(path),
  250. Attr: filer.Attr{
  251. Mtime: time.Now(),
  252. Crtime: time.Now(),
  253. Mode: os.FileMode(mode) | os.ModeDir,
  254. Uid: OS_UID,
  255. Gid: OS_GID,
  256. },
  257. }
  258. filerResult = &FilerPostResult{
  259. Name: util.FullPath(path).Name(),
  260. }
  261. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil {
  262. replyerr = dbErr
  263. filerResult.Error = dbErr.Error()
  264. log.Infof("failing to create dir %s on filer server : %v", path, dbErr)
  265. }
  266. return filerResult, replyerr
  267. }
  268. func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
  269. if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
  270. entry.Extended[xhttp.AmzStorageClass] = []byte(sc)
  271. }
  272. if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
  273. for _, v := range strings.Split(tags, "&") {
  274. tag := strings.Split(v, "=")
  275. if len(tag) == 2 {
  276. entry.Extended[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
  277. }
  278. }
  279. }
  280. for header, values := range r.Header {
  281. if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
  282. for _, value := range values {
  283. entry.Extended[header] = []byte(value)
  284. }
  285. }
  286. }
  287. }