You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

352 lines
10 KiB

5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
6 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "crypto/md5"
  5. "fmt"
  6. "hash"
  7. "io"
  8. "io/ioutil"
  9. "net/http"
  10. "os"
  11. "path"
  12. "strconv"
  13. "strings"
  14. "time"
  15. "github.com/chrislusf/seaweedfs/weed/filer"
  16. "github.com/chrislusf/seaweedfs/weed/glog"
  17. "github.com/chrislusf/seaweedfs/weed/operation"
  18. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  19. xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
  20. "github.com/chrislusf/seaweedfs/weed/security"
  21. "github.com/chrislusf/seaweedfs/weed/stats"
  22. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  23. "github.com/chrislusf/seaweedfs/weed/util"
  24. )
  25. func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
  26. // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
  27. query := r.URL.Query()
  28. parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
  29. maxMB := int32(parsedMaxMB)
  30. if maxMB <= 0 && fs.option.MaxMB > 0 {
  31. maxMB = int32(fs.option.MaxMB)
  32. }
  33. chunkSize := 1024 * 1024 * maxMB
  34. stats.FilerRequestCounter.WithLabelValues("postAutoChunk").Inc()
  35. start := time.Now()
  36. defer func() {
  37. stats.FilerRequestHistogram.WithLabelValues("postAutoChunk").Observe(time.Since(start).Seconds())
  38. }()
  39. var reply *FilerPostResult
  40. var err error
  41. var md5bytes []byte
  42. if r.Method == "POST" {
  43. if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
  44. reply, err = fs.mkdir(ctx, w, r)
  45. } else {
  46. reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, so)
  47. }
  48. } else {
  49. reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, so)
  50. }
  51. if err != nil {
  52. if strings.HasPrefix(err.Error(), "read input:") {
  53. writeJsonError(w, r, 499, err)
  54. } else {
  55. writeJsonError(w, r, http.StatusInternalServerError, err)
  56. }
  57. } else if reply != nil {
  58. if len(md5bytes) > 0 {
  59. w.Header().Set("Content-MD5", util.Base64Encode(md5bytes))
  60. }
  61. writeJsonQuiet(w, r, http.StatusCreated, reply)
  62. }
  63. }
  64. func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  65. multipartReader, multipartReaderErr := r.MultipartReader()
  66. if multipartReaderErr != nil {
  67. return nil, nil, multipartReaderErr
  68. }
  69. part1, part1Err := multipartReader.NextPart()
  70. if part1Err != nil {
  71. return nil, nil, part1Err
  72. }
  73. fileName := part1.FileName()
  74. if fileName != "" {
  75. fileName = path.Base(fileName)
  76. }
  77. contentType := part1.Header.Get("Content-Type")
  78. if contentType == "application/octet-stream" {
  79. contentType = ""
  80. }
  81. fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, so)
  82. if err != nil {
  83. return nil, nil, err
  84. }
  85. fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks)
  86. if replyerr != nil {
  87. glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr)
  88. return
  89. }
  90. md5bytes = md5Hash.Sum(nil)
  91. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
  92. return
  93. }
  94. func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  95. fileName := ""
  96. contentType := ""
  97. fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, so)
  98. if err != nil {
  99. return nil, nil, err
  100. }
  101. fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks)
  102. if replyerr != nil {
  103. glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr)
  104. return
  105. }
  106. md5bytes = md5Hash.Sum(nil)
  107. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
  108. return
  109. }
  110. func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) {
  111. // detect file mode
  112. modeStr := r.URL.Query().Get("mode")
  113. if modeStr == "" {
  114. modeStr = "0660"
  115. }
  116. mode, err := strconv.ParseUint(modeStr, 8, 32)
  117. if err != nil {
  118. glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  119. mode = 0660
  120. }
  121. // fix the path
  122. path := r.URL.Path
  123. if strings.HasSuffix(path, "/") {
  124. if fileName != "" {
  125. path += fileName
  126. }
  127. }
  128. glog.V(4).Infoln("saving", path)
  129. entry := &filer.Entry{
  130. FullPath: util.FullPath(path),
  131. Attr: filer.Attr{
  132. Mtime: time.Now(),
  133. Crtime: time.Now(),
  134. Mode: os.FileMode(mode),
  135. Uid: OS_UID,
  136. Gid: OS_GID,
  137. Replication: so.Replication,
  138. Collection: so.Collection,
  139. TtlSec: so.TtlSeconds,
  140. Mime: contentType,
  141. Md5: md5bytes,
  142. FileSize: uint64(chunkOffset),
  143. },
  144. Chunks: fileChunks,
  145. Content: content,
  146. }
  147. filerResult = &FilerPostResult{
  148. Name: fileName,
  149. Size: chunkOffset,
  150. }
  151. if entry.Extended == nil {
  152. entry.Extended = make(map[string][]byte)
  153. }
  154. fs.saveAmzMetaData(r, entry)
  155. for k, v := range r.Header {
  156. if len(v) > 0 && strings.HasPrefix(k, needle.PairNamePrefix) {
  157. entry.Extended[k] = []byte(v[0])
  158. }
  159. }
  160. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil {
  161. fs.filer.DeleteChunks(entry.Chunks)
  162. replyerr = dbErr
  163. filerResult.Error = dbErr.Error()
  164. glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr)
  165. }
  166. return filerResult, replyerr
  167. }
  168. func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error, []byte) {
  169. var fileChunks []*filer_pb.FileChunk
  170. md5Hash := md5.New()
  171. var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))
  172. chunkOffset := int64(0)
  173. var smallContent, content []byte
  174. for {
  175. limitedReader := io.LimitReader(partReader, int64(chunkSize))
  176. // assign one file id for one chunk
  177. fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
  178. if assignErr != nil {
  179. return nil, nil, 0, assignErr, nil
  180. }
  181. // upload the chunk to the volume server
  182. uploadResult, uploadErr, data := fs.doUpload(urlLocation, w, r, limitedReader, fileName, contentType, nil, auth)
  183. if uploadErr != nil {
  184. return nil, nil, 0, uploadErr, nil
  185. }
  186. content = data
  187. // if last chunk exhausted the reader exactly at the border
  188. if uploadResult.Size == 0 {
  189. break
  190. }
  191. // Save to chunk manifest structure
  192. fileChunks = append(fileChunks, uploadResult.ToPbFileChunk(fileId, chunkOffset))
  193. glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, len(fileChunks), fileId, chunkOffset, chunkOffset+int64(uploadResult.Size))
  194. // reset variables for the next chunk
  195. chunkOffset = chunkOffset + int64(uploadResult.Size)
  196. // if last chunk was not at full chunk size, but already exhausted the reader
  197. if int64(uploadResult.Size) < int64(chunkSize) {
  198. break
  199. }
  200. }
  201. if chunkOffset < fs.option.CacheToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && chunkOffset < 4*1024 {
  202. smallContent = content
  203. }
  204. return fileChunks, md5Hash, chunkOffset, nil, smallContent
  205. }
  206. func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
  207. stats.FilerRequestCounter.WithLabelValues("postAutoChunkUpload").Inc()
  208. start := time.Now()
  209. defer func() {
  210. stats.FilerRequestHistogram.WithLabelValues("postAutoChunkUpload").Observe(time.Since(start).Seconds())
  211. }()
  212. uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
  213. return uploadResult, err, data
  214. }
  215. func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
  216. return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
  217. // assign one file id for one chunk
  218. fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
  219. if assignErr != nil {
  220. return nil, "", "", assignErr
  221. }
  222. // upload the chunk to the volume server
  223. uploadResult, uploadErr, _ := operation.Upload(urlLocation, name, fs.option.Cipher, reader, false, "", nil, auth)
  224. if uploadErr != nil {
  225. return nil, "", "", uploadErr
  226. }
  227. return uploadResult.ToPbFileChunk(fileId, offset), so.Collection, so.Replication, nil
  228. }
  229. }
  230. func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http.Request) (filerResult *FilerPostResult, replyerr error) {
  231. // detect file mode
  232. modeStr := r.URL.Query().Get("mode")
  233. if modeStr == "" {
  234. modeStr = "0660"
  235. }
  236. mode, err := strconv.ParseUint(modeStr, 8, 32)
  237. if err != nil {
  238. glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  239. mode = 0660
  240. }
  241. // fix the path
  242. path := r.URL.Path
  243. if strings.HasSuffix(path, "/") {
  244. path = path[:len(path)-1]
  245. }
  246. existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
  247. if err == nil && existingEntry != nil {
  248. replyerr = fmt.Errorf("dir %s already exists", path)
  249. return
  250. }
  251. glog.V(4).Infoln("mkdir", path)
  252. entry := &filer.Entry{
  253. FullPath: util.FullPath(path),
  254. Attr: filer.Attr{
  255. Mtime: time.Now(),
  256. Crtime: time.Now(),
  257. Mode: os.FileMode(mode) | os.ModeDir,
  258. Uid: OS_UID,
  259. Gid: OS_GID,
  260. },
  261. }
  262. filerResult = &FilerPostResult{
  263. Name: util.FullPath(path).Name(),
  264. }
  265. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil {
  266. replyerr = dbErr
  267. filerResult.Error = dbErr.Error()
  268. glog.V(0).Infof("failing to create dir %s on filer server : %v", path, dbErr)
  269. }
  270. return filerResult, replyerr
  271. }
  272. func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
  273. if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
  274. entry.Extended[xhttp.AmzStorageClass] = []byte(sc)
  275. }
  276. if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
  277. for _, v := range strings.Split(tags, "&") {
  278. tag := strings.Split(v, "=")
  279. if len(tag) == 2 {
  280. entry.Extended[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
  281. }
  282. }
  283. }
  284. for header, values := range r.Header {
  285. if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
  286. for _, value := range values {
  287. entry.Extended[header] = []byte(value)
  288. }
  289. }
  290. }
  291. }