You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

343 lines
9.6 KiB

5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "fmt"
  5. "io"
  6. "net/http"
  7. "os"
  8. "path"
  9. "strconv"
  10. "strings"
  11. "time"
  12. "github.com/chrislusf/seaweedfs/weed/filer"
  13. "github.com/chrislusf/seaweedfs/weed/glog"
  14. "github.com/chrislusf/seaweedfs/weed/operation"
  15. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  16. xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
  17. "github.com/chrislusf/seaweedfs/weed/stats"
  18. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  19. "github.com/chrislusf/seaweedfs/weed/util"
  20. )
  21. func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, contentLength int64, so *operation.StorageOption) {
  22. // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
  23. query := r.URL.Query()
  24. parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
  25. maxMB := int32(parsedMaxMB)
  26. if maxMB <= 0 && fs.option.MaxMB > 0 {
  27. maxMB = int32(fs.option.MaxMB)
  28. }
  29. chunkSize := 1024 * 1024 * maxMB
  30. stats.FilerRequestCounter.WithLabelValues("chunk").Inc()
  31. start := time.Now()
  32. defer func() {
  33. stats.FilerRequestHistogram.WithLabelValues("chunk").Observe(time.Since(start).Seconds())
  34. }()
  35. var reply *FilerPostResult
  36. var err error
  37. var md5bytes []byte
  38. if r.Method == "POST" {
  39. if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
  40. reply, err = fs.mkdir(ctx, w, r)
  41. } else {
  42. reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, contentLength, so)
  43. }
  44. } else {
  45. reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, contentLength, so)
  46. }
  47. if err != nil {
  48. if strings.HasPrefix(err.Error(), "read input:") {
  49. writeJsonError(w, r, 499, err)
  50. } else if strings.HasSuffix(err.Error(), "is a file") {
  51. writeJsonError(w, r, http.StatusConflict, err)
  52. } else {
  53. writeJsonError(w, r, http.StatusInternalServerError, err)
  54. }
  55. } else if reply != nil {
  56. if len(md5bytes) > 0 {
  57. w.Header().Set("Content-MD5", util.Base64Encode(md5bytes))
  58. }
  59. writeJsonQuiet(w, r, http.StatusCreated, reply)
  60. }
  61. }
  62. func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  63. multipartReader, multipartReaderErr := r.MultipartReader()
  64. if multipartReaderErr != nil {
  65. return nil, nil, multipartReaderErr
  66. }
  67. part1, part1Err := multipartReader.NextPart()
  68. if part1Err != nil {
  69. return nil, nil, part1Err
  70. }
  71. fileName := part1.FileName()
  72. if fileName != "" {
  73. fileName = path.Base(fileName)
  74. }
  75. contentType := part1.Header.Get("Content-Type")
  76. if contentType == "application/octet-stream" {
  77. contentType = ""
  78. }
  79. fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, contentLength, so)
  80. if err != nil {
  81. return nil, nil, err
  82. }
  83. md5bytes = md5Hash.Sum(nil)
  84. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
  85. return
  86. }
  87. func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  88. fileName := path.Base(r.URL.Path)
  89. contentType := r.Header.Get("Content-Type")
  90. if contentType == "application/octet-stream" {
  91. contentType = ""
  92. }
  93. fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, contentLength, so)
  94. if err != nil {
  95. return nil, nil, err
  96. }
  97. md5bytes = md5Hash.Sum(nil)
  98. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
  99. return
  100. }
  101. func isAppend(r *http.Request) bool {
  102. return r.URL.Query().Get("op") == "append"
  103. }
  104. func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) {
  105. // detect file mode
  106. modeStr := r.URL.Query().Get("mode")
  107. if modeStr == "" {
  108. modeStr = "0660"
  109. }
  110. mode, err := strconv.ParseUint(modeStr, 8, 32)
  111. if err != nil {
  112. glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  113. mode = 0660
  114. }
  115. // fix the path
  116. path := r.URL.Path
  117. if strings.HasSuffix(path, "/") {
  118. if fileName != "" {
  119. path += fileName
  120. }
  121. } else {
  122. if fileName != "" {
  123. if possibleDirEntry, findDirErr := fs.filer.FindEntry(ctx, util.FullPath(path)); findDirErr == nil {
  124. if possibleDirEntry.IsDirectory() {
  125. path += "/" + fileName
  126. }
  127. }
  128. }
  129. }
  130. var entry *filer.Entry
  131. var mergedChunks []*filer_pb.FileChunk
  132. // when it is an append
  133. if isAppend(r) {
  134. existingEntry, findErr := fs.filer.FindEntry(ctx, util.FullPath(path))
  135. if findErr != nil && findErr != filer_pb.ErrNotFound {
  136. glog.V(0).Infof("failing to find %s: %v", path, findErr)
  137. }
  138. entry = existingEntry
  139. }
  140. if entry != nil {
  141. entry.Mtime = time.Now()
  142. entry.Md5 = nil
  143. // adjust chunk offsets
  144. for _, chunk := range fileChunks {
  145. chunk.Offset += int64(entry.FileSize)
  146. }
  147. mergedChunks = append(entry.Chunks, fileChunks...)
  148. entry.FileSize += uint64(chunkOffset)
  149. // TODO
  150. if len(entry.Content) > 0 {
  151. replyerr = fmt.Errorf("append to small file is not supported yet")
  152. return
  153. }
  154. } else {
  155. glog.V(4).Infoln("saving", path)
  156. mergedChunks = fileChunks
  157. entry = &filer.Entry{
  158. FullPath: util.FullPath(path),
  159. Attr: filer.Attr{
  160. Mtime: time.Now(),
  161. Crtime: time.Now(),
  162. Mode: os.FileMode(mode),
  163. Uid: OS_UID,
  164. Gid: OS_GID,
  165. Replication: so.Replication,
  166. Collection: so.Collection,
  167. TtlSec: so.TtlSeconds,
  168. DiskType: so.DiskType,
  169. Mime: contentType,
  170. Md5: md5bytes,
  171. FileSize: uint64(chunkOffset),
  172. },
  173. Content: content,
  174. }
  175. }
  176. // maybe compact entry chunks
  177. mergedChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), mergedChunks)
  178. if replyerr != nil {
  179. glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr)
  180. return
  181. }
  182. entry.Chunks = mergedChunks
  183. filerResult = &FilerPostResult{
  184. Name: fileName,
  185. Size: int64(entry.FileSize),
  186. }
  187. entry.Extended = SaveAmzMetaData(r, entry.Extended, false)
  188. for k, v := range r.Header {
  189. if len(v) > 0 && (strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires") {
  190. entry.Extended[k] = []byte(v[0])
  191. }
  192. }
  193. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil {
  194. fs.filer.DeleteChunks(fileChunks)
  195. replyerr = dbErr
  196. filerResult.Error = dbErr.Error()
  197. glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr)
  198. }
  199. return filerResult, replyerr
  200. }
  201. func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
  202. return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
  203. // assign one file id for one chunk
  204. fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
  205. if assignErr != nil {
  206. return nil, "", "", assignErr
  207. }
  208. // upload the chunk to the volume server
  209. uploadOption := &operation.UploadOption{
  210. UploadUrl: urlLocation,
  211. Filename: name,
  212. Cipher: fs.option.Cipher,
  213. IsInputCompressed: false,
  214. MimeType: "",
  215. PairMap: nil,
  216. Jwt: auth,
  217. }
  218. uploadResult, uploadErr, _ := operation.Upload(reader, uploadOption)
  219. if uploadErr != nil {
  220. return nil, "", "", uploadErr
  221. }
  222. return uploadResult.ToPbFileChunk(fileId, offset), so.Collection, so.Replication, nil
  223. }
  224. }
  225. func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http.Request) (filerResult *FilerPostResult, replyerr error) {
  226. // detect file mode
  227. modeStr := r.URL.Query().Get("mode")
  228. if modeStr == "" {
  229. modeStr = "0660"
  230. }
  231. mode, err := strconv.ParseUint(modeStr, 8, 32)
  232. if err != nil {
  233. glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  234. mode = 0660
  235. }
  236. // fix the path
  237. path := r.URL.Path
  238. if strings.HasSuffix(path, "/") {
  239. path = path[:len(path)-1]
  240. }
  241. existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
  242. if err == nil && existingEntry != nil {
  243. replyerr = fmt.Errorf("dir %s already exists", path)
  244. return
  245. }
  246. glog.V(4).Infoln("mkdir", path)
  247. entry := &filer.Entry{
  248. FullPath: util.FullPath(path),
  249. Attr: filer.Attr{
  250. Mtime: time.Now(),
  251. Crtime: time.Now(),
  252. Mode: os.FileMode(mode) | os.ModeDir,
  253. Uid: OS_UID,
  254. Gid: OS_GID,
  255. },
  256. }
  257. filerResult = &FilerPostResult{
  258. Name: util.FullPath(path).Name(),
  259. }
  260. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil {
  261. replyerr = dbErr
  262. filerResult.Error = dbErr.Error()
  263. glog.V(0).Infof("failing to create dir %s on filer server : %v", path, dbErr)
  264. }
  265. return filerResult, replyerr
  266. }
  267. func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool) (metadata map[string][]byte) {
  268. metadata = make(map[string][]byte)
  269. if !isReplace {
  270. for k, v := range existing {
  271. metadata[k] = v
  272. }
  273. }
  274. if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
  275. metadata[xhttp.AmzStorageClass] = []byte(sc)
  276. }
  277. if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
  278. for _, v := range strings.Split(tags, "&") {
  279. tag := strings.Split(v, "=")
  280. if len(tag) == 2 {
  281. metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
  282. }
  283. }
  284. }
  285. for header, values := range r.Header {
  286. if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
  287. for _, value := range values {
  288. metadata[header] = []byte(value)
  289. }
  290. }
  291. }
  292. return
  293. }