You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

374 lines
10 KiB

5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "fmt"
  5. "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
  6. "io"
  7. "net/http"
  8. "os"
  9. "path"
  10. "strconv"
  11. "strings"
  12. "time"
  13. "github.com/chrislusf/seaweedfs/weed/filer"
  14. "github.com/chrislusf/seaweedfs/weed/glog"
  15. "github.com/chrislusf/seaweedfs/weed/operation"
  16. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  17. "github.com/chrislusf/seaweedfs/weed/stats"
  18. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  19. "github.com/chrislusf/seaweedfs/weed/util"
  20. )
  21. func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, contentLength int64, so *operation.StorageOption) {
  22. // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
  23. query := r.URL.Query()
  24. parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
  25. maxMB := int32(parsedMaxMB)
  26. if maxMB <= 0 && fs.option.MaxMB > 0 {
  27. maxMB = int32(fs.option.MaxMB)
  28. }
  29. chunkSize := 1024 * 1024 * maxMB
  30. stats.FilerRequestCounter.WithLabelValues("chunk").Inc()
  31. start := time.Now()
  32. defer func() {
  33. stats.FilerRequestHistogram.WithLabelValues("chunk").Observe(time.Since(start).Seconds())
  34. }()
  35. var reply *FilerPostResult
  36. var err error
  37. var md5bytes []byte
  38. if r.Method == "POST" {
  39. if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
  40. reply, err = fs.mkdir(ctx, w, r)
  41. } else {
  42. reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, contentLength, so)
  43. }
  44. } else {
  45. reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, contentLength, so)
  46. }
  47. if err != nil {
  48. if strings.HasPrefix(err.Error(), "read input:") {
  49. writeJsonError(w, r, 499, err)
  50. } else if strings.HasSuffix(err.Error(), "is a file") {
  51. writeJsonError(w, r, http.StatusConflict, err)
  52. } else {
  53. writeJsonError(w, r, http.StatusInternalServerError, err)
  54. }
  55. } else if reply != nil {
  56. if len(md5bytes) > 0 {
  57. md5InBase64 := util.Base64Encode(md5bytes)
  58. w.Header().Set("Content-MD5", md5InBase64)
  59. }
  60. writeJsonQuiet(w, r, http.StatusCreated, reply)
  61. }
  62. }
  63. func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  64. multipartReader, multipartReaderErr := r.MultipartReader()
  65. if multipartReaderErr != nil {
  66. return nil, nil, multipartReaderErr
  67. }
  68. part1, part1Err := multipartReader.NextPart()
  69. if part1Err != nil {
  70. return nil, nil, part1Err
  71. }
  72. fileName := part1.FileName()
  73. if fileName != "" {
  74. fileName = path.Base(fileName)
  75. }
  76. contentType := part1.Header.Get("Content-Type")
  77. if contentType == "application/octet-stream" {
  78. contentType = ""
  79. }
  80. fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, contentLength, so)
  81. if err != nil {
  82. return nil, nil, err
  83. }
  84. md5bytes = md5Hash.Sum(nil)
  85. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
  86. if replyerr != nil {
  87. fs.filer.DeleteChunks(fileChunks)
  88. }
  89. return
  90. }
  91. func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  92. fileName := path.Base(r.URL.Path)
  93. contentType := r.Header.Get("Content-Type")
  94. if contentType == "application/octet-stream" {
  95. contentType = ""
  96. }
  97. fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, contentLength, so)
  98. if err != nil {
  99. return nil, nil, err
  100. }
  101. md5bytes = md5Hash.Sum(nil)
  102. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
  103. if replyerr != nil {
  104. fs.filer.DeleteChunks(fileChunks)
  105. }
  106. return
  107. }
  108. func isAppend(r *http.Request) bool {
  109. return r.URL.Query().Get("op") == "append"
  110. }
  111. func skipCheckParentDirEntry(r *http.Request) bool {
  112. return r.URL.Query().Get("skipCheckParentDir") == "true"
  113. }
  114. func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) {
  115. // detect file mode
  116. modeStr := r.URL.Query().Get("mode")
  117. if modeStr == "" {
  118. modeStr = "0660"
  119. }
  120. mode, err := strconv.ParseUint(modeStr, 8, 32)
  121. if err != nil {
  122. glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  123. mode = 0660
  124. }
  125. // fix the path
  126. path := r.URL.Path
  127. if strings.HasSuffix(path, "/") {
  128. if fileName != "" {
  129. path += fileName
  130. }
  131. } else {
  132. if fileName != "" {
  133. if possibleDirEntry, findDirErr := fs.filer.FindEntry(ctx, util.FullPath(path)); findDirErr == nil {
  134. if possibleDirEntry.IsDirectory() {
  135. path += "/" + fileName
  136. }
  137. }
  138. }
  139. }
  140. var entry *filer.Entry
  141. var newChunks []*filer_pb.FileChunk
  142. var mergedChunks []*filer_pb.FileChunk
  143. isAppend := isAppend(r)
  144. isOffsetWrite := len(fileChunks) > 0 && fileChunks[0].Offset > 0
  145. // when it is an append
  146. if isAppend || isOffsetWrite {
  147. existingEntry, findErr := fs.filer.FindEntry(ctx, util.FullPath(path))
  148. if findErr != nil && findErr != filer_pb.ErrNotFound {
  149. glog.V(0).Infof("failing to find %s: %v", path, findErr)
  150. }
  151. entry = existingEntry
  152. }
  153. if entry != nil {
  154. entry.Mtime = time.Now()
  155. entry.Md5 = nil
  156. // adjust chunk offsets
  157. if isAppend {
  158. for _, chunk := range fileChunks {
  159. chunk.Offset += int64(entry.FileSize)
  160. }
  161. entry.FileSize += uint64(chunkOffset)
  162. }
  163. newChunks = append(entry.Chunks, fileChunks...)
  164. // TODO
  165. if len(entry.Content) > 0 {
  166. replyerr = fmt.Errorf("append to small file is not supported yet")
  167. return
  168. }
  169. } else {
  170. glog.V(4).Infoln("saving", path)
  171. newChunks = fileChunks
  172. entry = &filer.Entry{
  173. FullPath: util.FullPath(path),
  174. Attr: filer.Attr{
  175. Mtime: time.Now(),
  176. Crtime: time.Now(),
  177. Mode: os.FileMode(mode),
  178. Uid: OS_UID,
  179. Gid: OS_GID,
  180. TtlSec: so.TtlSeconds,
  181. Mime: contentType,
  182. Md5: md5bytes,
  183. FileSize: uint64(chunkOffset),
  184. },
  185. Content: content,
  186. }
  187. }
  188. // maybe concatenate small chunks into one whole chunk
  189. mergedChunks, replyerr = fs.maybeMergeChunks(so, newChunks)
  190. if replyerr != nil {
  191. glog.V(0).Infof("merge chunks %s: %v", r.RequestURI, replyerr)
  192. mergedChunks = newChunks
  193. }
  194. // maybe compact entry chunks
  195. mergedChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), mergedChunks)
  196. if replyerr != nil {
  197. glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr)
  198. return
  199. }
  200. entry.Chunks = mergedChunks
  201. if isOffsetWrite {
  202. entry.Md5 = nil
  203. entry.FileSize = entry.Size()
  204. }
  205. filerResult = &FilerPostResult{
  206. Name: fileName,
  207. Size: int64(entry.FileSize),
  208. }
  209. entry.Extended = SaveAmzMetaData(r, entry.Extended, false)
  210. for k, v := range r.Header {
  211. if len(v) > 0 && len(v[0]) > 0 {
  212. if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" {
  213. entry.Extended[k] = []byte(v[0])
  214. }
  215. if k == "Response-Content-Disposition" {
  216. entry.Extended["Content-Disposition"] = []byte(v[0])
  217. }
  218. }
  219. }
  220. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, skipCheckParentDirEntry(r)); dbErr != nil {
  221. replyerr = dbErr
  222. filerResult.Error = dbErr.Error()
  223. glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr)
  224. }
  225. return filerResult, replyerr
  226. }
  227. func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
  228. return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
  229. // assign one file id for one chunk
  230. fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
  231. if assignErr != nil {
  232. return nil, "", "", assignErr
  233. }
  234. // upload the chunk to the volume server
  235. uploadOption := &operation.UploadOption{
  236. UploadUrl: urlLocation,
  237. Filename: name,
  238. Cipher: fs.option.Cipher,
  239. IsInputCompressed: false,
  240. MimeType: "",
  241. PairMap: nil,
  242. Jwt: auth,
  243. }
  244. uploadResult, uploadErr, _ := operation.Upload(reader, uploadOption)
  245. if uploadErr != nil {
  246. return nil, "", "", uploadErr
  247. }
  248. return uploadResult.ToPbFileChunk(fileId, offset), so.Collection, so.Replication, nil
  249. }
  250. }
  251. func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http.Request) (filerResult *FilerPostResult, replyerr error) {
  252. // detect file mode
  253. modeStr := r.URL.Query().Get("mode")
  254. if modeStr == "" {
  255. modeStr = "0660"
  256. }
  257. mode, err := strconv.ParseUint(modeStr, 8, 32)
  258. if err != nil {
  259. glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  260. mode = 0660
  261. }
  262. // fix the path
  263. path := r.URL.Path
  264. if strings.HasSuffix(path, "/") {
  265. path = path[:len(path)-1]
  266. }
  267. existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
  268. if err == nil && existingEntry != nil {
  269. replyerr = fmt.Errorf("dir %s already exists", path)
  270. return
  271. }
  272. glog.V(4).Infoln("mkdir", path)
  273. entry := &filer.Entry{
  274. FullPath: util.FullPath(path),
  275. Attr: filer.Attr{
  276. Mtime: time.Now(),
  277. Crtime: time.Now(),
  278. Mode: os.FileMode(mode) | os.ModeDir,
  279. Uid: OS_UID,
  280. Gid: OS_GID,
  281. },
  282. }
  283. filerResult = &FilerPostResult{
  284. Name: util.FullPath(path).Name(),
  285. }
  286. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, false); dbErr != nil {
  287. replyerr = dbErr
  288. filerResult.Error = dbErr.Error()
  289. glog.V(0).Infof("failing to create dir %s on filer server : %v", path, dbErr)
  290. }
  291. return filerResult, replyerr
  292. }
  293. func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool) (metadata map[string][]byte) {
  294. metadata = make(map[string][]byte)
  295. if !isReplace {
  296. for k, v := range existing {
  297. metadata[k] = v
  298. }
  299. }
  300. if sc := r.Header.Get(s3_constants.AmzStorageClass); sc != "" {
  301. metadata[s3_constants.AmzStorageClass] = []byte(sc)
  302. }
  303. if tags := r.Header.Get(s3_constants.AmzObjectTagging); tags != "" {
  304. for _, v := range strings.Split(tags, "&") {
  305. tag := strings.Split(v, "=")
  306. if len(tag) == 2 {
  307. metadata[s3_constants.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
  308. } else if len(tag) == 1 {
  309. metadata[s3_constants.AmzObjectTagging+"-"+tag[0]] = nil
  310. }
  311. }
  312. }
  313. for header, values := range r.Header {
  314. if strings.HasPrefix(header, s3_constants.AmzUserMetaPrefix) {
  315. for _, value := range values {
  316. metadata[header] = []byte(value)
  317. }
  318. }
  319. }
  320. return
  321. }