You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

369 lines
10 KiB

5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "fmt"
  5. "io"
  6. "net/http"
  7. "os"
  8. "path"
  9. "strconv"
  10. "strings"
  11. "time"
  12. "github.com/chrislusf/seaweedfs/weed/filer"
  13. "github.com/chrislusf/seaweedfs/weed/glog"
  14. "github.com/chrislusf/seaweedfs/weed/operation"
  15. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  16. xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
  17. "github.com/chrislusf/seaweedfs/weed/stats"
  18. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  19. "github.com/chrislusf/seaweedfs/weed/util"
  20. )
  21. func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, contentLength int64, so *operation.StorageOption) {
  22. // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
  23. query := r.URL.Query()
  24. parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
  25. maxMB := int32(parsedMaxMB)
  26. if maxMB <= 0 && fs.option.MaxMB > 0 {
  27. maxMB = int32(fs.option.MaxMB)
  28. }
  29. chunkSize := 1024 * 1024 * maxMB
  30. stats.FilerRequestCounter.WithLabelValues("chunk").Inc()
  31. start := time.Now()
  32. defer func() {
  33. stats.FilerRequestHistogram.WithLabelValues("chunk").Observe(time.Since(start).Seconds())
  34. }()
  35. var reply *FilerPostResult
  36. var err error
  37. var md5bytes []byte
  38. if r.Method == "POST" {
  39. if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
  40. reply, err = fs.mkdir(ctx, w, r)
  41. } else {
  42. reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, contentLength, so)
  43. }
  44. } else {
  45. reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, contentLength, so)
  46. }
  47. if err != nil {
  48. if strings.HasPrefix(err.Error(), "read input:") {
  49. writeJsonError(w, r, 499, err)
  50. } else if strings.HasSuffix(err.Error(), "is a file") {
  51. writeJsonError(w, r, http.StatusConflict, err)
  52. } else {
  53. writeJsonError(w, r, http.StatusInternalServerError, err)
  54. }
  55. } else if reply != nil {
  56. if len(md5bytes) > 0 {
  57. md5InBase64 := util.Base64Encode(md5bytes)
  58. w.Header().Set("Content-MD5", md5InBase64)
  59. }
  60. writeJsonQuiet(w, r, http.StatusCreated, reply)
  61. }
  62. }
  63. func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  64. multipartReader, multipartReaderErr := r.MultipartReader()
  65. if multipartReaderErr != nil {
  66. return nil, nil, multipartReaderErr
  67. }
  68. part1, part1Err := multipartReader.NextPart()
  69. if part1Err != nil {
  70. return nil, nil, part1Err
  71. }
  72. fileName := part1.FileName()
  73. if fileName != "" {
  74. fileName = path.Base(fileName)
  75. }
  76. contentType := part1.Header.Get("Content-Type")
  77. if contentType == "application/octet-stream" {
  78. contentType = ""
  79. }
  80. fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, contentLength, so)
  81. if err != nil {
  82. return nil, nil, err
  83. }
  84. md5bytes = md5Hash.Sum(nil)
  85. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
  86. if replyerr != nil {
  87. fs.filer.DeleteChunks(fileChunks)
  88. }
  89. return
  90. }
  91. func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
  92. fileName := path.Base(r.URL.Path)
  93. contentType := r.Header.Get("Content-Type")
  94. if contentType == "application/octet-stream" {
  95. contentType = ""
  96. }
  97. fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, contentLength, so)
  98. if err != nil {
  99. return nil, nil, err
  100. }
  101. md5bytes = md5Hash.Sum(nil)
  102. filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
  103. if replyerr != nil {
  104. fs.filer.DeleteChunks(fileChunks)
  105. }
  106. return
  107. }
  108. func isAppend(r *http.Request) bool {
  109. return r.URL.Query().Get("op") == "append"
  110. }
  111. func skipCheckParentDirEntry(r *http.Request) bool {
  112. return r.URL.Query().Get("skipCheckParentDir") != "true"
  113. }
  114. func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) {
  115. // detect file mode
  116. modeStr := r.URL.Query().Get("mode")
  117. if modeStr == "" {
  118. modeStr = "0660"
  119. }
  120. mode, err := strconv.ParseUint(modeStr, 8, 32)
  121. if err != nil {
  122. glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  123. mode = 0660
  124. }
  125. // fix the path
  126. path := r.URL.Path
  127. if strings.HasSuffix(path, "/") {
  128. if fileName != "" {
  129. path += fileName
  130. }
  131. } else {
  132. if fileName != "" {
  133. if possibleDirEntry, findDirErr := fs.filer.FindEntry(ctx, util.FullPath(path)); findDirErr == nil {
  134. if possibleDirEntry.IsDirectory() {
  135. path += "/" + fileName
  136. }
  137. }
  138. }
  139. }
  140. var entry *filer.Entry
  141. var mergedChunks []*filer_pb.FileChunk
  142. isAppend := isAppend(r)
  143. isOffsetWrite := len(fileChunks) > 0 && fileChunks[0].Offset > 0
  144. // when it is an append
  145. if isAppend || isOffsetWrite {
  146. existingEntry, findErr := fs.filer.FindEntry(ctx, util.FullPath(path))
  147. if findErr != nil && findErr != filer_pb.ErrNotFound {
  148. glog.V(0).Infof("failing to find %s: %v", path, findErr)
  149. }
  150. entry = existingEntry
  151. }
  152. if entry != nil {
  153. entry.Mtime = time.Now()
  154. entry.Md5 = nil
  155. // adjust chunk offsets
  156. if isAppend {
  157. for _, chunk := range fileChunks {
  158. chunk.Offset += int64(entry.FileSize)
  159. }
  160. entry.FileSize += uint64(chunkOffset)
  161. }
  162. mergedChunks = append(entry.Chunks, fileChunks...)
  163. // TODO
  164. if len(entry.Content) > 0 {
  165. replyerr = fmt.Errorf("append to small file is not supported yet")
  166. return
  167. }
  168. } else {
  169. glog.V(4).Infoln("saving", path)
  170. mergedChunks = fileChunks
  171. entry = &filer.Entry{
  172. FullPath: util.FullPath(path),
  173. Attr: filer.Attr{
  174. Mtime: time.Now(),
  175. Crtime: time.Now(),
  176. Mode: os.FileMode(mode),
  177. Uid: OS_UID,
  178. Gid: OS_GID,
  179. Replication: so.Replication,
  180. Collection: so.Collection,
  181. TtlSec: so.TtlSeconds,
  182. DiskType: so.DiskType,
  183. Mime: contentType,
  184. Md5: md5bytes,
  185. FileSize: uint64(chunkOffset),
  186. },
  187. Content: content,
  188. }
  189. }
  190. // maybe compact entry chunks
  191. mergedChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), mergedChunks)
  192. if replyerr != nil {
  193. glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr)
  194. return
  195. }
  196. entry.Chunks = mergedChunks
  197. if isOffsetWrite {
  198. entry.Md5 = nil
  199. entry.FileSize = entry.Size()
  200. }
  201. filerResult = &FilerPostResult{
  202. Name: fileName,
  203. Size: int64(entry.FileSize),
  204. }
  205. entry.Extended = SaveAmzMetaData(r, entry.Extended, false)
  206. for k, v := range r.Header {
  207. if len(v) > 0 && len(v[0]) > 0 {
  208. if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" {
  209. entry.Extended[k] = []byte(v[0])
  210. }
  211. if k == "Response-Content-Disposition" {
  212. entry.Extended["Content-Disposition"] = []byte(v[0])
  213. }
  214. }
  215. }
  216. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, skipCheckParentDirEntry(r)); dbErr != nil {
  217. replyerr = dbErr
  218. filerResult.Error = dbErr.Error()
  219. glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr)
  220. }
  221. return filerResult, replyerr
  222. }
  223. func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
  224. return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
  225. // assign one file id for one chunk
  226. fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
  227. if assignErr != nil {
  228. return nil, "", "", assignErr
  229. }
  230. // upload the chunk to the volume server
  231. uploadOption := &operation.UploadOption{
  232. UploadUrl: urlLocation,
  233. Filename: name,
  234. Cipher: fs.option.Cipher,
  235. IsInputCompressed: false,
  236. MimeType: "",
  237. PairMap: nil,
  238. Jwt: auth,
  239. }
  240. uploadResult, uploadErr, _ := operation.Upload(reader, uploadOption)
  241. if uploadErr != nil {
  242. return nil, "", "", uploadErr
  243. }
  244. return uploadResult.ToPbFileChunk(fileId, offset), so.Collection, so.Replication, nil
  245. }
  246. }
  247. func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http.Request) (filerResult *FilerPostResult, replyerr error) {
  248. // detect file mode
  249. modeStr := r.URL.Query().Get("mode")
  250. if modeStr == "" {
  251. modeStr = "0660"
  252. }
  253. mode, err := strconv.ParseUint(modeStr, 8, 32)
  254. if err != nil {
  255. glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
  256. mode = 0660
  257. }
  258. // fix the path
  259. path := r.URL.Path
  260. if strings.HasSuffix(path, "/") {
  261. path = path[:len(path)-1]
  262. }
  263. existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
  264. if err == nil && existingEntry != nil {
  265. replyerr = fmt.Errorf("dir %s already exists", path)
  266. return
  267. }
  268. glog.V(4).Infoln("mkdir", path)
  269. entry := &filer.Entry{
  270. FullPath: util.FullPath(path),
  271. Attr: filer.Attr{
  272. Mtime: time.Now(),
  273. Crtime: time.Now(),
  274. Mode: os.FileMode(mode) | os.ModeDir,
  275. Uid: OS_UID,
  276. Gid: OS_GID,
  277. },
  278. }
  279. filerResult = &FilerPostResult{
  280. Name: util.FullPath(path).Name(),
  281. }
  282. if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, false); dbErr != nil {
  283. replyerr = dbErr
  284. filerResult.Error = dbErr.Error()
  285. glog.V(0).Infof("failing to create dir %s on filer server : %v", path, dbErr)
  286. }
  287. return filerResult, replyerr
  288. }
  289. func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool) (metadata map[string][]byte) {
  290. metadata = make(map[string][]byte)
  291. if !isReplace {
  292. for k, v := range existing {
  293. metadata[k] = v
  294. }
  295. }
  296. if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
  297. metadata[xhttp.AmzStorageClass] = []byte(sc)
  298. }
  299. if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
  300. for _, v := range strings.Split(tags, "&") {
  301. tag := strings.Split(v, "=")
  302. if len(tag) == 2 {
  303. metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
  304. } else if len(tag) == 1 {
  305. metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = nil
  306. }
  307. }
  308. }
  309. for header, values := range r.Header {
  310. if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
  311. for _, value := range values {
  312. metadata[header] = []byte(value)
  313. }
  314. }
  315. }
  316. return
  317. }