You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

207 lines
5.3 KiB

4 years ago
  1. package needle
  2. import (
  3. "bytes"
  4. "crypto/md5"
  5. "encoding/base64"
  6. "fmt"
  7. "io"
  8. "mime"
  9. "net/http"
  10. "path"
  11. "path/filepath"
  12. "strconv"
  13. "strings"
  14. "github.com/seaweedfs/seaweedfs/weed/glog"
  15. "github.com/seaweedfs/seaweedfs/weed/util"
  16. )
  17. type ParsedUpload struct {
  18. FileName string
  19. Data []byte
  20. bytesBuffer *bytes.Buffer
  21. MimeType string
  22. PairMap map[string]string
  23. IsGzipped bool
  24. // IsZstd bool
  25. OriginalDataSize int
  26. ModifiedTime uint64
  27. Ttl *TTL
  28. IsChunkedFile bool
  29. UncompressedData []byte
  30. ContentMd5 string
  31. }
  32. func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (pu *ParsedUpload, e error) {
  33. bytesBuffer.Reset()
  34. pu = &ParsedUpload{bytesBuffer: bytesBuffer}
  35. pu.PairMap = make(map[string]string)
  36. for k, v := range r.Header {
  37. if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
  38. pu.PairMap[k] = v[0]
  39. }
  40. }
  41. if r.Method == "POST" {
  42. e = parseMultipart(r, sizeLimit, pu)
  43. } else {
  44. e = parsePut(r, sizeLimit, pu)
  45. }
  46. if e != nil {
  47. return
  48. }
  49. pu.ModifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
  50. pu.Ttl, _ = ReadTTL(r.FormValue("ttl"))
  51. pu.OriginalDataSize = len(pu.Data)
  52. pu.UncompressedData = pu.Data
  53. // println("received data", len(pu.Data), "isGzipped", pu.IsGzipped, "mime", pu.MimeType, "name", pu.FileName)
  54. if pu.IsGzipped {
  55. if unzipped, e := util.DecompressData(pu.Data); e == nil {
  56. pu.OriginalDataSize = len(unzipped)
  57. pu.UncompressedData = unzipped
  58. // println("ungzipped data size", len(unzipped))
  59. }
  60. } else {
  61. ext := filepath.Base(pu.FileName)
  62. mimeType := pu.MimeType
  63. if mimeType == "" {
  64. mimeType = http.DetectContentType(pu.Data)
  65. }
  66. // println("detected mimetype to", pu.MimeType)
  67. if mimeType == "application/octet-stream" {
  68. mimeType = ""
  69. }
  70. if shouldBeCompressed, iAmSure := util.IsCompressableFileType(ext, mimeType); shouldBeCompressed && iAmSure {
  71. // println("ext", ext, "iAmSure", iAmSure, "shouldBeCompressed", shouldBeCompressed, "mimeType", pu.MimeType)
  72. if compressedData, err := util.GzipData(pu.Data); err == nil {
  73. if len(compressedData)*10 < len(pu.Data)*9 {
  74. pu.Data = compressedData
  75. pu.IsGzipped = true
  76. }
  77. // println("gzipped data size", len(compressedData))
  78. }
  79. }
  80. }
  81. // md5
  82. h := md5.New()
  83. h.Write(pu.UncompressedData)
  84. pu.ContentMd5 = base64.StdEncoding.EncodeToString(h.Sum(nil))
  85. if expectedChecksum := r.Header.Get("Content-MD5"); expectedChecksum != "" {
  86. if expectedChecksum != pu.ContentMd5 {
  87. e = fmt.Errorf("Content-MD5 did not match md5 of file data expected [%s] received [%s] size %d", expectedChecksum, pu.ContentMd5, len(pu.UncompressedData))
  88. return
  89. }
  90. }
  91. return
  92. }
  93. func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error {
  94. pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
  95. // pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
  96. pu.MimeType = r.Header.Get("Content-Type")
  97. pu.FileName = ""
  98. dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
  99. if err == io.EOF || dataSize == sizeLimit+1 {
  100. io.Copy(io.Discard, r.Body)
  101. }
  102. pu.Data = pu.bytesBuffer.Bytes()
  103. r.Body.Close()
  104. return nil
  105. }
  106. func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
  107. defer func() {
  108. if e != nil && r.Body != nil {
  109. io.Copy(io.Discard, r.Body)
  110. r.Body.Close()
  111. }
  112. }()
  113. form, fe := r.MultipartReader()
  114. if fe != nil {
  115. glog.V(0).Infoln("MultipartReader [ERROR]", fe)
  116. e = fe
  117. return
  118. }
  119. // first multi-part item
  120. part, fe := form.NextPart()
  121. if fe != nil {
  122. glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
  123. e = fe
  124. return
  125. }
  126. pu.FileName = part.FileName()
  127. if pu.FileName != "" {
  128. pu.FileName = path.Base(pu.FileName)
  129. }
  130. var dataSize int64
  131. dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
  132. if e != nil {
  133. glog.V(0).Infoln("Reading Content [ERROR]", e)
  134. return
  135. }
  136. if dataSize == sizeLimit+1 {
  137. e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
  138. return
  139. }
  140. pu.Data = pu.bytesBuffer.Bytes()
  141. // if the filename is empty string, do a search on the other multi-part items
  142. for pu.FileName == "" {
  143. part2, fe := form.NextPart()
  144. if fe != nil {
  145. break // no more or on error, just safely break
  146. }
  147. fName := part2.FileName()
  148. // found the first <file type> multi-part has filename
  149. if fName != "" {
  150. pu.bytesBuffer.Reset()
  151. dataSize2, fe2 := pu.bytesBuffer.ReadFrom(io.LimitReader(part2, sizeLimit+1))
  152. if fe2 != nil {
  153. glog.V(0).Infoln("Reading Content [ERROR]", fe2)
  154. e = fe2
  155. return
  156. }
  157. if dataSize2 == sizeLimit+1 {
  158. e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
  159. return
  160. }
  161. // update
  162. pu.Data = pu.bytesBuffer.Bytes()
  163. pu.FileName = path.Base(fName)
  164. break
  165. }
  166. }
  167. pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
  168. if !pu.IsChunkedFile {
  169. dotIndex := strings.LastIndex(pu.FileName, ".")
  170. ext, mtype := "", ""
  171. if dotIndex > 0 {
  172. ext = strings.ToLower(pu.FileName[dotIndex:])
  173. mtype = mime.TypeByExtension(ext)
  174. }
  175. contentType := part.Header.Get("Content-Type")
  176. if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
  177. pu.MimeType = contentType // only return mime type if not deductable
  178. mtype = contentType
  179. }
  180. }
  181. pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
  182. // pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
  183. return
  184. }