You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

310 lines
8.9 KiB

3 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
5 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
12 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
  1. package operation
  2. import (
  3. "bytes"
  4. "encoding/json"
  5. "fmt"
  6. "github.com/seaweedfs/seaweedfs/weed/glog"
  7. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  8. "github.com/seaweedfs/seaweedfs/weed/security"
  9. "github.com/seaweedfs/seaweedfs/weed/util"
  10. "io"
  11. "mime"
  12. "mime/multipart"
  13. "net"
  14. "net/http"
  15. "net/textproto"
  16. "path/filepath"
  17. "strings"
  18. "time"
  19. )
  20. type UploadOption struct {
  21. UploadUrl string
  22. Filename string
  23. Cipher bool
  24. IsInputCompressed bool
  25. MimeType string
  26. PairMap map[string]string
  27. Jwt security.EncodedJwt
  28. }
  29. type UploadResult struct {
  30. Name string `json:"name,omitempty"`
  31. Size uint32 `json:"size,omitempty"`
  32. Error string `json:"error,omitempty"`
  33. ETag string `json:"eTag,omitempty"`
  34. CipherKey []byte `json:"cipherKey,omitempty"`
  35. Mime string `json:"mime,omitempty"`
  36. Gzip uint32 `json:"gzip,omitempty"`
  37. ContentMd5 string `json:"contentMd5,omitempty"`
  38. RetryCount int `json:"-"`
  39. }
  40. func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk {
  41. fid, _ := filer_pb.ToFileIdObject(fileId)
  42. return &filer_pb.FileChunk{
  43. FileId: fileId,
  44. Offset: offset,
  45. Size: uint64(uploadResult.Size),
  46. Mtime: time.Now().UnixNano(),
  47. ETag: uploadResult.ContentMd5,
  48. CipherKey: uploadResult.CipherKey,
  49. IsCompressed: uploadResult.Gzip > 0,
  50. Fid: fid,
  51. }
  52. }
  53. // HTTPClient interface for testing
  54. type HTTPClient interface {
  55. Do(req *http.Request) (*http.Response, error)
  56. }
  57. var (
  58. HttpClient HTTPClient
  59. )
  60. func init() {
  61. HttpClient = &http.Client{Transport: &http.Transport{
  62. DialContext: (&net.Dialer{
  63. Timeout: 10 * time.Second,
  64. KeepAlive: 10 * time.Second,
  65. }).DialContext,
  66. MaxIdleConns: 1024,
  67. MaxIdleConnsPerHost: 1024,
  68. }}
  69. }
  70. var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "")
  71. // Upload sends a POST request to a volume server to upload the content with adjustable compression level
  72. func UploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
  73. uploadResult, err = retriedUploadData(data, option)
  74. return
  75. }
  76. // Upload sends a POST request to a volume server to upload the content with fast compression
  77. func Upload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
  78. uploadResult, err, data = doUpload(reader, option)
  79. return
  80. }
  81. func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
  82. bytesReader, ok := reader.(*util.BytesReader)
  83. if ok {
  84. data = bytesReader.Bytes
  85. } else {
  86. data, err = io.ReadAll(reader)
  87. if err != nil {
  88. err = fmt.Errorf("read input: %v", err)
  89. return
  90. }
  91. }
  92. uploadResult, uploadErr := retriedUploadData(data, option)
  93. return uploadResult, uploadErr, data
  94. }
  95. func retriedUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
  96. for i := 0; i < 3; i++ {
  97. uploadResult, err = doUploadData(data, option)
  98. if err == nil {
  99. uploadResult.RetryCount = i
  100. return
  101. } else {
  102. glog.Warningf("uploading to %s: %v", option.UploadUrl, err)
  103. }
  104. time.Sleep(time.Millisecond * time.Duration(237*(i+1)))
  105. }
  106. return
  107. }
  108. func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
  109. contentIsGzipped := option.IsInputCompressed
  110. shouldGzipNow := false
  111. if !option.IsInputCompressed {
  112. if option.MimeType == "" {
  113. option.MimeType = http.DetectContentType(data)
  114. // println("detect1 mimetype to", MimeType)
  115. if option.MimeType == "application/octet-stream" {
  116. option.MimeType = ""
  117. }
  118. }
  119. if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
  120. shouldGzipNow = true
  121. } else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
  122. var compressed []byte
  123. compressed, err = util.GzipData(data[0:128])
  124. shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
  125. }
  126. }
  127. var clearDataLen int
  128. // gzip if possible
  129. // this could be double copying
  130. clearDataLen = len(data)
  131. clearData := data
  132. if shouldGzipNow && !option.Cipher {
  133. compressed, compressErr := util.GzipData(data)
  134. // fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
  135. if compressErr == nil {
  136. data = compressed
  137. contentIsGzipped = true
  138. }
  139. } else if option.IsInputCompressed {
  140. // just to get the clear data length
  141. clearData, err = util.DecompressData(data)
  142. if err == nil {
  143. clearDataLen = len(clearData)
  144. }
  145. }
  146. if option.Cipher {
  147. // encrypt(gzip(data))
  148. // encrypt
  149. cipherKey := util.GenCipherKey()
  150. encryptedData, encryptionErr := util.Encrypt(clearData, cipherKey)
  151. if encryptionErr != nil {
  152. err = fmt.Errorf("encrypt input: %v", encryptionErr)
  153. return
  154. }
  155. // upload data
  156. uploadResult, err = upload_content(func(w io.Writer) (err error) {
  157. _, err = w.Write(encryptedData)
  158. return
  159. }, len(encryptedData), &UploadOption{
  160. UploadUrl: option.UploadUrl,
  161. Filename: "",
  162. Cipher: false,
  163. IsInputCompressed: false,
  164. MimeType: "",
  165. PairMap: nil,
  166. Jwt: option.Jwt,
  167. })
  168. if uploadResult == nil {
  169. return
  170. }
  171. uploadResult.Name = option.Filename
  172. uploadResult.Mime = option.MimeType
  173. uploadResult.CipherKey = cipherKey
  174. uploadResult.Size = uint32(clearDataLen)
  175. } else {
  176. // upload data
  177. uploadResult, err = upload_content(func(w io.Writer) (err error) {
  178. _, err = w.Write(data)
  179. return
  180. }, len(data), &UploadOption{
  181. UploadUrl: option.UploadUrl,
  182. Filename: option.Filename,
  183. Cipher: false,
  184. IsInputCompressed: contentIsGzipped,
  185. MimeType: option.MimeType,
  186. PairMap: option.PairMap,
  187. Jwt: option.Jwt,
  188. })
  189. if uploadResult == nil {
  190. return
  191. }
  192. uploadResult.Size = uint32(clearDataLen)
  193. if contentIsGzipped {
  194. uploadResult.Gzip = 1
  195. }
  196. }
  197. return uploadResult, err
  198. }
  199. func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize int, option *UploadOption) (*UploadResult, error) {
  200. buf := GetBuffer()
  201. defer PutBuffer(buf)
  202. body_writer := multipart.NewWriter(buf)
  203. h := make(textproto.MIMEHeader)
  204. filename := fileNameEscaper.Replace(option.Filename)
  205. h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
  206. h.Set("Idempotency-Key", option.UploadUrl)
  207. if option.MimeType == "" {
  208. option.MimeType = mime.TypeByExtension(strings.ToLower(filepath.Ext(option.Filename)))
  209. }
  210. if option.MimeType != "" {
  211. h.Set("Content-Type", option.MimeType)
  212. }
  213. if option.IsInputCompressed {
  214. h.Set("Content-Encoding", "gzip")
  215. }
  216. file_writer, cp_err := body_writer.CreatePart(h)
  217. if cp_err != nil {
  218. glog.V(0).Infoln("error creating form file", cp_err.Error())
  219. return nil, cp_err
  220. }
  221. if err := fillBufferFunction(file_writer); err != nil {
  222. glog.V(0).Infoln("error copying data", err)
  223. return nil, err
  224. }
  225. content_type := body_writer.FormDataContentType()
  226. if err := body_writer.Close(); err != nil {
  227. glog.V(0).Infoln("error closing body", err)
  228. return nil, err
  229. }
  230. req, postErr := http.NewRequest("POST", option.UploadUrl, bytes.NewReader(buf.Bytes()))
  231. if postErr != nil {
  232. glog.V(1).Infof("create upload request %s: %v", option.UploadUrl, postErr)
  233. return nil, fmt.Errorf("create upload request %s: %v", option.UploadUrl, postErr)
  234. }
  235. req.Header.Set("Content-Type", content_type)
  236. for k, v := range option.PairMap {
  237. req.Header.Set(k, v)
  238. }
  239. if option.Jwt != "" {
  240. req.Header.Set("Authorization", "BEARER "+string(option.Jwt))
  241. }
  242. // print("+")
  243. resp, post_err := HttpClient.Do(req)
  244. if post_err != nil {
  245. if strings.Contains(post_err.Error(), "connection reset by peer") ||
  246. strings.Contains(post_err.Error(), "use of closed network connection") {
  247. glog.V(1).Infof("repeat error upload request %s: %v", option.UploadUrl, postErr)
  248. resp, post_err = HttpClient.Do(req)
  249. }
  250. }
  251. if post_err != nil {
  252. return nil, fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err)
  253. }
  254. // print("-")
  255. defer util.CloseResponse(resp)
  256. var ret UploadResult
  257. etag := getEtag(resp)
  258. if resp.StatusCode == http.StatusNoContent {
  259. ret.ETag = etag
  260. return &ret, nil
  261. }
  262. resp_body, ra_err := io.ReadAll(resp.Body)
  263. if ra_err != nil {
  264. return nil, fmt.Errorf("read response body %v: %v", option.UploadUrl, ra_err)
  265. }
  266. unmarshal_err := json.Unmarshal(resp_body, &ret)
  267. if unmarshal_err != nil {
  268. glog.Errorf("unmarshal %s: %v", option.UploadUrl, string(resp_body))
  269. return nil, fmt.Errorf("unmarshal %v: %v", option.UploadUrl, unmarshal_err)
  270. }
  271. if ret.Error != "" {
  272. return nil, fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)
  273. }
  274. ret.ETag = etag
  275. ret.ContentMd5 = resp.Header.Get("Content-MD5")
  276. return &ret, nil
  277. }
  278. func getEtag(r *http.Response) (etag string) {
  279. etag = r.Header.Get("ETag")
  280. if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
  281. etag = etag[1 : len(etag)-1]
  282. }
  283. return
  284. }