You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

189 lines
4.9 KiB

7 years ago
12 years ago
12 years ago
12 years ago
9 years ago
9 years ago
7 years ago
9 years ago
12 years ago
  1. package storage
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "net/http"
  6. "strconv"
  7. "strings"
  8. "time"
  9. "github.com/chrislusf/seaweedfs/weed/images"
  10. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  11. "io/ioutil"
  12. )
  13. const (
  14. NeedleChecksumSize = 4
  15. PairNamePrefix = "Seaweed-"
  16. )
  17. /*
  18. * A Needle means a uploaded and stored file.
  19. * Needle file size is limited to 4GB for now.
  20. */
  21. type Needle struct {
  22. Cookie Cookie `comment:"random number to mitigate brute force lookups"`
  23. Id NeedleId `comment:"needle id"`
  24. Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
  25. DataSize uint32 `comment:"Data size"` //version2
  26. Data []byte `comment:"The actual file data"`
  27. Flags byte `comment:"boolean flags"` //version2
  28. NameSize uint8 //version2
  29. Name []byte `comment:"maximum 256 characters"` //version2
  30. MimeSize uint8 //version2
  31. Mime []byte `comment:"maximum 256 characters"` //version2
  32. PairsSize uint16 //version2
  33. Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"`
  34. LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
  35. Ttl *TTL
  36. Checksum CRC `comment:"CRC32 to check integrity"`
  37. Padding []byte `comment:"Aligned to 8 bytes"`
  38. }
  39. func (n *Needle) String() (str string) {
  40. str = fmt.Sprintf("%s Size:%d, DataSize:%d, Name:%s, Mime:%s", formatNeedleIdCookie(n.Id, n.Cookie), n.Size, n.DataSize, n.Name, n.Mime)
  41. return
  42. }
  43. func ParseUpload(r *http.Request) (
  44. fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool,
  45. modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
  46. pairMap = make(map[string]string)
  47. for k, v := range r.Header {
  48. if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
  49. pairMap[k] = v[0]
  50. }
  51. }
  52. isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
  53. if r.Method == "POST" {
  54. fileName, data, mimeType, isGzipped, e = parseMultipart(r, isChunkedFile)
  55. } else {
  56. isGzipped = false
  57. mimeType = r.Header.Get("Content-Type")
  58. fileName = ""
  59. data, e = ioutil.ReadAll(r.Body)
  60. }
  61. if e != nil {
  62. return
  63. }
  64. modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
  65. ttl, _ = ReadTTL(r.FormValue("ttl"))
  66. return
  67. }
  68. func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
  69. var pairMap map[string]string
  70. fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
  71. n = new(Needle)
  72. fname, n.Data, mimeType, pairMap, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
  73. if e != nil {
  74. return
  75. }
  76. if len(fname) < 256 {
  77. n.Name = []byte(fname)
  78. n.SetHasName()
  79. }
  80. if len(mimeType) < 256 {
  81. n.Mime = []byte(mimeType)
  82. n.SetHasMime()
  83. }
  84. if len(pairMap) != 0 {
  85. trimmedPairMap := make(map[string]string)
  86. for k, v := range pairMap {
  87. trimmedPairMap[k[len(PairNamePrefix):]] = v
  88. }
  89. pairs, _ := json.Marshal(trimmedPairMap)
  90. if len(pairs) < 65536 {
  91. n.Pairs = pairs
  92. n.PairsSize = uint16(len(pairs))
  93. n.SetHasPairs()
  94. }
  95. }
  96. if isGzipped {
  97. n.SetGzipped()
  98. }
  99. if n.LastModified == 0 {
  100. n.LastModified = uint64(time.Now().Unix())
  101. }
  102. n.SetHasLastModifiedDate()
  103. if n.Ttl != EMPTY_TTL {
  104. n.SetHasTtl()
  105. }
  106. if isChunkedFile {
  107. n.SetIsChunkManifest()
  108. }
  109. if fixJpgOrientation {
  110. loweredName := strings.ToLower(fname)
  111. if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
  112. n.Data = images.FixJpgOrientation(n.Data)
  113. }
  114. }
  115. n.Checksum = NewCRC(n.Data)
  116. commaSep := strings.LastIndex(r.URL.Path, ",")
  117. dotSep := strings.LastIndex(r.URL.Path, ".")
  118. fid := r.URL.Path[commaSep+1:]
  119. if dotSep > 0 {
  120. fid = r.URL.Path[commaSep+1 : dotSep]
  121. }
  122. e = n.ParsePath(fid)
  123. return
  124. }
  125. func (n *Needle) ParsePath(fid string) (err error) {
  126. length := len(fid)
  127. if length <= CookieSize*2 {
  128. return fmt.Errorf("Invalid fid: %s", fid)
  129. }
  130. delta := ""
  131. deltaIndex := strings.LastIndex(fid, "_")
  132. if deltaIndex > 0 {
  133. fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
  134. }
  135. n.Id, n.Cookie, err = ParseNeedleIdCookie(fid)
  136. if err != nil {
  137. return err
  138. }
  139. if delta != "" {
  140. if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
  141. n.Id += NeedleId(d)
  142. } else {
  143. return e
  144. }
  145. }
  146. return err
  147. }
  148. func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error) {
  149. if len(key_hash_string) <= CookieSize*2 {
  150. return 0, 0, fmt.Errorf("KeyHash is too short.")
  151. }
  152. if len(key_hash_string) > (NeedleIdSize+CookieSize)*2 {
  153. return 0, 0, fmt.Errorf("KeyHash is too long.")
  154. }
  155. split := len(key_hash_string) - CookieSize*2
  156. needleId, err := ParseNeedleId(key_hash_string[:split])
  157. if err != nil {
  158. return 0, 0, fmt.Errorf("Parse needleId error: %v", err)
  159. }
  160. cookie, err := ParseCookie(key_hash_string[split:])
  161. if err != nil {
  162. return 0, 0, fmt.Errorf("Parse cookie error: %v", err)
  163. }
  164. return needleId, cookie, nil
  165. }
  166. func (n *Needle) LastModifiedString() string {
  167. return time.Unix(int64(n.LastModified), 0).Format("2006-01-02T15:04:05")
  168. }