You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

228 lines
5.7 KiB

9 years ago
12 years ago
12 years ago
12 years ago
12 years ago
9 years ago
9 years ago
12 years ago
12 years ago
  1. package storage
  2. import (
  3. "encoding/hex"
  4. "errors"
  5. "fmt"
  6. "io/ioutil"
  7. "mime"
  8. "net/http"
  9. "path"
  10. "strconv"
  11. "strings"
  12. "time"
  13. "github.com/chrislusf/seaweedfs/go/glog"
  14. "github.com/chrislusf/seaweedfs/go/images"
  15. "github.com/chrislusf/seaweedfs/go/operation"
  16. "github.com/chrislusf/seaweedfs/go/util"
  17. )
  18. const (
  19. NeedleHeaderSize = 16 //should never change this
  20. NeedlePaddingSize = 8
  21. NeedleChecksumSize = 4
  22. MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
  23. )
  24. /*
  25. * A Needle means a uploaded and stored file.
  26. * Needle file size is limited to 4GB for now.
  27. */
  28. type Needle struct {
  29. Cookie uint32 `comment:"random number to mitigate brute force lookups"`
  30. Id uint64 `comment:"needle id"`
  31. Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
  32. DataSize uint32 `comment:"Data size"` //version2
  33. Data []byte `comment:"The actual file data"`
  34. Flags byte `comment:"boolean flags"` //version2
  35. NameSize uint8 //version2
  36. Name []byte `comment:"maximum 256 characters"` //version2
  37. MimeSize uint8 //version2
  38. Mime []byte `comment:"maximum 256 characters"` //version2
  39. LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
  40. Ttl *TTL
  41. Checksum CRC `comment:"CRC32 to check integrity"`
  42. Padding []byte `comment:"Aligned to 8 bytes"`
  43. }
  44. func (n *Needle) String() (str string) {
  45. str = fmt.Sprintf("Cookie:%d, Id:%d, Size:%d, DataSize:%d, Name: %s, Mime: %s", n.Cookie, n.Id, n.Size, n.DataSize, n.Name, n.Mime)
  46. return
  47. }
  48. func ParseUpload(r *http.Request) (
  49. fileName string, data []byte, mimeType string, isGzipped bool,
  50. modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
  51. form, fe := r.MultipartReader()
  52. if fe != nil {
  53. glog.V(0).Infoln("MultipartReader [ERROR]", fe)
  54. e = fe
  55. return
  56. }
  57. //first multi-part item
  58. part, fe := form.NextPart()
  59. if fe != nil {
  60. glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
  61. e = fe
  62. return
  63. }
  64. fileName = part.FileName()
  65. if fileName != "" {
  66. fileName = path.Base(fileName)
  67. }
  68. data, e = ioutil.ReadAll(part)
  69. if e != nil {
  70. glog.V(0).Infoln("Reading Content [ERROR]", e)
  71. return
  72. }
  73. //if the filename is empty string, do a search on the other multi-part items
  74. for fileName == "" {
  75. part2, fe := form.NextPart()
  76. if fe != nil {
  77. break // no more or on error, just safely break
  78. }
  79. fName := part2.FileName()
  80. //found the first <file type> multi-part has filename
  81. if fName != "" {
  82. data2, fe2 := ioutil.ReadAll(part2)
  83. if fe2 != nil {
  84. glog.V(0).Infoln("Reading Content [ERROR]", fe2)
  85. e = fe2
  86. return
  87. }
  88. //update
  89. data = data2
  90. fileName = path.Base(fName)
  91. break
  92. }
  93. }
  94. dotIndex := strings.LastIndex(fileName, ".")
  95. ext, mtype := "", ""
  96. if dotIndex > 0 {
  97. ext = strings.ToLower(fileName[dotIndex:])
  98. mtype = mime.TypeByExtension(ext)
  99. }
  100. contentType := part.Header.Get("Content-Type")
  101. if contentType != "" && mtype != contentType {
  102. mimeType = contentType //only return mime type if not deductable
  103. mtype = contentType
  104. }
  105. if part.Header.Get("Content-Encoding") == "gzip" {
  106. isGzipped = true
  107. } else if operation.IsGzippable(ext, mtype) {
  108. if data, e = operation.GzipData(data); e != nil {
  109. return
  110. }
  111. isGzipped = true
  112. }
  113. if ext == ".gz" {
  114. isGzipped = true
  115. }
  116. if strings.HasSuffix(fileName, ".gz") &&
  117. !strings.HasSuffix(fileName, ".tar.gz") {
  118. fileName = fileName[:len(fileName)-3]
  119. }
  120. modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
  121. ttl, _ = ReadTTL(r.FormValue("ttl"))
  122. isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
  123. return
  124. }
  125. func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
  126. fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
  127. n = new(Needle)
  128. fname, n.Data, mimeType, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
  129. if e != nil {
  130. return
  131. }
  132. if len(fname) < 256 {
  133. n.Name = []byte(fname)
  134. n.SetHasName()
  135. }
  136. if len(mimeType) < 256 {
  137. n.Mime = []byte(mimeType)
  138. n.SetHasMime()
  139. }
  140. if isGzipped {
  141. n.SetGzipped()
  142. }
  143. if n.LastModified == 0 {
  144. n.LastModified = uint64(time.Now().Unix())
  145. }
  146. n.SetHasLastModifiedDate()
  147. if n.Ttl != EMPTY_TTL {
  148. n.SetHasTtl()
  149. }
  150. if isChunkedFile {
  151. n.SetIsChunkManifest()
  152. }
  153. if fixJpgOrientation {
  154. loweredName := strings.ToLower(fname)
  155. if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
  156. n.Data = images.FixJpgOrientation(n.Data)
  157. }
  158. }
  159. n.Checksum = NewCRC(n.Data)
  160. commaSep := strings.LastIndex(r.URL.Path, ",")
  161. dotSep := strings.LastIndex(r.URL.Path, ".")
  162. fid := r.URL.Path[commaSep+1:]
  163. if dotSep > 0 {
  164. fid = r.URL.Path[commaSep+1 : dotSep]
  165. }
  166. e = n.ParsePath(fid)
  167. return
  168. }
  169. func (n *Needle) ParsePath(fid string) (err error) {
  170. length := len(fid)
  171. if length <= 8 {
  172. return errors.New("Invalid fid:" + fid)
  173. }
  174. delta := ""
  175. deltaIndex := strings.LastIndex(fid, "_")
  176. if deltaIndex > 0 {
  177. fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
  178. }
  179. n.Id, n.Cookie, err = ParseKeyHash(fid)
  180. if err != nil {
  181. return err
  182. }
  183. if delta != "" {
  184. if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
  185. n.Id += d
  186. } else {
  187. return e
  188. }
  189. }
  190. return err
  191. }
  192. func ParseKeyHash(key_hash_string string) (uint64, uint32, error) {
  193. if len(key_hash_string)%2 == 1 {
  194. key_hash_string = "0" + key_hash_string
  195. }
  196. key_hash_bytes, khe := hex.DecodeString(key_hash_string)
  197. key_hash_len := len(key_hash_bytes)
  198. if khe != nil || key_hash_len <= 4 {
  199. glog.V(0).Infoln("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe)
  200. return 0, 0, errors.New("Invalid key and hash:" + key_hash_string)
  201. }
  202. key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4])
  203. hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len])
  204. return key, hash, nil
  205. }