You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

216 lines
5.3 KiB

12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
  1. package storage
  2. import (
  3. "encoding/hex"
  4. "errors"
  5. "fmt"
  6. "io/ioutil"
  7. "mime"
  8. "net/http"
  9. "path"
  10. "strconv"
  11. "strings"
  12. "time"
  13. "github.com/chrislusf/weed-fs/go/glog"
  14. "github.com/chrislusf/weed-fs/go/images"
  15. "github.com/chrislusf/weed-fs/go/util"
  16. )
  17. const (
  18. NeedleHeaderSize = 16 //should never change this
  19. NeedlePaddingSize = 8
  20. NeedleChecksumSize = 4
  21. MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
  22. )
  23. /*
  24. * A Needle means a uploaded and stored file.
  25. * Needle file size is limited to 4GB for now.
  26. */
  27. type Needle struct {
  28. Cookie uint32 `comment:"random number to mitigate brute force lookups"`
  29. Id uint64 `comment:"needle id"`
  30. Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
  31. DataSize uint32 `comment:"Data size"` //version2
  32. Data []byte `comment:"The actual file data"`
  33. Flags byte `comment:"boolean flags"` //version2
  34. NameSize uint8 //version2
  35. Name []byte `comment:"maximum 256 characters"` //version2
  36. MimeSize uint8 //version2
  37. Mime []byte `comment:"maximum 256 characters"` //version2
  38. LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
  39. Ttl *TTL
  40. Checksum CRC `comment:"CRC32 to check integrity"`
  41. Padding []byte `comment:"Aligned to 8 bytes"`
  42. }
  43. func (n *Needle) String() (str string) {
  44. str = fmt.Sprintf("Cookie:%d, Id:%d, Size:%d, DataSize:%d, Name: %s, Mime: %s", n.Cookie, n.Id, n.Size, n.DataSize, n.Name, n.Mime)
  45. return
  46. }
  47. func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, e error) {
  48. form, fe := r.MultipartReader()
  49. if fe != nil {
  50. glog.V(0).Infoln("MultipartReader [ERROR]", fe)
  51. e = fe
  52. return
  53. }
  54. //first multi-part item
  55. part, fe := form.NextPart()
  56. if fe != nil {
  57. glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
  58. e = fe
  59. return
  60. }
  61. fileName = part.FileName()
  62. if fileName != "" {
  63. fileName = path.Base(fileName)
  64. }
  65. data, e = ioutil.ReadAll(part)
  66. if e != nil {
  67. glog.V(0).Infoln("Reading Content [ERROR]", e)
  68. return
  69. }
  70. //if the filename is empty string, do a search on the other multi-part items
  71. for fileName == "" {
  72. part2, fe := form.NextPart()
  73. if fe != nil {
  74. break // no more or on error, just safely break
  75. }
  76. fName := part2.FileName()
  77. //found the first <file type> multi-part has filename
  78. if fName != "" {
  79. data2, fe2 := ioutil.ReadAll(part2)
  80. if fe2 != nil {
  81. glog.V(0).Infoln("Reading Content [ERROR]", fe2)
  82. e = fe2
  83. return
  84. }
  85. //update
  86. data = data2
  87. fileName = path.Base(fName)
  88. break
  89. }
  90. }
  91. dotIndex := strings.LastIndex(fileName, ".")
  92. ext, mtype := "", ""
  93. if dotIndex > 0 {
  94. ext = strings.ToLower(fileName[dotIndex:])
  95. mtype = mime.TypeByExtension(ext)
  96. }
  97. contentType := part.Header.Get("Content-Type")
  98. if contentType != "" && mtype != contentType {
  99. mimeType = contentType //only return mime type if not deductable
  100. mtype = contentType
  101. }
  102. if part.Header.Get("Content-Encoding") == "gzip" {
  103. isGzipped = true
  104. } else if IsGzippable(ext, mtype) {
  105. if data, e = GzipData(data); e != nil {
  106. return
  107. }
  108. isGzipped = true
  109. }
  110. if ext == ".gz" {
  111. isGzipped = true
  112. }
  113. if strings.HasSuffix(fileName, ".gz") {
  114. fileName = fileName[:len(fileName)-3]
  115. }
  116. modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
  117. ttl, _ = ReadTTL(r.FormValue("ttl"))
  118. return
  119. }
  120. func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
  121. fname, mimeType, isGzipped := "", "", false
  122. n = new(Needle)
  123. fname, n.Data, mimeType, isGzipped, n.LastModified, n.Ttl, e = ParseUpload(r)
  124. if e != nil {
  125. return
  126. }
  127. if len(fname) < 256 {
  128. n.Name = []byte(fname)
  129. n.SetHasName()
  130. }
  131. if len(mimeType) < 256 {
  132. n.Mime = []byte(mimeType)
  133. n.SetHasMime()
  134. }
  135. if isGzipped {
  136. n.SetGzipped()
  137. }
  138. if n.LastModified == 0 {
  139. n.LastModified = uint64(time.Now().Unix())
  140. }
  141. n.SetHasLastModifiedDate()
  142. if n.Ttl != EMPTY_TTL {
  143. n.SetHasTtl()
  144. }
  145. if fixJpgOrientation {
  146. loweredName := strings.ToLower(fname)
  147. if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
  148. n.Data = images.FixJpgOrientation(n.Data)
  149. }
  150. }
  151. n.Checksum = NewCRC(n.Data)
  152. commaSep := strings.LastIndex(r.URL.Path, ",")
  153. dotSep := strings.LastIndex(r.URL.Path, ".")
  154. fid := r.URL.Path[commaSep+1:]
  155. if dotSep > 0 {
  156. fid = r.URL.Path[commaSep+1 : dotSep]
  157. }
  158. e = n.ParsePath(fid)
  159. return
  160. }
  161. func (n *Needle) ParsePath(fid string) (err error) {
  162. length := len(fid)
  163. if length <= 8 {
  164. return errors.New("Invalid fid:" + fid)
  165. }
  166. delta := ""
  167. deltaIndex := strings.LastIndex(fid, "_")
  168. if deltaIndex > 0 {
  169. fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
  170. }
  171. n.Id, n.Cookie, err = ParseKeyHash(fid)
  172. if err != nil {
  173. return err
  174. }
  175. if delta != "" {
  176. if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
  177. n.Id += d
  178. } else {
  179. return e
  180. }
  181. }
  182. return err
  183. }
  184. func ParseKeyHash(key_hash_string string) (uint64, uint32, error) {
  185. key_hash_bytes, khe := hex.DecodeString(key_hash_string)
  186. key_hash_len := len(key_hash_bytes)
  187. if khe != nil || key_hash_len <= 4 {
  188. glog.V(0).Infoln("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe)
  189. return 0, 0, errors.New("Invalid key and hash:" + key_hash_string)
  190. }
  191. key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4])
  192. hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len])
  193. return key, hash, nil
  194. }