You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

166 lines
4.2 KiB

12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
  1. package storage
  2. import (
  3. "code.google.com/p/weed-fs/go/glog"
  4. "code.google.com/p/weed-fs/go/util"
  5. "encoding/hex"
  6. "errors"
  7. "io/ioutil"
  8. "mime"
  9. "net/http"
  10. "path"
  11. "strconv"
  12. "strings"
  13. "time"
  14. )
  15. const (
  16. NeedleHeaderSize = 16 //should never change this
  17. NeedlePaddingSize = 8
  18. NeedleChecksumSize = 4
  19. MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
  20. )
  21. /*
  22. * Needle file size is limited to 4GB for now.
  23. */
  24. type Needle struct {
  25. Cookie uint32 `comment:"random number to mitigate brute force lookups"`
  26. Id uint64 `comment:"needle id"`
  27. Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
  28. DataSize uint32 `comment:"Data size"` //version2
  29. Data []byte `comment:"The actual file data"`
  30. Flags byte `comment:"boolean flags"` //version2
  31. NameSize uint8 //version2
  32. Name []byte `comment:"maximum 256 characters"` //version2
  33. MimeSize uint8 //version2
  34. Mime []byte `comment:"maximum 256 characters"` //version2
  35. LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
  36. Checksum CRC `comment:"CRC32 to check integrity"`
  37. Padding []byte `comment:"Aligned to 8 bytes"`
  38. }
  39. func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, e error) {
  40. form, fe := r.MultipartReader()
  41. if fe != nil {
  42. glog.V(0).Infoln("MultipartReader [ERROR]", fe)
  43. e = fe
  44. return
  45. }
  46. part, fe := form.NextPart()
  47. if fe != nil {
  48. glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
  49. e = fe
  50. return
  51. }
  52. fileName = part.FileName()
  53. if fileName != "" {
  54. fileName = path.Base(fileName)
  55. }
  56. data, e = ioutil.ReadAll(part)
  57. if e != nil {
  58. glog.V(0).Infoln("Reading Content [ERROR]", e)
  59. return
  60. }
  61. dotIndex := strings.LastIndex(fileName, ".")
  62. ext, mtype := "", ""
  63. if dotIndex > 0 {
  64. ext = strings.ToLower(fileName[dotIndex:])
  65. mtype = mime.TypeByExtension(ext)
  66. }
  67. contentType := part.Header.Get("Content-Type")
  68. if contentType != "" && mtype != contentType {
  69. mimeType = contentType //only return mime type if not deductable
  70. mtype = contentType
  71. }
  72. if part.Header.Get("Content-Encoding") == "gzip" {
  73. isGzipped = true
  74. } else if IsGzippable(ext, mtype) {
  75. if data, e = GzipData(data); e != nil {
  76. return
  77. }
  78. isGzipped = true
  79. }
  80. if ext == ".gz" {
  81. isGzipped = true
  82. }
  83. if strings.HasSuffix(fileName, ".gz") {
  84. fileName = fileName[:len(fileName)-3]
  85. }
  86. modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
  87. return
  88. }
  89. func NewNeedle(r *http.Request) (n *Needle, e error) {
  90. fname, mimeType, isGzipped := "", "", false
  91. n = new(Needle)
  92. fname, n.Data, mimeType, isGzipped, n.LastModified, e = ParseUpload(r)
  93. if e != nil {
  94. return
  95. }
  96. if len(fname) < 256 {
  97. n.Name = []byte(fname)
  98. n.SetHasName()
  99. }
  100. if len(mimeType) < 256 {
  101. n.Mime = []byte(mimeType)
  102. n.SetHasMime()
  103. }
  104. if isGzipped {
  105. n.SetGzipped()
  106. }
  107. if n.LastModified == 0 {
  108. n.LastModified = uint64(time.Now().Unix())
  109. }
  110. n.SetHasLastModifiedDate()
  111. n.Checksum = NewCRC(n.Data)
  112. commaSep := strings.LastIndex(r.URL.Path, ",")
  113. dotSep := strings.LastIndex(r.URL.Path, ".")
  114. fid := r.URL.Path[commaSep+1:]
  115. if dotSep > 0 {
  116. fid = r.URL.Path[commaSep+1 : dotSep]
  117. }
  118. e = n.ParsePath(fid)
  119. return
  120. }
  121. func (n *Needle) ParsePath(fid string) (err error) {
  122. length := len(fid)
  123. if length <= 8 {
  124. return errors.New("Invalid fid:" + fid)
  125. }
  126. delta := ""
  127. deltaIndex := strings.LastIndex(fid, "_")
  128. if deltaIndex > 0 {
  129. fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
  130. }
  131. n.Id, n.Cookie, err = ParseKeyHash(fid)
  132. if err != nil {
  133. return err
  134. }
  135. if delta != "" {
  136. if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
  137. n.Id += d
  138. } else {
  139. return e
  140. }
  141. }
  142. return err
  143. }
  144. func ParseKeyHash(key_hash_string string) (uint64, uint32, error) {
  145. key_hash_bytes, khe := hex.DecodeString(key_hash_string)
  146. key_hash_len := len(key_hash_bytes)
  147. if khe != nil || key_hash_len <= 4 {
  148. glog.V(0).Infoln("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe)
  149. return 0, 0, errors.New("Invalid key and hash:" + key_hash_string)
  150. }
  151. key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4])
  152. hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len])
  153. return key, hash, nil
  154. }