You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

300 lines
7.9 KiB

6 years ago
5 years ago
5 years ago
12 years ago
5 years ago
12 years ago
5 years ago
9 years ago
7 years ago
7 years ago
7 years ago
5 years ago
5 years ago
9 years ago
7 years ago
5 years ago
12 years ago
  1. package needle
  2. import (
  3. "bytes"
  4. "encoding/json"
  5. "fmt"
  6. "io"
  7. "io/ioutil"
  8. "net/http"
  9. "strconv"
  10. "strings"
  11. "time"
  12. "github.com/valyala/fasthttp"
  13. "github.com/chrislusf/seaweedfs/weed/images"
  14. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  15. )
  16. const (
  17. NeedleChecksumSize = 4
  18. PairNamePrefix = "Seaweed-"
  19. )
  20. var (
  21. PairNamePrefixBytes = []byte("Seaweed-")
  22. )
  23. /*
  24. * A Needle means a uploaded and stored file.
  25. * Needle file size is limited to 4GB for now.
  26. */
  27. type Needle struct {
  28. Cookie Cookie `comment:"random number to mitigate brute force lookups"`
  29. Id NeedleId `comment:"needle id"`
  30. Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
  31. DataSize uint32 `comment:"Data size"` //version2
  32. Data []byte `comment:"The actual file data"`
  33. Flags byte `comment:"boolean flags"` //version2
  34. NameSize uint8 //version2
  35. Name []byte `comment:"maximum 256 characters"` //version2
  36. MimeSize uint8 //version2
  37. Mime []byte `comment:"maximum 256 characters"` //version2
  38. PairsSize uint16 //version2
  39. Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"`
  40. LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
  41. Ttl *TTL
  42. Checksum CRC `comment:"CRC32 to check integrity"`
  43. AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
  44. Padding []byte `comment:"Aligned to 8 bytes"`
  45. }
  46. func (n *Needle) String() (str string) {
  47. str = fmt.Sprintf("%s Size:%d, DataSize:%d, Name:%s, Mime:%s", formatNeedleIdCookie(n.Id, n.Cookie), n.Size, n.DataSize, n.Name, n.Mime)
  48. return
  49. }
  50. func OldParseUpload(r *http.Request, sizeLimit int64) (
  51. fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool, originalDataSize int,
  52. modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
  53. pairMap = make(map[string]string)
  54. for k, v := range r.Header {
  55. if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
  56. pairMap[k] = v[0]
  57. }
  58. }
  59. if r.Method == "POST" {
  60. fileName, data, mimeType, isGzipped, originalDataSize, isChunkedFile, e = parseMultipart(r, sizeLimit)
  61. } else {
  62. isGzipped = false
  63. mimeType = r.Header.Get("Content-Type")
  64. fileName = ""
  65. data, e = ioutil.ReadAll(io.LimitReader(r.Body, sizeLimit+1))
  66. originalDataSize = len(data)
  67. if e == io.EOF || int64(originalDataSize) == sizeLimit+1 {
  68. io.Copy(ioutil.Discard, r.Body)
  69. }
  70. r.Body.Close()
  71. }
  72. if e != nil {
  73. return
  74. }
  75. modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
  76. ttl, _ = ReadTTL(r.FormValue("ttl"))
  77. return
  78. }
  79. func ParseUpload(ctx *fasthttp.RequestCtx, sizeLimit int64) (
  80. fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool, originalDataSize int,
  81. modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
  82. pairMap = make(map[string]string)
  83. ctx.Request.Header.VisitAll(func(k, v []byte) {
  84. if len(v) > 0 && bytes.HasPrefix(k, PairNamePrefixBytes) {
  85. pairMap[string(k)] = string(v)
  86. }
  87. })
  88. if ctx.IsPost() {
  89. fileName, data, mimeType, isGzipped, originalDataSize, isChunkedFile, e = parseMultipart(r, sizeLimit)
  90. } else {
  91. isGzipped = false
  92. mimeType = string(ctx.Request.Header.Peek("Content-Type"))
  93. fileName = ""
  94. data, e = ioutil.ReadAll(io.LimitReader(ctx.PostBody(), sizeLimit+1))
  95. originalDataSize = len(data)
  96. if e == io.EOF || int64(originalDataSize) == sizeLimit+1 {
  97. io.Copy(ioutil.Discard, r.Body)
  98. }
  99. r.Body.Close()
  100. }
  101. if e != nil {
  102. return
  103. }
  104. modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
  105. ttl, _ = ReadTTL(r.FormValue("ttl"))
  106. return
  107. }
  108. func OldCreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit int64) (n *Needle, originalSize int, e error) {
  109. var pairMap map[string]string
  110. fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
  111. n = new(Needle)
  112. fname, n.Data, mimeType, pairMap, isGzipped, originalSize, n.LastModified, n.Ttl, isChunkedFile, e = OldParseUpload(r, sizeLimit)
  113. if e != nil {
  114. return
  115. }
  116. if len(fname) < 256 {
  117. n.Name = []byte(fname)
  118. n.SetHasName()
  119. }
  120. if len(mimeType) < 256 {
  121. n.Mime = []byte(mimeType)
  122. n.SetHasMime()
  123. }
  124. if len(pairMap) != 0 {
  125. trimmedPairMap := make(map[string]string)
  126. for k, v := range pairMap {
  127. trimmedPairMap[k[len(PairNamePrefix):]] = v
  128. }
  129. pairs, _ := json.Marshal(trimmedPairMap)
  130. if len(pairs) < 65536 {
  131. n.Pairs = pairs
  132. n.PairsSize = uint16(len(pairs))
  133. n.SetHasPairs()
  134. }
  135. }
  136. if isGzipped {
  137. n.SetGzipped()
  138. }
  139. if n.LastModified == 0 {
  140. n.LastModified = uint64(time.Now().Unix())
  141. }
  142. n.SetHasLastModifiedDate()
  143. if n.Ttl != EMPTY_TTL {
  144. n.SetHasTtl()
  145. }
  146. if isChunkedFile {
  147. n.SetIsChunkManifest()
  148. }
  149. if fixJpgOrientation {
  150. loweredName := strings.ToLower(fname)
  151. if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
  152. n.Data = images.FixJpgOrientation(n.Data)
  153. }
  154. }
  155. n.Checksum = NewCRC(n.Data)
  156. commaSep := strings.LastIndex(r.URL.Path, ",")
  157. dotSep := strings.LastIndex(r.URL.Path, ".")
  158. fid := r.URL.Path[commaSep+1:]
  159. if dotSep > 0 {
  160. fid = r.URL.Path[commaSep+1 : dotSep]
  161. }
  162. e = n.ParsePath(fid)
  163. return
  164. }
  165. func CreateNeedleFromRequest(ctx *fasthttp.RequestCtx, fixJpgOrientation bool, sizeLimit int64) (n *Needle, originalSize int, e error) {
  166. var pairMap map[string]string
  167. fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
  168. n = new(Needle)
  169. fname, n.Data, mimeType, pairMap, isGzipped, originalSize, n.LastModified, n.Ttl, isChunkedFile, e = OldParseUpload(r, sizeLimit)
  170. if e != nil {
  171. return
  172. }
  173. if len(fname) < 256 {
  174. n.Name = []byte(fname)
  175. n.SetHasName()
  176. }
  177. if len(mimeType) < 256 {
  178. n.Mime = []byte(mimeType)
  179. n.SetHasMime()
  180. }
  181. if len(pairMap) != 0 {
  182. trimmedPairMap := make(map[string]string)
  183. for k, v := range pairMap {
  184. trimmedPairMap[k[len(PairNamePrefix):]] = v
  185. }
  186. pairs, _ := json.Marshal(trimmedPairMap)
  187. if len(pairs) < 65536 {
  188. n.Pairs = pairs
  189. n.PairsSize = uint16(len(pairs))
  190. n.SetHasPairs()
  191. }
  192. }
  193. if isGzipped {
  194. n.SetGzipped()
  195. }
  196. if n.LastModified == 0 {
  197. n.LastModified = uint64(time.Now().Unix())
  198. }
  199. n.SetHasLastModifiedDate()
  200. if n.Ttl != EMPTY_TTL {
  201. n.SetHasTtl()
  202. }
  203. if isChunkedFile {
  204. n.SetIsChunkManifest()
  205. }
  206. if fixJpgOrientation {
  207. loweredName := strings.ToLower(fname)
  208. if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
  209. n.Data = images.FixJpgOrientation(n.Data)
  210. }
  211. }
  212. n.Checksum = NewCRC(n.Data)
  213. commaSep := strings.LastIndex(r.URL.Path, ",")
  214. dotSep := strings.LastIndex(r.URL.Path, ".")
  215. fid := r.URL.Path[commaSep+1:]
  216. if dotSep > 0 {
  217. fid = r.URL.Path[commaSep+1 : dotSep]
  218. }
  219. e = n.ParsePath(fid)
  220. return
  221. }
  222. func (n *Needle) ParsePath(fid string) (err error) {
  223. length := len(fid)
  224. if length <= CookieSize*2 {
  225. return fmt.Errorf("Invalid fid: %s", fid)
  226. }
  227. delta := ""
  228. deltaIndex := strings.LastIndex(fid, "_")
  229. if deltaIndex > 0 {
  230. fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
  231. }
  232. n.Id, n.Cookie, err = ParseNeedleIdCookie(fid)
  233. if err != nil {
  234. return err
  235. }
  236. if delta != "" {
  237. if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
  238. n.Id += Uint64ToNeedleId(d)
  239. } else {
  240. return e
  241. }
  242. }
  243. return err
  244. }
  245. func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error) {
  246. if len(key_hash_string) <= CookieSize*2 {
  247. return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too short.")
  248. }
  249. if len(key_hash_string) > (NeedleIdSize+CookieSize)*2 {
  250. return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too long.")
  251. }
  252. split := len(key_hash_string) - CookieSize*2
  253. needleId, err := ParseNeedleId(key_hash_string[:split])
  254. if err != nil {
  255. return NeedleIdEmpty, 0, fmt.Errorf("Parse needleId error: %v", err)
  256. }
  257. cookie, err := ParseCookie(key_hash_string[split:])
  258. if err != nil {
  259. return NeedleIdEmpty, 0, fmt.Errorf("Parse cookie error: %v", err)
  260. }
  261. return needleId, cookie, nil
  262. }
  263. func (n *Needle) LastModifiedString() string {
  264. return time.Unix(int64(n.LastModified), 0).Format("2006-01-02T15:04:05")
  265. }