You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

326 lines
9.0 KiB

12 years ago
12 years ago
12 years ago
7 years ago
12 years ago
9 years ago
10 years ago
12 years ago
12 years ago
7 years ago
12 years ago
7 years ago
7 years ago
12 years ago
12 years ago
12 years ago
7 years ago
12 years ago
7 years ago
12 years ago
7 years ago
12 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
9 years ago
9 years ago
  1. package storage
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "os"
  7. "github.com/chrislusf/seaweedfs/weed/glog"
  8. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  9. "github.com/chrislusf/seaweedfs/weed/util"
  10. )
  11. const (
  12. FlagGzip = 0x01
  13. FlagHasName = 0x02
  14. FlagHasMime = 0x04
  15. FlagHasLastModifiedDate = 0x08
  16. FlagHasTtl = 0x10
  17. FlagHasPairs = 0x20
  18. FlagIsChunkManifest = 0x80
  19. LastModifiedBytesLength = 5
  20. TtlBytesLength = 2
  21. )
  22. func (n *Needle) DiskSize() int64 {
  23. return getActualSize(n.Size)
  24. }
  25. func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize int64, err error) {
  26. if s, ok := w.(io.Seeker); ok {
  27. if end, e := s.Seek(0, 1); e == nil {
  28. defer func(s io.Seeker, off int64) {
  29. if err != nil {
  30. if _, e = s.Seek(off, 0); e != nil {
  31. glog.V(0).Infof("Failed to seek %s back to %d with error: %v", w, off, e)
  32. }
  33. }
  34. }(s, end)
  35. } else {
  36. err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
  37. return
  38. }
  39. }
  40. switch version {
  41. case Version1:
  42. header := make([]byte, NeedleEntrySize)
  43. CookieToBytes(header[0:CookieSize], n.Cookie)
  44. NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
  45. n.Size = uint32(len(n.Data))
  46. size = n.Size
  47. util.Uint32toBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
  48. if _, err = w.Write(header); err != nil {
  49. return
  50. }
  51. if _, err = w.Write(n.Data); err != nil {
  52. return
  53. }
  54. actualSize = NeedleEntrySize + int64(n.Size)
  55. padding := NeedlePaddingSize - ((NeedleEntrySize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
  56. util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
  57. _, err = w.Write(header[0 : NeedleChecksumSize+padding])
  58. return
  59. case Version2:
  60. header := make([]byte, NeedleEntrySize)
  61. CookieToBytes(header[0:CookieSize], n.Cookie)
  62. NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
  63. n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime))
  64. if n.DataSize > 0 {
  65. n.Size = 4 + n.DataSize + 1
  66. if n.HasName() {
  67. n.Size = n.Size + 1 + uint32(n.NameSize)
  68. }
  69. if n.HasMime() {
  70. n.Size = n.Size + 1 + uint32(n.MimeSize)
  71. }
  72. if n.HasLastModifiedDate() {
  73. n.Size = n.Size + LastModifiedBytesLength
  74. }
  75. if n.HasTtl() {
  76. n.Size = n.Size + TtlBytesLength
  77. }
  78. if n.HasPairs() {
  79. n.Size += 2 + uint32(n.PairsSize)
  80. }
  81. } else {
  82. n.Size = 0
  83. }
  84. size = n.DataSize
  85. util.Uint32toBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
  86. if _, err = w.Write(header); err != nil {
  87. return
  88. }
  89. if n.DataSize > 0 {
  90. util.Uint32toBytes(header[0:4], n.DataSize)
  91. if _, err = w.Write(header[0:4]); err != nil {
  92. return
  93. }
  94. if _, err = w.Write(n.Data); err != nil {
  95. return
  96. }
  97. util.Uint8toBytes(header[0:1], n.Flags)
  98. if _, err = w.Write(header[0:1]); err != nil {
  99. return
  100. }
  101. if n.HasName() {
  102. util.Uint8toBytes(header[0:1], n.NameSize)
  103. if _, err = w.Write(header[0:1]); err != nil {
  104. return
  105. }
  106. if _, err = w.Write(n.Name); err != nil {
  107. return
  108. }
  109. }
  110. if n.HasMime() {
  111. util.Uint8toBytes(header[0:1], n.MimeSize)
  112. if _, err = w.Write(header[0:1]); err != nil {
  113. return
  114. }
  115. if _, err = w.Write(n.Mime); err != nil {
  116. return
  117. }
  118. }
  119. if n.HasLastModifiedDate() {
  120. util.Uint64toBytes(header[0:8], n.LastModified)
  121. if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil {
  122. return
  123. }
  124. }
  125. if n.HasTtl() && n.Ttl != nil {
  126. n.Ttl.ToBytes(header[0:TtlBytesLength])
  127. if _, err = w.Write(header[0:TtlBytesLength]); err != nil {
  128. return
  129. }
  130. }
  131. if n.HasPairs() {
  132. util.Uint16toBytes(header[0:2], n.PairsSize)
  133. if _, err = w.Write(header[0:2]); err != nil {
  134. return
  135. }
  136. if _, err = w.Write(n.Pairs); err != nil {
  137. return
  138. }
  139. }
  140. }
  141. padding := NeedlePaddingSize - ((NeedleEntrySize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
  142. util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
  143. _, err = w.Write(header[0 : NeedleChecksumSize+padding])
  144. return n.DataSize, getActualSize(n.Size), err
  145. }
  146. return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
  147. }
  148. func ReadNeedleBlob(r *os.File, offset int64, size uint32) (dataSlice []byte, err error) {
  149. dataSlice = make([]byte, int(getActualSize(size)))
  150. _, err = r.ReadAt(dataSlice, offset)
  151. return dataSlice, err
  152. }
  153. func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
  154. bytes, err := ReadNeedleBlob(r, offset, size)
  155. if err != nil {
  156. return err
  157. }
  158. n.ParseNeedleHeader(bytes)
  159. if n.Size != size {
  160. return fmt.Errorf("File Entry Not Found. Needle id %d expected size %d Memory %d", n.Id, n.Size, size)
  161. }
  162. switch version {
  163. case Version1:
  164. n.Data = bytes[NeedleEntrySize : NeedleEntrySize+size]
  165. case Version2:
  166. n.readNeedleDataVersion2(bytes[NeedleEntrySize : NeedleEntrySize+int(n.Size)])
  167. }
  168. if size == 0 {
  169. return nil
  170. }
  171. checksum := util.BytesToUint32(bytes[NeedleEntrySize+size : NeedleEntrySize+size+NeedleChecksumSize])
  172. newChecksum := NewCRC(n.Data)
  173. if checksum != newChecksum.Value() {
  174. return errors.New("CRC error! Data On Disk Corrupted")
  175. }
  176. n.Checksum = newChecksum
  177. return nil
  178. }
  179. func (n *Needle) ParseNeedleHeader(bytes []byte) {
  180. n.Cookie = BytesToCookie(bytes[0:CookieSize])
  181. n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize])
  182. n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize : NeedleEntrySize])
  183. }
  184. func (n *Needle) readNeedleDataVersion2(bytes []byte) {
  185. index, lenBytes := 0, len(bytes)
  186. if index < lenBytes {
  187. n.DataSize = util.BytesToUint32(bytes[index : index+4])
  188. index = index + 4
  189. if int(n.DataSize)+index > lenBytes {
  190. // this if clause is due to bug #87 and #93, fixed in v0.69
  191. // remove this clause later
  192. return
  193. }
  194. n.Data = bytes[index : index+int(n.DataSize)]
  195. index = index + int(n.DataSize)
  196. n.Flags = bytes[index]
  197. index = index + 1
  198. }
  199. if index < lenBytes && n.HasName() {
  200. n.NameSize = uint8(bytes[index])
  201. index = index + 1
  202. n.Name = bytes[index : index+int(n.NameSize)]
  203. index = index + int(n.NameSize)
  204. }
  205. if index < lenBytes && n.HasMime() {
  206. n.MimeSize = uint8(bytes[index])
  207. index = index + 1
  208. n.Mime = bytes[index : index+int(n.MimeSize)]
  209. index = index + int(n.MimeSize)
  210. }
  211. if index < lenBytes && n.HasLastModifiedDate() {
  212. n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
  213. index = index + LastModifiedBytesLength
  214. }
  215. if index < lenBytes && n.HasTtl() {
  216. n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
  217. index = index + TtlBytesLength
  218. }
  219. if index < lenBytes && n.HasPairs() {
  220. n.PairsSize = util.BytesToUint16(bytes[index : index+2])
  221. index += 2
  222. end := index + int(n.PairsSize)
  223. n.Pairs = bytes[index:end]
  224. index = end
  225. }
  226. }
  227. func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bodyLength int64, err error) {
  228. n = new(Needle)
  229. if version == Version1 || version == Version2 {
  230. bytes := make([]byte, NeedleEntrySize)
  231. var count int
  232. count, err = r.ReadAt(bytes, offset)
  233. if count <= 0 || err != nil {
  234. return nil, 0, err
  235. }
  236. n.ParseNeedleHeader(bytes)
  237. padding := NeedlePaddingSize - ((n.Size + NeedleEntrySize + NeedleChecksumSize) % NeedlePaddingSize)
  238. bodyLength = int64(n.Size) + NeedleChecksumSize + int64(padding)
  239. }
  240. return
  241. }
  242. //n should be a needle already read the header
  243. //the input stream will read until next file entry
  244. func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength int64) (err error) {
  245. if bodyLength <= 0 {
  246. return nil
  247. }
  248. switch version {
  249. case Version1:
  250. bytes := make([]byte, bodyLength)
  251. if _, err = r.ReadAt(bytes, offset); err != nil {
  252. return
  253. }
  254. n.Data = bytes[:n.Size]
  255. n.Checksum = NewCRC(n.Data)
  256. case Version2:
  257. bytes := make([]byte, bodyLength)
  258. if _, err = r.ReadAt(bytes, offset); err != nil {
  259. return
  260. }
  261. n.readNeedleDataVersion2(bytes[0:n.Size])
  262. n.Checksum = NewCRC(n.Data)
  263. default:
  264. err = fmt.Errorf("Unsupported Version! (%d)", version)
  265. }
  266. return
  267. }
  268. func (n *Needle) IsGzipped() bool {
  269. return n.Flags&FlagGzip > 0
  270. }
  271. func (n *Needle) SetGzipped() {
  272. n.Flags = n.Flags | FlagGzip
  273. }
  274. func (n *Needle) HasName() bool {
  275. return n.Flags&FlagHasName > 0
  276. }
  277. func (n *Needle) SetHasName() {
  278. n.Flags = n.Flags | FlagHasName
  279. }
  280. func (n *Needle) HasMime() bool {
  281. return n.Flags&FlagHasMime > 0
  282. }
  283. func (n *Needle) SetHasMime() {
  284. n.Flags = n.Flags | FlagHasMime
  285. }
  286. func (n *Needle) HasLastModifiedDate() bool {
  287. return n.Flags&FlagHasLastModifiedDate > 0
  288. }
  289. func (n *Needle) SetHasLastModifiedDate() {
  290. n.Flags = n.Flags | FlagHasLastModifiedDate
  291. }
  292. func (n *Needle) HasTtl() bool {
  293. return n.Flags&FlagHasTtl > 0
  294. }
  295. func (n *Needle) SetHasTtl() {
  296. n.Flags = n.Flags | FlagHasTtl
  297. }
  298. func (n *Needle) IsChunkedManifest() bool {
  299. return n.Flags&FlagIsChunkManifest > 0
  300. }
  301. func (n *Needle) SetIsChunkManifest() {
  302. n.Flags = n.Flags | FlagIsChunkManifest
  303. }
  304. func (n *Needle) HasPairs() bool {
  305. return n.Flags&FlagHasPairs != 0
  306. }
  307. func (n *Needle) SetHasPairs() {
  308. n.Flags = n.Flags | FlagHasPairs
  309. }