You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

294 lines
8.4 KiB

12 years ago
12 years ago
12 years ago
12 years ago
10 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
  1. package storage
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "os"
  7. "github.com/chrislusf/seaweedfs/go/glog"
  8. "github.com/chrislusf/seaweedfs/go/util"
  9. )
  10. const (
  11. FlagGzip = 0x01
  12. FlagHasName = 0x02
  13. FlagHasMime = 0x04
  14. FlagHasLastModifiedDate = 0x08
  15. FlagHasTtl = 0x10
  16. LastModifiedBytesLength = 5
  17. TtlBytesLength = 2
  18. )
  19. func (n *Needle) DiskSize() int64 {
  20. padding := NeedlePaddingSize - ((NeedleHeaderSize + int64(n.Size) + NeedleChecksumSize) % NeedlePaddingSize)
  21. return NeedleHeaderSize + int64(n.Size) + padding + NeedleChecksumSize
  22. }
  23. func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) {
  24. if s, ok := w.(io.Seeker); ok {
  25. if end, e := s.Seek(0, 1); e == nil {
  26. defer func(s io.Seeker, off int64) {
  27. if err != nil {
  28. if _, e = s.Seek(off, 0); e != nil {
  29. glog.V(0).Infof("Failed to seek %s back to %d with error: %v", w, off, e)
  30. }
  31. }
  32. }(s, end)
  33. } else {
  34. err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
  35. return
  36. }
  37. }
  38. switch version {
  39. case Version1:
  40. header := make([]byte, NeedleHeaderSize)
  41. util.Uint32toBytes(header[0:4], n.Cookie)
  42. util.Uint64toBytes(header[4:12], n.Id)
  43. n.Size = uint32(len(n.Data))
  44. size = n.Size
  45. util.Uint32toBytes(header[12:16], n.Size)
  46. if _, err = w.Write(header); err != nil {
  47. return
  48. }
  49. if _, err = w.Write(n.Data); err != nil {
  50. return
  51. }
  52. padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
  53. util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
  54. _, err = w.Write(header[0 : NeedleChecksumSize+padding])
  55. return
  56. case Version2:
  57. header := make([]byte, NeedleHeaderSize)
  58. util.Uint32toBytes(header[0:4], n.Cookie)
  59. util.Uint64toBytes(header[4:12], n.Id)
  60. n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime))
  61. if n.DataSize > 0 {
  62. n.Size = 4 + n.DataSize + 1
  63. if n.HasName() {
  64. n.Size = n.Size + 1 + uint32(n.NameSize)
  65. }
  66. if n.HasMime() {
  67. n.Size = n.Size + 1 + uint32(n.MimeSize)
  68. }
  69. if n.HasLastModifiedDate() {
  70. n.Size = n.Size + LastModifiedBytesLength
  71. }
  72. if n.HasTtl() {
  73. n.Size = n.Size + TtlBytesLength
  74. }
  75. } else {
  76. n.Size = 0
  77. }
  78. size = n.DataSize
  79. util.Uint32toBytes(header[12:16], n.Size)
  80. if _, err = w.Write(header); err != nil {
  81. return
  82. }
  83. if n.DataSize > 0 {
  84. util.Uint32toBytes(header[0:4], n.DataSize)
  85. if _, err = w.Write(header[0:4]); err != nil {
  86. return
  87. }
  88. if _, err = w.Write(n.Data); err != nil {
  89. return
  90. }
  91. util.Uint8toBytes(header[0:1], n.Flags)
  92. if _, err = w.Write(header[0:1]); err != nil {
  93. return
  94. }
  95. if n.HasName() {
  96. util.Uint8toBytes(header[0:1], n.NameSize)
  97. if _, err = w.Write(header[0:1]); err != nil {
  98. return
  99. }
  100. if _, err = w.Write(n.Name); err != nil {
  101. return
  102. }
  103. }
  104. if n.HasMime() {
  105. util.Uint8toBytes(header[0:1], n.MimeSize)
  106. if _, err = w.Write(header[0:1]); err != nil {
  107. return
  108. }
  109. if _, err = w.Write(n.Mime); err != nil {
  110. return
  111. }
  112. }
  113. if n.HasLastModifiedDate() {
  114. util.Uint64toBytes(header[0:8], n.LastModified)
  115. if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil {
  116. return
  117. }
  118. }
  119. if n.HasTtl() {
  120. n.Ttl.ToBytes(header[0:TtlBytesLength])
  121. if _, err = w.Write(header[0:TtlBytesLength]); err != nil {
  122. return
  123. }
  124. }
  125. }
  126. padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
  127. util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
  128. _, err = w.Write(header[0 : NeedleChecksumSize+padding])
  129. return n.DataSize, err
  130. }
  131. return 0, fmt.Errorf("Unsupported Version! (%d)", version)
  132. }
  133. func (n *Needle) Read(r *os.File, offset int64, size uint32, version Version) (ret int, err error) {
  134. switch version {
  135. case Version1:
  136. bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
  137. if ret, err = r.ReadAt(bytes, offset); err != nil {
  138. return
  139. }
  140. n.readNeedleHeader(bytes)
  141. n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
  142. checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
  143. newChecksum := NewCRC(n.Data)
  144. if checksum != newChecksum.Value() {
  145. return 0, errors.New("CRC error! Data On Disk Corrupted")
  146. }
  147. n.Checksum = newChecksum
  148. return
  149. case Version2:
  150. if size == 0 {
  151. return 0, nil
  152. }
  153. bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
  154. if ret, err = r.ReadAt(bytes, offset); err != nil {
  155. return
  156. }
  157. if ret != int(NeedleHeaderSize+size+NeedleChecksumSize) {
  158. return 0, errors.New("File Entry Not Found")
  159. }
  160. n.readNeedleHeader(bytes)
  161. if n.Size != size {
  162. return 0, fmt.Errorf("File Entry Not Found. Needle %d Memory %d", n.Size, size)
  163. }
  164. n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
  165. checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+NeedleChecksumSize])
  166. newChecksum := NewCRC(n.Data)
  167. if checksum != newChecksum.Value() {
  168. return 0, errors.New("CRC Found Data On Disk Corrupted")
  169. }
  170. n.Checksum = newChecksum
  171. return
  172. }
  173. return 0, fmt.Errorf("Unsupported Version! (%d)", version)
  174. }
  175. func (n *Needle) readNeedleHeader(bytes []byte) {
  176. n.Cookie = util.BytesToUint32(bytes[0:4])
  177. n.Id = util.BytesToUint64(bytes[4:12])
  178. n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize])
  179. }
  180. func (n *Needle) readNeedleDataVersion2(bytes []byte) {
  181. index, lenBytes := 0, len(bytes)
  182. if index < lenBytes {
  183. n.DataSize = util.BytesToUint32(bytes[index : index+4])
  184. index = index + 4
  185. if int(n.DataSize)+index > lenBytes {
  186. // this if clause is due to bug #87 and #93, fixed in v0.69
  187. // remove this clause later
  188. return
  189. }
  190. n.Data = bytes[index : index+int(n.DataSize)]
  191. index = index + int(n.DataSize)
  192. n.Flags = bytes[index]
  193. index = index + 1
  194. }
  195. if index < lenBytes && n.HasName() {
  196. n.NameSize = uint8(bytes[index])
  197. index = index + 1
  198. n.Name = bytes[index : index+int(n.NameSize)]
  199. index = index + int(n.NameSize)
  200. }
  201. if index < lenBytes && n.HasMime() {
  202. n.MimeSize = uint8(bytes[index])
  203. index = index + 1
  204. n.Mime = bytes[index : index+int(n.MimeSize)]
  205. index = index + int(n.MimeSize)
  206. }
  207. if index < lenBytes && n.HasLastModifiedDate() {
  208. n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
  209. index = index + LastModifiedBytesLength
  210. }
  211. if index < lenBytes && n.HasTtl() {
  212. n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
  213. index = index + TtlBytesLength
  214. }
  215. }
  216. func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bodyLength uint32, err error) {
  217. n = new(Needle)
  218. if version == Version1 || version == Version2 {
  219. bytes := make([]byte, NeedleHeaderSize)
  220. var count int
  221. count, err = r.ReadAt(bytes, offset)
  222. if count <= 0 || err != nil {
  223. return nil, 0, err
  224. }
  225. n.readNeedleHeader(bytes)
  226. padding := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + NeedleChecksumSize) % NeedlePaddingSize)
  227. bodyLength = n.Size + NeedleChecksumSize + padding
  228. }
  229. return
  230. }
  231. //n should be a needle already read the header
  232. //the input stream will read until next file entry
  233. func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength uint32) (err error) {
  234. if bodyLength <= 0 {
  235. return nil
  236. }
  237. switch version {
  238. case Version1:
  239. bytes := make([]byte, bodyLength)
  240. if _, err = r.ReadAt(bytes, offset); err != nil {
  241. return
  242. }
  243. n.Data = bytes[:n.Size]
  244. n.Checksum = NewCRC(n.Data)
  245. case Version2:
  246. bytes := make([]byte, bodyLength)
  247. if _, err = r.ReadAt(bytes, offset); err != nil {
  248. return
  249. }
  250. n.readNeedleDataVersion2(bytes[0:n.Size])
  251. n.Checksum = NewCRC(n.Data)
  252. default:
  253. err = fmt.Errorf("Unsupported Version! (%d)", version)
  254. }
  255. return
  256. }
  257. func (n *Needle) IsGzipped() bool {
  258. return n.Flags&FlagGzip > 0
  259. }
  260. func (n *Needle) SetGzipped() {
  261. n.Flags = n.Flags | FlagGzip
  262. }
  263. func (n *Needle) HasName() bool {
  264. return n.Flags&FlagHasName > 0
  265. }
  266. func (n *Needle) SetHasName() {
  267. n.Flags = n.Flags | FlagHasName
  268. }
  269. func (n *Needle) HasMime() bool {
  270. return n.Flags&FlagHasMime > 0
  271. }
  272. func (n *Needle) SetHasMime() {
  273. n.Flags = n.Flags | FlagHasMime
  274. }
  275. func (n *Needle) HasLastModifiedDate() bool {
  276. return n.Flags&FlagHasLastModifiedDate > 0
  277. }
  278. func (n *Needle) SetHasLastModifiedDate() {
  279. n.Flags = n.Flags | FlagHasLastModifiedDate
  280. }
  281. func (n *Needle) HasTtl() bool {
  282. return n.Flags&FlagHasTtl > 0
  283. }
  284. func (n *Needle) SetHasTtl() {
  285. n.Flags = n.Flags | FlagHasTtl
  286. }