You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

265 lines
7.6 KiB

12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
  1. package storage
  2. import (
  3. "code.google.com/p/weed-fs/go/glog"
  4. "code.google.com/p/weed-fs/go/util"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "os"
  9. )
  10. const (
  11. FlagGzip = 0x01
  12. FlagHasName = 0x02
  13. FlagHasMime = 0x04
  14. FlagHasLastModifiedDate = 0x08
  15. LastModifiedBytesLength = 5
  16. )
  17. func (n *Needle) DiskSize() int64 {
  18. padding := NeedlePaddingSize - ((NeedleHeaderSize + int64(n.Size) + NeedleChecksumSize) % NeedlePaddingSize)
  19. return NeedleHeaderSize + int64(n.Size) + padding + NeedleChecksumSize
  20. }
  21. func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) {
  22. if s, ok := w.(io.Seeker); ok {
  23. if end, e := s.Seek(0, 1); e == nil {
  24. defer func(s io.Seeker, off int64) {
  25. if err != nil {
  26. if _, e = s.Seek(off, 0); e != nil {
  27. glog.V(0).Infof("Failed to seek %s back to %d with error: %s", w, off, e.Error())
  28. }
  29. }
  30. }(s, end)
  31. } else {
  32. err = fmt.Errorf("Cnnot Read Current Volume Position: %s", e.Error())
  33. return
  34. }
  35. }
  36. switch version {
  37. case Version1:
  38. header := make([]byte, NeedleHeaderSize)
  39. util.Uint32toBytes(header[0:4], n.Cookie)
  40. util.Uint64toBytes(header[4:12], n.Id)
  41. n.Size = uint32(len(n.Data))
  42. size = n.Size
  43. util.Uint32toBytes(header[12:16], n.Size)
  44. if _, err = w.Write(header); err != nil {
  45. return
  46. }
  47. if _, err = w.Write(n.Data); err != nil {
  48. return
  49. }
  50. padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
  51. util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
  52. _, err = w.Write(header[0 : NeedleChecksumSize+padding])
  53. return
  54. case Version2:
  55. header := make([]byte, NeedleHeaderSize)
  56. util.Uint32toBytes(header[0:4], n.Cookie)
  57. util.Uint64toBytes(header[4:12], n.Id)
  58. n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime))
  59. if n.DataSize > 0 {
  60. n.Size = 4 + n.DataSize + 1
  61. if n.HasName() {
  62. n.Size = n.Size + 1 + uint32(n.NameSize)
  63. }
  64. if n.HasMime() {
  65. n.Size = n.Size + 1 + uint32(n.MimeSize)
  66. }
  67. if n.HasLastModifiedDate() {
  68. n.Size = n.Size + LastModifiedBytesLength
  69. }
  70. }
  71. size = n.DataSize
  72. util.Uint32toBytes(header[12:16], n.Size)
  73. if _, err = w.Write(header); err != nil {
  74. return
  75. }
  76. if n.DataSize > 0 {
  77. util.Uint32toBytes(header[0:4], n.DataSize)
  78. if _, err = w.Write(header[0:4]); err != nil {
  79. return
  80. }
  81. if _, err = w.Write(n.Data); err != nil {
  82. return
  83. }
  84. util.Uint8toBytes(header[0:1], n.Flags)
  85. if _, err = w.Write(header[0:1]); err != nil {
  86. return
  87. }
  88. if n.HasName() {
  89. util.Uint8toBytes(header[0:1], n.NameSize)
  90. if _, err = w.Write(header[0:1]); err != nil {
  91. return
  92. }
  93. if _, err = w.Write(n.Name); err != nil {
  94. return
  95. }
  96. }
  97. if n.HasMime() {
  98. util.Uint8toBytes(header[0:1], n.MimeSize)
  99. if _, err = w.Write(header[0:1]); err != nil {
  100. return
  101. }
  102. if _, err = w.Write(n.Mime); err != nil {
  103. return
  104. }
  105. }
  106. if n.HasLastModifiedDate() {
  107. util.Uint64toBytes(header[0:8], n.LastModified)
  108. if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil {
  109. return
  110. }
  111. }
  112. }
  113. padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
  114. util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
  115. _, err = w.Write(header[0 : NeedleChecksumSize+padding])
  116. return n.DataSize, err
  117. }
  118. return 0, fmt.Errorf("Unsupported Version! (%d)", version)
  119. }
  120. func (n *Needle) Read(r io.Reader, size uint32, version Version) (ret int, err error) {
  121. switch version {
  122. case Version1:
  123. bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
  124. if ret, err = r.Read(bytes); err != nil {
  125. return
  126. }
  127. n.readNeedleHeader(bytes)
  128. n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
  129. checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
  130. newChecksum := NewCRC(n.Data)
  131. if checksum != newChecksum.Value() {
  132. return 0, errors.New("CRC error! Data On Disk Corrupted!")
  133. }
  134. n.Checksum = newChecksum
  135. return
  136. case Version2:
  137. if size == 0 {
  138. return 0, nil
  139. }
  140. bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
  141. if ret, err = r.Read(bytes); err != nil {
  142. return
  143. }
  144. if ret != int(NeedleHeaderSize+size+NeedleChecksumSize) {
  145. return 0, errors.New("File Entry Not Found!")
  146. }
  147. n.readNeedleHeader(bytes)
  148. if n.Size != size {
  149. return 0, fmt.Errorf("File Entry Not Found! Needle %d Memory %d", n.Size, size)
  150. }
  151. n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
  152. checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+NeedleChecksumSize])
  153. newChecksum := NewCRC(n.Data)
  154. if checksum != newChecksum.Value() {
  155. return 0, errors.New("CRC error! Data On Disk Corrupted!")
  156. }
  157. n.Checksum = newChecksum
  158. return
  159. }
  160. return 0, fmt.Errorf("Unsupported Version! (%d)", version)
  161. }
  162. func (n *Needle) readNeedleHeader(bytes []byte) {
  163. n.Cookie = util.BytesToUint32(bytes[0:4])
  164. n.Id = util.BytesToUint64(bytes[4:12])
  165. n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize])
  166. }
  167. func (n *Needle) readNeedleDataVersion2(bytes []byte) {
  168. index, lenBytes := 0, len(bytes)
  169. if index < lenBytes {
  170. n.DataSize = util.BytesToUint32(bytes[index : index+4])
  171. index = index + 4
  172. n.Data = bytes[index : index+int(n.DataSize)]
  173. index = index + int(n.DataSize)
  174. n.Flags = bytes[index]
  175. index = index + 1
  176. }
  177. if index < lenBytes && n.HasName() {
  178. n.NameSize = uint8(bytes[index])
  179. index = index + 1
  180. n.Name = bytes[index : index+int(n.NameSize)]
  181. index = index + int(n.NameSize)
  182. }
  183. if index < lenBytes && n.HasMime() {
  184. n.MimeSize = uint8(bytes[index])
  185. index = index + 1
  186. n.Mime = bytes[index : index+int(n.MimeSize)]
  187. index = index + int(n.MimeSize)
  188. }
  189. if index < lenBytes && n.HasLastModifiedDate() {
  190. n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
  191. index = index + LastModifiedBytesLength
  192. }
  193. }
  194. func ReadNeedleHeader(r *os.File, version Version) (n *Needle, bodyLength uint32, err error) {
  195. n = new(Needle)
  196. if version == Version1 || version == Version2 {
  197. bytes := make([]byte, NeedleHeaderSize)
  198. var count int
  199. count, err = r.Read(bytes)
  200. if count <= 0 || err != nil {
  201. return nil, 0, err
  202. }
  203. n.readNeedleHeader(bytes)
  204. padding := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + NeedleChecksumSize) % NeedlePaddingSize)
  205. bodyLength = n.Size + NeedleChecksumSize + padding
  206. }
  207. return
  208. }
  209. //n should be a needle already read the header
  210. //the input stream will read until next file entry
  211. func (n *Needle) ReadNeedleBody(r *os.File, version Version, bodyLength uint32) (err error) {
  212. if bodyLength <= 0 {
  213. return nil
  214. }
  215. switch version {
  216. case Version1:
  217. bytes := make([]byte, bodyLength)
  218. if _, err = r.Read(bytes); err != nil {
  219. return
  220. }
  221. n.Data = bytes[:n.Size]
  222. n.Checksum = NewCRC(n.Data)
  223. case Version2:
  224. bytes := make([]byte, bodyLength)
  225. if _, err = r.Read(bytes); err != nil {
  226. return
  227. }
  228. n.readNeedleDataVersion2(bytes[0:n.Size])
  229. n.Checksum = NewCRC(n.Data)
  230. default:
  231. err = fmt.Errorf("Unsupported Version! (%d)", version)
  232. }
  233. return
  234. }
  235. func (n *Needle) IsGzipped() bool {
  236. return n.Flags&FlagGzip > 0
  237. }
  238. func (n *Needle) SetGzipped() {
  239. n.Flags = n.Flags | FlagGzip
  240. }
  241. func (n *Needle) HasName() bool {
  242. return n.Flags&FlagHasName > 0
  243. }
  244. func (n *Needle) SetHasName() {
  245. n.Flags = n.Flags | FlagHasName
  246. }
  247. func (n *Needle) HasMime() bool {
  248. return n.Flags&FlagHasMime > 0
  249. }
  250. func (n *Needle) SetHasMime() {
  251. n.Flags = n.Flags | FlagHasMime
  252. }
  253. func (n *Needle) HasLastModifiedDate() bool {
  254. return n.Flags&FlagHasLastModifiedDate > 0
  255. }
  256. func (n *Needle) SetHasLastModifiedDate() {
  257. n.Flags = n.Flags | FlagHasLastModifiedDate
  258. }