You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

402 lines
12 KiB

6 years ago
12 years ago
12 years ago
6 years ago
12 years ago
5 years ago
9 years ago
12 years ago
6 years ago
12 years ago
12 years ago
12 years ago
12 years ago
7 years ago
12 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
5 years ago
5 years ago
9 years ago
9 years ago
6 years ago
6 years ago
  1. package needle
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "math"
  7. "github.com/chrislusf/seaweedfs/weed/util/log"
  8. "github.com/chrislusf/seaweedfs/weed/storage/backend"
  9. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  10. "github.com/chrislusf/seaweedfs/weed/util"
  11. )
  12. const (
  13. FlagIsCompressed = 0x01
  14. FlagHasName = 0x02
  15. FlagHasMime = 0x04
  16. FlagHasLastModifiedDate = 0x08
  17. FlagHasTtl = 0x10
  18. FlagHasPairs = 0x20
  19. FlagIsChunkManifest = 0x80
  20. LastModifiedBytesLength = 5
  21. TtlBytesLength = 2
  22. )
  23. var ErrorSizeMismatch = errors.New("size mismatch")
  24. func (n *Needle) DiskSize(version Version) int64 {
  25. return GetActualSize(n.Size, version)
  26. }
  27. func (n *Needle) prepareWriteBuffer(version Version) ([]byte, Size, int64, error) {
  28. writeBytes := make([]byte, 0)
  29. switch version {
  30. case Version1:
  31. header := make([]byte, NeedleHeaderSize)
  32. CookieToBytes(header[0:CookieSize], n.Cookie)
  33. NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
  34. n.Size = Size(len(n.Data))
  35. SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
  36. size := n.Size
  37. actualSize := NeedleHeaderSize + int64(n.Size)
  38. writeBytes = append(writeBytes, header...)
  39. writeBytes = append(writeBytes, n.Data...)
  40. padding := PaddingLength(n.Size, version)
  41. util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
  42. writeBytes = append(writeBytes, header[0:NeedleChecksumSize+padding]...)
  43. return writeBytes, size, actualSize, nil
  44. case Version2, Version3:
  45. header := make([]byte, NeedleHeaderSize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation
  46. CookieToBytes(header[0:CookieSize], n.Cookie)
  47. NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
  48. if len(n.Name) >= math.MaxUint8 {
  49. n.NameSize = math.MaxUint8
  50. } else {
  51. n.NameSize = uint8(len(n.Name))
  52. }
  53. n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime))
  54. if n.DataSize > 0 {
  55. n.Size = 4 + Size(n.DataSize) + 1
  56. if n.HasName() {
  57. n.Size = n.Size + 1 + Size(n.NameSize)
  58. }
  59. if n.HasMime() {
  60. n.Size = n.Size + 1 + Size(n.MimeSize)
  61. }
  62. if n.HasLastModifiedDate() {
  63. n.Size = n.Size + LastModifiedBytesLength
  64. }
  65. if n.HasTtl() {
  66. n.Size = n.Size + TtlBytesLength
  67. }
  68. if n.HasPairs() {
  69. n.Size += 2 + Size(n.PairsSize)
  70. }
  71. } else {
  72. n.Size = 0
  73. }
  74. SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
  75. writeBytes = append(writeBytes, header[0:NeedleHeaderSize]...)
  76. if n.DataSize > 0 {
  77. util.Uint32toBytes(header[0:4], n.DataSize)
  78. writeBytes = append(writeBytes, header[0:4]...)
  79. writeBytes = append(writeBytes, n.Data...)
  80. util.Uint8toBytes(header[0:1], n.Flags)
  81. writeBytes = append(writeBytes, header[0:1]...)
  82. if n.HasName() {
  83. util.Uint8toBytes(header[0:1], n.NameSize)
  84. writeBytes = append(writeBytes, header[0:1]...)
  85. writeBytes = append(writeBytes, n.Name[:n.NameSize]...)
  86. }
  87. if n.HasMime() {
  88. util.Uint8toBytes(header[0:1], n.MimeSize)
  89. writeBytes = append(writeBytes, header[0:1]...)
  90. writeBytes = append(writeBytes, n.Mime...)
  91. }
  92. if n.HasLastModifiedDate() {
  93. util.Uint64toBytes(header[0:8], n.LastModified)
  94. writeBytes = append(writeBytes, header[8-LastModifiedBytesLength:8]...)
  95. }
  96. if n.HasTtl() && n.Ttl != nil {
  97. n.Ttl.ToBytes(header[0:TtlBytesLength])
  98. writeBytes = append(writeBytes, header[0:TtlBytesLength]...)
  99. }
  100. if n.HasPairs() {
  101. util.Uint16toBytes(header[0:2], n.PairsSize)
  102. writeBytes = append(writeBytes, header[0:2]...)
  103. writeBytes = append(writeBytes, n.Pairs...)
  104. }
  105. }
  106. padding := PaddingLength(n.Size, version)
  107. util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
  108. if version == Version2 {
  109. writeBytes = append(writeBytes, header[0:NeedleChecksumSize+padding]...)
  110. } else {
  111. // version3
  112. util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs)
  113. writeBytes = append(writeBytes, header[0:NeedleChecksumSize+TimestampSize+padding]...)
  114. }
  115. return writeBytes, Size(n.DataSize), GetActualSize(n.Size, version), nil
  116. }
  117. return writeBytes, 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
  118. }
  119. func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error) {
  120. if end, _, e := w.GetStat(); e == nil {
  121. defer func(w backend.BackendStorageFile, off int64) {
  122. if err != nil {
  123. if te := w.Truncate(end); te != nil {
  124. log.Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
  125. }
  126. }
  127. }(w, end)
  128. offset = uint64(end)
  129. } else {
  130. err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
  131. return
  132. }
  133. if offset >= MaxPossibleVolumeSize {
  134. err = fmt.Errorf("Volume Size %d Exeededs %d", offset, MaxPossibleVolumeSize)
  135. return
  136. }
  137. bytesToWrite, size, actualSize, err := n.prepareWriteBuffer(version)
  138. if err == nil {
  139. _, err = w.WriteAt(bytesToWrite, int64(offset))
  140. }
  141. return offset, size, actualSize, err
  142. }
  143. func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, version Version) (dataSlice []byte, err error) {
  144. dataSize := GetActualSize(size, version)
  145. dataSlice = make([]byte, int(dataSize))
  146. _, err = r.ReadAt(dataSlice, offset)
  147. return dataSlice, err
  148. }
  149. // ReadBytes hydrates the needle from the bytes buffer, with only n.Id is set.
  150. func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Version) (err error) {
  151. n.ParseNeedleHeader(bytes)
  152. if n.Size != size {
  153. // cookie is not always passed in for this API. Use size to do preliminary checking.
  154. if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) {
  155. log.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
  156. return ErrorSizeMismatch
  157. }
  158. return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
  159. }
  160. switch version {
  161. case Version1:
  162. n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
  163. case Version2, Version3:
  164. err = n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
  165. }
  166. if err != nil && err != io.EOF {
  167. return err
  168. }
  169. if size > 0 {
  170. checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
  171. newChecksum := NewCRC(n.Data)
  172. if checksum != newChecksum.Value() {
  173. return errors.New("CRC error! Data On Disk Corrupted")
  174. }
  175. n.Checksum = newChecksum
  176. }
  177. if version == Version3 {
  178. tsOffset := NeedleHeaderSize + size + NeedleChecksumSize
  179. n.AppendAtNs = util.BytesToUint64(bytes[tsOffset : tsOffset+TimestampSize])
  180. }
  181. return nil
  182. }
  183. // ReadData hydrates the needle from the file, with only n.Id is set.
  184. func (n *Needle) ReadData(r backend.BackendStorageFile, offset int64, size Size, version Version) (err error) {
  185. bytes, err := ReadNeedleBlob(r, offset, size, version)
  186. if err != nil {
  187. return err
  188. }
  189. return n.ReadBytes(bytes, offset, size, version)
  190. }
  191. func (n *Needle) ParseNeedleHeader(bytes []byte) {
  192. n.Cookie = BytesToCookie(bytes[0:CookieSize])
  193. n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize])
  194. n.Size = BytesToSize(bytes[CookieSize+NeedleIdSize : NeedleHeaderSize])
  195. }
  196. func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
  197. index, lenBytes := 0, len(bytes)
  198. if index < lenBytes {
  199. n.DataSize = util.BytesToUint32(bytes[index : index+4])
  200. index = index + 4
  201. if int(n.DataSize)+index > lenBytes {
  202. return fmt.Errorf("index out of range %d", 1)
  203. }
  204. n.Data = bytes[index : index+int(n.DataSize)]
  205. index = index + int(n.DataSize)
  206. n.Flags = bytes[index]
  207. index = index + 1
  208. }
  209. if index < lenBytes && n.HasName() {
  210. n.NameSize = uint8(bytes[index])
  211. index = index + 1
  212. if int(n.NameSize)+index > lenBytes {
  213. return fmt.Errorf("index out of range %d", 2)
  214. }
  215. n.Name = bytes[index : index+int(n.NameSize)]
  216. index = index + int(n.NameSize)
  217. }
  218. if index < lenBytes && n.HasMime() {
  219. n.MimeSize = uint8(bytes[index])
  220. index = index + 1
  221. if int(n.MimeSize)+index > lenBytes {
  222. return fmt.Errorf("index out of range %d", 3)
  223. }
  224. n.Mime = bytes[index : index+int(n.MimeSize)]
  225. index = index + int(n.MimeSize)
  226. }
  227. if index < lenBytes && n.HasLastModifiedDate() {
  228. if LastModifiedBytesLength+index > lenBytes {
  229. return fmt.Errorf("index out of range %d", 4)
  230. }
  231. n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
  232. index = index + LastModifiedBytesLength
  233. }
  234. if index < lenBytes && n.HasTtl() {
  235. if TtlBytesLength+index > lenBytes {
  236. return fmt.Errorf("index out of range %d", 5)
  237. }
  238. n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
  239. index = index + TtlBytesLength
  240. }
  241. if index < lenBytes && n.HasPairs() {
  242. if 2+index > lenBytes {
  243. return fmt.Errorf("index out of range %d", 6)
  244. }
  245. n.PairsSize = util.BytesToUint16(bytes[index : index+2])
  246. index += 2
  247. if int(n.PairsSize)+index > lenBytes {
  248. return fmt.Errorf("index out of range %d", 7)
  249. }
  250. end := index + int(n.PairsSize)
  251. n.Pairs = bytes[index:end]
  252. index = end
  253. }
  254. return nil
  255. }
  256. func ReadNeedleHeader(r backend.BackendStorageFile, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error) {
  257. n = new(Needle)
  258. if version == Version1 || version == Version2 || version == Version3 {
  259. bytes = make([]byte, NeedleHeaderSize)
  260. var count int
  261. count, err = r.ReadAt(bytes, offset)
  262. if count <= 0 || err != nil {
  263. return nil, bytes, 0, err
  264. }
  265. n.ParseNeedleHeader(bytes)
  266. bodyLength = NeedleBodyLength(n.Size, version)
  267. }
  268. return
  269. }
  270. func PaddingLength(needleSize Size, version Version) Size {
  271. if version == Version3 {
  272. // this is same value as version2, but just listed here for clarity
  273. return NeedlePaddingSize - ((NeedleHeaderSize + needleSize + NeedleChecksumSize + TimestampSize) % NeedlePaddingSize)
  274. }
  275. return NeedlePaddingSize - ((NeedleHeaderSize + needleSize + NeedleChecksumSize) % NeedlePaddingSize)
  276. }
  277. func NeedleBodyLength(needleSize Size, version Version) int64 {
  278. if version == Version3 {
  279. return int64(needleSize) + NeedleChecksumSize + TimestampSize + int64(PaddingLength(needleSize, version))
  280. }
  281. return int64(needleSize) + NeedleChecksumSize + int64(PaddingLength(needleSize, version))
  282. }
  283. //n should be a needle already read the header
  284. //the input stream will read until next file entry
  285. func (n *Needle) ReadNeedleBody(r backend.BackendStorageFile, version Version, offset int64, bodyLength int64) (bytes []byte, err error) {
  286. if bodyLength <= 0 {
  287. return nil, nil
  288. }
  289. bytes = make([]byte, bodyLength)
  290. if _, err = r.ReadAt(bytes, offset); err != nil {
  291. return
  292. }
  293. err = n.ReadNeedleBodyBytes(bytes, version)
  294. return
  295. }
  296. func (n *Needle) ReadNeedleBodyBytes(needleBody []byte, version Version) (err error) {
  297. if len(needleBody) <= 0 {
  298. return nil
  299. }
  300. switch version {
  301. case Version1:
  302. n.Data = needleBody[:n.Size]
  303. n.Checksum = NewCRC(n.Data)
  304. case Version2, Version3:
  305. err = n.readNeedleDataVersion2(needleBody[0:n.Size])
  306. n.Checksum = NewCRC(n.Data)
  307. if version == Version3 {
  308. tsOffset := n.Size + NeedleChecksumSize
  309. n.AppendAtNs = util.BytesToUint64(needleBody[tsOffset : tsOffset+TimestampSize])
  310. }
  311. default:
  312. err = fmt.Errorf("unsupported version %d!", version)
  313. }
  314. return
  315. }
  316. func (n *Needle) IsCompressed() bool {
  317. return n.Flags&FlagIsCompressed > 0
  318. }
  319. func (n *Needle) SetIsCompressed() {
  320. n.Flags = n.Flags | FlagIsCompressed
  321. }
  322. func (n *Needle) HasName() bool {
  323. return n.Flags&FlagHasName > 0
  324. }
  325. func (n *Needle) SetHasName() {
  326. n.Flags = n.Flags | FlagHasName
  327. }
  328. func (n *Needle) HasMime() bool {
  329. return n.Flags&FlagHasMime > 0
  330. }
  331. func (n *Needle) SetHasMime() {
  332. n.Flags = n.Flags | FlagHasMime
  333. }
  334. func (n *Needle) HasLastModifiedDate() bool {
  335. return n.Flags&FlagHasLastModifiedDate > 0
  336. }
  337. func (n *Needle) SetHasLastModifiedDate() {
  338. n.Flags = n.Flags | FlagHasLastModifiedDate
  339. }
  340. func (n *Needle) HasTtl() bool {
  341. return n.Flags&FlagHasTtl > 0
  342. }
  343. func (n *Needle) SetHasTtl() {
  344. n.Flags = n.Flags | FlagHasTtl
  345. }
  346. func (n *Needle) IsChunkedManifest() bool {
  347. return n.Flags&FlagIsChunkManifest > 0
  348. }
  349. func (n *Needle) SetIsChunkManifest() {
  350. n.Flags = n.Flags | FlagIsChunkManifest
  351. }
  352. func (n *Needle) HasPairs() bool {
  353. return n.Flags&FlagHasPairs != 0
  354. }
  355. func (n *Needle) SetHasPairs() {
  356. n.Flags = n.Flags | FlagHasPairs
  357. }
  358. func GetActualSize(size Size, version Version) int64 {
  359. return NeedleHeaderSize + NeedleBodyLength(size, version)
  360. }