You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

127 lines
2.9 KiB

  1. package storage
  2. import (
  3. "fmt"
  4. "io/ioutil"
  5. "os"
  6. "sync"
  7. "github.com/chrislusf/seaweedfs/weed/util"
  8. )
  9. type NeedleMapType int
  10. const (
  11. NeedleMapInMemory NeedleMapType = iota
  12. NeedleMapLevelDb
  13. NeedleMapBoltDb
  14. )
  15. const (
  16. NeedleIndexSize = 16
  17. )
  18. type NeedleMapper interface {
  19. Put(key uint64, offset uint32, size uint32) error
  20. Get(key uint64) (element *NeedleValue, ok bool)
  21. Delete(key uint64, offset uint32) error
  22. Close()
  23. Destroy() error
  24. ContentSize() uint64
  25. DeletedSize() uint64
  26. FileCount() int
  27. DeletedCount() int
  28. MaxFileKey() uint64
  29. IndexFileSize() uint64
  30. IndexFileContent() ([]byte, error)
  31. IndexFileName() string
  32. }
  33. type baseNeedleMapper struct {
  34. indexFile *os.File
  35. indexFileAccessLock sync.Mutex
  36. mapMetric
  37. }
  38. func (nm *baseNeedleMapper) IndexFileSize() uint64 {
  39. stat, err := nm.indexFile.Stat()
  40. if err == nil {
  41. return uint64(stat.Size())
  42. }
  43. return 0
  44. }
  45. func (nm *baseNeedleMapper) IndexFileName() string {
  46. return nm.indexFile.Name()
  47. }
  48. func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) {
  49. key = util.BytesToUint64(bytes[:8])
  50. offset = util.BytesToUint32(bytes[8:12])
  51. size = util.BytesToUint32(bytes[12:16])
  52. return
  53. }
  54. func (nm *baseNeedleMapper) appendToIndexFile(key uint64, offset uint32, size uint32) error {
  55. bytes := make([]byte, 16)
  56. util.Uint64toBytes(bytes[0:8], key)
  57. util.Uint32toBytes(bytes[8:12], offset)
  58. util.Uint32toBytes(bytes[12:16], size)
  59. nm.indexFileAccessLock.Lock()
  60. defer nm.indexFileAccessLock.Unlock()
  61. if _, err := nm.indexFile.Seek(0, 2); err != nil {
  62. return fmt.Errorf("cannot seek end of indexfile %s: %v",
  63. nm.indexFile.Name(), err)
  64. }
  65. _, err := nm.indexFile.Write(bytes)
  66. return err
  67. }
  68. func (nm *baseNeedleMapper) IndexFileContent() ([]byte, error) {
  69. nm.indexFileAccessLock.Lock()
  70. defer nm.indexFileAccessLock.Unlock()
  71. return ioutil.ReadFile(nm.indexFile.Name())
  72. }
  73. type mapMetric struct {
  74. indexFile *os.File
  75. DeletionCounter int `json:"DeletionCounter"`
  76. FileCounter int `json:"FileCounter"`
  77. DeletionByteCounter uint64 `json:"DeletionByteCounter"`
  78. FileByteCounter uint64 `json:"FileByteCounter"`
  79. MaximumFileKey uint64 `json:"MaxFileKey"`
  80. }
  81. func (mm *mapMetric) logDelete(deletedByteCount uint32) {
  82. mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(deletedByteCount)
  83. mm.DeletionCounter++
  84. }
  85. func (mm *mapMetric) logPut(key uint64, oldSize uint32, newSize uint32) {
  86. if key > mm.MaximumFileKey {
  87. mm.MaximumFileKey = key
  88. }
  89. mm.FileCounter++
  90. mm.FileByteCounter = mm.FileByteCounter + uint64(newSize)
  91. if oldSize > 0 {
  92. mm.DeletionCounter++
  93. mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(oldSize)
  94. }
  95. }
  96. func (mm mapMetric) ContentSize() uint64 {
  97. return mm.FileByteCounter
  98. }
  99. func (mm mapMetric) DeletedSize() uint64 {
  100. return mm.DeletionByteCounter
  101. }
  102. func (mm mapMetric) FileCount() int {
  103. return mm.FileCounter
  104. }
  105. func (mm mapMetric) DeletedCount() int {
  106. return mm.DeletionCounter
  107. }
  108. func (mm mapMetric) MaxFileKey() uint64 {
  109. return mm.MaximumFileKey
  110. }