You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

123 lines
2.9 KiB

  1. package storage
  2. import (
  3. "fmt"
  4. "io/ioutil"
  5. "os"
  6. "sync"
  7. "github.com/chrislusf/seaweedfs/go/util"
  8. )
  9. type NeedleMapType int
  10. const (
  11. NeedleMapInMemory NeedleMapType = iota
  12. NeedleMapLevelDb
  13. NeedleMapBoltDb
  14. )
  15. type NeedleMapper interface {
  16. Put(key uint64, offset uint32, size uint32) error
  17. Get(key uint64) (element *NeedleValue, ok bool)
  18. Delete(key uint64) error
  19. Close()
  20. Destroy() error
  21. ContentSize() uint64
  22. DeletedSize() uint64
  23. FileCount() int
  24. DeletedCount() int
  25. MaxFileKey() uint64
  26. IndexFileSize() uint64
  27. IndexFileContent() ([]byte, error)
  28. IndexFileName() string
  29. }
  30. type baseNeedleMapper struct {
  31. indexFile *os.File
  32. indexFileAccessLock sync.Mutex
  33. mapMetric
  34. }
  35. func (nm baseNeedleMapper) IndexFileSize() uint64 {
  36. stat, err := nm.indexFile.Stat()
  37. if err == nil {
  38. return uint64(stat.Size())
  39. }
  40. return 0
  41. }
  42. func (nm baseNeedleMapper) IndexFileName() string {
  43. return nm.indexFile.Name()
  44. }
  45. func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) {
  46. key = util.BytesToUint64(bytes[:8])
  47. offset = util.BytesToUint32(bytes[8:12])
  48. size = util.BytesToUint32(bytes[12:16])
  49. return
  50. }
  51. func (nm baseNeedleMapper) appendToIndexFile(key uint64, offset uint32, size uint32) error {
  52. bytes := make([]byte, 16)
  53. util.Uint64toBytes(bytes[0:8], key)
  54. util.Uint32toBytes(bytes[8:12], offset)
  55. util.Uint32toBytes(bytes[12:16], size)
  56. nm.indexFileAccessLock.Lock()
  57. defer nm.indexFileAccessLock.Unlock()
  58. if _, err := nm.indexFile.Seek(0, 2); err != nil {
  59. return fmt.Errorf("cannot seek end of indexfile %s: %v",
  60. nm.indexFile.Name(), err)
  61. }
  62. _, err := nm.indexFile.Write(bytes)
  63. return err
  64. }
  65. func (nm baseNeedleMapper) IndexFileContent() ([]byte, error) {
  66. nm.indexFileAccessLock.Lock()
  67. defer nm.indexFileAccessLock.Unlock()
  68. return ioutil.ReadFile(nm.indexFile.Name())
  69. }
  70. type mapMetric struct {
  71. indexFile *os.File
  72. DeletionCounter int `json:"DeletionCounter"`
  73. FileCounter int `json:"FileCounter"`
  74. DeletionByteCounter uint64 `json:"DeletionByteCounter"`
  75. FileByteCounter uint64 `json:"FileByteCounter"`
  76. MaximumFileKey uint64 `json:"MaxFileKey"`
  77. }
  78. func (mm *mapMetric) logDelete(deletedByteCount uint32) {
  79. mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(deletedByteCount)
  80. mm.DeletionCounter++
  81. }
  82. func (mm *mapMetric) logPut(key uint64, oldSize uint32, newSize uint32) {
  83. if key > mm.MaximumFileKey {
  84. mm.MaximumFileKey = key
  85. }
  86. mm.FileCounter++
  87. mm.FileByteCounter = mm.FileByteCounter + uint64(newSize)
  88. if oldSize > 0 {
  89. mm.DeletionCounter++
  90. mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(oldSize)
  91. }
  92. }
  93. func (mm mapMetric) ContentSize() uint64 {
  94. return mm.FileByteCounter
  95. }
  96. func (mm mapMetric) DeletedSize() uint64 {
  97. return mm.DeletionByteCounter
  98. }
  99. func (mm mapMetric) FileCount() int {
  100. return mm.FileCounter
  101. }
  102. func (mm mapMetric) DeletedCount() int {
  103. return mm.DeletionCounter
  104. }
  105. func (mm mapMetric) MaxFileKey() uint64 {
  106. return mm.MaximumFileKey
  107. }