You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

124 lines
3.3 KiB

  1. package storage
  2. import (
  3. "io"
  4. "os"
  5. "github.com/chrislusf/seaweedfs/weed/glog"
  6. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  7. )
  8. type NeedleMap struct {
  9. m needle.NeedleValueMap
  10. baseNeedleMapper
  11. }
  12. func NewCompactNeedleMap(file *os.File) *NeedleMap {
  13. nm := &NeedleMap{
  14. m: needle.NewCompactMap(),
  15. }
  16. nm.indexFile = file
  17. return nm
  18. }
  19. func NewBtreeNeedleMap(file *os.File) *NeedleMap {
  20. nm := &NeedleMap{
  21. m: needle.NewBtreeMap(),
  22. }
  23. nm.indexFile = file
  24. return nm
  25. }
  26. const (
  27. RowsToRead = 1024
  28. )
  29. func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
  30. nm := NewCompactNeedleMap(file)
  31. return doLoading(file, nm)
  32. }
  33. func LoadBtreeNeedleMap(file *os.File) (*NeedleMap, error) {
  34. nm := NewBtreeNeedleMap(file)
  35. return doLoading(file, nm)
  36. }
  37. func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
  38. e := WalkIndexFile(file, func(key uint64, offset, size uint32) error {
  39. if key > nm.MaximumFileKey {
  40. nm.MaximumFileKey = key
  41. }
  42. if offset > 0 && size != TombstoneFileSize {
  43. nm.FileCounter++
  44. nm.FileByteCounter = nm.FileByteCounter + uint64(size)
  45. oldOffset, oldSize := nm.m.Set(needle.Key(key), offset, size)
  46. // glog.V(3).Infoln("reading key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
  47. if oldOffset > 0 && oldSize != TombstoneFileSize {
  48. nm.DeletionCounter++
  49. nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
  50. }
  51. } else {
  52. oldSize := nm.m.Delete(needle.Key(key))
  53. // glog.V(3).Infoln("removing key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
  54. nm.DeletionCounter++
  55. nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
  56. }
  57. return nil
  58. })
  59. glog.V(1).Infof("max file key: %d for file: %s", nm.MaximumFileKey, file.Name())
  60. return nm, e
  61. }
  62. // walks through the index file, calls fn function with each key, offset, size
  63. // stops with the error returned by the fn function
  64. func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) error {
  65. var readerOffset int64
  66. bytes := make([]byte, 16*RowsToRead)
  67. count, e := r.ReadAt(bytes, readerOffset)
  68. glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
  69. readerOffset += int64(count)
  70. var (
  71. key uint64
  72. offset, size uint32
  73. i int
  74. )
  75. for count > 0 && e == nil || e == io.EOF {
  76. for i = 0; i+16 <= count; i += 16 {
  77. key, offset, size = idxFileEntry(bytes[i: i+16])
  78. if e = fn(key, offset, size); e != nil {
  79. return e
  80. }
  81. }
  82. if e == io.EOF {
  83. return nil
  84. }
  85. count, e = r.ReadAt(bytes, readerOffset)
  86. glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
  87. readerOffset += int64(count)
  88. }
  89. return e
  90. }
  91. func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error {
  92. _, oldSize := nm.m.Set(needle.Key(key), offset, size)
  93. nm.logPut(key, oldSize, size)
  94. return nm.appendToIndexFile(key, offset, size)
  95. }
  96. func (nm *NeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) {
  97. element, ok = nm.m.Get(needle.Key(key))
  98. return
  99. }
  100. func (nm *NeedleMap) Delete(key uint64, offset uint32) error {
  101. deletedBytes := nm.m.Delete(needle.Key(key))
  102. nm.logDelete(deletedBytes)
  103. return nm.appendToIndexFile(key, offset, TombstoneFileSize)
  104. }
  105. func (nm *NeedleMap) Close() {
  106. _ = nm.indexFile.Close()
  107. }
  108. func (nm *NeedleMap) Destroy() error {
  109. nm.Close()
  110. return os.Remove(nm.indexFile.Name())
  111. }