You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

87 lines
2.5 KiB

6 years ago
6 years ago
6 years ago
6 years ago
  1. package storage
  2. import (
  3. "os"
  4. "github.com/chrislusf/seaweedfs/weed/glog"
  5. "github.com/chrislusf/seaweedfs/weed/storage/idx"
  6. "github.com/chrislusf/seaweedfs/weed/storage/needle_map"
  7. . "github.com/chrislusf/seaweedfs/weed/storage/types"
  8. )
  9. type NeedleMap struct {
  10. baseNeedleMapper
  11. m needle_map.NeedleValueMap
  12. }
  13. func NewCompactNeedleMap(file *os.File) *NeedleMap {
  14. nm := &NeedleMap{
  15. m: needle_map.NewCompactMap(),
  16. }
  17. nm.indexFile = file
  18. return nm
  19. }
  20. func NewBtreeNeedleMap(file *os.File) *NeedleMap {
  21. nm := &NeedleMap{
  22. m: needle_map.NewBtreeMap(),
  23. }
  24. nm.indexFile = file
  25. return nm
  26. }
  27. func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
  28. nm := NewCompactNeedleMap(file)
  29. return doLoading(file, nm)
  30. }
  31. func LoadBtreeNeedleMap(file *os.File) (*NeedleMap, error) {
  32. nm := NewBtreeNeedleMap(file)
  33. return doLoading(file, nm)
  34. }
  35. func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
  36. e := idx.WalkIndexFile(file, func(key NeedleId, offset Offset, size uint32) error {
  37. nm.MaybeSetMaxFileKey(key)
  38. if !offset.IsZero() && size != TombstoneFileSize {
  39. nm.FileCounter++
  40. nm.FileByteCounter = nm.FileByteCounter + uint64(size)
  41. oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size)
  42. // glog.V(3).Infoln("reading key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
  43. if !oldOffset.IsZero() && oldSize != TombstoneFileSize {
  44. nm.DeletionCounter++
  45. nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
  46. }
  47. } else {
  48. oldSize := nm.m.Delete(NeedleId(key))
  49. // glog.V(3).Infoln("removing key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
  50. nm.DeletionCounter++
  51. nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
  52. }
  53. return nil
  54. })
  55. glog.V(1).Infof("max file key: %d for file: %s", nm.MaxFileKey(), file.Name())
  56. return nm, e
  57. }
  58. func (nm *NeedleMap) Put(key NeedleId, offset Offset, size uint32) error {
  59. _, oldSize := nm.m.Set(NeedleId(key), offset, size)
  60. nm.logPut(key, oldSize, size)
  61. return nm.appendToIndexFile(key, offset, size)
  62. }
  63. func (nm *NeedleMap) Get(key NeedleId) (element *needle_map.NeedleValue, ok bool) {
  64. element, ok = nm.m.Get(NeedleId(key))
  65. return
  66. }
  67. func (nm *NeedleMap) Delete(key NeedleId, offset Offset) error {
  68. deletedBytes := nm.m.Delete(NeedleId(key))
  69. nm.logDelete(deletedBytes)
  70. return nm.appendToIndexFile(key, offset, TombstoneFileSize)
  71. }
  72. func (nm *NeedleMap) Close() {
  73. _ = nm.indexFile.Close()
  74. }
  75. func (nm *NeedleMap) Destroy() error {
  76. nm.Close()
  77. return os.Remove(nm.indexFile.Name())
  78. }