You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

81 lines
2.3 KiB

6 years ago
4 years ago
  1. package storage
  2. import (
  3. "os"
  4. "github.com/seaweedfs/seaweedfs/weed/glog"
  5. "github.com/seaweedfs/seaweedfs/weed/storage/idx"
  6. "github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
  7. . "github.com/seaweedfs/seaweedfs/weed/storage/types"
  8. )
  9. type NeedleMap struct {
  10. baseNeedleMapper
  11. m needle_map.NeedleValueMap
  12. }
  13. func NewCompactNeedleMap(file *os.File) *NeedleMap {
  14. nm := &NeedleMap{
  15. m: needle_map.NewCompactMap(),
  16. }
  17. nm.indexFile = file
  18. stat, err := file.Stat()
  19. if err != nil {
  20. glog.Fatalf("stat file %s: %v", file.Name(), err)
  21. }
  22. nm.indexFileOffset = stat.Size()
  23. return nm
  24. }
  25. func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
  26. nm := NewCompactNeedleMap(file)
  27. return doLoading(file, nm)
  28. }
  29. func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
  30. e := idx.WalkIndexFile(file, 0, func(key NeedleId, offset Offset, size Size) error {
  31. nm.MaybeSetMaxFileKey(key)
  32. if !offset.IsZero() && size.IsValid() {
  33. nm.FileCounter++
  34. nm.FileByteCounter = nm.FileByteCounter + uint64(size)
  35. oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size)
  36. if !oldOffset.IsZero() && oldSize.IsValid() {
  37. nm.DeletionCounter++
  38. nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
  39. }
  40. } else {
  41. oldSize := nm.m.Delete(NeedleId(key))
  42. nm.DeletionCounter++
  43. nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
  44. }
  45. return nil
  46. })
  47. glog.V(1).Infof("max file key: %d for file: %s", nm.MaxFileKey(), file.Name())
  48. return nm, e
  49. }
  50. func (nm *NeedleMap) Put(key NeedleId, offset Offset, size Size) error {
  51. _, oldSize := nm.m.Set(NeedleId(key), offset, size)
  52. nm.logPut(key, oldSize, size)
  53. return nm.appendToIndexFile(key, offset, size)
  54. }
  55. func (nm *NeedleMap) Get(key NeedleId) (element *needle_map.NeedleValue, ok bool) {
  56. element, ok = nm.m.Get(NeedleId(key))
  57. return
  58. }
  59. func (nm *NeedleMap) Delete(key NeedleId, offset Offset) error {
  60. deletedBytes := nm.m.Delete(NeedleId(key))
  61. nm.logDelete(deletedBytes)
  62. return nm.appendToIndexFile(key, offset, TombstoneFileSize)
  63. }
  64. func (nm *NeedleMap) Close() {
  65. indexFileName := nm.indexFile.Name()
  66. if err := nm.indexFile.Sync(); err != nil {
  67. glog.Warningf("sync file %s failed, %v", indexFileName, err)
  68. }
  69. _ = nm.indexFile.Close()
  70. }
  71. func (nm *NeedleMap) Destroy() error {
  72. nm.Close()
  73. return os.Remove(nm.indexFile.Name())
  74. }