You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

150 lines
5.3 KiB

6 years ago
6 years ago
  1. package storage
  2. import (
  3. "fmt"
  4. "os"
  5. "github.com/syndtr/goleveldb/leveldb/opt"
  6. "github.com/chrislusf/seaweedfs/weed/glog"
  7. "github.com/chrislusf/seaweedfs/weed/stats"
  8. "github.com/chrislusf/seaweedfs/weed/storage/backend"
  9. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  10. "github.com/chrislusf/seaweedfs/weed/storage/super_block"
  11. "github.com/chrislusf/seaweedfs/weed/util"
  12. )
  13. func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType) (v *Volume, err error) {
  14. v = &Volume{dir: dirname, Collection: collection, Id: id}
  15. v.SuperBlock = super_block.SuperBlock{}
  16. v.needleMapKind = needleMapKind
  17. err = v.load(false, false, needleMapKind, 0)
  18. return
  19. }
  20. func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64) (err error) {
  21. fileName := v.FileName()
  22. alreadyHasSuperBlock := false
  23. hasVolumeInfoFile := v.maybeLoadVolumeInfo() && v.volumeInfo.Version != 0
  24. if v.HasRemoteFile() {
  25. v.noWriteCanDelete = true
  26. v.noWriteOrDelete = false
  27. glog.V(0).Infof("loading volume %d from remote %v", v.Id, v.volumeInfo.Files)
  28. v.LoadRemoteFile()
  29. alreadyHasSuperBlock = true
  30. } else if exists, canRead, canWrite, modifiedTime, fileSize := util.CheckFile(fileName + ".dat"); exists {
  31. // open dat file
  32. if !canRead {
  33. return fmt.Errorf("cannot read Volume Data file %s.dat", fileName)
  34. }
  35. var dataFile *os.File
  36. if canWrite {
  37. dataFile, err = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
  38. } else {
  39. glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode")
  40. dataFile, err = os.Open(fileName + ".dat")
  41. v.noWriteOrDelete = true
  42. }
  43. v.lastModifiedTsSeconds = uint64(modifiedTime.Unix())
  44. if fileSize >= super_block.SuperBlockSize {
  45. alreadyHasSuperBlock = true
  46. }
  47. v.DataBackend = backend.NewDiskFile(dataFile)
  48. } else {
  49. if createDatIfMissing {
  50. v.DataBackend, err = createVolumeFile(fileName+".dat", preallocate, v.MemoryMapMaxSizeMb)
  51. } else {
  52. return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
  53. }
  54. }
  55. if err != nil {
  56. if !os.IsPermission(err) {
  57. return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, err)
  58. } else {
  59. return fmt.Errorf("load data file %s.dat: %v", fileName, err)
  60. }
  61. }
  62. if alreadyHasSuperBlock {
  63. err = v.readSuperBlock()
  64. } else {
  65. if !v.SuperBlock.Initialized() {
  66. return fmt.Errorf("volume %s.dat not initialized", fileName)
  67. }
  68. err = v.maybeWriteSuperBlock()
  69. }
  70. if err == nil && alsoLoadIndex {
  71. var indexFile *os.File
  72. if v.noWriteOrDelete {
  73. glog.V(0).Infoln("open to read file", fileName+".idx")
  74. if indexFile, err = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); err != nil {
  75. return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, err)
  76. }
  77. } else {
  78. glog.V(1).Infoln("open to write file", fileName+".idx")
  79. if indexFile, err = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); err != nil {
  80. return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, err)
  81. }
  82. }
  83. if v.lastAppendAtNs, err = CheckVolumeDataIntegrity(v, indexFile); err != nil {
  84. v.noWriteOrDelete = true
  85. glog.V(0).Infof("volumeDataIntegrityChecking failed %v", err)
  86. }
  87. if v.IsReadOnly() {
  88. if v.nm, err = NewSortedFileNeedleMap(fileName, indexFile); err != nil {
  89. glog.V(0).Infof("loading sorted db %s error: %v", fileName+".sdx", err)
  90. }
  91. } else {
  92. switch needleMapKind {
  93. case NeedleMapInMemory:
  94. glog.V(0).Infoln("loading index", fileName+".idx", "to memory")
  95. if v.nm, err = LoadCompactNeedleMap(indexFile); err != nil {
  96. glog.V(0).Infof("loading index %s to memory error: %v", fileName+".idx", err)
  97. }
  98. case NeedleMapLevelDb:
  99. glog.V(0).Infoln("loading leveldb", fileName+".ldb")
  100. opts := &opt.Options{
  101. BlockCacheCapacity: 2 * 1024 * 1024, // default value is 8MiB
  102. WriteBuffer: 1 * 1024 * 1024, // default value is 4MiB
  103. CompactionTableSizeMultiplier: 10, // default value is 1
  104. }
  105. if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil {
  106. glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err)
  107. }
  108. case NeedleMapLevelDbMedium:
  109. glog.V(0).Infoln("loading leveldb medium", fileName+".ldb")
  110. opts := &opt.Options{
  111. BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB
  112. WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB
  113. CompactionTableSizeMultiplier: 10, // default value is 1
  114. }
  115. if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil {
  116. glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err)
  117. }
  118. case NeedleMapLevelDbLarge:
  119. glog.V(0).Infoln("loading leveldb large", fileName+".ldb")
  120. opts := &opt.Options{
  121. BlockCacheCapacity: 8 * 1024 * 1024, // default value is 8MiB
  122. WriteBuffer: 4 * 1024 * 1024, // default value is 4MiB
  123. CompactionTableSizeMultiplier: 10, // default value is 1
  124. }
  125. if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil {
  126. glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err)
  127. }
  128. }
  129. }
  130. }
  131. if !hasVolumeInfoFile {
  132. v.volumeInfo.Version = uint32(v.SuperBlock.Version)
  133. v.SaveVolumeInfo()
  134. }
  135. stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Inc()
  136. return err
  137. }