|
|
@ -13,16 +13,15 @@ import ( |
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
|
|
|
) |
|
|
|
|
|
|
|
func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType) (v *Volume, e error) { |
|
|
|
func loadVolumeWithoutIndex(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType) (v *Volume, err error) { |
|
|
|
v = &Volume{dir: dirname, Collection: collection, Id: id} |
|
|
|
v.SuperBlock = SuperBlock{} |
|
|
|
v.needleMapKind = needleMapKind |
|
|
|
e = v.load(false, false, needleMapKind, 0) |
|
|
|
err = v.load(false, false, needleMapKind, 0) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64) error { |
|
|
|
var e error |
|
|
|
func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64) (err error) { |
|
|
|
fileName := v.FileName() |
|
|
|
alreadyHasSuperBlock := false |
|
|
|
|
|
|
@ -37,10 +36,10 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind |
|
|
|
} |
|
|
|
var dataFile *os.File |
|
|
|
if canWrite { |
|
|
|
dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644) |
|
|
|
dataFile, err = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644) |
|
|
|
} else { |
|
|
|
glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode") |
|
|
|
dataFile, e = os.Open(fileName + ".dat") |
|
|
|
dataFile, err = os.Open(fileName + ".dat") |
|
|
|
v.noWriteOrDelete = true |
|
|
|
} |
|
|
|
v.lastModifiedTsSeconds = uint64(modifiedTime.Unix()) |
|
|
@ -50,56 +49,56 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind |
|
|
|
v.DataBackend = backend.NewDiskFile(dataFile) |
|
|
|
} else { |
|
|
|
if createDatIfMissing { |
|
|
|
v.DataBackend, e = createVolumeFile(fileName+".dat", preallocate, v.MemoryMapMaxSizeMb) |
|
|
|
v.DataBackend, err = createVolumeFile(fileName+".dat", preallocate, v.MemoryMapMaxSizeMb) |
|
|
|
} else { |
|
|
|
return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if e != nil { |
|
|
|
if !os.IsPermission(e) { |
|
|
|
return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e) |
|
|
|
if err != nil { |
|
|
|
if !os.IsPermission(err) { |
|
|
|
return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, err) |
|
|
|
} else { |
|
|
|
return fmt.Errorf("load data file %s.dat: %v", fileName, e) |
|
|
|
return fmt.Errorf("load data file %s.dat: %v", fileName, err) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if alreadyHasSuperBlock { |
|
|
|
e = v.readSuperBlock() |
|
|
|
err = v.readSuperBlock() |
|
|
|
} else { |
|
|
|
if !v.SuperBlock.Initialized() { |
|
|
|
return fmt.Errorf("volume %s.dat not initialized", fileName) |
|
|
|
} |
|
|
|
e = v.maybeWriteSuperBlock() |
|
|
|
err = v.maybeWriteSuperBlock() |
|
|
|
} |
|
|
|
if e == nil && alsoLoadIndex { |
|
|
|
if err == nil && alsoLoadIndex { |
|
|
|
var indexFile *os.File |
|
|
|
if v.noWriteOrDelete { |
|
|
|
glog.V(1).Infoln("open to read file", fileName+".idx") |
|
|
|
if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil { |
|
|
|
return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e) |
|
|
|
if indexFile, err = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); err != nil { |
|
|
|
return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, err) |
|
|
|
} |
|
|
|
} else { |
|
|
|
glog.V(1).Infoln("open to write file", fileName+".idx") |
|
|
|
if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil { |
|
|
|
return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e) |
|
|
|
if indexFile, err = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); err != nil { |
|
|
|
return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, err) |
|
|
|
} |
|
|
|
} |
|
|
|
if v.lastAppendAtNs, e = CheckVolumeDataIntegrity(v, indexFile); e != nil { |
|
|
|
if v.lastAppendAtNs, err = CheckVolumeDataIntegrity(v, indexFile); err != nil { |
|
|
|
v.noWriteOrDelete = true |
|
|
|
glog.V(0).Infof("volumeDataIntegrityChecking failed %v", e) |
|
|
|
glog.V(0).Infof("volumeDataIntegrityChecking failed %v", err) |
|
|
|
} |
|
|
|
|
|
|
|
if v.noWriteOrDelete || v.noWriteCanDelete { |
|
|
|
if v.nm, e = NewSortedFileNeedleMap(fileName, indexFile); e != nil { |
|
|
|
glog.V(0).Infof("loading sorted db %s error: %v", fileName+".sdb", e) |
|
|
|
if v.nm, err = NewSortedFileNeedleMap(fileName, indexFile); err != nil { |
|
|
|
glog.V(0).Infof("loading sorted db %s error: %v", fileName+".sdb", err) |
|
|
|
} |
|
|
|
} else { |
|
|
|
switch needleMapKind { |
|
|
|
case NeedleMapInMemory: |
|
|
|
glog.V(0).Infoln("loading index", fileName+".idx", "to memory") |
|
|
|
if v.nm, e = LoadCompactNeedleMap(indexFile); e != nil { |
|
|
|
glog.V(0).Infof("loading index %s to memory error: %v", fileName+".idx", e) |
|
|
|
if v.nm, err = LoadCompactNeedleMap(indexFile); err != nil { |
|
|
|
glog.V(0).Infof("loading index %s to memory error: %v", fileName+".idx", err) |
|
|
|
} |
|
|
|
case NeedleMapLevelDb: |
|
|
|
glog.V(0).Infoln("loading leveldb", fileName+".ldb") |
|
|
@ -108,8 +107,8 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind |
|
|
|
WriteBuffer: 1 * 1024 * 1024, // default value is 4MiB
|
|
|
|
CompactionTableSizeMultiplier: 10, // default value is 1
|
|
|
|
} |
|
|
|
if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil { |
|
|
|
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e) |
|
|
|
if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil { |
|
|
|
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err) |
|
|
|
} |
|
|
|
case NeedleMapLevelDbMedium: |
|
|
|
glog.V(0).Infoln("loading leveldb medium", fileName+".ldb") |
|
|
@ -118,8 +117,8 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind |
|
|
|
WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB
|
|
|
|
CompactionTableSizeMultiplier: 10, // default value is 1
|
|
|
|
} |
|
|
|
if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil { |
|
|
|
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e) |
|
|
|
if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil { |
|
|
|
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err) |
|
|
|
} |
|
|
|
case NeedleMapLevelDbLarge: |
|
|
|
glog.V(0).Infoln("loading leveldb large", fileName+".ldb") |
|
|
@ -128,8 +127,8 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind |
|
|
|
WriteBuffer: 4 * 1024 * 1024, // default value is 4MiB
|
|
|
|
CompactionTableSizeMultiplier: 10, // default value is 1
|
|
|
|
} |
|
|
|
if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil { |
|
|
|
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e) |
|
|
|
if v.nm, err = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); err != nil { |
|
|
|
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", err) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
@ -137,7 +136,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind |
|
|
|
|
|
|
|
stats.VolumeServerVolumeCounter.WithLabelValues(v.Collection, "volume").Inc() |
|
|
|
|
|
|
|
return e |
|
|
|
return err |
|
|
|
} |
|
|
|
|
|
|
|
func checkFile(filename string) (exists, canRead, canWrite bool, modTime time.Time, fileSize int64) { |
|
|
|