|
|
@ -74,8 +74,13 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI |
|
|
|
|
|
|
|
func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool { |
|
|
|
name := fileInfo.Name() |
|
|
|
if !fileInfo.IsDir() && strings.HasSuffix(name, ".idx") { |
|
|
|
name := name[:len(name)-len(".idx")] |
|
|
|
if fileInfo.IsDir() { |
|
|
|
return false |
|
|
|
} |
|
|
|
if !strings.HasSuffix(name, ".idx") { |
|
|
|
return false |
|
|
|
} |
|
|
|
name = name[:len(name)-len(".idx")] |
|
|
|
noteFile := l.Directory + "/" + name + ".note" |
|
|
|
if util.FileExists(noteFile) { |
|
|
|
note, _ := ioutil.ReadFile(noteFile) |
|
|
@ -111,16 +116,14 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne |
|
|
|
l.Directory+"/"+name+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String()) |
|
|
|
return true |
|
|
|
} |
|
|
|
return false |
|
|
|
} |
|
|
|
|
|
|
|
func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, concurrency int) { |
|
|
|
|
|
|
|
task_queue := make(chan os.FileInfo, 10*concurrency) |
|
|
|
go func() { |
|
|
|
if dirs, err := ioutil.ReadDir(l.Directory); err == nil { |
|
|
|
for _, dir := range dirs { |
|
|
|
task_queue <- dir |
|
|
|
if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil { |
|
|
|
for _, fi := range fileInfos { |
|
|
|
task_queue <- fi |
|
|
|
} |
|
|
|
} |
|
|
|
close(task_queue) |
|
|
@ -131,8 +134,8 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con |
|
|
|
wg.Add(1) |
|
|
|
go func() { |
|
|
|
defer wg.Done() |
|
|
|
for dir := range task_queue { |
|
|
|
_ = l.loadExistingVolume(dir, needleMapKind) |
|
|
|
for fi := range task_queue { |
|
|
|
_ = l.loadExistingVolume(fi, needleMapKind) |
|
|
|
} |
|
|
|
}() |
|
|
|
} |
|
|
|