concurrent loading volume

weed/storage/disk_location.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"io/ioutil"
+	"os"
 	"strings"
 	"sync"
 
@@ -25,8 +26,24 @@ func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
 	l.Lock()
 	defer l.Unlock()
 
+	task_queue := make(chan os.FileInfo, 100)
+	go func() {
 		if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
 			for _, dir := range dirs {
+				task_queue <- dir
+			}
+		}
+		close(task_queue)
+	}()
+
+	const concurrency int = 10
+	var wg sync.WaitGroup
+	var mutex sync.RWMutex
+	for workerNum := 0; workerNum < concurrency; workerNum++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for dir := range task_queue {
 				name := dir.Name()
 				if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
 					collection := ""
@@ -36,9 +53,14 @@ func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
 						collection, base = base[0:i], base[i+1:]
 					}
 					if vid, err := NewVolumeId(base); err == nil {
-					if l.volumes[vid] == nil {
+						mutex.RLock()
+						_, found := l.volumes[vid]
+						mutex.RUnlock()
+						if !found {
 							if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil); e == nil {
+								mutex.Lock()
 								l.volumes[vid] = v
+								mutex.Unlock()
 								glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s", l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), v.Size(), v.Ttl.String())
 							} else {
 								glog.V(0).Infof("new volume %s error %s", name, e)
@@ -47,7 +69,10 @@ func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
 					}
 				}
 			}
+		}()
 	}
+	wg.Wait()
+
 	glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
 }
 

weed/storage/needle_map_memory.go

@@ -48,7 +48,7 @@ func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
 		}
 		return nil
 	})
-	glog.V(1).Infoln("max file key:", nm.MaximumFileKey)
+	glog.V(1).Infof("max file key: %d for file: %s", nm.MaximumFileKey, file.Name())
 	return nm, e
 }