Merge branch 'master' into support_ssd_volume

docker/Dockerfile.go_build

@@ -2,8 +2,8 @@ FROM frolvlad/alpine-glibc as builder
 RUN apk add git go g++
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
-ARG branch=${branch:-master}
-RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $ARG
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
   && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
   && go install -ldflags "${LDFLAGS}"

docker/Dockerfile.go_build_large

@@ -2,8 +2,8 @@ FROM frolvlad/alpine-glibc as builder
 RUN apk add git go g++
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
-ARG branch=${branch:-master}
-RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $ARG
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
 RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
   && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
   && go install -tags 5BytesOffset -ldflags "${LDFLAGS}"

weed/storage/disk_location.go

@@ -55,7 +55,7 @@ func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32
 }
 
 func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
-	if strings.HasSuffix(filename, ".idx") || strings.HasSuffix(filename, ".vif") {
+	if isValidVolume(filename) {
 		base := filename[:len(filename)-4]
 		collection, volumeId, err := parseCollectionVolumeId(base)
 		return volumeId, collection, err
@@ -73,15 +73,26 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI
 	return collection, vol, err
 }
 
+func isValidVolume(basename string) bool {
+	return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
+}
+
+func getValidVolumeName(basename string) string {
+	if isValidVolume(basename) {
+		return basename[:len(basename)-4]
+	}
+	return ""
+}
+
 func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool {
 	basename := fileInfo.Name()
 	if fileInfo.IsDir() {
 		return false
 	}
-	if !strings.HasSuffix(basename, ".idx") && !strings.HasSuffix(basename, ".vif") {
+	volumeName := getValidVolumeName(basename)
+	if volumeName == "" {
 		return false
 	}
-	volumeName := basename[:len(basename)-4]
 
 	// check for incomplete volume
 	noteFile := l.Directory + "/" + volumeName + ".note"
@@ -128,11 +139,19 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con
 
 	task_queue := make(chan os.FileInfo, 10*concurrency)
 	go func() {
+		foundVolumeNames := make(map[string]bool)
 		if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
 			for _, fi := range fileInfos {
+				volumeName := getValidVolumeName(fi.Name())
+				if volumeName == "" {
+					continue
+				}
+				if _, found := foundVolumeNames[volumeName]; !found {
+					foundVolumeNames[volumeName] = true
 					task_queue <- fi
 				}
 			}
+		}
 		close(task_queue)
 	}()