EC: with multiple volume locations, the ec rebuilding may fail

weed/server/volume_grpc_erasure_coding.go

@@ -90,6 +90,15 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s
 	var rebuiltShardIds []uint32
 
 	for _, location := range vs.store.Locations {
+		_, _, existingShardCount, err := checkEcVolumeStatus(bName, location)
+		if err != nil {
+			return nil, err
+		}
+
+		if existingShardCount == 0 {
+			continue
+		}
+
 		if util.FileExists(path.Join(location.IdxDirectory, baseFileName+".ecx")) {
 			// write .ec00 ~ .ec13 files
 			dataBaseFileName := path.Join(location.Directory, baseFileName)
@@ -206,19 +215,36 @@ func deleteEcShardIdsForEachLocation(bName string, location *storage.DiskLocatio
 		return nil
 	}
 
-	// check whether to delete the .ecx and .ecj file also
-	hasEcxFile := false
-	hasIdxFile := false
-	existingShardCount := 0
+	hasEcxFile, hasIdxFile, existingShardCount, err := checkEcVolumeStatus(bName, location)
+	if err != nil {
+		return err
+	}
 
+	if hasEcxFile && existingShardCount == 0 {
+		if err := os.Remove(indexBaseFilename + ".ecx"); err != nil {
+			return err
+		}
+		os.Remove(indexBaseFilename + ".ecj")
+
+		if !hasIdxFile {
+			// .vif is used for ec volumes and normal volumes
+			os.Remove(dataBaseFilename + ".vif")
+		}
+	}
+
+	return nil
+}
+
+func checkEcVolumeStatus(bName string, location *storage.DiskLocation) (hasEcxFile bool, hasIdxFile bool, existingShardCount int, err error) {
+	// check whether to delete the .ecx and .ecj file also
 	fileInfos, err := os.ReadDir(location.Directory)
 	if err != nil {
-		return err
+		return false, false, 0, err
 	}
 	if location.IdxDirectory != location.Directory {
 		idxFileInfos, err := os.ReadDir(location.IdxDirectory)
 		if err != nil {
-			return err
+			return false, false, 0, err
 		}
 		fileInfos = append(fileInfos, idxFileInfos...)
 	}
@@ -235,20 +261,7 @@ func deleteEcShardIdsForEachLocation(bName string, location *storage.DiskLocatio
 			existingShardCount++
 		}
 	}
-
-	if hasEcxFile && existingShardCount == 0 {
-		if err := os.Remove(indexBaseFilename + ".ecx"); err != nil {
-			return err
-		}
-		os.Remove(indexBaseFilename + ".ecj")
-
-		if !hasIdxFile {
-			// .vif is used for ec volumes and normal volumes
-			os.Remove(dataBaseFilename + ".vif")
-		}
-	}
-
-	return nil
+	return hasEcxFile, hasIdxFile, existingShardCount, nil
 }
 
 func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest) (*volume_server_pb.VolumeEcShardsMountResponse, error) {