reduce concurrent volume grow requests

weed/server/master_grpc_server_volume.go

@@ -54,6 +54,7 @@ func (ms *MasterServer) ProcessGrowRequest() {
 					start := time.Now()
 					_, err := ms.vg.AutomaticGrowByType(req.Option, ms.grpcDialOption, ms.Topo, req.Count)
 					glog.V(1).Infoln("finished automatic volume grow, cost ", time.Now().Sub(start))
+					vl.DoneGrowRequest()
 
 					if req.ErrCh != nil {
 						req.ErrCh <- err
@@ -135,10 +136,11 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 
 	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType)
 
-	if vl.ShouldGrowVolumes(option) {
+	if !vl.HasGrowRequest() && vl.ShouldGrowVolumes(option) {
 		if ms.Topo.AvailableSpaceFor(option) <= 0 {
 			return nil, fmt.Errorf("no free volumes left for " + option.String())
 		}
+		vl.AddGrowRequest()
 		ms.vgCh <- &topology.VolumeGrowRequest{
 			Option: option,
 			Count:  int(req.WritableVolumeCount),

weed/server/master_server_handlers.go

@@ -115,13 +115,14 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
 
 	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType)
 
-	if vl.ShouldGrowVolumes(option) {
+	if !vl.HasGrowRequest() && vl.ShouldGrowVolumes(option) {
 		glog.V(0).Infof("dirAssign volume growth %v from %v", option.String(), r.RemoteAddr)
 		if ms.Topo.AvailableSpaceFor(option) <= 0 {
 			writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left for " + option.String()})
 			return
 		}
 		errCh := make(chan error, 1)
+		vl.AddGrowRequest()
 		ms.vgCh <- &topology.VolumeGrowRequest{
 			Option: option,
 			Count:  writableVolumeCount,

weed/topology/volume_layout.go

@@ -114,6 +114,8 @@ type VolumeLayout struct {
 	volumeSizeLimit  uint64
 	replicationAsMin bool
 	accessLock       sync.RWMutex
+	growRequestCount int
+	growRequestTime  time.Time
 }
 
 type VolumeLayoutStats struct {
@@ -310,6 +312,21 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (*n
 	return &vid, count, locationList, nil
 }
 
+func (vl *VolumeLayout) HasGrowRequest() bool {
+	if vl.growRequestCount > 0 && vl.growRequestTime.Add(time.Minute).After(time.Now()) {
+		return true
+	}
+	return false
+}
+func (vl *VolumeLayout) AddGrowRequest() {
+	vl.growRequestTime = time.Now()
+	vl.growRequestCount++
+}
+func (vl *VolumeLayout) DoneGrowRequest() {
+	vl.growRequestTime = time.Unix(0,0)
+	vl.growRequestCount = 0
+}
+
 func (vl *VolumeLayout) ShouldGrowVolumes(option *VolumeGrowOption) bool {
 	active, crowded := vl.GetActiveVolumeCount(option)
 	//glog.V(0).Infof("active volume: %d, high usage volume: %d\n", active, high)