[master] avoid crowded more writable for auto grow (#6214)

avoid crowded more writable
https://github.com/seaweedfs/seaweedfs/issues/6121

weed/server/master_grpc_server_volume.go

@@ -43,12 +43,12 @@ func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) {
 func (ms *MasterServer) ProcessGrowRequest() {
 	go func() {
 		ctx := context.Background()
-		firstRun := true 
+		firstRun := true
 		for {
 			if firstRun {
-				firstRun = false 
+				firstRun = false
 			} else {
-				time.Sleep(14*time.Minute + time.Duration(120*rand.Float32())*time.Second)
+				time.Sleep(5*time.Minute + time.Duration(30*rand.Float32())*time.Second)
 			}
 			if !ms.Topo.IsLeader() {
 				continue
@@ -71,9 +71,6 @@ func (ms *MasterServer) ProcessGrowRequest() {
 				case mustGrow > 0:
 					vgr.WritableVolumeCount = uint32(mustGrow)
 					_, err = ms.VolumeGrow(ctx, vgr)
-				case crowded+volumeGrowStepCount >= writable:
-					vgr.WritableVolumeCount = volumeGrowStepCount
-					_, err = ms.VolumeGrow(ctx, vgr)
 				default:
 					for _, dc := range dcs {
 						if vl.ShouldGrowVolumesByDataNode("DataCenter", dc) {

weed/topology/volume_layout.go

@@ -407,10 +407,10 @@ func (vl *VolumeLayout) removeFromWritable(vid needle.VolumeId) bool {
 			break
 		}
 	}
+	vl.removeFromCrowded(vid)
 	if toDeleteIndex >= 0 {
 		glog.V(0).Infoln("Volume", vid, "becomes unwritable")
 		vl.writables = append(vl.writables[0:toDeleteIndex], vl.writables[toDeleteIndex+1:]...)
-		vl.removeFromCrowded(vid)
 		return true
 	}
 	return false
@@ -506,7 +506,10 @@ func (vl *VolumeLayout) SetVolumeCapacityFull(vid needle.VolumeId) bool {
 }
 
 func (vl *VolumeLayout) removeFromCrowded(vid needle.VolumeId) {
-	delete(vl.crowded, vid)
+	if _, ok := vl.crowded[vid]; ok {
+		glog.V(0).Infoln("Volume", vid, "becomes uncrowded")
+		delete(vl.crowded, vid)
+	}
 }
 
 func (vl *VolumeLayout) setVolumeCrowded(vid needle.VolumeId) {