recreate grpc connections if too many errors

address https://github.com/chrislusf/seaweedfs/issues/2098

weed/pb/grpc_client_server.go

@@ -32,6 +32,7 @@ var (
 type versionedGrpcClient struct {
 	*grpc.ClientConn
 	version  int
+	errCount int
 }
 
 func init() {
@@ -103,6 +104,7 @@ func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedG
 	vgc := &versionedGrpcClient{
 		grpcConnection,
 		rand.Int(),
+		0,
 	}
 	grpcClients[address] = vgc
 
@@ -116,7 +118,11 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts
 		return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
 	}
 	executionErr := fn(vgc.ClientConn)
-	if executionErr != nil && strings.Contains(executionErr.Error(), "transport") {
+	if executionErr != nil {
+		vgc.errCount++
+		if vgc.errCount > 3 ||
+			strings.Contains(executionErr.Error(), "transport") ||
+			strings.Contains(executionErr.Error(), "connection closed") {
 			grpcClientsLock.Lock()
 			if t, ok := grpcClients[address]; ok {
 				if t.version == vgc.version {
@@ -126,6 +132,7 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts
 			}
 			grpcClientsLock.Unlock()
 		}
+	}
 
 	return executionErr
 }