You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

258 lines
7.2 KiB

5 years ago
6 years ago
4 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
6 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
  1. package pb
  2. import (
  3. "context"
  4. "fmt"
  5. "github.com/chrislusf/seaweedfs/weed/glog"
  6. "math/rand"
  7. "net/http"
  8. "strconv"
  9. "strings"
  10. "sync"
  11. "time"
  12. "google.golang.org/grpc"
  13. "google.golang.org/grpc/keepalive"
  14. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  15. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  16. "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
  17. )
  18. const (
  19. Max_Message_Size = 1 << 30 // 1 GB
  20. )
  21. var (
  22. // cache grpc connections
  23. grpcClients = make(map[string]*versionedGrpcClient)
  24. grpcClientsLock sync.Mutex
  25. )
  26. type versionedGrpcClient struct {
  27. *grpc.ClientConn
  28. version int
  29. errCount int
  30. }
  31. func init() {
  32. http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
  33. http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
  34. }
  35. func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
  36. var options []grpc.ServerOption
  37. options = append(options,
  38. grpc.KeepaliveParams(keepalive.ServerParameters{
  39. Time: 10 * time.Second, // wait time before ping if no activity
  40. Timeout: 20 * time.Second, // ping timeout
  41. MaxConnectionAge: 10 * time.Hour,
  42. }),
  43. grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
  44. MinTime: 60 * time.Second, // min time a client should wait before sending a ping
  45. PermitWithoutStream: true,
  46. }),
  47. grpc.MaxRecvMsgSize(Max_Message_Size),
  48. grpc.MaxSendMsgSize(Max_Message_Size),
  49. )
  50. for _, opt := range opts {
  51. if opt != nil {
  52. options = append(options, opt)
  53. }
  54. }
  55. return grpc.NewServer(options...)
  56. }
  57. func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
  58. // opts = append(opts, grpc.WithBlock())
  59. // opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
  60. var options []grpc.DialOption
  61. options = append(options,
  62. // grpc.WithInsecure(),
  63. grpc.WithDefaultCallOptions(
  64. grpc.MaxCallSendMsgSize(Max_Message_Size),
  65. grpc.MaxCallRecvMsgSize(Max_Message_Size),
  66. ),
  67. grpc.WithKeepaliveParams(keepalive.ClientParameters{
  68. Time: 30 * time.Second, // client ping server if no activity for this long
  69. Timeout: 20 * time.Second,
  70. PermitWithoutStream: true,
  71. }))
  72. for _, opt := range opts {
  73. if opt != nil {
  74. options = append(options, opt)
  75. }
  76. }
  77. return grpc.DialContext(ctx, address, options...)
  78. }
  79. func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
  80. grpcClientsLock.Lock()
  81. defer grpcClientsLock.Unlock()
  82. existingConnection, found := grpcClients[address]
  83. if found {
  84. return existingConnection, nil
  85. }
  86. grpcConnection, err := GrpcDial(context.Background(), address, opts...)
  87. if err != nil {
  88. return nil, fmt.Errorf("fail to dial %s: %v", address, err)
  89. }
  90. vgc := &versionedGrpcClient{
  91. grpcConnection,
  92. rand.Int(),
  93. 0,
  94. }
  95. grpcClients[address] = vgc
  96. return vgc, nil
  97. }
  98. func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {
  99. vgc, err := getOrCreateConnection(address, opts...)
  100. if err != nil {
  101. return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
  102. }
  103. executionErr := fn(vgc.ClientConn)
  104. if executionErr != nil {
  105. vgc.errCount++
  106. if vgc.errCount > 3 ||
  107. strings.Contains(executionErr.Error(), "transport") ||
  108. strings.Contains(executionErr.Error(), "connection closed") {
  109. grpcClientsLock.Lock()
  110. if t, ok := grpcClients[address]; ok {
  111. if t.version == vgc.version {
  112. vgc.Close()
  113. delete(grpcClients, address)
  114. }
  115. }
  116. grpcClientsLock.Unlock()
  117. }
  118. }
  119. return executionErr
  120. }
  121. func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
  122. return ParseServerAddress(server, 10000)
  123. }
  124. func ParseServersToGrpcAddresses(servers []string) (serverGrpcAddresses []string, err error) {
  125. for _, server := range servers {
  126. if serverGrpcAddress, parseErr := ParseServerToGrpcAddress(server); parseErr == nil {
  127. serverGrpcAddresses = append(serverGrpcAddresses, serverGrpcAddress)
  128. } else {
  129. return nil, parseErr
  130. }
  131. }
  132. return
  133. }
  134. func ParseServerAddress(server string, deltaPort int) (newServerAddress string, err error) {
  135. host, port, parseErr := hostAndPort(server)
  136. if parseErr != nil {
  137. return "", fmt.Errorf("server port parse error: %v", parseErr)
  138. }
  139. newPort := int(port) + deltaPort
  140. return fmt.Sprintf("%s:%d", host, newPort), nil
  141. }
  142. func hostAndPort(address string) (host string, port uint64, err error) {
  143. colonIndex := strings.LastIndex(address, ":")
  144. if colonIndex < 0 {
  145. return "", 0, fmt.Errorf("server should have hostname:port format: %v", address)
  146. }
  147. port, err = strconv.ParseUint(address[colonIndex+1:], 10, 64)
  148. if err != nil {
  149. return "", 0, fmt.Errorf("server port parse error: %v", err)
  150. }
  151. return address[:colonIndex], port, err
  152. }
  153. func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
  154. host, port, parseErr := hostAndPort(server)
  155. if parseErr != nil {
  156. glog.Fatalf("server address %s parse error: %v", server, parseErr)
  157. }
  158. grpcPort := int(port) + 10000
  159. return fmt.Sprintf("%s:%d", host, grpcPort)
  160. }
  161. func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) {
  162. host, grpcPort, parseErr := hostAndPort(grpcAddress)
  163. if parseErr != nil {
  164. glog.Fatalf("server grpc address %s parse error: %v", grpcAddress, parseErr)
  165. }
  166. port := int(grpcPort) - 10000
  167. return fmt.Sprintf("%s:%d", host, port)
  168. }
  169. func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {
  170. masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master)
  171. if parseErr != nil {
  172. return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr)
  173. }
  174. return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  175. client := master_pb.NewSeaweedClient(grpcConnection)
  176. return fn(client)
  177. }, masterGrpcAddress, grpcDialOption)
  178. }
  179. func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {
  180. return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  181. client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
  182. return fn(client)
  183. }, brokerGrpcAddress, grpcDialOption)
  184. }
  185. func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
  186. filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer)
  187. if parseErr != nil {
  188. return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr)
  189. }
  190. return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn)
  191. }
  192. func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
  193. return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  194. client := filer_pb.NewSeaweedFilerClient(grpcConnection)
  195. return fn(client)
  196. }, filerGrpcAddress, grpcDialOption)
  197. }
  198. func WithOneOfGrpcFilerClients(filerGrpcAddresses []string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) (err error) {
  199. for _, filerGrpcAddress := range filerGrpcAddresses {
  200. err = WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  201. client := filer_pb.NewSeaweedFilerClient(grpcConnection)
  202. return fn(client)
  203. }, filerGrpcAddress, grpcDialOption)
  204. if err == nil {
  205. return nil
  206. }
  207. }
  208. return err
  209. }