You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

204 lines
5.9 KiB

5 years ago
6 years ago
4 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
  1. package pb
  2. import (
  3. "context"
  4. "fmt"
  5. "github.com/chrislusf/seaweedfs/weed/glog"
  6. "net/http"
  7. "strconv"
  8. "strings"
  9. "sync"
  10. "time"
  11. "google.golang.org/grpc"
  12. "google.golang.org/grpc/keepalive"
  13. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  14. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  15. "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
  16. )
  17. const (
  18. Max_Message_Size = 1 << 30 // 1 GB
  19. )
  20. var (
  21. // cache grpc connections
  22. grpcClients = make(map[string]*grpc.ClientConn)
  23. grpcClientsLock sync.Mutex
  24. )
  25. func init() {
  26. http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
  27. http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
  28. }
  29. func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
  30. var options []grpc.ServerOption
  31. options = append(options,
  32. grpc.KeepaliveParams(keepalive.ServerParameters{
  33. Time: 10 * time.Second, // wait time before ping if no activity
  34. Timeout: 20 * time.Second, // ping timeout
  35. MaxConnectionAge: 10 * time.Hour,
  36. }),
  37. grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
  38. MinTime: 60 * time.Second, // min time a client should wait before sending a ping
  39. PermitWithoutStream: false,
  40. }),
  41. grpc.MaxRecvMsgSize(Max_Message_Size),
  42. grpc.MaxSendMsgSize(Max_Message_Size),
  43. )
  44. for _, opt := range opts {
  45. if opt != nil {
  46. options = append(options, opt)
  47. }
  48. }
  49. return grpc.NewServer(options...)
  50. }
  51. func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
  52. // opts = append(opts, grpc.WithBlock())
  53. // opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
  54. var options []grpc.DialOption
  55. options = append(options,
  56. // grpc.WithInsecure(),
  57. grpc.WithDefaultCallOptions(
  58. grpc.MaxCallSendMsgSize(Max_Message_Size),
  59. grpc.MaxCallRecvMsgSize(Max_Message_Size),
  60. ),
  61. grpc.WithKeepaliveParams(keepalive.ClientParameters{
  62. Time: 30 * time.Second, // client ping server if no activity for this long
  63. Timeout: 20 * time.Second,
  64. PermitWithoutStream: false,
  65. }))
  66. for _, opt := range opts {
  67. if opt != nil {
  68. options = append(options, opt)
  69. }
  70. }
  71. return grpc.DialContext(ctx, address, options...)
  72. }
  73. func getOrCreateConnection(address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
  74. grpcClientsLock.Lock()
  75. defer grpcClientsLock.Unlock()
  76. existingConnection, found := grpcClients[address]
  77. if found {
  78. return existingConnection, nil
  79. }
  80. grpcConnection, err := GrpcDial(context.Background(), address, opts...)
  81. if err != nil {
  82. return nil, fmt.Errorf("fail to dial %s: %v", address, err)
  83. }
  84. grpcClients[address] = grpcConnection
  85. return grpcConnection, nil
  86. }
  87. func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {
  88. grpcConnection, err := getOrCreateConnection(address, opts...)
  89. if err != nil {
  90. return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
  91. }
  92. return fn(grpcConnection)
  93. }
  94. func ParseServerToGrpcAddress(server string) (serverGrpcAddress string, err error) {
  95. return ParseServerAddress(server, 10000)
  96. }
  97. func ParseServerAddress(server string, deltaPort int) (newServerAddress string, err error) {
  98. host, port, parseErr := hostAndPort(server)
  99. if parseErr != nil {
  100. return "", fmt.Errorf("server port parse error: %v", parseErr)
  101. }
  102. newPort := int(port) + deltaPort
  103. return fmt.Sprintf("%s:%d", host, newPort), nil
  104. }
  105. func hostAndPort(address string) (host string, port uint64, err error) {
  106. colonIndex := strings.LastIndex(address, ":")
  107. if colonIndex < 0 {
  108. return "", 0, fmt.Errorf("server should have hostname:port format: %v", address)
  109. }
  110. port, err = strconv.ParseUint(address[colonIndex+1:], 10, 64)
  111. if err != nil {
  112. return "", 0, fmt.Errorf("server port parse error: %v", err)
  113. }
  114. return address[:colonIndex], port, err
  115. }
  116. func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
  117. host, port, parseErr := hostAndPort(server)
  118. if parseErr != nil {
  119. glog.Fatalf("server address %s parse error: %v", server, parseErr)
  120. }
  121. grpcPort := int(port) + 10000
  122. return fmt.Sprintf("%s:%d", host, grpcPort)
  123. }
  124. func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) {
  125. host, grpcPort, parseErr := hostAndPort(grpcAddress)
  126. if parseErr != nil {
  127. glog.Fatalf("server grpc address %s parse error: %v", grpcAddress, parseErr)
  128. }
  129. port := int(grpcPort) - 10000
  130. return fmt.Sprintf("%s:%d", host, port)
  131. }
  132. func WithMasterClient(master string, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {
  133. masterGrpcAddress, parseErr := ParseServerToGrpcAddress(master)
  134. if parseErr != nil {
  135. return fmt.Errorf("failed to parse master grpc %v: %v", master, parseErr)
  136. }
  137. return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  138. client := master_pb.NewSeaweedClient(grpcConnection)
  139. return fn(client)
  140. }, masterGrpcAddress, grpcDialOption)
  141. }
  142. func WithBrokerGrpcClient(brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {
  143. return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  144. client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
  145. return fn(client)
  146. }, brokerGrpcAddress, grpcDialOption)
  147. }
  148. func WithFilerClient(filer string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
  149. filerGrpcAddress, parseErr := ParseServerToGrpcAddress(filer)
  150. if parseErr != nil {
  151. return fmt.Errorf("failed to parse filer grpc %v: %v", filer, parseErr)
  152. }
  153. return WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, fn)
  154. }
  155. func WithGrpcFilerClient(filerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
  156. return WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  157. client := filer_pb.NewSeaweedFilerClient(grpcConnection)
  158. return fn(client)
  159. }, filerGrpcAddress, grpcDialOption)
  160. }