You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

271 lines
8.0 KiB

5 years ago
6 years ago
4 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
3 years ago
  1. package pb
  2. import (
  3. "context"
  4. "fmt"
  5. "github.com/chrislusf/seaweedfs/weed/glog"
  6. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  7. "github.com/chrislusf/seaweedfs/weed/util"
  8. "math/rand"
  9. "net/http"
  10. "strconv"
  11. "strings"
  12. "sync"
  13. "time"
  14. "google.golang.org/grpc"
  15. "google.golang.org/grpc/keepalive"
  16. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  17. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  18. "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
  19. )
  20. const (
  21. Max_Message_Size = 1 << 30 // 1 GB
  22. )
  23. var (
  24. // cache grpc connections
  25. grpcClients = make(map[string]*versionedGrpcClient)
  26. grpcClientsLock sync.Mutex
  27. )
  28. type versionedGrpcClient struct {
  29. *grpc.ClientConn
  30. version int
  31. errCount int
  32. }
  33. func init() {
  34. http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 1024
  35. http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
  36. }
  37. func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
  38. var options []grpc.ServerOption
  39. options = append(options,
  40. grpc.KeepaliveParams(keepalive.ServerParameters{
  41. Time: 10 * time.Second, // wait time before ping if no activity
  42. Timeout: 20 * time.Second, // ping timeout
  43. // MaxConnectionAge: 10 * time.Hour,
  44. }),
  45. grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
  46. MinTime: 60 * time.Second, // min time a client should wait before sending a ping
  47. PermitWithoutStream: true,
  48. }),
  49. grpc.MaxRecvMsgSize(Max_Message_Size),
  50. grpc.MaxSendMsgSize(Max_Message_Size),
  51. )
  52. for _, opt := range opts {
  53. if opt != nil {
  54. options = append(options, opt)
  55. }
  56. }
  57. return grpc.NewServer(options...)
  58. }
  59. func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
  60. // opts = append(opts, grpc.WithBlock())
  61. // opts = append(opts, grpc.WithTimeout(time.Duration(5*time.Second)))
  62. var options []grpc.DialOption
  63. options = append(options,
  64. // grpc.WithInsecure(),
  65. grpc.WithDefaultCallOptions(
  66. grpc.MaxCallSendMsgSize(Max_Message_Size),
  67. grpc.MaxCallRecvMsgSize(Max_Message_Size),
  68. ),
  69. grpc.WithKeepaliveParams(keepalive.ClientParameters{
  70. Time: 30 * time.Second, // client ping server if no activity for this long
  71. Timeout: 20 * time.Second,
  72. PermitWithoutStream: true,
  73. }))
  74. for _, opt := range opts {
  75. if opt != nil {
  76. options = append(options, opt)
  77. }
  78. }
  79. return grpc.DialContext(ctx, address, options...)
  80. }
  81. func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedGrpcClient, error) {
  82. grpcClientsLock.Lock()
  83. defer grpcClientsLock.Unlock()
  84. existingConnection, found := grpcClients[address]
  85. if found {
  86. return existingConnection, nil
  87. }
  88. ctx := context.Background()
  89. grpcConnection, err := GrpcDial(ctx, address, opts...)
  90. if err != nil {
  91. return nil, fmt.Errorf("fail to dial %s: %v", address, err)
  92. }
  93. vgc := &versionedGrpcClient{
  94. grpcConnection,
  95. rand.Int(),
  96. 0,
  97. }
  98. grpcClients[address] = vgc
  99. return vgc, nil
  100. }
  101. // WithGrpcClient In streamingMode, always use a fresh connection. Otherwise, try to reuse an existing connection.
  102. func WithGrpcClient(streamingMode bool, fn func(*grpc.ClientConn) error, address string, opts ...grpc.DialOption) error {
  103. if !streamingMode {
  104. vgc, err := getOrCreateConnection(address, opts...)
  105. if err != nil {
  106. return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
  107. }
  108. executionErr := fn(vgc.ClientConn)
  109. if executionErr != nil {
  110. if strings.Contains(executionErr.Error(), "transport") ||
  111. strings.Contains(executionErr.Error(), "connection closed") {
  112. grpcClientsLock.Lock()
  113. if t, ok := grpcClients[address]; ok {
  114. if t.version == vgc.version {
  115. vgc.Close()
  116. delete(grpcClients, address)
  117. }
  118. }
  119. grpcClientsLock.Unlock()
  120. }
  121. }
  122. return executionErr
  123. } else {
  124. grpcConnection, err := GrpcDial(context.Background(), address, opts...)
  125. if err != nil {
  126. return fmt.Errorf("fail to dial %s: %v", address, err)
  127. }
  128. defer grpcConnection.Close()
  129. executionErr := fn(grpcConnection)
  130. if executionErr != nil {
  131. return executionErr
  132. }
  133. return nil
  134. }
  135. }
  136. func ParseServerAddress(server string, deltaPort int) (newServerAddress string, err error) {
  137. host, port, parseErr := hostAndPort(server)
  138. if parseErr != nil {
  139. return "", fmt.Errorf("server port parse error: %v", parseErr)
  140. }
  141. newPort := int(port) + deltaPort
  142. return util.JoinHostPort(host, newPort), nil
  143. }
  144. func hostAndPort(address string) (host string, port uint64, err error) {
  145. colonIndex := strings.LastIndex(address, ":")
  146. if colonIndex < 0 {
  147. return "", 0, fmt.Errorf("server should have hostname:port format: %v", address)
  148. }
  149. port, err = strconv.ParseUint(address[colonIndex+1:], 10, 64)
  150. if err != nil {
  151. return "", 0, fmt.Errorf("server port parse error: %v", err)
  152. }
  153. return address[:colonIndex], port, err
  154. }
  155. func ServerToGrpcAddress(server string) (serverGrpcAddress string) {
  156. host, port, parseErr := hostAndPort(server)
  157. if parseErr != nil {
  158. glog.Fatalf("server address %s parse error: %v", server, parseErr)
  159. }
  160. grpcPort := int(port) + 10000
  161. return util.JoinHostPort(host, grpcPort)
  162. }
  163. func GrpcAddressToServerAddress(grpcAddress string) (serverAddress string) {
  164. host, grpcPort, parseErr := hostAndPort(grpcAddress)
  165. if parseErr != nil {
  166. glog.Fatalf("server grpc address %s parse error: %v", grpcAddress, parseErr)
  167. }
  168. port := int(grpcPort) - 10000
  169. return util.JoinHostPort(host, port)
  170. }
  171. func WithMasterClient(streamingMode bool, master ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) error {
  172. return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
  173. client := master_pb.NewSeaweedClient(grpcConnection)
  174. return fn(client)
  175. }, master.ToGrpcAddress(), grpcDialOption)
  176. }
  177. func WithVolumeServerClient(streamingMode bool, volumeServer ServerAddress, grpcDialOption grpc.DialOption, fn func(client volume_server_pb.VolumeServerClient) error) error {
  178. return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
  179. client := volume_server_pb.NewVolumeServerClient(grpcConnection)
  180. return fn(client)
  181. }, volumeServer.ToGrpcAddress(), grpcDialOption)
  182. }
  183. func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses map[string]ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
  184. for _, masterGrpcAddress := range masterGrpcAddresses {
  185. err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
  186. client := master_pb.NewSeaweedClient(grpcConnection)
  187. return fn(client)
  188. }, masterGrpcAddress.ToGrpcAddress(), grpcDialOption)
  189. if err == nil {
  190. return nil
  191. }
  192. }
  193. return err
  194. }
  195. func WithBrokerGrpcClient(streamingMode bool, brokerGrpcAddress string, grpcDialOption grpc.DialOption, fn func(client messaging_pb.SeaweedMessagingClient) error) error {
  196. return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
  197. client := messaging_pb.NewSeaweedMessagingClient(grpcConnection)
  198. return fn(client)
  199. }, brokerGrpcAddress, grpcDialOption)
  200. }
  201. func WithFilerClient(streamingMode bool, filer ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
  202. return WithGrpcFilerClient(streamingMode, filer, grpcDialOption, fn)
  203. }
  204. func WithGrpcFilerClient(streamingMode bool, filerGrpcAddress ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) error {
  205. return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
  206. client := filer_pb.NewSeaweedFilerClient(grpcConnection)
  207. return fn(client)
  208. }, filerGrpcAddress.ToGrpcAddress(), grpcDialOption)
  209. }
  210. func WithOneOfGrpcFilerClients(streamingMode bool, filerAddresses []ServerAddress, grpcDialOption grpc.DialOption, fn func(client filer_pb.SeaweedFilerClient) error) (err error) {
  211. for _, filerAddress := range filerAddresses {
  212. err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
  213. client := filer_pb.NewSeaweedFilerClient(grpcConnection)
  214. return fn(client)
  215. }, filerAddress.ToGrpcAddress(), grpcDialOption)
  216. if err == nil {
  217. return nil
  218. }
  219. }
  220. return err
  221. }