You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

192 lines
6.1 KiB

5 years ago
5 years ago
5 years ago
  1. package weed_server
  2. import (
  3. "fmt"
  4. "net"
  5. "time"
  6. "google.golang.org/grpc"
  7. "github.com/chrislusf/seaweedfs/weed/pb"
  8. "github.com/chrislusf/seaweedfs/weed/security"
  9. "github.com/chrislusf/seaweedfs/weed/storage/backend"
  10. "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
  11. "golang.org/x/net/context"
  12. "github.com/chrislusf/seaweedfs/weed/glog"
  13. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  14. "github.com/chrislusf/seaweedfs/weed/util"
  15. )
  16. func (vs *VolumeServer) GetMaster() string {
  17. return vs.currentMaster
  18. }
  19. func (vs *VolumeServer) heartbeat() {
  20. glog.V(0).Infof("Volume server start with seed master nodes: %v", vs.SeedMasterNodes)
  21. vs.store.SetDataCenter(vs.dataCenter)
  22. vs.store.SetRack(vs.rack)
  23. grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.volume")
  24. var err error
  25. var newLeader string
  26. for {
  27. for _, master := range vs.SeedMasterNodes {
  28. if newLeader != "" {
  29. master = newLeader
  30. }
  31. masterGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(master)
  32. if parseErr != nil {
  33. glog.V(0).Infof("failed to parse master grpc %v: %v", masterGrpcAddress, parseErr)
  34. continue
  35. }
  36. vs.store.MasterAddress = master
  37. newLeader, err = vs.doHeartbeat(master, masterGrpcAddress, grpcDialOption, time.Duration(vs.pulseSeconds)*time.Second)
  38. if err != nil {
  39. glog.V(0).Infof("heartbeat error: %v", err)
  40. time.Sleep(time.Duration(vs.pulseSeconds) * time.Second)
  41. newLeader = ""
  42. vs.store.MasterAddress = ""
  43. }
  44. }
  45. }
  46. }
  47. func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDialOption grpc.DialOption, sleepInterval time.Duration) (newLeader string, err error) {
  48. grpcConection, err := pb.GrpcDial(context.Background(), masterGrpcAddress, grpcDialOption)
  49. if err != nil {
  50. return "", fmt.Errorf("fail to dial %s : %v", masterNode, err)
  51. }
  52. defer grpcConection.Close()
  53. client := master_pb.NewSeaweedClient(grpcConection)
  54. stream, err := client.SendHeartbeat(context.Background())
  55. if err != nil {
  56. glog.V(0).Infof("SendHeartbeat to %s: %v", masterNode, err)
  57. return "", err
  58. }
  59. glog.V(0).Infof("Heartbeat to: %v", masterNode)
  60. vs.currentMaster = masterNode
  61. doneChan := make(chan error, 1)
  62. go func() {
  63. for {
  64. in, err := stream.Recv()
  65. if err != nil {
  66. doneChan <- err
  67. return
  68. }
  69. if in.GetVolumeSizeLimit() != 0 {
  70. vs.store.SetVolumeSizeLimit(in.GetVolumeSizeLimit())
  71. }
  72. if in.GetLeader() != "" && masterNode != in.GetLeader() && !isSameIP(in.GetLeader(), masterNode) {
  73. glog.V(0).Infof("Volume Server found a new master newLeader: %v instead of %v", in.GetLeader(), masterNode)
  74. newLeader = in.GetLeader()
  75. doneChan <- nil
  76. return
  77. }
  78. if in.GetMetricsAddress() != "" && vs.MetricsAddress != in.GetMetricsAddress() {
  79. vs.MetricsAddress = in.GetMetricsAddress()
  80. vs.MetricsIntervalSec = int(in.GetMetricsIntervalSeconds())
  81. }
  82. if len(in.StorageBackends) > 0 {
  83. backend.LoadFromPbStorageBackends(in.StorageBackends)
  84. }
  85. }
  86. }()
  87. if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
  88. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  89. return "", err
  90. }
  91. if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
  92. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  93. return "", err
  94. }
  95. volumeTickChan := time.Tick(sleepInterval)
  96. ecShardTickChan := time.Tick(17 * sleepInterval)
  97. for {
  98. select {
  99. case volumeMessage := <-vs.store.NewVolumesChan:
  100. deltaBeat := &master_pb.Heartbeat{
  101. NewVolumes: []*master_pb.VolumeShortInformationMessage{
  102. &volumeMessage,
  103. },
  104. }
  105. glog.V(1).Infof("volume server %s:%d adds volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id)
  106. if err = stream.Send(deltaBeat); err != nil {
  107. glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
  108. return "", err
  109. }
  110. case ecShardMessage := <-vs.store.NewEcShardsChan:
  111. deltaBeat := &master_pb.Heartbeat{
  112. NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
  113. &ecShardMessage,
  114. },
  115. }
  116. glog.V(1).Infof("volume server %s:%d adds ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id,
  117. erasure_coding.ShardBits(ecShardMessage.EcIndexBits).ShardIds())
  118. if err = stream.Send(deltaBeat); err != nil {
  119. glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
  120. return "", err
  121. }
  122. case volumeMessage := <-vs.store.DeletedVolumesChan:
  123. deltaBeat := &master_pb.Heartbeat{
  124. DeletedVolumes: []*master_pb.VolumeShortInformationMessage{
  125. &volumeMessage,
  126. },
  127. }
  128. glog.V(1).Infof("volume server %s:%d deletes volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id)
  129. if err = stream.Send(deltaBeat); err != nil {
  130. glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
  131. return "", err
  132. }
  133. case ecShardMessage := <-vs.store.DeletedEcShardsChan:
  134. deltaBeat := &master_pb.Heartbeat{
  135. DeletedEcShards: []*master_pb.VolumeEcShardInformationMessage{
  136. &ecShardMessage,
  137. },
  138. }
  139. glog.V(1).Infof("volume server %s:%d deletes ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id,
  140. erasure_coding.ShardBits(ecShardMessage.EcIndexBits).ShardIds())
  141. if err = stream.Send(deltaBeat); err != nil {
  142. glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
  143. return "", err
  144. }
  145. case <-volumeTickChan:
  146. glog.V(4).Infof("volume server %s:%d heartbeat", vs.store.Ip, vs.store.Port)
  147. if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
  148. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  149. return "", err
  150. }
  151. case <-ecShardTickChan:
  152. glog.V(4).Infof("volume server %s:%d ec heartbeat", vs.store.Ip, vs.store.Port)
  153. if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
  154. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  155. return "", err
  156. }
  157. case err = <-doneChan:
  158. return
  159. }
  160. }
  161. }
  162. func isSameIP(ip string, host string) bool {
  163. ips, err := net.LookupIP(host)
  164. if err != nil {
  165. return false
  166. }
  167. for _, t := range ips {
  168. if ip == t.String() {
  169. return true
  170. }
  171. }
  172. return false
  173. }