You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

197 lines
6.4 KiB

5 years ago
5 years ago
5 years ago
  1. package weed_server
  2. import (
  3. "fmt"
  4. "net"
  5. "time"
  6. "google.golang.org/grpc"
  7. "github.com/chrislusf/seaweedfs/weed/pb"
  8. "github.com/chrislusf/seaweedfs/weed/security"
  9. "github.com/chrislusf/seaweedfs/weed/storage/backend"
  10. "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
  11. "golang.org/x/net/context"
  12. "github.com/chrislusf/seaweedfs/weed/glog"
  13. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  14. "github.com/chrislusf/seaweedfs/weed/util"
  15. )
  16. func (vs *VolumeServer) GetMaster() string {
  17. return vs.currentMaster
  18. }
  19. func (vs *VolumeServer) heartbeat() {
  20. glog.V(0).Infof("Volume server start with seed master nodes: %v", vs.SeedMasterNodes)
  21. vs.store.SetDataCenter(vs.dataCenter)
  22. vs.store.SetRack(vs.rack)
  23. grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.volume")
  24. var err error
  25. var newLeader string
  26. for {
  27. for _, master := range vs.SeedMasterNodes {
  28. if newLeader != "" {
  29. master = newLeader
  30. }
  31. masterGrpcAddress, parseErr := pb.ParseServerToGrpcAddress(master)
  32. if parseErr != nil {
  33. glog.V(0).Infof("failed to parse master grpc %v: %v", masterGrpcAddress, parseErr)
  34. continue
  35. }
  36. vs.store.MasterAddress = master
  37. newLeader, err = vs.doHeartbeat(master, masterGrpcAddress, grpcDialOption, time.Duration(vs.pulseSeconds)*time.Second)
  38. if err != nil {
  39. glog.V(0).Infof("heartbeat error: %v", err)
  40. time.Sleep(time.Duration(vs.pulseSeconds) * time.Second)
  41. newLeader = ""
  42. vs.store.MasterAddress = ""
  43. }
  44. }
  45. }
  46. }
  47. func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, grpcDialOption grpc.DialOption, sleepInterval time.Duration) (newLeader string, err error) {
  48. grpcConection, err := pb.GrpcDial(context.Background(), masterGrpcAddress, grpcDialOption)
  49. if err != nil {
  50. return "", fmt.Errorf("fail to dial %s : %v", masterNode, err)
  51. }
  52. defer grpcConection.Close()
  53. client := master_pb.NewSeaweedClient(grpcConection)
  54. stream, err := client.SendHeartbeat(context.Background())
  55. if err != nil {
  56. glog.V(0).Infof("SendHeartbeat to %s: %v", masterNode, err)
  57. return "", err
  58. }
  59. glog.V(0).Infof("Heartbeat to: %v", masterNode)
  60. vs.currentMaster = masterNode
  61. doneChan := make(chan error, 1)
  62. go func() {
  63. for {
  64. in, err := stream.Recv()
  65. if err != nil {
  66. doneChan <- err
  67. return
  68. }
  69. if in.GetVolumeSizeLimit() != 0 && vs.store.GetVolumeSizeLimit() != in.GetVolumeSizeLimit() {
  70. vs.store.SetVolumeSizeLimit(in.GetVolumeSizeLimit())
  71. if vs.store.MaybeAdjustVolumeMax() {
  72. if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
  73. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  74. }
  75. }
  76. }
  77. if in.GetLeader() != "" && masterNode != in.GetLeader() && !isSameIP(in.GetLeader(), masterNode) {
  78. glog.V(0).Infof("Volume Server found a new master newLeader: %v instead of %v", in.GetLeader(), masterNode)
  79. newLeader = in.GetLeader()
  80. doneChan <- nil
  81. return
  82. }
  83. if in.GetMetricsAddress() != "" && vs.MetricsAddress != in.GetMetricsAddress() {
  84. vs.MetricsAddress = in.GetMetricsAddress()
  85. vs.MetricsIntervalSec = int(in.GetMetricsIntervalSeconds())
  86. }
  87. if len(in.StorageBackends) > 0 {
  88. backend.LoadFromPbStorageBackends(in.StorageBackends)
  89. }
  90. }
  91. }()
  92. if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
  93. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  94. return "", err
  95. }
  96. if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
  97. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  98. return "", err
  99. }
  100. volumeTickChan := time.Tick(sleepInterval)
  101. ecShardTickChan := time.Tick(17 * sleepInterval)
  102. for {
  103. select {
  104. case volumeMessage := <-vs.store.NewVolumesChan:
  105. deltaBeat := &master_pb.Heartbeat{
  106. NewVolumes: []*master_pb.VolumeShortInformationMessage{
  107. &volumeMessage,
  108. },
  109. }
  110. glog.V(1).Infof("volume server %s:%d adds volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id)
  111. if err = stream.Send(deltaBeat); err != nil {
  112. glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
  113. return "", err
  114. }
  115. case ecShardMessage := <-vs.store.NewEcShardsChan:
  116. deltaBeat := &master_pb.Heartbeat{
  117. NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
  118. &ecShardMessage,
  119. },
  120. }
  121. glog.V(1).Infof("volume server %s:%d adds ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id,
  122. erasure_coding.ShardBits(ecShardMessage.EcIndexBits).ShardIds())
  123. if err = stream.Send(deltaBeat); err != nil {
  124. glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
  125. return "", err
  126. }
  127. case volumeMessage := <-vs.store.DeletedVolumesChan:
  128. deltaBeat := &master_pb.Heartbeat{
  129. DeletedVolumes: []*master_pb.VolumeShortInformationMessage{
  130. &volumeMessage,
  131. },
  132. }
  133. glog.V(1).Infof("volume server %s:%d deletes volume %d", vs.store.Ip, vs.store.Port, volumeMessage.Id)
  134. if err = stream.Send(deltaBeat); err != nil {
  135. glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
  136. return "", err
  137. }
  138. case ecShardMessage := <-vs.store.DeletedEcShardsChan:
  139. deltaBeat := &master_pb.Heartbeat{
  140. DeletedEcShards: []*master_pb.VolumeEcShardInformationMessage{
  141. &ecShardMessage,
  142. },
  143. }
  144. glog.V(1).Infof("volume server %s:%d deletes ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id,
  145. erasure_coding.ShardBits(ecShardMessage.EcIndexBits).ShardIds())
  146. if err = stream.Send(deltaBeat); err != nil {
  147. glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
  148. return "", err
  149. }
  150. case <-volumeTickChan:
  151. glog.V(4).Infof("volume server %s:%d heartbeat", vs.store.Ip, vs.store.Port)
  152. if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
  153. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  154. return "", err
  155. }
  156. case <-ecShardTickChan:
  157. glog.V(4).Infof("volume server %s:%d ec heartbeat", vs.store.Ip, vs.store.Port)
  158. if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
  159. glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
  160. return "", err
  161. }
  162. case err = <-doneChan:
  163. return
  164. }
  165. }
  166. }
  167. func isSameIP(ip string, host string) bool {
  168. ips, err := net.LookupIP(host)
  169. if err != nil {
  170. return false
  171. }
  172. for _, t := range ips {
  173. if ip == t.String() {
  174. return true
  175. }
  176. }
  177. return false
  178. }