You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

194 lines
4.8 KiB

6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
  1. package weed_server
  2. import (
  3. "fmt"
  4. "net"
  5. "strings"
  6. "time"
  7. "github.com/chrislusf/raft"
  8. "github.com/chrislusf/seaweedfs/weed/glog"
  9. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  10. "github.com/chrislusf/seaweedfs/weed/topology"
  11. "google.golang.org/grpc/peer"
  12. )
  13. func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
  14. var dn *topology.DataNode
  15. t := ms.Topo
  16. defer func() {
  17. if dn != nil {
  18. glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port)
  19. t.UnRegisterDataNode(dn)
  20. message := &master_pb.VolumeLocation{
  21. Url: dn.Url(),
  22. PublicUrl: dn.PublicUrl,
  23. }
  24. for _, v := range dn.GetVolumes() {
  25. message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
  26. }
  27. if len(message.DeletedVids) > 0 {
  28. ms.clientChansLock.RLock()
  29. for _, ch := range ms.clientChans {
  30. ch <- message
  31. }
  32. ms.clientChansLock.RUnlock()
  33. }
  34. }
  35. }()
  36. for {
  37. heartbeat, err := stream.Recv()
  38. if err != nil {
  39. return err
  40. }
  41. if dn == nil {
  42. t.Sequence.SetMax(heartbeat.MaxFileKey)
  43. if heartbeat.Ip == "" {
  44. if pr, ok := peer.FromContext(stream.Context()); ok {
  45. if pr.Addr != net.Addr(nil) {
  46. heartbeat.Ip = pr.Addr.String()[0:strings.LastIndex(pr.Addr.String(), ":")]
  47. glog.V(0).Infof("remote IP address is detected as %v", heartbeat.Ip)
  48. }
  49. }
  50. }
  51. dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
  52. dc := t.GetOrCreateDataCenter(dcName)
  53. rack := dc.GetOrCreateRack(rackName)
  54. dn = rack.GetOrCreateDataNode(heartbeat.Ip,
  55. int(heartbeat.Port), heartbeat.PublicUrl,
  56. int(heartbeat.MaxVolumeCount))
  57. glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
  58. if err := stream.Send(&master_pb.HeartbeatResponse{
  59. VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
  60. }); err != nil {
  61. return err
  62. }
  63. }
  64. message := &master_pb.VolumeLocation{
  65. Url: dn.Url(),
  66. PublicUrl: dn.PublicUrl,
  67. }
  68. if len(heartbeat.NewVids) > 0 || len(heartbeat.DeletedVids) > 0 {
  69. // process delta volume ids if exists for fast volume id updates
  70. message.NewVids = append(message.NewVids, heartbeat.NewVids...)
  71. message.DeletedVids = append(message.DeletedVids, heartbeat.DeletedVids...)
  72. } else {
  73. // process heartbeat.Volumes
  74. newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn)
  75. for _, v := range newVolumes {
  76. message.NewVids = append(message.NewVids, uint32(v.Id))
  77. }
  78. for _, v := range deletedVolumes {
  79. message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
  80. }
  81. }
  82. if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
  83. ms.clientChansLock.RLock()
  84. for _, ch := range ms.clientChans {
  85. ch <- message
  86. }
  87. ms.clientChansLock.RUnlock()
  88. }
  89. // tell the volume servers about the leader
  90. newLeader, err := t.Leader()
  91. if err != nil {
  92. return err
  93. }
  94. if err := stream.Send(&master_pb.HeartbeatResponse{
  95. Leader: newLeader,
  96. }); err != nil {
  97. return err
  98. }
  99. }
  100. }
  101. // KeepConnected keep a stream gRPC call to the master. Used by clients to know the master is up.
  102. // And clients gets the up-to-date list of volume locations
  103. func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {
  104. req, err := stream.Recv()
  105. if err != nil {
  106. return err
  107. }
  108. if !ms.Topo.IsLeader() {
  109. return raft.NotLeaderError
  110. }
  111. // remember client address
  112. ctx := stream.Context()
  113. // fmt.Printf("FromContext %+v\n", ctx)
  114. pr, ok := peer.FromContext(ctx)
  115. if !ok {
  116. glog.Error("failed to get peer from ctx")
  117. return fmt.Errorf("failed to get peer from ctx")
  118. }
  119. if pr.Addr == net.Addr(nil) {
  120. glog.Error("failed to get peer address")
  121. return fmt.Errorf("failed to get peer address")
  122. }
  123. clientName := req.Name + pr.Addr.String()
  124. glog.V(0).Infof("+ client %v", clientName)
  125. messageChan := make(chan *master_pb.VolumeLocation)
  126. stopChan := make(chan bool)
  127. ms.clientChansLock.Lock()
  128. ms.clientChans[clientName] = messageChan
  129. ms.clientChansLock.Unlock()
  130. defer func() {
  131. glog.V(0).Infof("- client %v", clientName)
  132. ms.clientChansLock.Lock()
  133. delete(ms.clientChans, clientName)
  134. ms.clientChansLock.Unlock()
  135. }()
  136. for _, message := range ms.Topo.ToVolumeLocations() {
  137. if err := stream.Send(message); err != nil {
  138. return err
  139. }
  140. }
  141. go func() {
  142. for {
  143. _, err := stream.Recv()
  144. if err != nil {
  145. glog.V(2).Infof("- client %v: %v", clientName, err)
  146. stopChan <- true
  147. break
  148. }
  149. }
  150. }()
  151. ticker := time.NewTicker(5 * time.Second)
  152. for {
  153. select {
  154. case message := <-messageChan:
  155. if err := stream.Send(message); err != nil {
  156. glog.V(0).Infof("=> client %v: %+v", clientName, message)
  157. return err
  158. }
  159. case <-ticker.C:
  160. if !ms.Topo.IsLeader() {
  161. return raft.NotLeaderError
  162. }
  163. case <-stopChan:
  164. return nil
  165. }
  166. }
  167. return nil
  168. }