You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

314 lines
8.8 KiB

7 years ago
7 years ago
5 years ago
6 years ago
5 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
  1. package weed_server
  2. import (
  3. "context"
  4. "fmt"
  5. "net"
  6. "strings"
  7. "time"
  8. "github.com/chrislusf/raft"
  9. "google.golang.org/grpc/peer"
  10. "github.com/chrislusf/seaweedfs/weed/glog"
  11. "github.com/chrislusf/seaweedfs/weed/pb"
  12. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  13. "github.com/chrislusf/seaweedfs/weed/storage/backend"
  14. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  15. "github.com/chrislusf/seaweedfs/weed/topology"
  16. )
  17. func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error {
  18. var dn *topology.DataNode
  19. t := ms.Topo
  20. defer func() {
  21. if dn != nil {
  22. glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port)
  23. t.UnRegisterDataNode(dn)
  24. message := &master_pb.VolumeLocation{
  25. Url: dn.Url(),
  26. PublicUrl: dn.PublicUrl,
  27. }
  28. for _, v := range dn.GetVolumes() {
  29. message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
  30. }
  31. for _, s := range dn.GetEcShards() {
  32. message.DeletedVids = append(message.DeletedVids, uint32(s.VolumeId))
  33. }
  34. if len(message.DeletedVids) > 0 {
  35. ms.clientChansLock.RLock()
  36. for _, ch := range ms.clientChans {
  37. ch <- message
  38. }
  39. ms.clientChansLock.RUnlock()
  40. }
  41. }
  42. }()
  43. for {
  44. heartbeat, err := stream.Recv()
  45. if err != nil {
  46. if dn != nil {
  47. glog.Warningf("SendHeartbeat.Recv server %s:%d : %v", dn.Ip, dn.Port, err)
  48. } else {
  49. glog.Warningf("SendHeartbeat.Recv: %v", err)
  50. }
  51. return err
  52. }
  53. t.Sequence.SetMax(heartbeat.MaxFileKey)
  54. if dn == nil {
  55. dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
  56. dc := t.GetOrCreateDataCenter(dcName)
  57. rack := dc.GetOrCreateRack(rackName)
  58. dn = rack.GetOrCreateDataNode(heartbeat.Ip,
  59. int(heartbeat.Port), heartbeat.PublicUrl,
  60. int64(heartbeat.MaxVolumeCount))
  61. glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
  62. if err := stream.Send(&master_pb.HeartbeatResponse{
  63. VolumeSizeLimit: uint64(ms.option.VolumeSizeLimitMB) * 1024 * 1024,
  64. MetricsAddress: ms.option.MetricsAddress,
  65. MetricsIntervalSeconds: uint32(ms.option.MetricsIntervalSec),
  66. StorageBackends: backend.ToPbStorageBackends(),
  67. }); err != nil {
  68. glog.Warningf("SendHeartbeat.Send volume size to %s:%d %v", dn.Ip, dn.Port, err)
  69. return err
  70. }
  71. }
  72. if heartbeat.MaxVolumeCount != 0 && dn.GetMaxVolumeCount() != int64(heartbeat.MaxVolumeCount) {
  73. delta := int64(heartbeat.MaxVolumeCount) - dn.GetMaxVolumeCount()
  74. dn.UpAdjustMaxVolumeCountDelta(delta)
  75. }
  76. glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
  77. message := &master_pb.VolumeLocation{
  78. Url: dn.Url(),
  79. PublicUrl: dn.PublicUrl,
  80. }
  81. if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
  82. // process delta volume ids if exists for fast volume id updates
  83. for _, volInfo := range heartbeat.NewVolumes {
  84. message.NewVids = append(message.NewVids, volInfo.Id)
  85. }
  86. for _, volInfo := range heartbeat.DeletedVolumes {
  87. message.DeletedVids = append(message.DeletedVids, volInfo.Id)
  88. }
  89. // update master internal volume layouts
  90. t.IncrementalSyncDataNodeRegistration(heartbeat.NewVolumes, heartbeat.DeletedVolumes, dn)
  91. }
  92. if len(heartbeat.Volumes) > 0 || heartbeat.HasNoVolumes {
  93. // process heartbeat.Volumes
  94. newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn)
  95. for _, v := range newVolumes {
  96. glog.V(0).Infof("master see new volume %d from %s", uint32(v.Id), dn.Url())
  97. message.NewVids = append(message.NewVids, uint32(v.Id))
  98. }
  99. for _, v := range deletedVolumes {
  100. glog.V(0).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url())
  101. message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
  102. }
  103. }
  104. if len(heartbeat.NewEcShards) > 0 || len(heartbeat.DeletedEcShards) > 0 {
  105. // update master internal volume layouts
  106. t.IncrementalSyncDataNodeEcShards(heartbeat.NewEcShards, heartbeat.DeletedEcShards, dn)
  107. for _, s := range heartbeat.NewEcShards {
  108. message.NewVids = append(message.NewVids, s.Id)
  109. }
  110. for _, s := range heartbeat.DeletedEcShards {
  111. if dn.HasVolumesById(needle.VolumeId(s.Id)) {
  112. continue
  113. }
  114. message.DeletedVids = append(message.DeletedVids, s.Id)
  115. }
  116. }
  117. if len(heartbeat.EcShards) > 0 || heartbeat.HasNoEcShards {
  118. glog.V(1).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards)
  119. newShards, deletedShards := t.SyncDataNodeEcShards(heartbeat.EcShards, dn)
  120. // broadcast the ec vid changes to master clients
  121. for _, s := range newShards {
  122. message.NewVids = append(message.NewVids, uint32(s.VolumeId))
  123. }
  124. for _, s := range deletedShards {
  125. if dn.HasVolumesById(s.VolumeId) {
  126. continue
  127. }
  128. message.DeletedVids = append(message.DeletedVids, uint32(s.VolumeId))
  129. }
  130. }
  131. if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
  132. ms.clientChansLock.RLock()
  133. for host, ch := range ms.clientChans {
  134. glog.V(0).Infof("master send to %s: %s", host, message.String())
  135. ch <- message
  136. }
  137. ms.clientChansLock.RUnlock()
  138. }
  139. // tell the volume servers about the leader
  140. newLeader, err := t.Leader()
  141. if err != nil {
  142. glog.Warningf("SendHeartbeat find leader: %v", err)
  143. return err
  144. }
  145. if err := stream.Send(&master_pb.HeartbeatResponse{
  146. Leader: newLeader,
  147. }); err != nil {
  148. glog.Warningf("SendHeartbeat.Send response to to %s:%d %v", dn.Ip, dn.Port, err)
  149. return err
  150. }
  151. }
  152. }
  153. // KeepConnected keep a stream gRPC call to the master. Used by clients to know the master is up.
  154. // And clients gets the up-to-date list of volume locations
  155. func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {
  156. req, err := stream.Recv()
  157. if err != nil {
  158. return err
  159. }
  160. if !ms.Topo.IsLeader() {
  161. return ms.informNewLeader(stream)
  162. }
  163. peerAddress := findClientAddress(stream.Context(), req.GrpcPort)
  164. // only one shell can be connected at any time
  165. if req.Name == pb.AdminShellClient {
  166. if ms.currentAdminShellClient == "" {
  167. ms.currentAdminShellClient = peerAddress
  168. defer func() {
  169. ms.currentAdminShellClient = ""
  170. }()
  171. } else {
  172. return fmt.Errorf("only one concurrent shell allowed, but another shell is already connected from %s", peerAddress)
  173. }
  174. }
  175. stopChan := make(chan bool)
  176. clientName, messageChan := ms.addClient(req.Name, peerAddress)
  177. defer ms.deleteClient(clientName)
  178. for _, message := range ms.Topo.ToVolumeLocations() {
  179. if err := stream.Send(message); err != nil {
  180. return err
  181. }
  182. }
  183. go func() {
  184. for {
  185. _, err := stream.Recv()
  186. if err != nil {
  187. glog.V(2).Infof("- client %v: %v", clientName, err)
  188. stopChan <- true
  189. break
  190. }
  191. }
  192. }()
  193. ticker := time.NewTicker(5 * time.Second)
  194. for {
  195. select {
  196. case message := <-messageChan:
  197. if err := stream.Send(message); err != nil {
  198. glog.V(0).Infof("=> client %v: %+v", clientName, message)
  199. return err
  200. }
  201. case <-ticker.C:
  202. if !ms.Topo.IsLeader() {
  203. return ms.informNewLeader(stream)
  204. }
  205. case <-stopChan:
  206. return nil
  207. }
  208. }
  209. }
  210. func (ms *MasterServer) informNewLeader(stream master_pb.Seaweed_KeepConnectedServer) error {
  211. leader, err := ms.Topo.Leader()
  212. if err != nil {
  213. glog.Errorf("topo leader: %v", err)
  214. return raft.NotLeaderError
  215. }
  216. if err := stream.Send(&master_pb.VolumeLocation{
  217. Leader: leader,
  218. }); err != nil {
  219. return err
  220. }
  221. return nil
  222. }
  223. func (ms *MasterServer) addClient(clientType string, clientAddress string) (clientName string, messageChan chan *master_pb.VolumeLocation) {
  224. clientName = clientType + "@" + clientAddress
  225. glog.V(0).Infof("+ client %v", clientName)
  226. messageChan = make(chan *master_pb.VolumeLocation)
  227. ms.clientChansLock.Lock()
  228. ms.clientChans[clientName] = messageChan
  229. ms.clientChansLock.Unlock()
  230. return
  231. }
  232. func (ms *MasterServer) deleteClient(clientName string) {
  233. glog.V(0).Infof("- client %v", clientName)
  234. ms.clientChansLock.Lock()
  235. delete(ms.clientChans, clientName)
  236. ms.clientChansLock.Unlock()
  237. }
  238. func findClientAddress(ctx context.Context, grpcPort uint32) string {
  239. // fmt.Printf("FromContext %+v\n", ctx)
  240. pr, ok := peer.FromContext(ctx)
  241. if !ok {
  242. glog.Error("failed to get peer from ctx")
  243. return ""
  244. }
  245. if pr.Addr == net.Addr(nil) {
  246. glog.Error("failed to get peer address")
  247. return ""
  248. }
  249. if grpcPort == 0 {
  250. return pr.Addr.String()
  251. }
  252. if tcpAddr, ok := pr.Addr.(*net.TCPAddr); ok {
  253. externalIP := tcpAddr.IP
  254. return fmt.Sprintf("%s:%d", externalIP, grpcPort)
  255. }
  256. return pr.Addr.String()
  257. }
  258. func (ms *MasterServer) ListMasterClients(ctx context.Context, req *master_pb.ListMasterClientsRequest) (*master_pb.ListMasterClientsResponse, error) {
  259. resp := &master_pb.ListMasterClientsResponse{}
  260. ms.clientChansLock.RLock()
  261. defer ms.clientChansLock.RUnlock()
  262. for k := range ms.clientChans {
  263. if strings.HasPrefix(k, req.ClientType+"@") {
  264. resp.GrpcAddresses = append(resp.GrpcAddresses, k[len(req.ClientType)+1:])
  265. }
  266. }
  267. return resp, nil
  268. }