You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

297 lines
9.7 KiB

7 years ago
5 years ago
3 years ago
7 years ago
3 years ago
2 years ago
4 years ago
3 years ago
6 years ago
  1. package wdclient
  2. import (
  3. "context"
  4. "fmt"
  5. "github.com/seaweedfs/seaweedfs/weed/stats"
  6. "math/rand"
  7. "time"
  8. "github.com/seaweedfs/seaweedfs/weed/util"
  9. "google.golang.org/grpc"
  10. "github.com/seaweedfs/seaweedfs/weed/glog"
  11. "github.com/seaweedfs/seaweedfs/weed/pb"
  12. "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
  13. )
  14. type MasterClient struct {
  15. FilerGroup string
  16. clientType string
  17. clientHost pb.ServerAddress
  18. rack string
  19. currentMaster pb.ServerAddress
  20. masters map[string]pb.ServerAddress
  21. grpcDialOption grpc.DialOption
  22. vidMap
  23. vidMapCacheSize int
  24. OnPeerUpdate func(update *master_pb.ClusterNodeUpdate, startFrom time.Time)
  25. }
  26. func NewMasterClient(grpcDialOption grpc.DialOption, filerGroup string, clientType string, clientHost pb.ServerAddress, clientDataCenter string, rack string, masters map[string]pb.ServerAddress) *MasterClient {
  27. return &MasterClient{
  28. FilerGroup: filerGroup,
  29. clientType: clientType,
  30. clientHost: clientHost,
  31. rack: rack,
  32. masters: masters,
  33. grpcDialOption: grpcDialOption,
  34. vidMap: newVidMap(clientDataCenter),
  35. vidMapCacheSize: 5,
  36. }
  37. }
  38. func (mc *MasterClient) GetLookupFileIdFunction() LookupFileIdFunctionType {
  39. return mc.LookupFileIdWithFallback
  40. }
  41. func (mc *MasterClient) LookupFileIdWithFallback(fileId string) (fullUrls []string, err error) {
  42. fullUrls, err = mc.vidMap.LookupFileId(fileId)
  43. if err == nil && len(fullUrls) > 0 {
  44. return
  45. }
  46. err = pb.WithMasterClient(false, mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
  47. resp, err := client.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{
  48. VolumeOrFileIds: []string{fileId},
  49. })
  50. if err != nil {
  51. return fmt.Errorf("LookupVolume failed: %v", err)
  52. }
  53. for vid, vidLocation := range resp.VolumeIdLocations {
  54. for _, vidLoc := range vidLocation.Locations {
  55. loc := Location{
  56. Url: vidLoc.Url,
  57. PublicUrl: vidLoc.PublicUrl,
  58. GrpcPort: int(vidLoc.GrpcPort),
  59. DataCenter: vidLoc.DataCenter,
  60. }
  61. mc.vidMap.addLocation(uint32(vid), loc)
  62. httpUrl := "http://" + loc.Url + "/" + fileId
  63. // Prefer same data center
  64. if mc.DataCenter != "" && mc.DataCenter == loc.DataCenter {
  65. fullUrls = append([]string{httpUrl}, fullUrls...)
  66. } else {
  67. fullUrls = append(fullUrls, httpUrl)
  68. }
  69. }
  70. }
  71. return nil
  72. })
  73. return
  74. }
  75. func (mc *MasterClient) GetMaster() pb.ServerAddress {
  76. mc.WaitUntilConnected()
  77. return mc.currentMaster
  78. }
  79. func (mc *MasterClient) GetMasters() map[string]pb.ServerAddress {
  80. mc.WaitUntilConnected()
  81. return mc.masters
  82. }
  83. func (mc *MasterClient) WaitUntilConnected() {
  84. for {
  85. if mc.currentMaster != "" {
  86. return
  87. }
  88. time.Sleep(time.Duration(rand.Int31n(200)) * time.Millisecond)
  89. }
  90. }
  91. func (mc *MasterClient) KeepConnectedToMaster() {
  92. glog.V(1).Infof("%s.%s masterClient bootstraps with masters %v", mc.FilerGroup, mc.clientType, mc.masters)
  93. for {
  94. mc.tryAllMasters()
  95. time.Sleep(time.Second)
  96. }
  97. }
  98. func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress pb.ServerAddress) (leader string) {
  99. for _, master := range mc.masters {
  100. if master == myMasterAddress {
  101. continue
  102. }
  103. if grpcErr := pb.WithMasterClient(false, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
  104. ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
  105. defer cancel()
  106. resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
  107. if err != nil {
  108. return err
  109. }
  110. leader = resp.Leader
  111. return nil
  112. }); grpcErr != nil {
  113. glog.V(0).Infof("connect to %s: %v", master, grpcErr)
  114. }
  115. if leader != "" {
  116. glog.V(0).Infof("existing leader is %s", leader)
  117. return
  118. }
  119. }
  120. glog.V(0).Infof("No existing leader found!")
  121. return
  122. }
  123. func (mc *MasterClient) tryAllMasters() {
  124. var nextHintedLeader pb.ServerAddress
  125. for _, master := range mc.masters {
  126. nextHintedLeader = mc.tryConnectToMaster(master)
  127. for nextHintedLeader != "" {
  128. nextHintedLeader = mc.tryConnectToMaster(nextHintedLeader)
  129. }
  130. mc.currentMaster = ""
  131. }
  132. }
  133. func (mc *MasterClient) tryConnectToMaster(master pb.ServerAddress) (nextHintedLeader pb.ServerAddress) {
  134. glog.V(1).Infof("%s.%s masterClient Connecting to master %v", mc.FilerGroup, mc.clientType, master)
  135. stats.MasterClientConnectCounter.WithLabelValues("total").Inc()
  136. gprcErr := pb.WithMasterClient(true, master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
  137. ctx, cancel := context.WithCancel(context.Background())
  138. defer cancel()
  139. stream, err := client.KeepConnected(ctx)
  140. if err != nil {
  141. glog.V(1).Infof("%s.%s masterClient failed to keep connected to %s: %v", mc.FilerGroup, mc.clientType, master, err)
  142. stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToKeepConnected).Inc()
  143. return err
  144. }
  145. if err = stream.Send(&master_pb.KeepConnectedRequest{
  146. FilerGroup: mc.FilerGroup,
  147. DataCenter: mc.DataCenter,
  148. Rack: mc.rack,
  149. ClientType: mc.clientType,
  150. ClientAddress: string(mc.clientHost),
  151. Version: util.Version(),
  152. }); err != nil {
  153. glog.V(0).Infof("%s.%s masterClient failed to send to %s: %v", mc.FilerGroup, mc.clientType, master, err)
  154. stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToSend).Inc()
  155. return err
  156. }
  157. glog.V(1).Infof("%s.%s masterClient Connected to %v", mc.FilerGroup, mc.clientType, master)
  158. resp, err := stream.Recv()
  159. if err != nil {
  160. glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
  161. stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
  162. return err
  163. }
  164. // check if it is the leader to determine whether to reset the vidMap
  165. if resp.VolumeLocation != nil {
  166. if resp.VolumeLocation.Leader != "" && string(master) != resp.VolumeLocation.Leader {
  167. glog.V(0).Infof("master %v redirected to leader %v", master, resp.VolumeLocation.Leader)
  168. nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
  169. stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToLeader).Inc()
  170. return nil
  171. }
  172. mc.resetVidMap()
  173. mc.updateVidMap(resp)
  174. } else {
  175. mc.resetVidMap()
  176. }
  177. mc.currentMaster = master
  178. for {
  179. resp, err := stream.Recv()
  180. if err != nil {
  181. glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
  182. stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
  183. return err
  184. }
  185. if resp.VolumeLocation != nil {
  186. // maybe the leader is changed
  187. if resp.VolumeLocation.Leader != "" && string(mc.currentMaster) != resp.VolumeLocation.Leader {
  188. glog.V(0).Infof("currentMaster %v redirected to leader %v", mc.currentMaster, resp.VolumeLocation.Leader)
  189. nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
  190. stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToLeader).Inc()
  191. return nil
  192. }
  193. mc.updateVidMap(resp)
  194. }
  195. if resp.ClusterNodeUpdate != nil {
  196. update := resp.ClusterNodeUpdate
  197. if mc.OnPeerUpdate != nil {
  198. if update.FilerGroup == mc.FilerGroup {
  199. if update.IsAdd {
  200. glog.V(0).Infof("+ %s.%s %s leader:%v\n", update.FilerGroup, update.NodeType, update.Address, update.IsLeader)
  201. } else {
  202. glog.V(0).Infof("- %s.%s %s leader:%v\n", update.FilerGroup, update.NodeType, update.Address, update.IsLeader)
  203. }
  204. stats.MasterClientConnectCounter.WithLabelValues(stats.OnPeerUpdate).Inc()
  205. mc.OnPeerUpdate(update, time.Now())
  206. }
  207. }
  208. }
  209. }
  210. })
  211. if gprcErr != nil {
  212. stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc()
  213. glog.V(1).Infof("%s.%s masterClient failed to connect with master %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
  214. }
  215. return
  216. }
  217. func (mc *MasterClient) updateVidMap(resp *master_pb.KeepConnectedResponse) {
  218. // process new volume location
  219. glog.V(1).Infof("updateVidMap() resp.VolumeLocation.DataCenter %v", resp.VolumeLocation.DataCenter)
  220. loc := Location{
  221. Url: resp.VolumeLocation.Url,
  222. PublicUrl: resp.VolumeLocation.PublicUrl,
  223. DataCenter: resp.VolumeLocation.DataCenter,
  224. GrpcPort: int(resp.VolumeLocation.GrpcPort),
  225. }
  226. for _, newVid := range resp.VolumeLocation.NewVids {
  227. glog.V(1).Infof("%s.%s: %s masterClient adds volume %d", mc.FilerGroup, mc.clientType, loc.Url, newVid)
  228. mc.addLocation(newVid, loc)
  229. }
  230. for _, deletedVid := range resp.VolumeLocation.DeletedVids {
  231. glog.V(1).Infof("%s.%s: %s masterClient removes volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedVid)
  232. mc.deleteLocation(deletedVid, loc)
  233. }
  234. for _, newEcVid := range resp.VolumeLocation.NewEcVids {
  235. glog.V(1).Infof("%s.%s: %s masterClient adds ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, newEcVid)
  236. mc.addEcLocation(newEcVid, loc)
  237. }
  238. for _, deletedEcVid := range resp.VolumeLocation.DeletedEcVids {
  239. glog.V(1).Infof("%s.%s: %s masterClient removes ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedEcVid)
  240. mc.deleteEcLocation(deletedEcVid, loc)
  241. }
  242. }
  243. func (mc *MasterClient) WithClient(streamingMode bool, fn func(client master_pb.SeaweedClient) error) error {
  244. return util.Retry("master grpc", func() error {
  245. for mc.currentMaster == "" {
  246. time.Sleep(3 * time.Second)
  247. }
  248. return pb.WithMasterClient(streamingMode, mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
  249. return fn(client)
  250. })
  251. })
  252. }
  253. func (mc *MasterClient) resetVidMap() {
  254. tail := &vidMap{
  255. vid2Locations: mc.vid2Locations,
  256. ecVid2Locations: mc.ecVid2Locations,
  257. DataCenter: mc.DataCenter,
  258. cache: mc.cache,
  259. }
  260. mc.vidMap = newVidMap(mc.DataCenter)
  261. mc.vidMap.cache = tail
  262. for i := 0; i < mc.vidMapCacheSize && tail.cache != nil; i++ {
  263. if i == mc.vidMapCacheSize-1 {
  264. tail.cache = nil
  265. } else {
  266. tail = tail.cache
  267. }
  268. }
  269. }