You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

233 lines
8.9 KiB

5 years ago
6 years ago
5 years ago
5 years ago
6 years ago
6 years ago
12 years ago
6 years ago
6 years ago
5 years ago
6 years ago
6 years ago
6 years ago
12 years ago
6 years ago
4 years ago
6 years ago
5 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
6 years ago
  1. package command
  2. import (
  3. "github.com/chrislusf/raft/protobuf"
  4. stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
  5. "github.com/gorilla/mux"
  6. "google.golang.org/grpc/reflection"
  7. "net/http"
  8. "os"
  9. "sort"
  10. "strings"
  11. "time"
  12. "github.com/chrislusf/seaweedfs/weed/util/grace"
  13. "github.com/chrislusf/seaweedfs/weed/glog"
  14. "github.com/chrislusf/seaweedfs/weed/pb"
  15. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  16. "github.com/chrislusf/seaweedfs/weed/security"
  17. "github.com/chrislusf/seaweedfs/weed/server"
  18. "github.com/chrislusf/seaweedfs/weed/storage/backend"
  19. "github.com/chrislusf/seaweedfs/weed/util"
  20. )
  21. var (
  22. m MasterOptions
  23. )
  24. type MasterOptions struct {
  25. port *int
  26. portGrpc *int
  27. ip *string
  28. ipBind *string
  29. metaFolder *string
  30. peers *string
  31. volumeSizeLimitMB *uint
  32. volumePreallocate *bool
  33. // pulseSeconds *int
  34. defaultReplication *string
  35. garbageThreshold *float64
  36. whiteList *string
  37. disableHttp *bool
  38. metricsAddress *string
  39. metricsIntervalSec *int
  40. raftResumeState *bool
  41. metricsHttpPort *int
  42. heartbeatInterval *time.Duration
  43. electionTimeout *time.Duration
  44. }
  45. func init() {
  46. cmdMaster.Run = runMaster // break init cycle
  47. m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
  48. m.portGrpc = cmdMaster.Flag.Int("port.grpc", 0, "grpc listen port")
  49. m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
  50. m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
  51. m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
  52. m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
  53. m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
  54. m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
  55. // m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
  56. m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "", "Default replication type if not specified.")
  57. m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
  58. m.whiteList = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  59. m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
  60. m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address <host>:<port>")
  61. m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
  62. m.metricsHttpPort = cmdMaster.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
  63. m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server")
  64. m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
  65. m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers")
  66. }
  67. var cmdMaster = &Command{
  68. UsageLine: "master -port=9333",
  69. Short: "start a master server",
  70. Long: `start a master server to provide volume=>location mapping service and sequence number of file ids
  71. The configuration file "security.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
  72. The example security.toml configuration file can be generated by "weed scaffold -config=security"
  73. `,
  74. }
  75. var (
  76. masterCpuProfile = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
  77. masterMemProfile = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
  78. )
  79. func runMaster(cmd *Command, args []string) bool {
  80. util.LoadConfiguration("security", false)
  81. util.LoadConfiguration("master", false)
  82. grace.SetupProfiling(*masterCpuProfile, *masterMemProfile)
  83. parent, _ := util.FullPath(*m.metaFolder).DirAndName()
  84. if util.FileExists(string(parent)) && !util.FileExists(*m.metaFolder) {
  85. os.MkdirAll(*m.metaFolder, 0755)
  86. }
  87. if err := util.TestFolderWritable(util.ResolvePath(*m.metaFolder)); err != nil {
  88. glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *m.metaFolder, err)
  89. }
  90. var masterWhiteList []string
  91. if *m.whiteList != "" {
  92. masterWhiteList = strings.Split(*m.whiteList, ",")
  93. }
  94. if *m.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
  95. glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
  96. }
  97. go stats_collect.StartMetricsServer(*m.metricsHttpPort)
  98. startMaster(m, masterWhiteList)
  99. return true
  100. }
  101. func startMaster(masterOption MasterOptions, masterWhiteList []string) {
  102. backend.LoadConfiguration(util.GetViper())
  103. if *masterOption.portGrpc == 0 {
  104. *masterOption.portGrpc = 10000 + *masterOption.port
  105. }
  106. myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
  107. r := mux.NewRouter()
  108. ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
  109. listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
  110. glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
  111. masterListener, e := util.NewListener(listeningAddress, 0)
  112. if e != nil {
  113. glog.Fatalf("Master startup error: %v", e)
  114. }
  115. // start raftServer
  116. raftServerOption := &weed_server.RaftServerOption{
  117. GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"),
  118. Peers: peers,
  119. ServerAddr: myMasterAddress,
  120. DataDir: util.ResolvePath(*masterOption.metaFolder),
  121. Topo: ms.Topo,
  122. RaftResumeState: *masterOption.raftResumeState,
  123. HeartbeatInterval: *masterOption.heartbeatInterval,
  124. ElectionTimeout: *masterOption.electionTimeout,
  125. }
  126. raftServer, err := weed_server.NewRaftServer(raftServerOption)
  127. if raftServer == nil {
  128. glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
  129. }
  130. ms.SetRaftServer(raftServer)
  131. r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
  132. // starting grpc server
  133. grpcPort := *masterOption.portGrpc
  134. grpcL, err := util.NewListener(util.JoinHostPort(*masterOption.ipBind, grpcPort), 0)
  135. if err != nil {
  136. glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
  137. }
  138. grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
  139. master_pb.RegisterSeaweedServer(grpcS, ms)
  140. protobuf.RegisterRaftServer(grpcS, raftServer)
  141. reflection.Register(grpcS)
  142. glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
  143. go grpcS.Serve(grpcL)
  144. go func() {
  145. time.Sleep(1500 * time.Millisecond)
  146. if ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty() && isTheFirstOne(myMasterAddress, peers) {
  147. if ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" {
  148. raftServer.DoJoinCommand()
  149. }
  150. }
  151. }()
  152. go ms.MasterClient.KeepConnectedToMaster()
  153. // start http server
  154. httpS := &http.Server{Handler: r}
  155. go httpS.Serve(masterListener)
  156. select {}
  157. }
  158. func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers string) (masterAddress pb.ServerAddress, cleanedPeers []pb.ServerAddress) {
  159. glog.V(0).Infof("current: %s:%d peers:%s", masterIp, masterPort, peers)
  160. masterAddress = pb.NewServerAddress(masterIp, masterPort, masterGrpcPort)
  161. cleanedPeers = pb.ServerAddresses(peers).ToAddresses()
  162. hasSelf := false
  163. for _, peer := range cleanedPeers {
  164. if peer.ToHttpAddress() == masterAddress.ToHttpAddress() {
  165. hasSelf = true
  166. break
  167. }
  168. }
  169. if !hasSelf {
  170. cleanedPeers = append(cleanedPeers, masterAddress)
  171. }
  172. if len(cleanedPeers)%2 == 0 {
  173. glog.Fatalf("Only odd number of masters are supported: %+v", cleanedPeers)
  174. }
  175. return
  176. }
  177. func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
  178. sort.Slice(peers, func(i, j int) bool {
  179. return strings.Compare(string(peers[i]), string(peers[j])) < 0
  180. })
  181. if len(peers) <= 0 {
  182. return true
  183. }
  184. return self == peers[0]
  185. }
  186. func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
  187. masterAddress := pb.NewServerAddress(*m.ip, *m.port, *m.portGrpc)
  188. return &weed_server.MasterOption{
  189. Master: masterAddress,
  190. MetaFolder: *m.metaFolder,
  191. VolumeSizeLimitMB: uint32(*m.volumeSizeLimitMB),
  192. VolumePreallocate: *m.volumePreallocate,
  193. // PulseSeconds: *m.pulseSeconds,
  194. DefaultReplicaPlacement: *m.defaultReplication,
  195. GarbageThreshold: *m.garbageThreshold,
  196. WhiteList: whiteList,
  197. DisableHttp: *m.disableHttp,
  198. MetricsAddress: *m.metricsAddress,
  199. MetricsIntervalSec: *m.metricsIntervalSec,
  200. }
  201. }