You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

223 lines
9.0 KiB

6 years ago
11 years ago
11 years ago
10 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
  1. package command
  2. import (
  3. "github.com/chrislusf/raft/protobuf"
  4. "github.com/chrislusf/seaweedfs/weed/security"
  5. "github.com/spf13/viper"
  6. "net/http"
  7. "os"
  8. "runtime"
  9. "runtime/pprof"
  10. "strconv"
  11. "strings"
  12. "sync"
  13. "time"
  14. "github.com/chrislusf/seaweedfs/weed/glog"
  15. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  16. "github.com/chrislusf/seaweedfs/weed/server"
  17. "github.com/chrislusf/seaweedfs/weed/util"
  18. "github.com/gorilla/mux"
  19. "google.golang.org/grpc/reflection"
  20. )
  21. type ServerOptions struct {
  22. cpuprofile *string
  23. v VolumeServerOptions
  24. }
  25. var (
  26. serverOptions ServerOptions
  27. filerOptions FilerOptions
  28. )
  29. func init() {
  30. cmdServer.Run = runServer // break init cycle
  31. }
  32. var cmdServer = &Command{
  33. UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
  34. Short: "start a server, including volume server, and automatically elect a master server",
  35. Long: `start both a volume server to provide storage spaces
  36. and a master server to provide volume=>location mapping service and sequence number of file ids
  37. This is provided as a convenient way to start both volume server and master server.
  38. The servers are exactly the same as starting them separately.
  39. So other volume servers can use this embedded master server also.
  40. Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
  41. They run with meta data on disk, not shared. So each filer server is different.
  42. `,
  43. }
  44. var (
  45. serverIp = cmdServer.Flag.String("ip", "localhost", "ip or server name")
  46. serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
  47. serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
  48. serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
  49. serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
  50. serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
  51. serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  52. serverPeers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
  53. serverGarbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
  54. masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
  55. masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
  56. masterVolumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
  57. masterVolumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
  58. masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
  59. volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
  60. volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
  61. pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
  62. isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer")
  63. serverWhiteList []string
  64. )
  65. func init() {
  66. serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
  67. filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
  68. filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
  69. filerOptions.grpcPort = cmdServer.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to http port + 10000")
  70. filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
  71. filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
  72. filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
  73. filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
  74. filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
  75. filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
  76. serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
  77. serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
  78. serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
  79. serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
  80. serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
  81. serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
  82. }
  83. func runServer(cmd *Command, args []string) bool {
  84. weed_server.LoadConfiguration("security", false)
  85. if *serverOptions.cpuprofile != "" {
  86. f, err := os.Create(*serverOptions.cpuprofile)
  87. if err != nil {
  88. glog.Fatal(err)
  89. }
  90. pprof.StartCPUProfile(f)
  91. defer pprof.StopCPUProfile()
  92. }
  93. if *filerOptions.redirectOnRead {
  94. *isStartingFiler = true
  95. }
  96. master := *serverIp + ":" + strconv.Itoa(*masterPort)
  97. filerOptions.ip = serverIp
  98. serverOptions.v.ip = serverIp
  99. serverOptions.v.bindIp = serverBindIp
  100. serverOptions.v.masters = &master
  101. serverOptions.v.idleConnectionTimeout = serverTimeout
  102. serverOptions.v.maxCpu = serverMaxCpu
  103. serverOptions.v.dataCenter = serverDataCenter
  104. serverOptions.v.rack = serverRack
  105. serverOptions.v.pulseSeconds = pulseSeconds
  106. filerOptions.dataCenter = serverDataCenter
  107. if *filerOptions.defaultReplicaPlacement == "" {
  108. *filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
  109. }
  110. if *serverMaxCpu < 1 {
  111. *serverMaxCpu = runtime.NumCPU()
  112. }
  113. runtime.GOMAXPROCS(*serverMaxCpu)
  114. folders := strings.Split(*volumeDataFolders, ",")
  115. if *masterVolumeSizeLimitMB > 30*1000 {
  116. glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
  117. }
  118. if *masterMetaFolder == "" {
  119. *masterMetaFolder = folders[0]
  120. }
  121. if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
  122. glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
  123. }
  124. filerOptions.defaultLevelDbDirectory = masterMetaFolder
  125. if *serverWhiteListOption != "" {
  126. serverWhiteList = strings.Split(*serverWhiteListOption, ",")
  127. }
  128. if *isStartingFiler {
  129. go func() {
  130. time.Sleep(1 * time.Second)
  131. filerOptions.startFiler()
  132. }()
  133. }
  134. var volumeWait sync.WaitGroup
  135. volumeWait.Add(1)
  136. go func() {
  137. r := mux.NewRouter()
  138. ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
  139. *masterVolumeSizeLimitMB, *masterVolumePreallocate,
  140. *pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
  141. serverWhiteList,
  142. )
  143. glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
  144. masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
  145. if e != nil {
  146. glog.Fatalf("Master startup error: %v", e)
  147. }
  148. go func() {
  149. // start raftServer
  150. myMasterAddress, peers := checkPeers(*masterIp, *mport, *masterPeers)
  151. raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
  152. peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
  153. ms.SetRaftServer(raftServer)
  154. // starting grpc server
  155. grpcPort := *masterPort + 10000
  156. grpcL, err := util.NewListener(*serverIp+":"+strconv.Itoa(grpcPort), 0)
  157. if err != nil {
  158. glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
  159. }
  160. // Create your protocol servers.
  161. glog.V(0).Infof("grpc config %+v", viper.Sub("grpc"))
  162. grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
  163. master_pb.RegisterSeaweedServer(grpcS, ms)
  164. protobuf.RegisterRaftServer(grpcS, raftServer)
  165. reflection.Register(grpcS)
  166. glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *serverIp, grpcPort)
  167. grpcS.Serve(grpcL)
  168. }()
  169. volumeWait.Done()
  170. // start http server
  171. httpS := &http.Server{Handler: r}
  172. if err := httpS.Serve(masterListener); err != nil {
  173. glog.Fatalf("master server failed to serve: %v", err)
  174. }
  175. }()
  176. volumeWait.Wait()
  177. time.Sleep(100 * time.Millisecond)
  178. serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption)
  179. return true
  180. }