You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

253 lines
10 KiB

6 years ago
11 years ago
11 years ago
10 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
  1. package command
  2. import (
  3. "fmt"
  4. "net/http"
  5. "os"
  6. "runtime"
  7. "runtime/pprof"
  8. "strconv"
  9. "strings"
  10. "sync"
  11. "time"
  12. "github.com/chrislusf/raft/protobuf"
  13. "github.com/chrislusf/seaweedfs/weed/security"
  14. "github.com/spf13/viper"
  15. "github.com/chrislusf/seaweedfs/weed/glog"
  16. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  17. "github.com/chrislusf/seaweedfs/weed/server"
  18. "github.com/chrislusf/seaweedfs/weed/util"
  19. "github.com/gorilla/mux"
  20. "google.golang.org/grpc/reflection"
  21. )
  22. type ServerOptions struct {
  23. cpuprofile *string
  24. v VolumeServerOptions
  25. }
  26. var (
  27. serverOptions ServerOptions
  28. filerOptions FilerOptions
  29. s3Options S3Options
  30. )
  31. func init() {
  32. cmdServer.Run = runServer // break init cycle
  33. }
  34. var cmdServer = &Command{
  35. UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
  36. Short: "start a master server, a volume server, and optionally a filer and a S3 gateway",
  37. Long: `start both a volume server to provide storage spaces
  38. and a master server to provide volume=>location mapping service and sequence number of file ids
  39. This is provided as a convenient way to start both volume server and master server.
  40. The servers acts exactly the same as starting them separately.
  41. So other volume servers can connect to this master server also.
  42. Optionally, a filer server can be started.
  43. Also optionally, a S3 gateway can be started.
  44. `,
  45. }
  46. var (
  47. serverIp = cmdServer.Flag.String("ip", "localhost", "ip or server name")
  48. serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
  49. serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
  50. serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
  51. serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
  52. serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
  53. serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  54. serverDisableHttp = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
  55. serverPeers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
  56. serverGarbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
  57. masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
  58. masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
  59. masterVolumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
  60. masterVolumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
  61. masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
  62. volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
  63. volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
  64. pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
  65. isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer")
  66. isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
  67. serverWhiteList []string
  68. )
  69. func init() {
  70. serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
  71. filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
  72. filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
  73. filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
  74. filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
  75. filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
  76. filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
  77. filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
  78. filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
  79. serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
  80. serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
  81. serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
  82. serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
  83. serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
  84. serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
  85. serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
  86. s3Options.filerBucketsPath = cmdServer.Flag.String("s3.filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
  87. s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
  88. s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
  89. s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
  90. s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
  91. }
  92. func runServer(cmd *Command, args []string) bool {
  93. util.LoadConfiguration("security", false)
  94. util.LoadConfiguration("master", false)
  95. if *serverOptions.cpuprofile != "" {
  96. f, err := os.Create(*serverOptions.cpuprofile)
  97. if err != nil {
  98. glog.Fatal(err)
  99. }
  100. pprof.StartCPUProfile(f)
  101. defer pprof.StopCPUProfile()
  102. }
  103. if *filerOptions.redirectOnRead {
  104. *isStartingFiler = true
  105. }
  106. if *isStartingS3 {
  107. *isStartingFiler = true
  108. }
  109. master := *serverIp + ":" + strconv.Itoa(*masterPort)
  110. filerOptions.masters = &master
  111. filerOptions.ip = serverBindIp
  112. serverOptions.v.ip = serverIp
  113. serverOptions.v.bindIp = serverBindIp
  114. serverOptions.v.masters = &master
  115. serverOptions.v.idleConnectionTimeout = serverTimeout
  116. serverOptions.v.maxCpu = serverMaxCpu
  117. serverOptions.v.dataCenter = serverDataCenter
  118. serverOptions.v.rack = serverRack
  119. serverOptions.v.pulseSeconds = pulseSeconds
  120. filerOptions.dataCenter = serverDataCenter
  121. filerOptions.disableHttp = serverDisableHttp
  122. filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
  123. s3Options.filer = &filerAddress
  124. if *filerOptions.defaultReplicaPlacement == "" {
  125. *filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
  126. }
  127. if *serverMaxCpu < 1 {
  128. *serverMaxCpu = runtime.NumCPU()
  129. }
  130. runtime.GOMAXPROCS(*serverMaxCpu)
  131. folders := strings.Split(*volumeDataFolders, ",")
  132. if *masterVolumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
  133. glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
  134. }
  135. if *masterMetaFolder == "" {
  136. *masterMetaFolder = folders[0]
  137. }
  138. if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
  139. glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
  140. }
  141. filerOptions.defaultLevelDbDirectory = masterMetaFolder
  142. if *serverWhiteListOption != "" {
  143. serverWhiteList = strings.Split(*serverWhiteListOption, ",")
  144. }
  145. if *isStartingFiler {
  146. go func() {
  147. time.Sleep(1 * time.Second)
  148. filerOptions.startFiler()
  149. }()
  150. }
  151. if *isStartingS3 {
  152. go func() {
  153. time.Sleep(2 * time.Second)
  154. s3Options.startS3Server()
  155. }()
  156. }
  157. var volumeWait sync.WaitGroup
  158. volumeWait.Add(1)
  159. go func() {
  160. r := mux.NewRouter()
  161. ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
  162. *masterVolumeSizeLimitMB, *masterVolumePreallocate,
  163. *pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
  164. serverWhiteList, *serverDisableHttp,
  165. )
  166. glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
  167. masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
  168. if e != nil {
  169. glog.Fatalf("Master startup error: %v", e)
  170. }
  171. go func() {
  172. // start raftServer
  173. myMasterAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
  174. raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
  175. peers, myMasterAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
  176. ms.SetRaftServer(raftServer)
  177. r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
  178. // starting grpc server
  179. grpcPort := *masterPort + 10000
  180. grpcL, err := util.NewListener(*serverBindIp+":"+strconv.Itoa(grpcPort), 0)
  181. if err != nil {
  182. glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
  183. }
  184. // Create your protocol servers.
  185. glog.V(1).Infof("grpc config %+v", viper.Sub("grpc"))
  186. grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
  187. master_pb.RegisterSeaweedServer(grpcS, ms)
  188. protobuf.RegisterRaftServer(grpcS, raftServer)
  189. reflection.Register(grpcS)
  190. glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *serverIp, grpcPort)
  191. grpcS.Serve(grpcL)
  192. }()
  193. volumeWait.Done()
  194. // start http server
  195. httpS := &http.Server{Handler: r}
  196. if err := httpS.Serve(masterListener); err != nil {
  197. glog.Fatalf("master server failed to serve: %v", err)
  198. }
  199. }()
  200. volumeWait.Wait()
  201. time.Sleep(100 * time.Millisecond)
  202. serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption)
  203. return true
  204. }