You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

289 lines
11 KiB

11 years ago
11 years ago
10 years ago
10 years ago
11 years ago
11 years ago
11 years ago
10 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
  1. package command
  2. import (
  3. "net/http"
  4. "os"
  5. "runtime"
  6. "runtime/pprof"
  7. "strconv"
  8. "strings"
  9. "sync"
  10. "time"
  11. "github.com/chrislusf/seaweedfs/weed/glog"
  12. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  13. "github.com/chrislusf/seaweedfs/weed/server"
  14. "github.com/chrislusf/seaweedfs/weed/storage"
  15. "github.com/chrislusf/seaweedfs/weed/util"
  16. "github.com/gorilla/mux"
  17. "github.com/soheilhy/cmux"
  18. "google.golang.org/grpc"
  19. "google.golang.org/grpc/reflection"
  20. )
  21. type ServerOptions struct {
  22. cpuprofile *string
  23. }
  24. var (
  25. serverOptions ServerOptions
  26. filerOptions FilerOptions
  27. )
  28. func init() {
  29. cmdServer.Run = runServer // break init cycle
  30. }
  31. var cmdServer = &Command{
  32. UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
  33. Short: "start a server, including volume server, and automatically elect a master server",
  34. Long: `start both a volume server to provide storage spaces
  35. and a master server to provide volume=>location mapping service and sequence number of file ids
  36. This is provided as a convenient way to start both volume server and master server.
  37. The servers are exactly the same as starting them separately.
  38. So other volume servers can use this embedded master server also.
  39. Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
  40. They run with meta data on disk, not shared. So each filer server is different.
  41. `,
  42. }
  43. var (
  44. serverIp = cmdServer.Flag.String("ip", "localhost", "ip or server name")
  45. serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
  46. serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
  47. serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
  48. serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
  49. serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
  50. serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  51. serverPeers = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list")
  52. serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
  53. serverGarbageThreshold = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
  54. masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
  55. masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
  56. masterVolumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
  57. masterVolumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
  58. masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
  59. volumePort = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
  60. volumePublicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
  61. volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
  62. volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
  63. volumePulse = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
  64. volumeIndexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
  65. volumeFixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", true, "Adjust jpg orientation when uploading.")
  66. volumeReadRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
  67. volumeServerPublicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
  68. isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer")
  69. serverWhiteList []string
  70. )
  71. func init() {
  72. serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
  73. filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
  74. filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
  75. filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
  76. filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
  77. filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
  78. filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
  79. filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
  80. }
  81. func runServer(cmd *Command, args []string) bool {
  82. filerOptions.secretKey = serverSecureKey
  83. if *serverOptions.cpuprofile != "" {
  84. f, err := os.Create(*serverOptions.cpuprofile)
  85. if err != nil {
  86. glog.Fatal(err)
  87. }
  88. pprof.StartCPUProfile(f)
  89. defer pprof.StopCPUProfile()
  90. }
  91. if *filerOptions.redirectOnRead {
  92. *isStartingFiler = true
  93. }
  94. master := *serverIp + ":" + strconv.Itoa(*masterPort)
  95. filerOptions.ip = serverIp
  96. if *filerOptions.defaultReplicaPlacement == "" {
  97. *filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
  98. }
  99. if *volumePublicPort == 0 {
  100. *volumePublicPort = *volumePort
  101. }
  102. if *serverMaxCpu < 1 {
  103. *serverMaxCpu = runtime.NumCPU()
  104. }
  105. runtime.GOMAXPROCS(*serverMaxCpu)
  106. folders := strings.Split(*volumeDataFolders, ",")
  107. maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
  108. var maxCounts []int
  109. for _, maxString := range maxCountStrings {
  110. if max, e := strconv.Atoi(maxString); e == nil {
  111. maxCounts = append(maxCounts, max)
  112. } else {
  113. glog.Fatalf("The max specified in -max not a valid number %s", maxString)
  114. }
  115. }
  116. if len(folders) != len(maxCounts) {
  117. glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
  118. }
  119. for _, folder := range folders {
  120. if err := util.TestFolderWritable(folder); err != nil {
  121. glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
  122. }
  123. }
  124. if *masterVolumeSizeLimitMB > 30*1000 {
  125. glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
  126. }
  127. if *masterMetaFolder == "" {
  128. *masterMetaFolder = folders[0]
  129. }
  130. if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
  131. glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
  132. }
  133. if *serverWhiteListOption != "" {
  134. serverWhiteList = strings.Split(*serverWhiteListOption, ",")
  135. }
  136. if *isStartingFiler {
  137. go func() {
  138. time.Sleep(1 * time.Second)
  139. filerOptions.start()
  140. }()
  141. }
  142. var raftWaitForMaster sync.WaitGroup
  143. var volumeWait sync.WaitGroup
  144. raftWaitForMaster.Add(1)
  145. volumeWait.Add(1)
  146. go func() {
  147. r := mux.NewRouter()
  148. ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
  149. *masterVolumeSizeLimitMB, *masterVolumePreallocate,
  150. *volumePulse, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
  151. serverWhiteList, *serverSecureKey,
  152. )
  153. glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
  154. masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
  155. if e != nil {
  156. glog.Fatalf("Master startup error: %v", e)
  157. }
  158. go func() {
  159. raftWaitForMaster.Wait()
  160. time.Sleep(100 * time.Millisecond)
  161. myAddress := *serverIp + ":" + strconv.Itoa(*masterPort)
  162. var peers []string
  163. if *serverPeers != "" {
  164. peers = strings.Split(*serverPeers, ",")
  165. }
  166. raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse)
  167. ms.SetRaftServer(raftServer)
  168. volumeWait.Done()
  169. }()
  170. raftWaitForMaster.Done()
  171. // start grpc and http server
  172. m := cmux.New(masterListener)
  173. grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
  174. httpL := m.Match(cmux.Any())
  175. // Create your protocol servers.
  176. grpcS := grpc.NewServer()
  177. master_pb.RegisterSeaweedServer(grpcS, ms)
  178. reflection.Register(grpcS)
  179. httpS := &http.Server{Handler: r}
  180. go grpcS.Serve(grpcL)
  181. go httpS.Serve(httpL)
  182. if err := m.Serve(); err != nil {
  183. glog.Fatalf("master server failed to serve: %v", err)
  184. }
  185. }()
  186. volumeWait.Wait()
  187. time.Sleep(100 * time.Millisecond)
  188. if *volumePublicPort == 0 {
  189. *volumePublicPort = *volumePort
  190. }
  191. if *volumeServerPublicUrl == "" {
  192. *volumeServerPublicUrl = *serverIp + ":" + strconv.Itoa(*volumePublicPort)
  193. }
  194. isSeperatedPublicPort := *volumePublicPort != *volumePort
  195. volumeMux := http.NewServeMux()
  196. publicVolumeMux := volumeMux
  197. if isSeperatedPublicPort {
  198. publicVolumeMux = http.NewServeMux()
  199. }
  200. volumeNeedleMapKind := storage.NeedleMapInMemory
  201. switch *volumeIndexType {
  202. case "leveldb":
  203. volumeNeedleMapKind = storage.NeedleMapLevelDb
  204. case "boltdb":
  205. volumeNeedleMapKind = storage.NeedleMapBoltDb
  206. case "btree":
  207. volumeNeedleMapKind = storage.NeedleMapBtree
  208. }
  209. volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
  210. *serverIp, *volumePort, *volumeServerPublicUrl,
  211. folders, maxCounts,
  212. volumeNeedleMapKind,
  213. []string{master}, *volumePulse, *serverDataCenter, *serverRack,
  214. serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect,
  215. )
  216. glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
  217. volumeListener, eListen := util.NewListener(
  218. *serverBindIp+":"+strconv.Itoa(*volumePort),
  219. time.Duration(*serverTimeout)*time.Second,
  220. )
  221. if eListen != nil {
  222. glog.Fatalf("Volume server listener error: %v", eListen)
  223. }
  224. if isSeperatedPublicPort {
  225. publicListeningAddress := *serverIp + ":" + strconv.Itoa(*volumePublicPort)
  226. glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "public at", publicListeningAddress)
  227. publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*serverTimeout)*time.Second)
  228. if e != nil {
  229. glog.Fatalf("Volume server listener error:%v", e)
  230. }
  231. go func() {
  232. if e := http.Serve(publicListener, publicVolumeMux); e != nil {
  233. glog.Fatalf("Volume server fail to serve public: %v", e)
  234. }
  235. }()
  236. }
  237. util.OnInterrupt(func() {
  238. volumeServer.Shutdown()
  239. pprof.StopCPUProfile()
  240. })
  241. if e := http.Serve(volumeListener, volumeMux); e != nil {
  242. glog.Fatalf("Volume server fail to serve:%v", e)
  243. }
  244. return true
  245. }