You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

335 lines
12 KiB

5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
5 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
12 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package command
  2. import (
  3. "fmt"
  4. "net/http"
  5. httppprof "net/http/pprof"
  6. "os"
  7. "runtime"
  8. "runtime/pprof"
  9. "strconv"
  10. "strings"
  11. "time"
  12. "github.com/spf13/viper"
  13. "google.golang.org/grpc"
  14. "github.com/chrislusf/seaweedfs/weed/util/grace"
  15. "github.com/chrislusf/seaweedfs/weed/pb"
  16. "github.com/chrislusf/seaweedfs/weed/security"
  17. "github.com/chrislusf/seaweedfs/weed/util/httpdown"
  18. "google.golang.org/grpc/reflection"
  19. "github.com/chrislusf/seaweedfs/weed/glog"
  20. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  21. "github.com/chrislusf/seaweedfs/weed/server"
  22. "github.com/chrislusf/seaweedfs/weed/storage"
  23. "github.com/chrislusf/seaweedfs/weed/util"
  24. )
  25. var (
  26. v VolumeServerOptions
  27. )
  28. type VolumeServerOptions struct {
  29. port *int
  30. publicPort *int
  31. folders []string
  32. folderMaxLimits []int
  33. ip *string
  34. publicUrl *string
  35. bindIp *string
  36. masters *string
  37. idleConnectionTimeout *int
  38. dataCenter *string
  39. rack *string
  40. whiteList []string
  41. indexType *string
  42. readRedirect *bool
  43. cpuProfile *string
  44. memProfile *string
  45. compactionMBPerSecond *int
  46. fileSizeLimitMB *int
  47. minFreeSpacePercent []float32
  48. pprof *bool
  49. // pulseSeconds *int
  50. }
  51. func init() {
  52. cmdVolume.Run = runVolume // break init cycle
  53. v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
  54. v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
  55. v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
  56. v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
  57. v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
  58. v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
  59. // v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
  60. v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
  61. v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
  62. v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
  63. v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
  64. v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
  65. v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
  66. v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
  67. v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
  68. v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
  69. v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
  70. }
  71. var cmdVolume = &Command{
  72. UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
  73. Short: "start a volume server",
  74. Long: `start a volume server to provide storage spaces
  75. `,
  76. }
  77. var (
  78. volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
  79. maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
  80. volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  81. minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "0", "minimum free disk space(in percents). Low disk space will mark all volumes as ReadOnly.")
  82. )
  83. func runVolume(cmd *Command, args []string) bool {
  84. util.LoadConfiguration("security", false)
  85. runtime.GOMAXPROCS(runtime.NumCPU())
  86. // If --pprof is set we assume the caller wants to be able to collect
  87. // cpu and memory profiles via go tool pprof
  88. if !*v.pprof {
  89. grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
  90. }
  91. v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, *minFreeSpacePercent)
  92. return true
  93. }
  94. func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpacePercent string) {
  95. // Set multiple folders and each folder's max volume count limit'
  96. v.folders = strings.Split(volumeFolders, ",")
  97. for _, folder := range v.folders {
  98. if err := util.TestFolderWritable(folder); err != nil {
  99. glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
  100. }
  101. }
  102. // set max
  103. maxCountStrings := strings.Split(maxVolumeCounts, ",")
  104. for _, maxString := range maxCountStrings {
  105. if max, e := strconv.Atoi(maxString); e == nil {
  106. v.folderMaxLimits = append(v.folderMaxLimits, max)
  107. } else {
  108. glog.Fatalf("The max specified in -max not a valid number %s", maxString)
  109. }
  110. }
  111. if len(v.folders) != len(v.folderMaxLimits) {
  112. glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
  113. }
  114. // set minFreeSpacePercent
  115. minFreeSpacePercentStrings := strings.Split(minFreeSpacePercent, ",")
  116. for _, freeString := range minFreeSpacePercentStrings {
  117. if value, e := strconv.ParseFloat(freeString, 32); e == nil {
  118. v.minFreeSpacePercent = append(v.minFreeSpacePercent, float32(value))
  119. } else {
  120. glog.Fatalf("The value specified in -minFreeSpacePercent not a valid value %s", freeString)
  121. }
  122. }
  123. if len(v.minFreeSpacePercent) == 1 && len(v.folders) > 1 {
  124. for i := 0; i < len(v.folders)-1; i++ {
  125. v.minFreeSpacePercent = append(v.minFreeSpacePercent, v.minFreeSpacePercent[0])
  126. }
  127. }
  128. if len(v.folders) != len(v.minFreeSpacePercent) {
  129. glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpacePercent))
  130. }
  131. // security related white list configuration
  132. if volumeWhiteListOption != "" {
  133. v.whiteList = strings.Split(volumeWhiteListOption, ",")
  134. }
  135. if *v.ip == "" {
  136. *v.ip = util.DetectedHostAddress()
  137. glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
  138. }
  139. if *v.publicPort == 0 {
  140. *v.publicPort = *v.port
  141. }
  142. if *v.publicUrl == "" {
  143. *v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort)
  144. }
  145. volumeMux := http.NewServeMux()
  146. publicVolumeMux := volumeMux
  147. if v.isSeparatedPublicPort() {
  148. publicVolumeMux = http.NewServeMux()
  149. }
  150. if *v.pprof {
  151. volumeMux.HandleFunc("/debug/pprof/", httppprof.Index)
  152. volumeMux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
  153. volumeMux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
  154. volumeMux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
  155. volumeMux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
  156. }
  157. volumeNeedleMapKind := storage.NeedleMapInMemory
  158. switch *v.indexType {
  159. case "leveldb":
  160. volumeNeedleMapKind = storage.NeedleMapLevelDb
  161. case "leveldbMedium":
  162. volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
  163. case "leveldbLarge":
  164. volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
  165. }
  166. masters := *v.masters
  167. volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
  168. *v.ip, *v.port, *v.publicUrl,
  169. v.folders, v.folderMaxLimits, v.minFreeSpacePercent,
  170. volumeNeedleMapKind,
  171. strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
  172. v.whiteList,
  173. *v.readRedirect,
  174. *v.compactionMBPerSecond,
  175. *v.fileSizeLimitMB,
  176. )
  177. // starting grpc server
  178. grpcS := v.startGrpcService(volumeServer)
  179. // starting public http server
  180. var publicHttpDown httpdown.Server
  181. if v.isSeparatedPublicPort() {
  182. publicHttpDown = v.startPublicHttpService(publicVolumeMux)
  183. if nil == publicHttpDown {
  184. glog.Fatalf("start public http service failed")
  185. }
  186. }
  187. // starting the cluster http server
  188. clusterHttpServer := v.startClusterHttpService(volumeMux)
  189. stopChain := make(chan struct{})
  190. grace.OnInterrupt(func() {
  191. fmt.Println("volume server has be killed")
  192. var startTime time.Time
  193. // firstly, stop the public http service to prevent from receiving new user request
  194. if nil != publicHttpDown {
  195. startTime = time.Now()
  196. if err := publicHttpDown.Stop(); err != nil {
  197. glog.Warningf("stop the public http server failed, %v", err)
  198. }
  199. delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
  200. glog.V(0).Infof("stop public http server, elapsed %dms", delta)
  201. }
  202. startTime = time.Now()
  203. if err := clusterHttpServer.Stop(); err != nil {
  204. glog.Warningf("stop the cluster http server failed, %v", err)
  205. }
  206. delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
  207. glog.V(0).Infof("graceful stop cluster http server, elapsed [%d]", delta)
  208. startTime = time.Now()
  209. grpcS.GracefulStop()
  210. delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
  211. glog.V(0).Infof("graceful stop gRPC, elapsed [%d]", delta)
  212. startTime = time.Now()
  213. volumeServer.Shutdown()
  214. delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
  215. glog.V(0).Infof("stop volume server, elapsed [%d]", delta)
  216. pprof.StopCPUProfile()
  217. close(stopChain) // notify exit
  218. })
  219. select {
  220. case <-stopChain:
  221. }
  222. glog.Warningf("the volume server exit.")
  223. }
  224. // check whether configure the public port
  225. func (v VolumeServerOptions) isSeparatedPublicPort() bool {
  226. return *v.publicPort != *v.port
  227. }
  228. func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
  229. grpcPort := *v.port + 10000
  230. grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
  231. if err != nil {
  232. glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
  233. }
  234. grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
  235. volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
  236. reflection.Register(grpcS)
  237. go func() {
  238. if err := grpcS.Serve(grpcL); err != nil {
  239. glog.Fatalf("start gRPC service failed, %s", err)
  240. }
  241. }()
  242. return grpcS
  243. }
  244. func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
  245. publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort)
  246. glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
  247. publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
  248. if e != nil {
  249. glog.Fatalf("Volume server listener error:%v", e)
  250. }
  251. pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
  252. publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
  253. go func() {
  254. if err := publicHttpDown.Wait(); err != nil {
  255. glog.Errorf("public http down wait failed, %v", err)
  256. }
  257. }()
  258. return publicHttpDown
  259. }
  260. func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
  261. var (
  262. certFile, keyFile string
  263. )
  264. if viper.GetString("https.volume.key") != "" {
  265. certFile = viper.GetString("https.volume.cert")
  266. keyFile = viper.GetString("https.volume.key")
  267. }
  268. listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
  269. glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
  270. listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
  271. if e != nil {
  272. glog.Fatalf("Volume server listener error:%v", e)
  273. }
  274. httpDown := httpdown.HTTP{
  275. KillTimeout: 5 * time.Minute,
  276. StopTimeout: 5 * time.Minute,
  277. CertFile: certFile,
  278. KeyFile: keyFile}
  279. clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
  280. go func() {
  281. if e := clusterHttpServer.Wait(); e != nil {
  282. glog.Fatalf("Volume server fail to serve: %v", e)
  283. }
  284. }()
  285. return clusterHttpServer
  286. }