You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

297 lines
10 KiB

5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
12 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package command
  2. import (
  3. "fmt"
  4. "net/http"
  5. "os"
  6. "runtime"
  7. "runtime/pprof"
  8. "strconv"
  9. "strings"
  10. "time"
  11. "github.com/spf13/viper"
  12. "google.golang.org/grpc"
  13. "github.com/chrislusf/seaweedfs/weed/pb"
  14. "github.com/chrislusf/seaweedfs/weed/security"
  15. "github.com/chrislusf/seaweedfs/weed/util/httpdown"
  16. "google.golang.org/grpc/reflection"
  17. "github.com/chrislusf/seaweedfs/weed/glog"
  18. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  19. "github.com/chrislusf/seaweedfs/weed/server"
  20. "github.com/chrislusf/seaweedfs/weed/storage"
  21. "github.com/chrislusf/seaweedfs/weed/util"
  22. )
  23. var (
  24. v VolumeServerOptions
  25. )
  26. type VolumeServerOptions struct {
  27. port *int
  28. publicPort *int
  29. folders []string
  30. folderMaxLimits []int
  31. ip *string
  32. publicUrl *string
  33. bindIp *string
  34. masters *string
  35. pulseSeconds *int
  36. idleConnectionTimeout *int
  37. dataCenter *string
  38. rack *string
  39. whiteList []string
  40. indexType *string
  41. fixJpgOrientation *bool
  42. readRedirect *bool
  43. cpuProfile *string
  44. memProfile *string
  45. compactionMBPerSecond *int
  46. fileSizeLimitMB *int
  47. }
  48. func init() {
  49. cmdVolume.Run = runVolume // break init cycle
  50. v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
  51. v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
  52. v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
  53. v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
  54. v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
  55. v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
  56. v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
  57. v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
  58. v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
  59. v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
  60. v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
  61. v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
  62. v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
  63. v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
  64. v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
  65. v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
  66. v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
  67. }
  68. var cmdVolume = &Command{
  69. UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
  70. Short: "start a volume server",
  71. Long: `start a volume server to provide storage spaces
  72. `,
  73. }
  74. var (
  75. volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
  76. maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]... If set to zero on non-windows OS, the limit will be auto configured.")
  77. volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  78. )
  79. func runVolume(cmd *Command, args []string) bool {
  80. util.LoadConfiguration("security", false)
  81. runtime.GOMAXPROCS(runtime.NumCPU())
  82. util.SetupProfiling(*v.cpuProfile, *v.memProfile)
  83. v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption)
  84. return true
  85. }
  86. func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string) {
  87. // Set multiple folders and each folder's max volume count limit'
  88. v.folders = strings.Split(volumeFolders, ",")
  89. maxCountStrings := strings.Split(maxVolumeCounts, ",")
  90. for _, maxString := range maxCountStrings {
  91. if max, e := strconv.Atoi(maxString); e == nil {
  92. v.folderMaxLimits = append(v.folderMaxLimits, max)
  93. } else {
  94. glog.Fatalf("The max specified in -max not a valid number %s", maxString)
  95. }
  96. }
  97. if len(v.folders) != len(v.folderMaxLimits) {
  98. glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
  99. }
  100. for _, folder := range v.folders {
  101. if err := util.TestFolderWritable(folder); err != nil {
  102. glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
  103. }
  104. }
  105. // security related white list configuration
  106. if volumeWhiteListOption != "" {
  107. v.whiteList = strings.Split(volumeWhiteListOption, ",")
  108. }
  109. if *v.ip == "" {
  110. *v.ip = util.DetectedHostAddress()
  111. glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
  112. }
  113. if *v.publicPort == 0 {
  114. *v.publicPort = *v.port
  115. }
  116. if *v.publicUrl == "" {
  117. *v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort)
  118. }
  119. volumeMux := http.NewServeMux()
  120. publicVolumeMux := volumeMux
  121. if v.isSeparatedPublicPort() {
  122. publicVolumeMux = http.NewServeMux()
  123. }
  124. volumeNeedleMapKind := storage.NeedleMapInMemory
  125. switch *v.indexType {
  126. case "leveldb":
  127. volumeNeedleMapKind = storage.NeedleMapLevelDb
  128. case "leveldbMedium":
  129. volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
  130. case "leveldbLarge":
  131. volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
  132. }
  133. masters := *v.masters
  134. volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
  135. *v.ip, *v.port, *v.publicUrl,
  136. v.folders, v.folderMaxLimits,
  137. volumeNeedleMapKind,
  138. strings.Split(masters, ","), *v.pulseSeconds, *v.dataCenter, *v.rack,
  139. v.whiteList,
  140. *v.fixJpgOrientation, *v.readRedirect,
  141. *v.compactionMBPerSecond,
  142. *v.fileSizeLimitMB,
  143. )
  144. // starting grpc server
  145. grpcS := v.startGrpcService(volumeServer)
  146. // starting public http server
  147. var publicHttpDown httpdown.Server
  148. if v.isSeparatedPublicPort() {
  149. publicHttpDown = v.startPublicHttpService(publicVolumeMux)
  150. if nil == publicHttpDown {
  151. glog.Fatalf("start public http service failed")
  152. }
  153. }
  154. // starting the cluster http server
  155. clusterHttpServer := v.startClusterHttpService(volumeMux)
  156. stopChain := make(chan struct{})
  157. util.OnInterrupt(func() {
  158. fmt.Println("volume server has be killed")
  159. var startTime time.Time
  160. // firstly, stop the public http service to prevent from receiving new user request
  161. if nil != publicHttpDown {
  162. startTime = time.Now()
  163. if err := publicHttpDown.Stop(); err != nil {
  164. glog.Warningf("stop the public http server failed, %v", err)
  165. }
  166. delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
  167. glog.V(0).Infof("stop public http server, elapsed %dms", delta)
  168. }
  169. startTime = time.Now()
  170. if err := clusterHttpServer.Stop(); err != nil {
  171. glog.Warningf("stop the cluster http server failed, %v", err)
  172. }
  173. delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
  174. glog.V(0).Infof("graceful stop cluster http server, elapsed [%d]", delta)
  175. startTime = time.Now()
  176. grpcS.GracefulStop()
  177. delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
  178. glog.V(0).Infof("graceful stop gRPC, elapsed [%d]", delta)
  179. startTime = time.Now()
  180. volumeServer.Shutdown()
  181. delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
  182. glog.V(0).Infof("stop volume server, elapsed [%d]", delta)
  183. pprof.StopCPUProfile()
  184. close(stopChain) // notify exit
  185. })
  186. select {
  187. case <-stopChain:
  188. }
  189. glog.Warningf("the volume server exit.")
  190. }
  191. // check whether configure the public port
  192. func (v VolumeServerOptions) isSeparatedPublicPort() bool {
  193. return *v.publicPort != *v.port
  194. }
  195. func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
  196. grpcPort := *v.port + 10000
  197. grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
  198. if err != nil {
  199. glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
  200. }
  201. grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
  202. volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
  203. reflection.Register(grpcS)
  204. go func() {
  205. if err := grpcS.Serve(grpcL); err != nil {
  206. glog.Fatalf("start gRPC service failed, %s", err)
  207. }
  208. }()
  209. return grpcS
  210. }
  211. func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
  212. publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort)
  213. glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "public at", publicListeningAddress)
  214. publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
  215. if e != nil {
  216. glog.Fatalf("Volume server listener error:%v", e)
  217. }
  218. pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
  219. publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
  220. go func() {
  221. if err := publicHttpDown.Wait(); err != nil {
  222. glog.Errorf("public http down wait failed, %v", err)
  223. }
  224. }()
  225. return publicHttpDown
  226. }
  227. func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
  228. var (
  229. certFile, keyFile string
  230. )
  231. if viper.GetString("https.volume.key") != "" {
  232. certFile = viper.GetString("https.volume.cert")
  233. keyFile = viper.GetString("https.volume.key")
  234. }
  235. listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
  236. glog.V(0).Infof("Start Seaweed volume server %s at %s", util.VERSION, listeningAddress)
  237. listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
  238. if e != nil {
  239. glog.Fatalf("Volume server listener error:%v", e)
  240. }
  241. httpDown := httpdown.HTTP{
  242. KillTimeout: 5 * time.Minute,
  243. StopTimeout: 5 * time.Minute,
  244. CertFile: certFile,
  245. KeyFile: keyFile}
  246. clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
  247. go func() {
  248. if e := clusterHttpServer.Wait(); e != nil {
  249. glog.Fatalf("Volume server fail to serve: %v", e)
  250. }
  251. }()
  252. return clusterHttpServer
  253. }