You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

396 lines
14 KiB

5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
12 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. package command
  2. import (
  3. "fmt"
  4. "github.com/chrislusf/seaweedfs/weed/storage/types"
  5. "net/http"
  6. httppprof "net/http/pprof"
  7. "os"
  8. "runtime/pprof"
  9. "strconv"
  10. "strings"
  11. "time"
  12. "github.com/spf13/viper"
  13. "google.golang.org/grpc"
  14. "github.com/chrislusf/seaweedfs/weed/util/grace"
  15. "github.com/chrislusf/seaweedfs/weed/pb"
  16. "github.com/chrislusf/seaweedfs/weed/security"
  17. "github.com/chrislusf/seaweedfs/weed/util/httpdown"
  18. "google.golang.org/grpc/reflection"
  19. "github.com/chrislusf/seaweedfs/weed/glog"
  20. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  21. "github.com/chrislusf/seaweedfs/weed/server"
  22. stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
  23. "github.com/chrislusf/seaweedfs/weed/storage"
  24. "github.com/chrislusf/seaweedfs/weed/util"
  25. )
  26. var (
  27. v VolumeServerOptions
  28. )
  29. type VolumeServerOptions struct {
  30. port *int
  31. publicPort *int
  32. folders []string
  33. folderMaxLimits []int
  34. idxFolder *string
  35. ip *string
  36. publicUrl *string
  37. bindIp *string
  38. masters *string
  39. idleConnectionTimeout *int
  40. dataCenter *string
  41. rack *string
  42. whiteList []string
  43. indexType *string
  44. diskType *string
  45. fixJpgOrientation *bool
  46. readRedirect *bool
  47. cpuProfile *string
  48. memProfile *string
  49. compactionMBPerSecond *int
  50. fileSizeLimitMB *int
  51. concurrentUploadLimitMB *int
  52. minFreeSpacePercents []float32
  53. pprof *bool
  54. preStopSeconds *int
  55. metricsHttpPort *int
  56. // pulseSeconds *int
  57. enableTcp *bool
  58. }
  59. func init() {
  60. cmdVolume.Run = runVolume // break init cycle
  61. v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
  62. v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
  63. v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
  64. v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
  65. v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
  66. v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
  67. v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
  68. // v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
  69. v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
  70. v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
  71. v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
  72. v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
  73. v.diskType = cmdVolume.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  74. v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
  75. v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
  76. v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
  77. v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
  78. v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
  79. v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
  80. v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
  81. v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
  82. v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
  83. v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
  84. v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<exprimental> enable tcp port")
  85. }
  86. var cmdVolume = &Command{
  87. UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
  88. Short: "start a volume server",
  89. Long: `start a volume server to provide storage spaces
  90. `,
  91. }
  92. var (
  93. volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
  94. maxVolumeCounts = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
  95. volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
  96. minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")
  97. )
  98. func runVolume(cmd *Command, args []string) bool {
  99. util.LoadConfiguration("security", false)
  100. // If --pprof is set we assume the caller wants to be able to collect
  101. // cpu and memory profiles via go tool pprof
  102. if !*v.pprof {
  103. grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
  104. }
  105. go stats_collect.StartMetricsServer(*v.metricsHttpPort)
  106. v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, *minFreeSpacePercent)
  107. return true
  108. }
  109. func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpacePercent string) {
  110. // Set multiple folders and each folder's max volume count limit'
  111. v.folders = strings.Split(volumeFolders, ",")
  112. for _, folder := range v.folders {
  113. if err := util.TestFolderWritable(util.ResolvePath(folder)); err != nil {
  114. glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
  115. }
  116. }
  117. // set max
  118. maxCountStrings := strings.Split(maxVolumeCounts, ",")
  119. for _, maxString := range maxCountStrings {
  120. if max, e := strconv.Atoi(maxString); e == nil {
  121. v.folderMaxLimits = append(v.folderMaxLimits, max)
  122. } else {
  123. glog.Fatalf("The max specified in -max not a valid number %s", maxString)
  124. }
  125. }
  126. if len(v.folderMaxLimits) == 1 && len(v.folders) > 1 {
  127. for i := 0; i < len(v.folders)-1; i++ {
  128. v.folderMaxLimits = append(v.folderMaxLimits, v.folderMaxLimits[0])
  129. }
  130. }
  131. if len(v.folders) != len(v.folderMaxLimits) {
  132. glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
  133. }
  134. // set minFreeSpacePercent
  135. minFreeSpacePercentStrings := strings.Split(minFreeSpacePercent, ",")
  136. for _, freeString := range minFreeSpacePercentStrings {
  137. if value, e := strconv.ParseFloat(freeString, 32); e == nil {
  138. v.minFreeSpacePercents = append(v.minFreeSpacePercents, float32(value))
  139. } else {
  140. glog.Fatalf("The value specified in -minFreeSpacePercent not a valid value %s", freeString)
  141. }
  142. }
  143. if len(v.minFreeSpacePercents) == 1 && len(v.folders) > 1 {
  144. for i := 0; i < len(v.folders)-1; i++ {
  145. v.minFreeSpacePercents = append(v.minFreeSpacePercents, v.minFreeSpacePercents[0])
  146. }
  147. }
  148. if len(v.folders) != len(v.minFreeSpacePercents) {
  149. glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpacePercents))
  150. }
  151. // set disk types
  152. var diskTypes []types.DiskType
  153. diskTypeStrings := strings.Split(*v.diskType, ",")
  154. for _, diskTypeString := range diskTypeStrings {
  155. diskTypes = append(diskTypes, types.ToDiskType(diskTypeString))
  156. }
  157. if len(diskTypes) == 1 && len(v.folders) > 1 {
  158. for i := 0; i < len(v.folders)-1; i++ {
  159. diskTypes = append(diskTypes, diskTypes[0])
  160. }
  161. }
  162. if len(v.folders) != len(diskTypes) {
  163. glog.Fatalf("%d directories by -dir, but only %d disk types is set by -disk", len(v.folders), len(diskTypes))
  164. }
  165. // security related white list configuration
  166. if volumeWhiteListOption != "" {
  167. v.whiteList = strings.Split(volumeWhiteListOption, ",")
  168. }
  169. if *v.ip == "" {
  170. *v.ip = util.DetectedHostAddress()
  171. glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
  172. }
  173. if *v.publicPort == 0 {
  174. *v.publicPort = *v.port
  175. }
  176. if *v.publicUrl == "" {
  177. *v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort)
  178. }
  179. volumeMux := http.NewServeMux()
  180. publicVolumeMux := volumeMux
  181. if v.isSeparatedPublicPort() {
  182. publicVolumeMux = http.NewServeMux()
  183. }
  184. if *v.pprof {
  185. volumeMux.HandleFunc("/debug/pprof/", httppprof.Index)
  186. volumeMux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
  187. volumeMux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
  188. volumeMux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
  189. volumeMux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
  190. }
  191. volumeNeedleMapKind := storage.NeedleMapInMemory
  192. switch *v.indexType {
  193. case "leveldb":
  194. volumeNeedleMapKind = storage.NeedleMapLevelDb
  195. case "leveldbMedium":
  196. volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
  197. case "leveldbLarge":
  198. volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
  199. }
  200. masters := *v.masters
  201. volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
  202. *v.ip, *v.port, *v.publicUrl,
  203. v.folders, v.folderMaxLimits, v.minFreeSpacePercents, diskTypes,
  204. *v.idxFolder,
  205. volumeNeedleMapKind,
  206. strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
  207. v.whiteList,
  208. *v.fixJpgOrientation, *v.readRedirect,
  209. *v.compactionMBPerSecond,
  210. *v.fileSizeLimitMB,
  211. int64(*v.concurrentUploadLimitMB)*1024*1024,
  212. )
  213. // starting grpc server
  214. grpcS := v.startGrpcService(volumeServer)
  215. // starting public http server
  216. var publicHttpDown httpdown.Server
  217. if v.isSeparatedPublicPort() {
  218. publicHttpDown = v.startPublicHttpService(publicVolumeMux)
  219. if nil == publicHttpDown {
  220. glog.Fatalf("start public http service failed")
  221. }
  222. }
  223. // starting tcp server
  224. if *v.enableTcp {
  225. go v.startTcpService(volumeServer)
  226. }
  227. // starting the cluster http server
  228. clusterHttpServer := v.startClusterHttpService(volumeMux)
  229. stopChan := make(chan bool)
  230. grace.OnInterrupt(func() {
  231. fmt.Println("volume server has be killed")
  232. // Stop heartbeats
  233. if !volumeServer.StopHeartbeat() {
  234. glog.V(0).Infof("stop send heartbeat and wait %d seconds until shutdown ...", *v.preStopSeconds)
  235. time.Sleep(time.Duration(*v.preStopSeconds) * time.Second)
  236. }
  237. shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
  238. stopChan <- true
  239. })
  240. select {
  241. case <-stopChan:
  242. }
  243. }
  244. func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer) {
  245. // firstly, stop the public http service to prevent from receiving new user request
  246. if nil != publicHttpDown {
  247. glog.V(0).Infof("stop public http server ... ")
  248. if err := publicHttpDown.Stop(); err != nil {
  249. glog.Warningf("stop the public http server failed, %v", err)
  250. }
  251. }
  252. glog.V(0).Infof("graceful stop cluster http server ... ")
  253. if err := clusterHttpServer.Stop(); err != nil {
  254. glog.Warningf("stop the cluster http server failed, %v", err)
  255. }
  256. glog.V(0).Infof("graceful stop gRPC ...")
  257. grpcS.GracefulStop()
  258. volumeServer.Shutdown()
  259. pprof.StopCPUProfile()
  260. }
  261. // check whether configure the public port
  262. func (v VolumeServerOptions) isSeparatedPublicPort() bool {
  263. return *v.publicPort != *v.port
  264. }
  265. func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
  266. grpcPort := *v.port + 10000
  267. grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
  268. if err != nil {
  269. glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
  270. }
  271. grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
  272. volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
  273. reflection.Register(grpcS)
  274. go func() {
  275. if err := grpcS.Serve(grpcL); err != nil {
  276. glog.Fatalf("start gRPC service failed, %s", err)
  277. }
  278. }()
  279. return grpcS
  280. }
  281. func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
  282. publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort)
  283. glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
  284. publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
  285. if e != nil {
  286. glog.Fatalf("Volume server listener error:%v", e)
  287. }
  288. pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
  289. publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
  290. go func() {
  291. if err := publicHttpDown.Wait(); err != nil {
  292. glog.Errorf("public http down wait failed, %v", err)
  293. }
  294. }()
  295. return publicHttpDown
  296. }
  297. func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
  298. var (
  299. certFile, keyFile string
  300. )
  301. if viper.GetString("https.volume.key") != "" {
  302. certFile = viper.GetString("https.volume.cert")
  303. keyFile = viper.GetString("https.volume.key")
  304. }
  305. listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
  306. glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
  307. listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
  308. if e != nil {
  309. glog.Fatalf("Volume server listener error:%v", e)
  310. }
  311. httpDown := httpdown.HTTP{
  312. KillTimeout: 5 * time.Minute,
  313. StopTimeout: 5 * time.Minute,
  314. CertFile: certFile,
  315. KeyFile: keyFile}
  316. clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
  317. go func() {
  318. if e := clusterHttpServer.Wait(); e != nil {
  319. glog.Fatalf("Volume server fail to serve: %v", e)
  320. }
  321. }()
  322. return clusterHttpServer
  323. }
  324. func (v VolumeServerOptions) startTcpService(volumeServer *weed_server.VolumeServer) {
  325. listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port+20000)
  326. glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "tcp at", listeningAddress)
  327. listener, e := util.NewListener(listeningAddress, 0)
  328. if e != nil {
  329. glog.Fatalf("Volume server listener error on %s:%v", listeningAddress, e)
  330. }
  331. defer listener.Close()
  332. for {
  333. c, err := listener.Accept()
  334. if err != nil {
  335. fmt.Println(err)
  336. return
  337. }
  338. go volumeServer.HandleTcpConnection(c)
  339. }
  340. }