You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

413 lines
16 KiB

11 years ago
11 years ago
11 years ago
11 years ago
4 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
4 years ago
3 years ago
4 years ago
11 years ago
11 years ago
11 years ago
11 years ago
4 years ago
4 years ago
4 years ago
4 years ago
11 years ago
11 years ago
11 years ago
11 years ago
5 years ago
11 years ago
11 years ago
  1. package command
  2. import (
  3. "context"
  4. "crypto/tls"
  5. "crypto/x509"
  6. "fmt"
  7. "net"
  8. "net/http"
  9. "os"
  10. "runtime"
  11. "sort"
  12. "strings"
  13. "time"
  14. "github.com/seaweedfs/seaweedfs/weed/filer"
  15. "github.com/seaweedfs/seaweedfs/weed/glog"
  16. "github.com/seaweedfs/seaweedfs/weed/pb"
  17. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  18. "github.com/seaweedfs/seaweedfs/weed/security"
  19. weed_server "github.com/seaweedfs/seaweedfs/weed/server"
  20. stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
  21. "github.com/seaweedfs/seaweedfs/weed/util"
  22. "github.com/spf13/viper"
  23. "google.golang.org/grpc/credentials/tls/certprovider"
  24. "google.golang.org/grpc/credentials/tls/certprovider/pemfile"
  25. "google.golang.org/grpc/reflection"
  26. )
  27. var (
  28. f FilerOptions
  29. filerStartS3 *bool
  30. filerS3Options S3Options
  31. filerStartWebDav *bool
  32. filerWebDavOptions WebDavOption
  33. filerStartIam *bool
  34. filerIamOptions IamOptions
  35. )
  36. type FilerOptions struct {
  37. masters *pb.ServerDiscovery
  38. mastersString *string
  39. ip *string
  40. bindIp *string
  41. port *int
  42. portGrpc *int
  43. publicPort *int
  44. filerGroup *string
  45. collection *string
  46. defaultReplicaPlacement *string
  47. disableDirListing *bool
  48. maxMB *int
  49. dirListingLimit *int
  50. dataCenter *string
  51. rack *string
  52. enableNotification *bool
  53. disableHttp *bool
  54. cipher *bool
  55. metricsHttpPort *int
  56. metricsHttpIp *string
  57. saveToFilerLimit *int
  58. defaultLevelDbDirectory *string
  59. concurrentUploadLimitMB *int
  60. debug *bool
  61. debugPort *int
  62. localSocket *string
  63. showUIDirectoryDelete *bool
  64. downloadMaxMBps *int
  65. diskType *string
  66. allowedOrigins *string
  67. exposeDirectoryData *bool
  68. certProvider certprovider.Provider
  69. }
  70. func init() {
  71. cmdFiler.Run = runFiler // break init cycle
  72. f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers or a single DNS SRV record of at least 1 master server, prepended with dnssrv+")
  73. f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
  74. f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
  75. f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
  76. f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
  77. f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
  78. f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
  79. f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
  80. f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
  81. f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
  82. f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
  83. f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
  84. f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
  85. f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
  86. f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
  87. f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
  88. f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
  89. f.metricsHttpIp = cmdFiler.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
  90. f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
  91. f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
  92. f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
  93. f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
  94. f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
  95. f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
  96. f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
  97. f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
  98. f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  99. f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
  100. f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")
  101. // start s3 on filer
  102. filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
  103. filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port")
  104. filerS3Options.portHttps = cmdFiler.Flag.Int("s3.port.https", 0, "s3 server https listen port")
  105. filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
  106. filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
  107. filerS3Options.allowedOrigins = cmdFiler.Flag.String("s3.allowedOrigins", "*", "comma separated list of allowed origins")
  108. filerS3Options.dataCenter = cmdFiler.Flag.String("s3.dataCenter", "", "prefer to read and write to volumes in this data center")
  109. filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
  110. filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
  111. filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
  112. filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
  113. filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
  114. filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
  115. filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
  116. // start webdav on filer
  117. filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
  118. filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port")
  119. filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files")
  120. filerWebDavOptions.replication = cmdFiler.Flag.String("webdav.replication", "", "replication to create the files")
  121. filerWebDavOptions.disk = cmdFiler.Flag.String("webdav.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  122. filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file")
  123. filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
  124. filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
  125. filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
  126. filerWebDavOptions.maxMB = cmdFiler.Flag.Int("webdav.maxMB", 4, "split files larger than the limit")
  127. filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
  128. // start iam on filer
  129. filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
  130. filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
  131. filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
  132. }
  133. func filerLongDesc() string {
  134. desc := `start a file server which accepts REST operation for any files.
  135. //create or overwrite the file, the directories /path/to will be automatically created
  136. POST /path/to/file
  137. //get the file content
  138. GET /path/to/file
  139. //create or overwrite the file, the filename in the multipart request will be used
  140. POST /path/to/
  141. //return a json format subdirectory and files listing
  142. GET /path/to/
  143. The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
  144. If the "filer.toml" is not found, an embedded filer store will be created under "-defaultStoreDir".
  145. The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
  146. Supported Filer Stores:
  147. `
  148. storeNames := make([]string, len(filer.Stores))
  149. for i, store := range filer.Stores {
  150. storeNames[i] = "\t" + store.GetName()
  151. }
  152. sort.Strings(storeNames)
  153. storeList := strings.Join(storeNames, "\n")
  154. return desc + storeList
  155. }
  156. var cmdFiler = &Command{
  157. UsageLine: "filer -port=8888 -master=<ip:port>[,<ip:port>]*",
  158. Short: "start a file server that points to a master server, or a list of master servers",
  159. Long: filerLongDesc(),
  160. }
  161. func runFiler(cmd *Command, args []string) bool {
  162. if *f.debug {
  163. go http.ListenAndServe(fmt.Sprintf(":%d", *f.debugPort), nil)
  164. }
  165. util.LoadSecurityConfiguration()
  166. switch {
  167. case *f.metricsHttpIp != "":
  168. // noting to do, use f.metricsHttpIp
  169. case *f.bindIp != "":
  170. *f.metricsHttpIp = *f.bindIp
  171. case *f.ip != "":
  172. *f.metricsHttpIp = *f.ip
  173. }
  174. go stats_collect.StartMetricsServer(*f.metricsHttpIp, *f.metricsHttpPort)
  175. filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
  176. startDelay := time.Duration(2)
  177. if *filerStartS3 {
  178. filerS3Options.filer = &filerAddress
  179. filerS3Options.bindIp = f.bindIp
  180. filerS3Options.localFilerSocket = f.localSocket
  181. if *f.dataCenter != "" && *filerS3Options.dataCenter == "" {
  182. filerS3Options.dataCenter = f.dataCenter
  183. }
  184. go func(delay time.Duration) {
  185. time.Sleep(delay * time.Second)
  186. filerS3Options.startS3Server()
  187. }(startDelay)
  188. startDelay++
  189. }
  190. if *filerStartWebDav {
  191. filerWebDavOptions.filer = &filerAddress
  192. if *filerWebDavOptions.disk == "" {
  193. filerWebDavOptions.disk = f.diskType
  194. }
  195. go func(delay time.Duration) {
  196. time.Sleep(delay * time.Second)
  197. filerWebDavOptions.startWebDav()
  198. }(startDelay)
  199. startDelay++
  200. }
  201. if *filerStartIam {
  202. filerIamOptions.filer = &filerAddress
  203. filerIamOptions.masters = f.mastersString
  204. go func(delay time.Duration) {
  205. time.Sleep(delay * time.Second)
  206. filerIamOptions.startIamServer()
  207. }(startDelay)
  208. }
  209. f.masters = pb.ServerAddresses(*f.mastersString).ToServiceDiscovery()
  210. f.startFiler()
  211. return true
  212. }
  213. // GetCertificateWithUpdate Auto refreshing TSL certificate
  214. func (fo *FilerOptions) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
  215. certs, err := fo.certProvider.KeyMaterial(context.Background())
  216. return &certs.Certs[0], err
  217. }
  218. func (fo *FilerOptions) startFiler() {
  219. defaultMux := http.NewServeMux()
  220. publicVolumeMux := defaultMux
  221. if *fo.publicPort != 0 {
  222. publicVolumeMux = http.NewServeMux()
  223. }
  224. if *fo.portGrpc == 0 {
  225. *fo.portGrpc = 10000 + *fo.port
  226. }
  227. if *fo.bindIp == "" {
  228. *fo.bindIp = *fo.ip
  229. }
  230. if *fo.allowedOrigins == "" {
  231. *fo.allowedOrigins = "*"
  232. }
  233. defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
  234. filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)
  235. fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
  236. Masters: fo.masters,
  237. FilerGroup: *fo.filerGroup,
  238. Collection: *fo.collection,
  239. DefaultReplication: *fo.defaultReplicaPlacement,
  240. DisableDirListing: *fo.disableDirListing,
  241. MaxMB: *fo.maxMB,
  242. DirListingLimit: *fo.dirListingLimit,
  243. DataCenter: *fo.dataCenter,
  244. Rack: *fo.rack,
  245. DefaultLevelDbDir: defaultLevelDbDirectory,
  246. DisableHttp: *fo.disableHttp,
  247. Host: filerAddress,
  248. Cipher: *fo.cipher,
  249. SaveToFilerLimit: int64(*fo.saveToFilerLimit),
  250. ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
  251. ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
  252. DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
  253. DiskType: *fo.diskType,
  254. AllowedOrigins: strings.Split(*fo.allowedOrigins, ","),
  255. })
  256. if nfs_err != nil {
  257. glog.Fatalf("Filer startup error: %v", nfs_err)
  258. }
  259. if *fo.publicPort != 0 {
  260. publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
  261. glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
  262. publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
  263. if e != nil {
  264. glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
  265. }
  266. go func() {
  267. if e := http.Serve(publicListener, publicVolumeMux); e != nil {
  268. glog.Fatalf("Volume server fail to serve public: %v", e)
  269. }
  270. }()
  271. if localPublicListener != nil {
  272. go func() {
  273. if e := http.Serve(localPublicListener, publicVolumeMux); e != nil {
  274. glog.Errorf("Volume server fail to serve public: %v", e)
  275. }
  276. }()
  277. }
  278. }
  279. glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
  280. filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
  281. *fo.bindIp, *fo.port,
  282. time.Duration(10)*time.Second,
  283. )
  284. if e != nil {
  285. glog.Fatalf("Filer listener error: %v", e)
  286. }
  287. // starting grpc server
  288. grpcPort := *fo.portGrpc
  289. grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
  290. if err != nil {
  291. glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
  292. }
  293. grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
  294. filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
  295. reflection.Register(grpcS)
  296. if grpcLocalL != nil {
  297. go grpcS.Serve(grpcLocalL)
  298. }
  299. go grpcS.Serve(grpcL)
  300. httpS := &http.Server{Handler: defaultMux}
  301. if runtime.GOOS != "windows" {
  302. localSocket := *fo.localSocket
  303. if localSocket == "" {
  304. localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
  305. }
  306. if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
  307. glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
  308. }
  309. go func() {
  310. // start on local unix socket
  311. filerSocketListener, err := net.Listen("unix", localSocket)
  312. if err != nil {
  313. glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
  314. }
  315. httpS.Serve(filerSocketListener)
  316. }()
  317. }
  318. if viper.GetString("https.filer.key") != "" {
  319. certFile := viper.GetString("https.filer.cert")
  320. keyFile := viper.GetString("https.filer.key")
  321. caCertFile := viper.GetString("https.filer.ca")
  322. disbaleTlsVerifyClientCert := viper.GetBool("https.filer.disable_tls_verify_client_cert")
  323. pemfileOptions := pemfile.Options{
  324. CertFile: certFile,
  325. KeyFile: keyFile,
  326. RefreshDuration: security.CredRefreshingInterval,
  327. }
  328. if fo.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
  329. glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
  330. }
  331. caCertPool := x509.NewCertPool()
  332. if caCertFile != "" {
  333. caCertFile, err := os.ReadFile(caCertFile)
  334. if err != nil {
  335. glog.Fatalf("error reading CA certificate: %v", err)
  336. }
  337. caCertPool.AppendCertsFromPEM(caCertFile)
  338. }
  339. clientAuth := tls.NoClientCert
  340. if !disbaleTlsVerifyClientCert {
  341. clientAuth = tls.RequireAndVerifyClientCert
  342. }
  343. httpS.TLSConfig = &tls.Config{
  344. GetCertificate: fo.GetCertificateWithUpdate,
  345. ClientAuth: clientAuth,
  346. ClientCAs: caCertPool,
  347. }
  348. if filerLocalListener != nil {
  349. go func() {
  350. if err := httpS.ServeTLS(filerLocalListener, "", ""); err != nil {
  351. glog.Errorf("Filer Fail to serve: %v", e)
  352. }
  353. }()
  354. }
  355. if err := httpS.ServeTLS(filerListener, "", ""); err != nil {
  356. glog.Fatalf("Filer Fail to serve: %v", e)
  357. }
  358. } else {
  359. if filerLocalListener != nil {
  360. go func() {
  361. if err := httpS.Serve(filerLocalListener); err != nil {
  362. glog.Errorf("Filer Fail to serve: %v", e)
  363. }
  364. }()
  365. }
  366. if err := httpS.Serve(filerListener); err != nil {
  367. glog.Fatalf("Filer Fail to serve: %v", e)
  368. }
  369. }
  370. }