You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

365 lines
12 KiB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
  1. package command
  2. import (
  3. "context"
  4. "crypto/tls"
  5. "crypto/x509"
  6. "fmt"
  7. "io/ioutil"
  8. "net"
  9. "net/http"
  10. "os"
  11. "runtime"
  12. "time"
  13. "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
  14. "google.golang.org/grpc/credentials/tls/certprovider"
  15. "google.golang.org/grpc/credentials/tls/certprovider/pemfile"
  16. "google.golang.org/grpc/reflection"
  17. "github.com/seaweedfs/seaweedfs/weed/pb"
  18. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  19. "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
  20. "github.com/seaweedfs/seaweedfs/weed/security"
  21. "github.com/gorilla/mux"
  22. "github.com/seaweedfs/seaweedfs/weed/glog"
  23. "github.com/seaweedfs/seaweedfs/weed/s3api"
  24. stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
  25. "github.com/seaweedfs/seaweedfs/weed/util"
  26. )
  27. var (
  28. s3StandaloneOptions S3Options
  29. )
  30. type S3Options struct {
  31. filer *string
  32. bindIp *string
  33. port *int
  34. portHttps *int
  35. portGrpc *int
  36. config *string
  37. domainName *string
  38. tlsPrivateKey *string
  39. tlsCertificate *string
  40. tlsCACertificate *string
  41. tlsVerifyClientCert *bool
  42. metricsHttpPort *int
  43. allowEmptyFolder *bool
  44. allowDeleteBucketNotEmpty *bool
  45. auditLogConfig *string
  46. localFilerSocket *string
  47. dataCenter *string
  48. localSocket *string
  49. certProvider certprovider.Provider
  50. }
  51. func init() {
  52. cmdS3.Run = runS3 // break init cycle
  53. s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
  54. s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.")
  55. s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
  56. s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port")
  57. s3StandaloneOptions.portGrpc = cmdS3.Flag.Int("port.grpc", 0, "s3 server grpc listen port")
  58. s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
  59. s3StandaloneOptions.dataCenter = cmdS3.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
  60. s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
  61. s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
  62. s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
  63. s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
  64. s3StandaloneOptions.tlsCACertificate = cmdS3.Flag.String("cacert.file", "", "path to the TLS CA certificate file")
  65. s3StandaloneOptions.tlsVerifyClientCert = cmdS3.Flag.Bool("tlsVerifyClientCert", false, "whether to verify the client's certificate")
  66. s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
  67. s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
  68. s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
  69. s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
  70. s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
  71. }
  72. var cmdS3 = &Command{
  73. UsageLine: "s3 [-port=8333] [-filer=<ip:port>] [-config=</path/to/config.json>]",
  74. Short: "start a s3 API compatible server that is backed by a filer",
  75. Long: `start a s3 API compatible server that is backed by a filer.
  76. By default, you can use any access key and secret key to access the S3 APIs.
  77. To enable credential based access, create a config.json file similar to this:
  78. {
  79. "identities": [
  80. {
  81. "name": "anonymous",
  82. "actions": [
  83. "Read"
  84. ]
  85. },
  86. {
  87. "name": "some_admin_user",
  88. "credentials": [
  89. {
  90. "accessKey": "some_access_key1",
  91. "secretKey": "some_secret_key1"
  92. }
  93. ],
  94. "actions": [
  95. "Admin",
  96. "Read",
  97. "List",
  98. "Tagging",
  99. "Write"
  100. ]
  101. },
  102. {
  103. "name": "some_read_only_user",
  104. "credentials": [
  105. {
  106. "accessKey": "some_access_key2",
  107. "secretKey": "some_secret_key2"
  108. }
  109. ],
  110. "actions": [
  111. "Read"
  112. ]
  113. },
  114. {
  115. "name": "some_normal_user",
  116. "credentials": [
  117. {
  118. "accessKey": "some_access_key3",
  119. "secretKey": "some_secret_key3"
  120. }
  121. ],
  122. "actions": [
  123. "Read",
  124. "List",
  125. "Tagging",
  126. "Write"
  127. ]
  128. },
  129. {
  130. "name": "user_limited_to_bucket1",
  131. "credentials": [
  132. {
  133. "accessKey": "some_access_key4",
  134. "secretKey": "some_secret_key4"
  135. }
  136. ],
  137. "actions": [
  138. "Read:bucket1",
  139. "List:bucket1",
  140. "Tagging:bucket1",
  141. "Write:bucket1"
  142. ]
  143. }
  144. ]
  145. }
  146. `,
  147. }
  148. func runS3(cmd *Command, args []string) bool {
  149. util.LoadConfiguration("security", false)
  150. go stats_collect.StartMetricsServer(*s3StandaloneOptions.bindIp, *s3StandaloneOptions.metricsHttpPort)
  151. return s3StandaloneOptions.startS3Server()
  152. }
  153. // GetCertificateWithUpdate Auto refreshing TSL certificate
  154. func (S3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
  155. certs, err := S3opt.certProvider.KeyMaterial(context.Background())
  156. return &certs.Certs[0], err
  157. }
  158. func (s3opt *S3Options) startS3Server() bool {
  159. filerAddress := pb.ServerAddress(*s3opt.filer)
  160. filerBucketsPath := "/buckets"
  161. filerGroup := ""
  162. grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
  163. // metrics read from the filer
  164. var metricsAddress string
  165. var metricsIntervalSec int
  166. for {
  167. err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  168. resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
  169. if err != nil {
  170. return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
  171. }
  172. filerBucketsPath = resp.DirBuckets
  173. filerGroup = resp.FilerGroup
  174. metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
  175. glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
  176. return nil
  177. })
  178. if err != nil {
  179. glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress())
  180. time.Sleep(time.Second)
  181. } else {
  182. glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress())
  183. break
  184. }
  185. }
  186. go stats_collect.LoopPushingMetric("s3", stats_collect.SourceName(uint32(*s3opt.port)), metricsAddress, metricsIntervalSec)
  187. router := mux.NewRouter().SkipClean(true)
  188. var localFilerSocket string
  189. if s3opt.localFilerSocket != nil {
  190. localFilerSocket = *s3opt.localFilerSocket
  191. }
  192. s3ApiServer, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
  193. Filer: filerAddress,
  194. Port: *s3opt.port,
  195. Config: *s3opt.config,
  196. DomainName: *s3opt.domainName,
  197. BucketsPath: filerBucketsPath,
  198. GrpcDialOption: grpcDialOption,
  199. AllowEmptyFolder: *s3opt.allowEmptyFolder,
  200. AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
  201. LocalFilerSocket: localFilerSocket,
  202. DataCenter: *s3opt.dataCenter,
  203. FilerGroup: filerGroup,
  204. })
  205. if s3ApiServer_err != nil {
  206. glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
  207. }
  208. httpS := &http.Server{Handler: router}
  209. if *s3opt.portGrpc == 0 {
  210. *s3opt.portGrpc = 10000 + *s3opt.port
  211. }
  212. if *s3opt.bindIp == "" {
  213. *s3opt.bindIp = "localhost"
  214. }
  215. if runtime.GOOS != "windows" {
  216. localSocket := *s3opt.localSocket
  217. if localSocket == "" {
  218. localSocket = fmt.Sprintf("/tmp/seaweedfs-s3-%d.sock", *s3opt.port)
  219. }
  220. if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
  221. glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
  222. }
  223. go func() {
  224. // start on local unix socket
  225. s3SocketListener, err := net.Listen("unix", localSocket)
  226. if err != nil {
  227. glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
  228. }
  229. httpS.Serve(s3SocketListener)
  230. }()
  231. }
  232. listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
  233. s3ApiListener, s3ApiLocalListener, err := util.NewIpAndLocalListeners(*s3opt.bindIp, *s3opt.port, time.Duration(10)*time.Second)
  234. if err != nil {
  235. glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
  236. }
  237. if len(*s3opt.auditLogConfig) > 0 {
  238. s3err.InitAuditLog(*s3opt.auditLogConfig)
  239. if s3err.Logger != nil {
  240. defer s3err.Logger.Close()
  241. }
  242. }
  243. // starting grpc server
  244. grpcPort := *s3opt.portGrpc
  245. grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*s3opt.bindIp, grpcPort, 0)
  246. if err != nil {
  247. glog.Fatalf("s3 failed to listen on grpc port %d: %v", grpcPort, err)
  248. }
  249. grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.s3"))
  250. s3_pb.RegisterSeaweedS3Server(grpcS, s3ApiServer)
  251. reflection.Register(grpcS)
  252. if grpcLocalL != nil {
  253. go grpcS.Serve(grpcLocalL)
  254. }
  255. go grpcS.Serve(grpcL)
  256. if *s3opt.tlsPrivateKey != "" {
  257. pemfileOptions := pemfile.Options{
  258. CertFile: *s3opt.tlsCertificate,
  259. KeyFile: *s3opt.tlsPrivateKey,
  260. RefreshDuration: security.CredRefreshingInterval,
  261. }
  262. if s3opt.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
  263. glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
  264. }
  265. caCertPool := x509.NewCertPool()
  266. if *s3Options.tlsCACertificate != "" {
  267. // load CA certificate file and add it to list of client CAs
  268. caCertFile, err := ioutil.ReadFile(*s3opt.tlsCACertificate)
  269. if err != nil {
  270. glog.Fatalf("error reading CA certificate: %v", err)
  271. }
  272. caCertPool.AppendCertsFromPEM(caCertFile)
  273. }
  274. clientAuth := tls.NoClientCert
  275. if *s3Options.tlsVerifyClientCert {
  276. clientAuth = tls.RequireAndVerifyClientCert
  277. }
  278. httpS.TLSConfig = &tls.Config{
  279. GetCertificate: s3opt.GetCertificateWithUpdate,
  280. ClientAuth: clientAuth,
  281. ClientCAs: caCertPool,
  282. }
  283. if *s3opt.portHttps == 0 {
  284. glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
  285. if s3ApiLocalListener != nil {
  286. go func() {
  287. if err = httpS.ServeTLS(s3ApiLocalListener, "", ""); err != nil {
  288. glog.Fatalf("S3 API Server Fail to serve: %v", err)
  289. }
  290. }()
  291. }
  292. if err = httpS.ServeTLS(s3ApiListener, "", ""); err != nil {
  293. glog.Fatalf("S3 API Server Fail to serve: %v", err)
  294. }
  295. } else {
  296. glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.portHttps)
  297. s3ApiListenerHttps, s3ApiLocalListenerHttps, _ := util.NewIpAndLocalListeners(
  298. *s3opt.bindIp, *s3opt.portHttps, time.Duration(10)*time.Second)
  299. if s3ApiLocalListenerHttps != nil {
  300. go func() {
  301. if err = httpS.ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {
  302. glog.Fatalf("S3 API Server Fail to serve: %v", err)
  303. }
  304. }()
  305. }
  306. go func() {
  307. if err = httpS.ServeTLS(s3ApiListenerHttps, "", ""); err != nil {
  308. glog.Fatalf("S3 API Server Fail to serve: %v", err)
  309. }
  310. }()
  311. }
  312. }
  313. if *s3opt.tlsPrivateKey == "" || *s3opt.portHttps > 0 {
  314. glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
  315. if s3ApiLocalListener != nil {
  316. go func() {
  317. if err = httpS.Serve(s3ApiLocalListener); err != nil {
  318. glog.Fatalf("S3 API Server Fail to serve: %v", err)
  319. }
  320. }()
  321. }
  322. if err = httpS.Serve(s3ApiListener); err != nil {
  323. glog.Fatalf("S3 API Server Fail to serve: %v", err)
  324. }
  325. }
  326. return true
  327. }