You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

195 lines
5.3 KiB

5 years ago
7 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
  1. package command
  2. import (
  3. "context"
  4. "fmt"
  5. "net/http"
  6. "time"
  7. "github.com/chrislusf/seaweedfs/weed/pb"
  8. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  9. "github.com/chrislusf/seaweedfs/weed/security"
  10. "github.com/gorilla/mux"
  11. "github.com/chrislusf/seaweedfs/weed/glog"
  12. "github.com/chrislusf/seaweedfs/weed/s3api"
  13. stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
  14. "github.com/chrislusf/seaweedfs/weed/util"
  15. )
  16. var (
  17. s3StandaloneOptions S3Options
  18. )
  19. type S3Options struct {
  20. filer *string
  21. port *int
  22. config *string
  23. domainName *string
  24. tlsPrivateKey *string
  25. tlsCertificate *string
  26. }
  27. func init() {
  28. cmdS3.Run = runS3 // break init cycle
  29. s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
  30. s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
  31. s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
  32. s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
  33. s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
  34. s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
  35. }
  36. var cmdS3 = &Command{
  37. UsageLine: "s3 [-port=8333] [-filer=<ip:port>] [-config=</path/to/config.json>]",
  38. Short: "start a s3 API compatible server that is backed by a filer",
  39. Long: `start a s3 API compatible server that is backed by a filer.
  40. By default, you can use any access key and secret key to access the S3 APIs.
  41. To enable credential based access, create a config.json file similar to this:
  42. {
  43. "identities": [
  44. {
  45. "name": "some_name",
  46. "credentials": [
  47. {
  48. "accessKey": "some_access_key1",
  49. "secretKey": "some_secret_key1"
  50. }
  51. ],
  52. "actions": [
  53. "Admin",
  54. "Read",
  55. "Write"
  56. ]
  57. },
  58. {
  59. "name": "some_read_only_user",
  60. "credentials": [
  61. {
  62. "accessKey": "some_access_key2",
  63. "secretKey": "some_secret_key2"
  64. }
  65. ],
  66. "actions": [
  67. "Read"
  68. ]
  69. },
  70. {
  71. "name": "some_normal_user",
  72. "credentials": [
  73. {
  74. "accessKey": "some_access_key3",
  75. "secretKey": "some_secret_key3"
  76. }
  77. ],
  78. "actions": [
  79. "Read",
  80. "Write"
  81. ]
  82. },
  83. {
  84. "name": "user_limited_to_bucket1",
  85. "credentials": [
  86. {
  87. "accessKey": "some_access_key4",
  88. "secretKey": "some_secret_key4"
  89. }
  90. ],
  91. "actions": [
  92. "Read:bucket1",
  93. "Write:bucket1"
  94. ]
  95. }
  96. ]
  97. }
  98. `,
  99. }
  100. func runS3(cmd *Command, args []string) bool {
  101. util.LoadConfiguration("security", false)
  102. return s3StandaloneOptions.startS3Server()
  103. }
  104. func (s3opt *S3Options) startS3Server() bool {
  105. filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*s3opt.filer)
  106. if err != nil {
  107. glog.Fatal(err)
  108. return false
  109. }
  110. filerBucketsPath := "/buckets"
  111. grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
  112. // metrics read from the filer
  113. var metricsAddress string
  114. var metricsIntervalSec int
  115. for {
  116. err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  117. resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
  118. if err != nil {
  119. return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
  120. }
  121. filerBucketsPath = resp.DirBuckets
  122. metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
  123. glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
  124. return nil
  125. })
  126. if err != nil {
  127. glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress)
  128. time.Sleep(time.Second)
  129. } else {
  130. glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress)
  131. break
  132. }
  133. }
  134. go stats_collect.LoopPushingMetric("s3", stats_collect.SourceName(uint32(*s3opt.port)), stats_collect.S3Gather, metricsAddress, metricsIntervalSec)
  135. router := mux.NewRouter().SkipClean(true)
  136. _, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
  137. Filer: *s3opt.filer,
  138. Port: *s3opt.port,
  139. FilerGrpcAddress: filerGrpcAddress,
  140. Config: *s3opt.config,
  141. DomainName: *s3opt.domainName,
  142. BucketsPath: filerBucketsPath,
  143. GrpcDialOption: grpcDialOption,
  144. })
  145. if s3ApiServer_err != nil {
  146. glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
  147. }
  148. httpS := &http.Server{Handler: router}
  149. listenAddress := fmt.Sprintf(":%d", *s3opt.port)
  150. s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
  151. if err != nil {
  152. glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
  153. }
  154. if *s3opt.tlsPrivateKey != "" {
  155. glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
  156. if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
  157. glog.Fatalf("S3 API Server Fail to serve: %v", err)
  158. }
  159. } else {
  160. glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
  161. if err = httpS.Serve(s3ApiListener); err != nil {
  162. glog.Fatalf("S3 API Server Fail to serve: %v", err)
  163. }
  164. }
  165. return true
  166. }