You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

298 lines
14 KiB

5 years ago
5 years ago
FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client - one JWT for reading and one for writing, analogous to how the JWT between Master and Volume Server works - I did not implement IP `whiteList` parameter on the filer Additionally, because http_util.DownloadFile now sets the JWT, the `download` command should now work when `jwt.signing.read` is configured. By looking at the code, I think this case did not work before. ## Docs to be adjusted after a release Page `Amazon-S3-API`: ``` # Authentication with Filer You can use mTLS for the gRPC connection between S3-API-Proxy and the filer, as explained in [Security-Configuration](Security-Configuration) - controlled by the `grpc.*` configuration in `security.toml`. Starting with version XX, it is also possible to authenticate the HTTP operations between the S3-API-Proxy and the Filer (especially uploading new files). This is configured by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. With both configurations (gRPC and JWT), it is possible to have Filer and S3 communicate in fully authenticated fashion; so Filer will reject any unauthenticated communication. ``` Page `Security Overview`: ``` The following items are not covered, yet: - master server http REST services Starting with version XX, the Filer HTTP REST services can be secured with a JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. ... Before version XX: "weed filer -disableHttp", disable http operations, only gRPC operations are allowed. This works with "weed mount" by FUSE. It does **not work** with the [S3 Gateway](Amazon S3 API), as this does HTTP calls to the Filer. Starting with version XX: secured by JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. **This now works with the [S3 Gateway](Amazon S3 API).** ... # Securing Filer HTTP with JWT To enable JWT-based access control for the Filer, 1. generate `security.toml` file by `weed scaffold -config=security` 2. set `filer_jwt.signing.key` to a secret string - and optionally filer_jwt.signing.read.key` as well to a secret string 3. copy the same `security.toml` file to the filers and all S3 proxies. If `filer_jwt.signing.key` is configured: When sending upload/update/delete HTTP operations to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.key`. If `filer_jwt.signing.read.key` is configured: When sending GET or HEAD requests to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.read.key`. The S3 API Gateway reads the above JWT keys and sends authenticated HTTP requests to the filer. ``` Page `Security Configuration`: ``` (update scaffold file) ... [filer_jwt.signing] key = "blahblahblahblah" [filer_jwt.signing.read] key = "blahblahblahblah" ``` Resolves: #158
3 years ago
1 year ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
  1. package s3api
  2. import (
  3. "context"
  4. "fmt"
  5. "net"
  6. "net/http"
  7. "strings"
  8. "time"
  9. "github.com/seaweedfs/seaweedfs/weed/filer"
  10. "github.com/seaweedfs/seaweedfs/weed/glog"
  11. "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
  12. "github.com/seaweedfs/seaweedfs/weed/util/grace"
  13. "github.com/gorilla/mux"
  14. "github.com/seaweedfs/seaweedfs/weed/pb"
  15. . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
  16. "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
  17. "github.com/seaweedfs/seaweedfs/weed/security"
  18. "github.com/seaweedfs/seaweedfs/weed/util"
  19. "google.golang.org/grpc"
  20. util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
  21. util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
  22. )
  23. type S3ApiServerOption struct {
  24. Filer pb.ServerAddress
  25. Port int
  26. Config string
  27. DomainName string
  28. AllowedOrigins []string
  29. BucketsPath string
  30. GrpcDialOption grpc.DialOption
  31. AllowEmptyFolder bool
  32. AllowDeleteBucketNotEmpty bool
  33. LocalFilerSocket string
  34. DataCenter string
  35. FilerGroup string
  36. }
  37. type S3ApiServer struct {
  38. s3_pb.UnimplementedSeaweedS3Server
  39. option *S3ApiServerOption
  40. iam *IdentityAccessManagement
  41. cb *CircuitBreaker
  42. randomClientId int32
  43. filerGuard *security.Guard
  44. client util_http_client.HTTPClientInterface
  45. bucketRegistry *BucketRegistry
  46. }
  47. func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
  48. v := util.GetViper()
  49. signingKey := v.GetString("jwt.filer_signing.key")
  50. v.SetDefault("jwt.filer_signing.expires_after_seconds", 10)
  51. expiresAfterSec := v.GetInt("jwt.filer_signing.expires_after_seconds")
  52. readSigningKey := v.GetString("jwt.filer_signing.read.key")
  53. v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60)
  54. readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds")
  55. v.SetDefault("cors.allowed_origins.values", "*")
  56. if (option.AllowedOrigins == nil) || (len(option.AllowedOrigins) == 0) {
  57. allowedOrigins := v.GetString("cors.allowed_origins.values")
  58. domains := strings.Split(allowedOrigins, ",")
  59. option.AllowedOrigins = domains
  60. }
  61. s3ApiServer = &S3ApiServer{
  62. option: option,
  63. iam: NewIdentityAccessManagement(option),
  64. randomClientId: util.RandomInt32(),
  65. filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec),
  66. cb: NewCircuitBreaker(option),
  67. }
  68. if option.Config != "" {
  69. grace.OnReload(func() {
  70. if err := s3ApiServer.iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
  71. glog.Errorf("fail to load config file %s: %v", option.Config, err)
  72. } else {
  73. glog.V(0).Infof("Loaded %d identities from config file %s", len(s3ApiServer.iam.identities), option.Config)
  74. }
  75. })
  76. }
  77. s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
  78. if option.LocalFilerSocket == "" {
  79. if s3ApiServer.client, err = util_http.NewGlobalHttpClient(); err != nil {
  80. return nil, err
  81. }
  82. } else {
  83. s3ApiServer.client = &http.Client{
  84. Transport: &http.Transport{
  85. DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
  86. return net.Dial("unix", option.LocalFilerSocket)
  87. },
  88. },
  89. }
  90. }
  91. s3ApiServer.registerRouter(router)
  92. go s3ApiServer.subscribeMetaEvents("s3", time.Now().UnixNano(), filer.DirectoryEtcRoot, []string{option.BucketsPath})
  93. return s3ApiServer, nil
  94. }
  95. func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
  96. // API Router
  97. apiRouter := router.PathPrefix("/").Subrouter()
  98. // Readiness Probe
  99. apiRouter.Methods(http.MethodGet).Path("/status").HandlerFunc(s3a.StatusHandler)
  100. apiRouter.Methods(http.MethodOptions).HandlerFunc(
  101. func(w http.ResponseWriter, r *http.Request) {
  102. origin := r.Header.Get("Origin")
  103. if origin != "" {
  104. if s3a.option.AllowedOrigins == nil || len(s3a.option.AllowedOrigins) == 0 || s3a.option.AllowedOrigins[0] == "*" {
  105. origin = "*"
  106. } else {
  107. originFound := false
  108. for _, allowedOrigin := range s3a.option.AllowedOrigins {
  109. if origin == allowedOrigin {
  110. originFound = true
  111. }
  112. }
  113. if !originFound {
  114. writeFailureResponse(w, r, http.StatusForbidden)
  115. return
  116. }
  117. }
  118. }
  119. w.Header().Set("Access-Control-Allow-Origin", origin)
  120. w.Header().Set("Access-Control-Expose-Headers", "*")
  121. w.Header().Set("Access-Control-Allow-Methods", "*")
  122. w.Header().Set("Access-Control-Allow-Headers", "*")
  123. writeSuccessResponseEmpty(w, r)
  124. })
  125. var routers []*mux.Router
  126. if s3a.option.DomainName != "" {
  127. domainNames := strings.Split(s3a.option.DomainName, ",")
  128. for _, domainName := range domainNames {
  129. routers = append(routers, apiRouter.Host(
  130. fmt.Sprintf("%s.%s:%d", "{bucket:.+}", domainName, s3a.option.Port)).Subrouter())
  131. routers = append(routers, apiRouter.Host(
  132. fmt.Sprintf("%s.%s", "{bucket:.+}", domainName)).Subrouter())
  133. }
  134. }
  135. routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
  136. for _, bucket := range routers {
  137. // each case should follow the next rule:
  138. // - requesting object with query must precede any other methods
  139. // - requesting object must precede any methods with buckets
  140. // - requesting bucket with query must precede raw methods with buckets
  141. // - requesting bucket must be processed in the end
  142. // objects with query
  143. // CopyObjectPart
  144. bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
  145. // PutObjectPart
  146. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
  147. // CompleteMultipartUpload
  148. bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}")
  149. // NewMultipartUpload
  150. bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "")
  151. // AbortMultipartUpload
  152. bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}")
  153. // ListObjectParts
  154. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}")
  155. // ListMultipartUploads
  156. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "")
  157. // GetObjectTagging
  158. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectTaggingHandler, ACTION_READ)), "GET")).Queries("tagging", "")
  159. // PutObjectTagging
  160. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectTaggingHandler, ACTION_TAGGING)), "PUT")).Queries("tagging", "")
  161. // DeleteObjectTagging
  162. bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectTaggingHandler, ACTION_TAGGING)), "DELETE")).Queries("tagging", "")
  163. // PutObjectACL
  164. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "")
  165. // PutObjectRetention
  166. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectRetentionHandler, ACTION_WRITE)), "PUT")).Queries("retention", "")
  167. // PutObjectLegalHold
  168. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLegalHoldHandler, ACTION_WRITE)), "PUT")).Queries("legal-hold", "")
  169. // PutObjectLockConfiguration
  170. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLockConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("object-lock", "")
  171. // GetObjectACL
  172. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "")
  173. // objects with query
  174. // raw objects
  175. // HeadObject
  176. bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadObjectHandler, ACTION_READ)), "GET"))
  177. // GetObject, but directory listing is not supported
  178. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectHandler, ACTION_READ)), "GET"))
  179. // CopyObject
  180. bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectHandler, ACTION_WRITE)), "COPY"))
  181. // PutObject
  182. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectHandler, ACTION_WRITE)), "PUT"))
  183. // DeleteObject
  184. bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectHandler, ACTION_WRITE)), "DELETE"))
  185. // raw objects
  186. // buckets with query
  187. // DeleteMultipleObjects
  188. bucket.Methods(http.MethodPost).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE)), "DELETE")).Queries("delete", "")
  189. // GetBucketACL
  190. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "")
  191. // PutBucketACL
  192. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "")
  193. // GetBucketPolicy
  194. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketPolicyHandler, ACTION_READ)), "GET")).Queries("policy", "")
  195. // PutBucketPolicy
  196. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketPolicyHandler, ACTION_WRITE)), "PUT")).Queries("policy", "")
  197. // DeleteBucketPolicy
  198. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketPolicyHandler, ACTION_WRITE)), "DELETE")).Queries("policy", "")
  199. // GetBucketCors
  200. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketCorsHandler, ACTION_READ)), "GET")).Queries("cors", "")
  201. // PutBucketCors
  202. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketCorsHandler, ACTION_WRITE)), "PUT")).Queries("cors", "")
  203. // DeleteBucketCors
  204. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketCorsHandler, ACTION_WRITE)), "DELETE")).Queries("cors", "")
  205. // GetBucketLifecycleConfiguration
  206. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "")
  207. // PutBucketLifecycleConfiguration
  208. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "")
  209. // DeleteBucketLifecycleConfiguration
  210. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "")
  211. // GetBucketLocation
  212. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "")
  213. // GetBucketRequestPayment
  214. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketRequestPaymentHandler, ACTION_READ)), "GET")).Queries("requestPayment", "")
  215. // GetBucketVersioning
  216. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketVersioningHandler, ACTION_READ)), "GET")).Queries("versioning", "")
  217. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketVersioningHandler, ACTION_WRITE)), "PUT")).Queries("versioning", "")
  218. // ListObjectsV2
  219. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")
  220. // buckets with query
  221. // PutBucketOwnershipControls
  222. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "")
  223. //GetBucketOwnershipControls
  224. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "")
  225. //DeleteBucketOwnershipControls
  226. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "")
  227. // raw buckets
  228. // PostPolicy
  229. bucket.Methods(http.MethodPost).HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PostPolicyBucketHandler, ACTION_WRITE)), "POST"))
  230. // HeadBucket
  231. bucket.Methods(http.MethodHead).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadBucketHandler, ACTION_READ)), "GET"))
  232. // PutBucket
  233. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketHandler, ACTION_ADMIN)), "PUT"))
  234. // DeleteBucket
  235. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE"))
  236. // ListObjectsV1 (Legacy)
  237. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST"))
  238. // raw buckets
  239. }
  240. // ListBuckets
  241. apiRouter.Methods(http.MethodGet).Path("/").HandlerFunc(track(s3a.ListBucketsHandler, "LIST"))
  242. // NotFound
  243. apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler)
  244. }