You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

301 lines
14 KiB

5 years ago
5 years ago
FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client - one JWT for reading and one for writing, analogous to how the JWT between Master and Volume Server works - I did not implement IP `whiteList` parameter on the filer Additionally, because http_util.DownloadFile now sets the JWT, the `download` command should now work when `jwt.signing.read` is configured. By looking at the code, I think this case did not work before. ## Docs to be adjusted after a release Page `Amazon-S3-API`: ``` # Authentication with Filer You can use mTLS for the gRPC connection between S3-API-Proxy and the filer, as explained in [Security-Configuration](Security-Configuration) - controlled by the `grpc.*` configuration in `security.toml`. Starting with version XX, it is also possible to authenticate the HTTP operations between the S3-API-Proxy and the Filer (especially uploading new files). This is configured by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. With both configurations (gRPC and JWT), it is possible to have Filer and S3 communicate in fully authenticated fashion; so Filer will reject any unauthenticated communication. ``` Page `Security Overview`: ``` The following items are not covered, yet: - master server http REST services Starting with version XX, the Filer HTTP REST services can be secured with a JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. ... Before version XX: "weed filer -disableHttp", disable http operations, only gRPC operations are allowed. This works with "weed mount" by FUSE. It does **not work** with the [S3 Gateway](Amazon S3 API), as this does HTTP calls to the Filer. Starting with version XX: secured by JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. **This now works with the [S3 Gateway](Amazon S3 API).** ... # Securing Filer HTTP with JWT To enable JWT-based access control for the Filer, 1. generate `security.toml` file by `weed scaffold -config=security` 2. set `filer_jwt.signing.key` to a secret string - and optionally filer_jwt.signing.read.key` as well to a secret string 3. copy the same `security.toml` file to the filers and all S3 proxies. If `filer_jwt.signing.key` is configured: When sending upload/update/delete HTTP operations to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.key`. If `filer_jwt.signing.read.key` is configured: When sending GET or HEAD requests to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.read.key`. The S3 API Gateway reads the above JWT keys and sends authenticated HTTP requests to the filer. ``` Page `Security Configuration`: ``` (update scaffold file) ... [filer_jwt.signing] key = "blahblahblahblah" [filer_jwt.signing.read] key = "blahblahblahblah" ``` Resolves: #158
3 years ago
1 year ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
  1. package s3api
  2. import (
  3. "context"
  4. "fmt"
  5. "net"
  6. "net/http"
  7. "strings"
  8. "time"
  9. "github.com/seaweedfs/seaweedfs/weed/filer"
  10. "github.com/seaweedfs/seaweedfs/weed/glog"
  11. "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
  12. "github.com/seaweedfs/seaweedfs/weed/util/grace"
  13. "github.com/gorilla/mux"
  14. "github.com/seaweedfs/seaweedfs/weed/pb"
  15. . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
  16. "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
  17. "github.com/seaweedfs/seaweedfs/weed/security"
  18. "github.com/seaweedfs/seaweedfs/weed/util"
  19. util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
  20. util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
  21. "google.golang.org/grpc"
  22. )
  23. type S3ApiServerOption struct {
  24. Filer pb.ServerAddress
  25. Port int
  26. Config string
  27. DomainName string
  28. AllowedOrigins []string
  29. BucketsPath string
  30. GrpcDialOption grpc.DialOption
  31. AllowEmptyFolder bool
  32. AllowDeleteBucketNotEmpty bool
  33. LocalFilerSocket string
  34. DataCenter string
  35. FilerGroup string
  36. }
  37. type S3ApiServer struct {
  38. s3_pb.UnimplementedSeaweedS3Server
  39. option *S3ApiServerOption
  40. iam *IdentityAccessManagement
  41. cb *CircuitBreaker
  42. randomClientId int32
  43. filerGuard *security.Guard
  44. client util_http_client.HTTPClientInterface
  45. bucketRegistry *BucketRegistry
  46. }
  47. func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
  48. startTsNs := time.Now().UnixNano()
  49. v := util.GetViper()
  50. signingKey := v.GetString("jwt.filer_signing.key")
  51. v.SetDefault("jwt.filer_signing.expires_after_seconds", 10)
  52. expiresAfterSec := v.GetInt("jwt.filer_signing.expires_after_seconds")
  53. readSigningKey := v.GetString("jwt.filer_signing.read.key")
  54. v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60)
  55. readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds")
  56. v.SetDefault("cors.allowed_origins.values", "*")
  57. if (option.AllowedOrigins == nil) || (len(option.AllowedOrigins) == 0) {
  58. allowedOrigins := v.GetString("cors.allowed_origins.values")
  59. domains := strings.Split(allowedOrigins, ",")
  60. option.AllowedOrigins = domains
  61. }
  62. s3ApiServer = &S3ApiServer{
  63. option: option,
  64. iam: NewIdentityAccessManagement(option),
  65. randomClientId: util.RandomInt32(),
  66. filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec),
  67. cb: NewCircuitBreaker(option),
  68. }
  69. if option.Config != "" {
  70. grace.OnReload(func() {
  71. if err := s3ApiServer.iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
  72. glog.Errorf("fail to load config file %s: %v", option.Config, err)
  73. } else {
  74. glog.V(0).Infof("Loaded %d identities from config file %s", len(s3ApiServer.iam.identities), option.Config)
  75. }
  76. })
  77. }
  78. s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
  79. if option.LocalFilerSocket == "" {
  80. if s3ApiServer.client, err = util_http.NewGlobalHttpClient(); err != nil {
  81. return nil, err
  82. }
  83. } else {
  84. s3ApiServer.client = &http.Client{
  85. Transport: &http.Transport{
  86. DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
  87. return net.Dial("unix", option.LocalFilerSocket)
  88. },
  89. },
  90. }
  91. }
  92. s3ApiServer.registerRouter(router)
  93. go s3ApiServer.subscribeMetaEvents("s3", startTsNs, filer.DirectoryEtcRoot, []string{option.BucketsPath})
  94. return s3ApiServer, nil
  95. }
  96. func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
  97. // API Router
  98. apiRouter := router.PathPrefix("/").Subrouter()
  99. // Readiness Probe
  100. apiRouter.Methods(http.MethodGet).Path("/status").HandlerFunc(s3a.StatusHandler)
  101. apiRouter.Methods(http.MethodGet).Path("/healthz").HandlerFunc(s3a.StatusHandler)
  102. apiRouter.Methods(http.MethodOptions).HandlerFunc(
  103. func(w http.ResponseWriter, r *http.Request) {
  104. origin := r.Header.Get("Origin")
  105. if origin != "" {
  106. if s3a.option.AllowedOrigins == nil || len(s3a.option.AllowedOrigins) == 0 || s3a.option.AllowedOrigins[0] == "*" {
  107. origin = "*"
  108. } else {
  109. originFound := false
  110. for _, allowedOrigin := range s3a.option.AllowedOrigins {
  111. if origin == allowedOrigin {
  112. originFound = true
  113. }
  114. }
  115. if !originFound {
  116. writeFailureResponse(w, r, http.StatusForbidden)
  117. return
  118. }
  119. }
  120. }
  121. w.Header().Set("Access-Control-Allow-Origin", origin)
  122. w.Header().Set("Access-Control-Expose-Headers", "*")
  123. w.Header().Set("Access-Control-Allow-Methods", "*")
  124. w.Header().Set("Access-Control-Allow-Headers", "*")
  125. writeSuccessResponseEmpty(w, r)
  126. })
  127. var routers []*mux.Router
  128. if s3a.option.DomainName != "" {
  129. domainNames := strings.Split(s3a.option.DomainName, ",")
  130. for _, domainName := range domainNames {
  131. routers = append(routers, apiRouter.Host(
  132. fmt.Sprintf("%s.%s:%d", "{bucket:.+}", domainName, s3a.option.Port)).Subrouter())
  133. routers = append(routers, apiRouter.Host(
  134. fmt.Sprintf("%s.%s", "{bucket:.+}", domainName)).Subrouter())
  135. }
  136. }
  137. routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
  138. for _, bucket := range routers {
  139. // each case should follow the next rule:
  140. // - requesting object with query must precede any other methods
  141. // - requesting object must precede any methods with buckets
  142. // - requesting bucket with query must precede raw methods with buckets
  143. // - requesting bucket must be processed in the end
  144. // objects with query
  145. // CopyObjectPart
  146. bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
  147. // PutObjectPart
  148. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
  149. // CompleteMultipartUpload
  150. bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}")
  151. // NewMultipartUpload
  152. bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "")
  153. // AbortMultipartUpload
  154. bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}")
  155. // ListObjectParts
  156. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}")
  157. // ListMultipartUploads
  158. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "")
  159. // GetObjectTagging
  160. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectTaggingHandler, ACTION_READ)), "GET")).Queries("tagging", "")
  161. // PutObjectTagging
  162. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectTaggingHandler, ACTION_TAGGING)), "PUT")).Queries("tagging", "")
  163. // DeleteObjectTagging
  164. bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectTaggingHandler, ACTION_TAGGING)), "DELETE")).Queries("tagging", "")
  165. // PutObjectACL
  166. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "")
  167. // PutObjectRetention
  168. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectRetentionHandler, ACTION_WRITE)), "PUT")).Queries("retention", "")
  169. // PutObjectLegalHold
  170. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLegalHoldHandler, ACTION_WRITE)), "PUT")).Queries("legal-hold", "")
  171. // PutObjectLockConfiguration
  172. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLockConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("object-lock", "")
  173. // GetObjectACL
  174. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "")
  175. // objects with query
  176. // raw objects
  177. // HeadObject
  178. bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadObjectHandler, ACTION_READ)), "GET"))
  179. // GetObject, but directory listing is not supported
  180. bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectHandler, ACTION_READ)), "GET"))
  181. // CopyObject
  182. bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectHandler, ACTION_WRITE)), "COPY"))
  183. // PutObject
  184. bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectHandler, ACTION_WRITE)), "PUT"))
  185. // DeleteObject
  186. bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectHandler, ACTION_WRITE)), "DELETE"))
  187. // raw objects
  188. // buckets with query
  189. // DeleteMultipleObjects
  190. bucket.Methods(http.MethodPost).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE)), "DELETE")).Queries("delete", "")
  191. // GetBucketACL
  192. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "")
  193. // PutBucketACL
  194. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "")
  195. // GetBucketPolicy
  196. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketPolicyHandler, ACTION_READ)), "GET")).Queries("policy", "")
  197. // PutBucketPolicy
  198. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketPolicyHandler, ACTION_WRITE)), "PUT")).Queries("policy", "")
  199. // DeleteBucketPolicy
  200. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketPolicyHandler, ACTION_WRITE)), "DELETE")).Queries("policy", "")
  201. // GetBucketCors
  202. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketCorsHandler, ACTION_READ)), "GET")).Queries("cors", "")
  203. // PutBucketCors
  204. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketCorsHandler, ACTION_WRITE)), "PUT")).Queries("cors", "")
  205. // DeleteBucketCors
  206. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketCorsHandler, ACTION_WRITE)), "DELETE")).Queries("cors", "")
  207. // GetBucketLifecycleConfiguration
  208. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "")
  209. // PutBucketLifecycleConfiguration
  210. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "")
  211. // DeleteBucketLifecycleConfiguration
  212. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "")
  213. // GetBucketLocation
  214. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "")
  215. // GetBucketRequestPayment
  216. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketRequestPaymentHandler, ACTION_READ)), "GET")).Queries("requestPayment", "")
  217. // GetBucketVersioning
  218. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketVersioningHandler, ACTION_READ)), "GET")).Queries("versioning", "")
  219. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketVersioningHandler, ACTION_WRITE)), "PUT")).Queries("versioning", "")
  220. // ListObjectsV2
  221. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")
  222. // buckets with query
  223. // PutBucketOwnershipControls
  224. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "")
  225. //GetBucketOwnershipControls
  226. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "")
  227. //DeleteBucketOwnershipControls
  228. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "")
  229. // raw buckets
  230. // PostPolicy
  231. bucket.Methods(http.MethodPost).HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PostPolicyBucketHandler, ACTION_WRITE)), "POST"))
  232. // HeadBucket
  233. bucket.Methods(http.MethodHead).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadBucketHandler, ACTION_READ)), "GET"))
  234. // PutBucket
  235. bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketHandler, ACTION_ADMIN)), "PUT"))
  236. // DeleteBucket
  237. bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE"))
  238. // ListObjectsV1 (Legacy)
  239. bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST"))
  240. // raw buckets
  241. }
  242. // ListBuckets
  243. apiRouter.Methods(http.MethodGet).Path("/").HandlerFunc(track(s3a.ListBucketsHandler, "LIST"))
  244. // NotFound
  245. apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler)
  246. }