From eda4c43a08e449a26fb1cc6d5bca17e2144b0add Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Wed, 29 Dec 2021 12:38:14 +0100 Subject: [PATCH 1/7] fix typo in error message --- weed/security/guard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/security/guard.go b/weed/security/guard.go index 87ec91ec1..8cb52620e 100644 --- a/weed/security/guard.go +++ b/weed/security/guard.go @@ -123,5 +123,5 @@ func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error { } glog.V(0).Infof("Not in whitelist: %s", r.RemoteAddr) - return fmt.Errorf("Not in whitelis: %s", r.RemoteAddr) + return fmt.Errorf("Not in whitelist: %s", r.RemoteAddr) } From d156d410efb8166ded4a0c6b206fd7b071b61c8b Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Wed, 29 Dec 2021 12:39:41 +0100 Subject: [PATCH 2/7] rename security.GenJwt to security.GenJwtForVolumeServer --- weed/security/jwt.go | 4 +++- weed/server/master_grpc_server_volume.go | 4 ++-- weed/server/master_server_handlers.go | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/weed/security/jwt.go b/weed/security/jwt.go index 7327f7b8b..f025af519 100644 --- a/weed/security/jwt.go +++ b/weed/security/jwt.go @@ -13,12 +13,14 @@ import ( type EncodedJwt string type SigningKey []byte +// SeaweedFileIdClaims is created by Master server(s) and consumed by Volume server(s), +// restricting the access this JWT allows to only a single file. type SeaweedFileIdClaims struct { Fid string `json:"fid"` jwt.StandardClaims } -func GenJwt(signingKey SigningKey, expiresAfterSec int, fileId string) EncodedJwt { +func GenJwtForVolumeServer(signingKey SigningKey, expiresAfterSec int, fileId string) EncodedJwt { if len(signingKey) == 0 { return "" } diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 551e59990..9389bceb8 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -86,7 +86,7 @@ func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupV } var auth string if strings.Contains(result.VolumeOrFileId, ",") { // this is a file id - auth = string(security.GenJwt(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, result.VolumeOrFileId)) + auth = string(security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, result.VolumeOrFileId)) } resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{ VolumeOrFileId: result.VolumeOrFileId, @@ -173,7 +173,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest GrpcPort: uint32(dn.GrpcPort), }, Count: count, - Auth: string(security.GenJwt(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fid)), + Auth: string(security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fid)), Replicas: replicas, }, nil } diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index 50a3f12f6..0b79c4ed5 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -149,9 +149,9 @@ func (ms *MasterServer) maybeAddJwtAuthorization(w http.ResponseWriter, fileId s } var encodedJwt security.EncodedJwt if isWrite { - encodedJwt = security.GenJwt(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fileId) + encodedJwt = security.GenJwtForVolumeServer(ms.guard.SigningKey, ms.guard.ExpiresAfterSec, fileId) } else { - encodedJwt = security.GenJwt(ms.guard.ReadSigningKey, ms.guard.ReadExpiresAfterSec, fileId) + encodedJwt = security.GenJwtForVolumeServer(ms.guard.ReadSigningKey, ms.guard.ReadExpiresAfterSec, fileId) } if encodedJwt == "" { return From fcc09cef6fa9f7baa87675b1513425c233dbfffe Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Wed, 29 Dec 2021 12:40:41 +0100 Subject: [PATCH 3/7] Refactor: pass in claim type into security.DecodeJwt --- weed/security/jwt.go | 4 ++-- weed/server/volume_server_handlers.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/security/jwt.go b/weed/security/jwt.go index f025af519..1976c8ffe 100644 --- a/weed/security/jwt.go +++ b/weed/security/jwt.go @@ -57,9 +57,9 @@ func GetJwt(r *http.Request) EncodedJwt { return EncodedJwt(tokenStr) } -func DecodeJwt(signingKey SigningKey, tokenString EncodedJwt) (token *jwt.Token, err error) { +func DecodeJwt(signingKey SigningKey, tokenString EncodedJwt, claims jwt.Claims) (token *jwt.Token, err error) { // check exp, nbf - return jwt.ParseWithClaims(string(tokenString), &SeaweedFileIdClaims{}, func(token *jwt.Token) (interface{}, error) { + return jwt.ParseWithClaims(string(tokenString), claims, func(token *jwt.Token) (interface{}, error) { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { return nil, fmt.Errorf("unknown token method") } diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index ff2eccc11..510902cf0 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -133,7 +133,7 @@ func (vs *VolumeServer) maybeCheckJwtAuthorization(r *http.Request, vid, fid str return false } - token, err := security.DecodeJwt(signingKey, tokenStr) + token, err := security.DecodeJwt(signingKey, tokenStr, &security.SeaweedFileIdClaims{}) if err != nil { glog.V(1).Infof("jwt verification error from %s: %v", r.RemoteAddr, err) return false From 10404c4275548f89a025820c3347505f17b22cd7 Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Wed, 29 Dec 2021 19:47:53 +0100 Subject: [PATCH 4/7] FEATURE: add JWT to HTTP endpoints of Filer and use them in S3 Client - one JWT for reading and one for writing, analogous to how the JWT between Master and Volume Server works - I did not implement IP `whiteList` parameter on the filer Additionally, because http_util.DownloadFile now sets the JWT, the `download` command should now work when `jwt.signing.read` is configured. By looking at the code, I think this case did not work before. ## Docs to be adjusted after a release Page `Amazon-S3-API`: ``` # Authentication with Filer You can use mTLS for the gRPC connection between S3-API-Proxy and the filer, as explained in [Security-Configuration](Security-Configuration) - controlled by the `grpc.*` configuration in `security.toml`. Starting with version XX, it is also possible to authenticate the HTTP operations between the S3-API-Proxy and the Filer (especially uploading new files). This is configured by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. With both configurations (gRPC and JWT), it is possible to have Filer and S3 communicate in fully authenticated fashion; so Filer will reject any unauthenticated communication. ``` Page `Security Overview`: ``` The following items are not covered, yet: - master server http REST services Starting with version XX, the Filer HTTP REST services can be secured with a JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. ... Before version XX: "weed filer -disableHttp", disable http operations, only gRPC operations are allowed. This works with "weed mount" by FUSE. It does **not work** with the [S3 Gateway](Amazon S3 API), as this does HTTP calls to the Filer. Starting with version XX: secured by JWT, by setting `filer_jwt.signing.key` and `filer_jwt.signing.read.key` in `security.toml`. **This now works with the [S3 Gateway](Amazon S3 API).** ... # Securing Filer HTTP with JWT To enable JWT-based access control for the Filer, 1. generate `security.toml` file by `weed scaffold -config=security` 2. set `filer_jwt.signing.key` to a secret string - and optionally filer_jwt.signing.read.key` as well to a secret string 3. copy the same `security.toml` file to the filers and all S3 proxies. If `filer_jwt.signing.key` is configured: When sending upload/update/delete HTTP operations to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.key`. If `filer_jwt.signing.read.key` is configured: When sending GET or HEAD requests to a filer server, the request header `Authorization` should be the JWT string (`Authorization: Bearer [JwtToken]`). The operation is authorized after the filer validates the JWT with `filer_jwt.signing.read.key`. The S3 API Gateway reads the above JWT keys and sends authenticated HTTP requests to the filer. ``` Page `Security Configuration`: ``` (update scaffold file) ... [filer_jwt.signing] key = "blahblahblahblah" [filer_jwt.signing.read] key = "blahblahblahblah" ``` Resolves: #158 --- weed/command/scaffold/security.toml | 30 +++++++-- weed/s3api/s3api_object_copy_handlers.go | 4 +- weed/s3api/s3api_object_handlers.go | 31 ++++++++-- weed/s3api/s3api_server.go | 21 +++++-- weed/security/jwt.go | 29 +++++++++ weed/server/filer_server.go | 13 +++- weed/server/filer_server_handlers.go | 78 +++++++++++++++++++++--- weed/util/http_util.go | 17 +++++- 8 files changed, 197 insertions(+), 26 deletions(-) diff --git a/weed/command/scaffold/security.toml b/weed/command/scaffold/security.toml index 93b4cc05f..f40f70250 100644 --- a/weed/command/scaffold/security.toml +++ b/weed/command/scaffold/security.toml @@ -4,24 +4,46 @@ # /etc/seaweedfs/security.toml # this file is read by master, volume server, and filer -# the jwt signing key is read by master and volume server. -# a jwt defaults to expire after 10 seconds. +# this jwt signing key is read by master and volume server, and it is used for write operations: +# - the Master server generates the JWT, which can be used to write a certain file on a volume server +# - the Volume server validates the JWT on writing +# the jwt defaults to expire after 10 seconds. [jwt.signing] key = "" expires_after_seconds = 10 # seconds # by default, if the signing key above is set, the Volume UI over HTTP is disabled. # by setting ui.access to true, you can re-enable the Volume UI. Despite -# some information leakage (as the UI is unauthenticted), this should not +# some information leakage (as the UI is not authenticated), this should not # pose a security risk. [access] ui = false -# jwt for read is only supported with master+volume setup. Filer does not support this mode. +# this jwt signing key is read by master and volume server, and it is used for read operations: +# - the Master server generates the JWT, which can be used to read a certain file on a volume server +# - the Volume server validates the JWT on reading +# NOTE: jwt for read is only supported with master+volume setup. Filer does not support this mode. [jwt.signing.read] key = "" expires_after_seconds = 10 # seconds + +# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT: +# - f.e. the S3 API Shim generates the JWT +# - the Filer server validates the JWT on writing +# the jwt defaults to expire after 10 seconds. +[filer_jwt.signing] +key = "" +expires_after_seconds = 10 # seconds + +# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT: +# - f.e. the S3 API Shim generates the JWT +# - the Filer server validates the JWT on writing +# the jwt defaults to expire after 10 seconds. +[filer_jwt.signing.read] +key = "" +expires_after_seconds = 10 # seconds + # all grpc tls authentications are mutual # the values for the following ca, cert, and key are paths to the PERM files. # the host name is not checked, so the PERM files can be shared. diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 7756e1348..8af0cacf1 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -74,7 +74,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request srcUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlPathEscape(srcObject)) - _, _, resp, err := util.DownloadFile(srcUrl, "") + _, _, resp, err := util.DownloadFile(srcUrl, s3a.maybeGetFilerJwtAuthorizationToken(false)) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return @@ -157,7 +157,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req srcUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlPathEscape(srcObject)) - dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader) + dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, s3a.maybeGetFilerJwtAuthorizationToken(false), rangeHeader) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 2ac9c8102..ef27f626a 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -6,6 +6,7 @@ import ( "encoding/json" "encoding/xml" "fmt" + "github.com/chrislusf/seaweedfs/weed/security" "io" "net/http" "net/url" @@ -143,7 +144,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) destUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object)) - s3a.proxyToFiler(w, r, destUrl, passThroughResponse) + s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse) } func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { @@ -154,7 +155,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request destUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object)) - s3a.proxyToFiler(w, r, destUrl, passThroughResponse) + s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse) } func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { @@ -165,7 +166,7 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object)) - s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) { + s3a.proxyToFiler(w, r, destUrl, true, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) { statusCode = http.StatusNoContent for k, v := range proxyResponse.Header { w.Header()[k] = v @@ -306,11 +307,12 @@ func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerCli return } -func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int)) { +func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, isWrite bool, responseFn func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int)) { glog.V(3).Infof("s3 proxying %s to %s", r.Method, destUrl) proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body) + s3a.maybeAddFilerJwtAuthorization(proxyReq, isWrite) if err != nil { glog.Errorf("NewRequest %s: %v", destUrl, err) @@ -374,6 +376,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader var body = io.TeeReader(dataReader, hash) proxyReq, err := http.NewRequest("PUT", uploadUrl, body) + s3a.maybeAddFilerJwtAuthorization(proxyReq, true) if err != nil { glog.Errorf("NewRequest %s: %v", uploadUrl, err) @@ -433,3 +436,23 @@ func filerErrorToS3Error(errString string) s3err.ErrorCode { } return s3err.ErrInternalError } + +func (s3a *S3ApiServer) maybeAddFilerJwtAuthorization(r *http.Request, isWrite bool) { + encodedJwt := s3a.maybeGetFilerJwtAuthorizationToken(isWrite) + + if encodedJwt == "" { + return + } + + r.Header.Add("Authorization", "BEARER "+string(encodedJwt)) +} + +func (s3a *S3ApiServer) maybeGetFilerJwtAuthorizationToken(isWrite bool) string { + var encodedJwt security.EncodedJwt + if isWrite { + encodedJwt = security.GenJwtForFilerServer(s3a.filerGuard.SigningKey, s3a.filerGuard.ExpiresAfterSec) + } else { + encodedJwt = security.GenJwtForFilerServer(s3a.filerGuard.ReadSigningKey, s3a.filerGuard.ReadExpiresAfterSec) + } + return string(encodedJwt) +} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 1abf9259d..b992fdf88 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -3,6 +3,8 @@ package s3api import ( "fmt" "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" "net/http" "strings" "time" @@ -25,14 +27,25 @@ type S3ApiServerOption struct { } type S3ApiServer struct { - option *S3ApiServerOption - iam *IdentityAccessManagement + option *S3ApiServerOption + iam *IdentityAccessManagement + filerGuard *security.Guard } func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) { + v := util.GetViper() + signingKey := v.GetString("jwt.filer_signing.key") + v.SetDefault("jwt.filer_signing.expires_after_seconds", 10) + expiresAfterSec := v.GetInt("jwt.filer_signing.expires_after_seconds") + + readSigningKey := v.GetString("jwt.filer_signing.read.key") + v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60) + readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds") + s3ApiServer = &S3ApiServer{ - option: option, - iam: NewIdentityAccessManagement(option), + option: option, + iam: NewIdentityAccessManagement(option), + filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec), } s3ApiServer.registerRouter(router) diff --git a/weed/security/jwt.go b/weed/security/jwt.go index 1976c8ffe..82ba0df12 100644 --- a/weed/security/jwt.go +++ b/weed/security/jwt.go @@ -20,6 +20,13 @@ type SeaweedFileIdClaims struct { jwt.StandardClaims } +// SeaweedFilerClaims is created e.g. by S3 proxy server and consumed by Filer server. +// Right now, it only contains the standard claims; but this might be extended later +// for more fine-grained permissions. +type SeaweedFilerClaims struct { + jwt.StandardClaims +} + func GenJwtForVolumeServer(signingKey SigningKey, expiresAfterSec int, fileId string) EncodedJwt { if len(signingKey) == 0 { return "" @@ -41,6 +48,28 @@ func GenJwtForVolumeServer(signingKey SigningKey, expiresAfterSec int, fileId st return EncodedJwt(encoded) } +// GenJwtForFilerServer creates a JSON-web-token for using the authenticated Filer API. Used f.e. inside +// the S3 API +func GenJwtForFilerServer(signingKey SigningKey, expiresAfterSec int) EncodedJwt { + if len(signingKey) == 0 { + return "" + } + + claims := SeaweedFilerClaims{ + jwt.StandardClaims{}, + } + if expiresAfterSec > 0 { + claims.ExpiresAt = time.Now().Add(time.Second * time.Duration(expiresAfterSec)).Unix() + } + t := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + encoded, e := t.SignedString([]byte(signingKey)) + if e != nil { + glog.V(0).Infof("Failed to sign claims %+v: %v", t.Claims, e) + return "" + } + return EncodedJwt(encoded) +} + func GetJwt(r *http.Request) EncodedJwt { // Get token from query params diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index c9343a9bf..ccb416f38 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -71,6 +71,7 @@ type FilerServer struct { option *FilerOption secret security.SigningKey filer *filer.Filer + filerGuard *security.Guard grpcDialOption grpc.DialOption // metrics read from the master @@ -90,6 +91,15 @@ type FilerServer struct { func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) { + v := util.GetViper() + signingKey := v.GetString("jwt.filer_signing.key") + v.SetDefault("jwt.filer_signing.expires_after_seconds", 10) + expiresAfterSec := v.GetInt("jwt.filer_signing.expires_after_seconds") + + readSigningKey := v.GetString("jwt.filer_signing.read.key") + v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60) + readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds") + fs = &FilerServer{ option: option, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"), @@ -106,13 +116,14 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs.listenersCond.Broadcast() }) fs.filer.Cipher = option.Cipher + // we do not support IP whitelist right now + fs.filerGuard = security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) fs.checkWithMaster() go stats.LoopPushingMetric("filer", string(fs.option.Host), fs.metricsAddress, fs.metricsIntervalSec) go fs.filer.KeepMasterClientConnected() - v := util.GetViper() if !util.LoadConfiguration("filer", false) { v.Set("leveldb2.enabled", true) v.Set("leveldb2.dir", option.DefaultLevelDbDir) diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 118646a04..6f0d0b7ca 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -1,7 +1,9 @@ package weed_server import ( + "errors" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" "net/http" "strings" @@ -15,6 +17,19 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { start := time.Now() + if r.Method == "OPTIONS" { + stats.FilerRequestCounter.WithLabelValues("options").Inc() + OptionsHandler(w, r, false) + stats.FilerRequestHistogram.WithLabelValues("options").Observe(time.Since(start).Seconds()) + return + } + + isReadHttpCall := r.Method == "GET" || r.Method == "HEAD" + if !fs.maybeCheckJwtAuthorization(r, !isReadHttpCall) { + writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt")) + return + } + // proxy to volume servers var fileId string if strings.HasPrefix(r.RequestURI, "/?proxyChunkId=") { @@ -78,20 +93,31 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { fs.PostHandler(w, r, contentLength) stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) } - case "OPTIONS": - stats.FilerRequestCounter.WithLabelValues("options").Inc() - OptionsHandler(w, r, false) - stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) } } func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) { + + start := time.Now() + + // We handle OPTIONS first because it never should be authenticated + if r.Method == "OPTIONS" { + stats.FilerRequestCounter.WithLabelValues("options").Inc() + OptionsHandler(w, r, true) + stats.FilerRequestHistogram.WithLabelValues("options").Observe(time.Since(start).Seconds()) + return + } + + if !fs.maybeCheckJwtAuthorization(r, false) { + writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt")) + return + } + w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION) if r.Header.Get("Origin") != "" { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Credentials", "true") } - start := time.Now() switch r.Method { case "GET": stats.FilerRequestCounter.WithLabelValues("get").Inc() @@ -101,10 +127,6 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque stats.FilerRequestCounter.WithLabelValues("head").Inc() fs.GetOrHeadHandler(w, r) stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) - case "OPTIONS": - stats.FilerRequestCounter.WithLabelValues("options").Inc() - OptionsHandler(w, r, true) - stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) } } @@ -116,3 +138,41 @@ func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool) { } w.Header().Add("Access-Control-Allow-Headers", "*") } + +// maybeCheckJwtAuthorization returns true if access should be granted, false if it should be denied +func (fs *FilerServer) maybeCheckJwtAuthorization(r *http.Request, isWrite bool) bool { + + var signingKey security.SigningKey + + if isWrite { + if len(fs.filerGuard.SigningKey) == 0 { + return true + } else { + signingKey = fs.filerGuard.SigningKey + } + } else { + if len(fs.filerGuard.ReadSigningKey) == 0 { + return true + } else { + signingKey = fs.filerGuard.ReadSigningKey + } + } + + tokenStr := security.GetJwt(r) + if tokenStr == "" { + glog.V(1).Infof("missing jwt from %s", r.RemoteAddr) + return false + } + + token, err := security.DecodeJwt(signingKey, tokenStr, &security.SeaweedFilerClaims{}) + if err != nil { + glog.V(1).Infof("jwt verification error from %s: %v", r.RemoteAddr, err) + return false + } + if !token.Valid { + glog.V(1).Infof("jwt invalid from %s: %v", r.RemoteAddr, tokenStr) + return false + } else { + return true + } +} diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 7bd6758eb..5c814a6d3 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -180,7 +180,16 @@ func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) e } func DownloadFile(fileUrl string, jwt string) (filename string, header http.Header, resp *http.Response, e error) { - response, err := client.Get(fileUrl) + req, err := http.NewRequest("GET", fileUrl, nil) + if err != nil { + return "", nil, nil, err + } + + if len(jwt) > 0 { + req.Header.Add("Authorization", "BEARER "+jwt) + } + + response, err := client.Do(req) if err != nil { return "", nil, nil, err } @@ -358,7 +367,7 @@ func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool return false, nil } -func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) { +func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (io.ReadCloser, error) { req, err := http.NewRequest("GET", fileUrl, nil) if err != nil { @@ -370,6 +379,10 @@ func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, e req.Header.Add("Accept-Encoding", "gzip") } + if len(jwt) > 0 { + req.Header.Add("Authorization", "BEARER "+jwt) + } + r, err := client.Do(req) if err != nil { return nil, err From 1cd3b6b4e12e4d25e66d1c6203ba1c58081b873b Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Fri, 31 Dec 2021 22:05:41 +0100 Subject: [PATCH 5/7] BUGFIX: security.toml contained wrong keys --- weed/command/scaffold/security.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/command/scaffold/security.toml b/weed/command/scaffold/security.toml index f40f70250..090f4f664 100644 --- a/weed/command/scaffold/security.toml +++ b/weed/command/scaffold/security.toml @@ -32,7 +32,7 @@ expires_after_seconds = 10 # seconds # - f.e. the S3 API Shim generates the JWT # - the Filer server validates the JWT on writing # the jwt defaults to expire after 10 seconds. -[filer_jwt.signing] +[jwt.filer_signing] key = "" expires_after_seconds = 10 # seconds @@ -40,7 +40,7 @@ expires_after_seconds = 10 # seconds # - f.e. the S3 API Shim generates the JWT # - the Filer server validates the JWT on writing # the jwt defaults to expire after 10 seconds. -[filer_jwt.signing.read] +[jwt.filer_signing.read] key = "" expires_after_seconds = 10 # seconds From c35660175d0ffc88a1e0097ec90bd7e000339d14 Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Fri, 31 Dec 2021 22:06:18 +0100 Subject: [PATCH 6/7] BUGFIX: ensure Authorization header is only added once --- weed/s3api/s3api_object_handlers.go | 11 +++++++---- weed/util/http_util.go | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index ef27f626a..13ce60945 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -312,7 +312,6 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des glog.V(3).Infof("s3 proxying %s to %s", r.Method, destUrl) proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body) - s3a.maybeAddFilerJwtAuthorization(proxyReq, isWrite) if err != nil { glog.Errorf("NewRequest %s: %v", destUrl, err) @@ -330,6 +329,9 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des proxyReq.Header[header] = values } + // ensure that the Authorization header is overriding any previous + // Authorization header which might be already present in proxyReq + s3a.maybeAddFilerJwtAuthorization(proxyReq, isWrite) resp, postErr := client.Do(proxyReq) if postErr != nil { @@ -376,7 +378,6 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader var body = io.TeeReader(dataReader, hash) proxyReq, err := http.NewRequest("PUT", uploadUrl, body) - s3a.maybeAddFilerJwtAuthorization(proxyReq, true) if err != nil { glog.Errorf("NewRequest %s: %v", uploadUrl, err) @@ -390,7 +391,9 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader proxyReq.Header.Add(header, value) } } - + // ensure that the Authorization header is overriding any previous + // Authorization header which might be already present in proxyReq + s3a.maybeAddFilerJwtAuthorization(proxyReq, true) resp, postErr := client.Do(proxyReq) if postErr != nil { @@ -444,7 +447,7 @@ func (s3a *S3ApiServer) maybeAddFilerJwtAuthorization(r *http.Request, isWrite b return } - r.Header.Add("Authorization", "BEARER "+string(encodedJwt)) + r.Header.Set("Authorization", "BEARER "+string(encodedJwt)) } func (s3a *S3ApiServer) maybeGetFilerJwtAuthorizationToken(isWrite bool) string { diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 5c814a6d3..e658ab66b 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -186,7 +186,7 @@ func DownloadFile(fileUrl string, jwt string) (filename string, header http.Head } if len(jwt) > 0 { - req.Header.Add("Authorization", "BEARER "+jwt) + req.Header.Set("Authorization", "BEARER "+jwt) } response, err := client.Do(req) @@ -380,7 +380,7 @@ func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (io.R } if len(jwt) > 0 { - req.Header.Add("Authorization", "BEARER "+jwt) + req.Header.Set("Authorization", "BEARER "+jwt) } r, err := client.Do(req) From 99abddf3769a5e4a25c72e67df9106e41b7aa8f3 Mon Sep 17 00:00:00 2001 From: Sebastian Kurfuerst Date: Fri, 31 Dec 2021 22:07:49 +0100 Subject: [PATCH 7/7] FEATURE: add test setup for running the CephFS S3 compatibility suite --- test/s3/compatibility/.gitignore | 2 + test/s3/compatibility/Dockerfile | 11 +++ test/s3/compatibility/README.md | 13 ++++ test/s3/compatibility/prepare.sh | 5 ++ test/s3/compatibility/run.sh | 24 +++++++ test/s3/compatibility/s3tests.conf | 109 +++++++++++++++++++++++++++++ 6 files changed, 164 insertions(+) create mode 100644 test/s3/compatibility/.gitignore create mode 100644 test/s3/compatibility/Dockerfile create mode 100644 test/s3/compatibility/README.md create mode 100755 test/s3/compatibility/prepare.sh create mode 100755 test/s3/compatibility/run.sh create mode 100644 test/s3/compatibility/s3tests.conf diff --git a/test/s3/compatibility/.gitignore b/test/s3/compatibility/.gitignore new file mode 100644 index 000000000..dc3cc5207 --- /dev/null +++ b/test/s3/compatibility/.gitignore @@ -0,0 +1,2 @@ +/s3-tests +/tmp diff --git a/test/s3/compatibility/Dockerfile b/test/s3/compatibility/Dockerfile new file mode 100644 index 000000000..b2a1040cb --- /dev/null +++ b/test/s3/compatibility/Dockerfile @@ -0,0 +1,11 @@ +# the tests only support python 3.6, not newer +FROM ubuntu:latest + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata +RUN git clone https://github.com/ceph/s3-tests.git +WORKDIR s3-tests + +# we pin a certain commit +RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e + +RUN ./bootstrap diff --git a/test/s3/compatibility/README.md b/test/s3/compatibility/README.md new file mode 100644 index 000000000..de1b6e9ec --- /dev/null +++ b/test/s3/compatibility/README.md @@ -0,0 +1,13 @@ +# Running S3 Compatibility tests against SeaweedFS + +This is using [the tests from CephFS](https://github.com/ceph/s3-tests). + +## Prerequisites + +- have Docker installed +- this has been executed on Mac. On Linux, the hostname in `s3tests.conf` needs to be adjusted. + +## Running tests + +- `./prepare.sh` to build the docker image +- `./run.sh` to execute all tests diff --git a/test/s3/compatibility/prepare.sh b/test/s3/compatibility/prepare.sh new file mode 100755 index 000000000..7f9c20746 --- /dev/null +++ b/test/s3/compatibility/prepare.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -ex + +docker build --progress=plain -t s3tests . diff --git a/test/s3/compatibility/run.sh b/test/s3/compatibility/run.sh new file mode 100755 index 000000000..96d630dd7 --- /dev/null +++ b/test/s3/compatibility/run.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -ex + +killall -9 weed || echo "already stopped" +rm -Rf tmp +mkdir tmp +docker stop s3test-instance || echo "already stopped" + +ulimit -n 10000 +../../../weed/weed server -filer -s3 -volume.max 0 -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log & + +until $(curl --output /dev/null --silent --head --fail http://127.0.0.1:9333); do + printf '.' + sleep 5 +done +sleep 3 + +rm -Rf logs-full.txt logs-summary.txt +# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v -a 'resource=object,!bucket-policy,!versioning,!encryption' +docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt + +docker stop s3test-instance || echo "already stopped" +killall -9 weed diff --git a/test/s3/compatibility/s3tests.conf b/test/s3/compatibility/s3tests.conf new file mode 100644 index 000000000..5adb61791 --- /dev/null +++ b/test/s3/compatibility/s3tests.conf @@ -0,0 +1,109 @@ +[DEFAULT] +## this section is just used for host, port and bucket_prefix + +# host set for rgw in vstart.sh +host = host.docker.internal + +# port set for rgw in vstart.sh +port = 8333 + +## say "False" to disable TLS +is_secure = False + +## say "False" to disable SSL Verify +ssl_verify = False + +[fixtures] +## all the buckets created will start with this prefix; +## {random} will be filled with random characters to pad +## the prefix to 30 characters long, and avoid collisions +bucket prefix = yournamehere-{random}- + +[s3 main] +# main display_name set in vstart.sh +display_name = M. Tester + +# main user_idname set in vstart.sh +user_id = testid + +# main email set in vstart.sh +email = tester@ceph.com + +# zonegroup api_name for bucket location +api_name = default + +## main AWS access key +access_key = 0555b35654ad1656d804 + +## main AWS secret key +secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== + +## replace with key id obtained when secret is created, or delete if KMS not tested +#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef + +[s3 alt] +# alt display_name set in vstart.sh +display_name = john.doe +## alt email set in vstart.sh +email = john.doe@example.com + +# alt user_id set in vstart.sh +user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 + +# alt AWS access key set in vstart.sh +access_key = NOPQRSTUVWXYZABCDEFG + +# alt AWS secret key set in vstart.sh +secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm + +[s3 tenant] +# tenant display_name set in vstart.sh +display_name = testx$tenanteduser + +# tenant user_id set in vstart.sh +user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef + +# tenant AWS secret key set in vstart.sh +access_key = HIJKLMNOPQRSTUVWXYZA + +# tenant AWS secret key set in vstart.sh +secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab + +# tenant email set in vstart.sh +email = tenanteduser@example.com + +#following section needs to be added for all sts-tests +[iam] +#used for iam operations in sts-tests +#email from vstart.sh +email = s3@example.com + +#user_id from vstart.sh +user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + +#access_key from vstart.sh +access_key = ABCDEFGHIJKLMNOPQRST + +#secret_key vstart.sh +secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmn + +#display_name from vstart.sh +display_name = youruseridhere + +#following section needs to be added when you want to run Assume Role With Webidentity test +[webidentity] +#used for assume role with web identity test in sts-tests +#all parameters will be obtained from ceph/qa/tasks/keycloak.py +token= + +aud= + +sub= + +azp= + +user_token=] + +thumbprint= + +KC_REALM=