From e9584d96615870176d9fd5317b31695e87ff7b7e Mon Sep 17 00:00:00 2001 From: LHHDZ Date: Sun, 2 Oct 2022 10:18:00 +0800 Subject: [PATCH] add ownership rest apis (#3765) --- weed/s3api/auth_credentials.go | 33 +++++- weed/s3api/auth_credentials_test.go | 94 +++++++++++++++++ weed/s3api/filer_util.go | 16 +++ weed/s3api/s3_constants/header.go | 1 + weed/s3api/s3api_acp.go | 28 +++++ weed/s3api/s3api_bucket_handlers.go | 158 ++++++++++++++++++++++++++++ weed/s3api/s3api_server.go | 8 ++ weed/s3api/s3err/error_handler.go | 11 ++ weed/s3api/s3err/s3api_errors.go | 8 ++ 9 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 weed/s3api/s3api_acp.go diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index a243d6222..46a66a427 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -16,6 +16,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) +var IdentityAnonymous *Identity + type Action string type Iam interface { @@ -32,10 +34,15 @@ type IdentityAccessManagement struct { type Identity struct { Name string + AccountId string Credentials []*Credential Actions []Action } +func (i *Identity) isAnonymous() bool { + return i.Name == AccountAnonymous.Name +} + type Credential struct { AccessKey string SecretKey string @@ -125,9 +132,23 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api for _, ident := range config.Identities { t := &Identity{ Name: ident.Name, + AccountId: AccountAdmin.Id, Credentials: nil, Actions: nil, } + + if ident.Name == AccountAnonymous.Name { + if ident.AccountId != "" && ident.AccountId != AccountAnonymous.Id { + glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid") + } + t.AccountId = AccountAnonymous.Id + IdentityAnonymous = t + } else { + if len(ident.AccountId) > 0 { + t.AccountId = ident.AccountId + } + } + for _, action := range ident.Actions { t.Actions = append(t.Actions, Action(action)) } @@ -139,6 +160,13 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api } identities = append(identities, t) } + + if IdentityAnonymous == nil { + IdentityAnonymous = &Identity{ + Name: AccountAnonymous.Name, + AccountId: AccountAnonymous.Id, + } + } iam.m.Lock() // atomically switch iam.identities = identities @@ -173,7 +201,7 @@ func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, foun iam.m.RLock() defer iam.m.RUnlock() for _, ident := range iam.identities { - if ident.Name == "anonymous" { + if ident.isAnonymous() { return ident, true } } @@ -259,6 +287,9 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) return identity, s3err.ErrAccessDenied } + if !identity.isAnonymous() { + r.Header.Set(s3_constants.AmzAccountId, identity.AccountId) + } return identity, s3err.ErrNone } diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go index d2fa5b216..51a163b98 100644 --- a/weed/s3api/auth_credentials_test.go +++ b/weed/s3api/auth_credentials_test.go @@ -3,6 +3,7 @@ package s3api import ( . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/stretchr/testify/assert" + "reflect" "testing" jsonpb "google.golang.org/protobuf/encoding/protojson" @@ -124,5 +125,98 @@ func TestCanDo(t *testing.T) { } assert.Equal(t, true, ident5.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt")) assert.Equal(t, true, ident5.canDo(ACTION_WRITE, "special_bucket", "/a/b/c/d.txt")) +} + +type LoadS3ApiConfigurationTestCase struct { + pbIdent *iam_pb.Identity + expectIdent *Identity +} + +func TestLoadS3ApiConfiguration(t *testing.T) { + testCases := map[string]*LoadS3ApiConfigurationTestCase{ + "notSpecifyAccountId": { + pbIdent: &iam_pb.Identity{ + Name: "notSpecifyAccountId", + Actions: []string{ + "Read", + "Write", + }, + Credentials: []*iam_pb.Credential{ + { + AccessKey: "some_access_key1", + SecretKey: "some_secret_key2", + }, + }, + }, + expectIdent: &Identity{ + Name: "notSpecifyAccountId", + AccountId: AccountAdmin.Id, + Actions: []Action{ + "Read", + "Write", + }, + Credentials: []*Credential{ + { + AccessKey: "some_access_key1", + SecretKey: "some_secret_key2", + }, + }, + }, + }, + "specifiedAccountID": { + pbIdent: &iam_pb.Identity{ + Name: "specifiedAccountID", + AccountId: "specifiedAccountID", + Actions: []string{ + "Read", + "Write", + }, + }, + expectIdent: &Identity{ + Name: "specifiedAccountID", + AccountId: "specifiedAccountID", + Actions: []Action{ + "Read", + "Write", + }, + }, + }, + "anonymous": { + pbIdent: &iam_pb.Identity{ + Name: "anonymous", + Actions: []string{ + "Read", + "Write", + }, + }, + expectIdent: &Identity{ + Name: "anonymous", + AccountId: "anonymous", + Actions: []Action{ + "Read", + "Write", + }, + }, + }, + } + + config := &iam_pb.S3ApiConfiguration{ + Identities: make([]*iam_pb.Identity, 0), + } + for _, v := range testCases { + config.Identities = append(config.Identities, v.pbIdent) + } + iam := IdentityAccessManagement{} + err := iam.loadS3ApiConfiguration(config) + if err != nil { + return + } + + for _, ident := range iam.identities { + tc := testCases[ident.Name] + if !reflect.DeepEqual(ident, tc.expectIdent) { + t.Error("not expect") + } + } } diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index aab190ff1..c2276b89a 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -91,6 +91,22 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry * return filer_pb.GetEntry(s3a, fullPath) } +func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_pb.Entry) error { + updateEntryRequest := &filer_pb.UpdateEntryRequest{ + Directory: parentDirectoryPath, + Entry: newEntry, + } + + err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + err := filer_pb.UpdateEntry(client, updateEntryRequest) + if err != nil { + return err + } + return nil + }) + return err +} + func objectKey(key *string) *string { if strings.HasPrefix(*key, "/") { t := (*key)[1:] diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index a18955185..5e19d67be 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -43,6 +43,7 @@ const ( // Non-Standard S3 HTTP request constants const ( AmzIdentityId = "s3-identity-id" + AmzAccountId = "s3-account-id" AmzAuthType = "s3-auth-type" AmzIsAdmin = "s3-is-admin" // only set to http request header as a context ) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go new file mode 100644 index 000000000..0a79990f5 --- /dev/null +++ b/weed/s3api/s3api_acp.go @@ -0,0 +1,28 @@ +package s3api + +import ( + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "net/http" +) + +func getAccountId(r *http.Request) string { + id := r.Header.Get(s3_constants.AmzAccountId) + if len(id) == 0 { + return AccountAnonymous.Id + } else { + return id + } +} + +func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode { + metadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return errCode + } + accountId := getAccountId(r) + if accountId == AccountAdmin.Id || accountId == *metadata.Owner.ID { + return s3err.ErrNone + } + return s3err.ErrAccessDenied +} diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index e25316838..9e215db9e 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -5,6 +5,8 @@ import ( "encoding/xml" "errors" "fmt" + "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" + "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" "time" @@ -343,3 +345,159 @@ func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http. func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) { writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"}) } + +// PutBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketOwnershipControls.html +func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *http.Request) { + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("PutBucketOwnershipControls %s", bucket) + + errCode := s3a.checkAccessByOwnership(r, bucket) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + if r.Body == nil || r.Body == http.NoBody { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + + var v s3.OwnershipControls + defer util.CloseRequest(r) + + err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "") + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + + if len(v.Rules) != 1 { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + + printOwnership := true + ownership := *v.Rules[0].ObjectOwnership + switch ownership { + case s3_constants.OwnershipObjectWriter: + case s3_constants.OwnershipBucketOwnerPreferred: + case s3_constants.OwnershipBucketOwnerEnforced: + printOwnership = false + default: + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + + bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket) + if err != nil { + if err == filer_pb.ErrNotFound { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) + return + } + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + + oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey] + if !ok || string(oldOwnership) != ownership { + if bucketEntry.Extended == nil { + bucketEntry.Extended = make(map[string][]byte) + } + bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership) + err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + } + + if printOwnership { + result := &s3.PutBucketOwnershipControlsInput{ + OwnershipControls: &v, + } + s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result) + } else { + writeSuccessResponseEmpty(w, r) + } +} + +// GetBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketOwnershipControls.html +func (s3a *S3ApiServer) GetBucketOwnershipControls(w http.ResponseWriter, r *http.Request) { + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("GetBucketOwnershipControls %s", bucket) + + errCode := s3a.checkAccessByOwnership(r, bucket) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket) + if err != nil { + if err == filer_pb.ErrNotFound { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) + return + } + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + + v, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey] + if !ok { + s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError) + return + } + ownership := string(v) + + result := &s3.PutBucketOwnershipControlsInput{ + OwnershipControls: &s3.OwnershipControls{ + Rules: []*s3.OwnershipControlsRule{ + { + ObjectOwnership: &ownership, + }, + }, + }, + } + + s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result) +} + +// DeleteBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketOwnershipControls.html +func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *http.Request) { + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("PutBucketOwnershipControls %s", bucket) + + errCode := s3a.checkAccessByOwnership(r, bucket) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket) + if err != nil { + if err == filer_pb.ErrNotFound { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) + return + } + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + + _, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey] + if !ok { + s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError) + return + } + + delete(bucketEntry.Extended, s3_constants.ExtOwnershipKey) + err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + + emptyOwnershipControls := &s3.OwnershipControls{ + Rules: []*s3.OwnershipControlsRule{}, + } + s3err.WriteAwsXMLResponse(w, r, http.StatusOK, emptyOwnershipControls) +} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index e94611d6a..2163e557d 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -216,6 +216,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2") // buckets with query + // PutBucketOwnershipControls + bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "") + + //GetBucketOwnershipControls + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "") + + //DeleteBucketOwnershipControls + bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "") // raw buckets diff --git a/weed/s3api/s3err/error_handler.go b/weed/s3api/s3err/error_handler.go index 6c3f13938..3fb04a313 100644 --- a/weed/s3api/s3err/error_handler.go +++ b/weed/s3api/s3err/error_handler.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/xml" "fmt" + "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/glog" "net/http" @@ -19,6 +20,16 @@ const ( MimeXML mimeType = "application/xml" ) +func WriteAwsXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, result interface{}) { + var bytesBuffer bytes.Buffer + err := xmlutil.BuildXML(result, xml.NewEncoder(&bytesBuffer)) + if err != nil { + WriteErrorResponse(w, r, ErrInternalError) + return + } + WriteResponse(w, r, statusCode, bytesBuffer.Bytes(), MimeXML) +} + func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) { WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML) } diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 57f269a2e..0348d4ddc 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -107,6 +107,8 @@ const ( ErrTooManyRequest ErrRequestBytesExceed + + OwnershipControlsNotFoundError ) // error code to APIError structure, these fields carry respective @@ -414,6 +416,12 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Simultaneous request bytes exceed limitations", HTTPStatusCode: http.StatusTooManyRequests, }, + + OwnershipControlsNotFoundError: { + Code: "OwnershipControlsNotFoundError", + Description: "The bucket ownership controls were not found", + HTTPStatusCode: http.StatusNotFound, + }, } // GetAPIError provides API Error for input API error code.