From 03c7953254e75994f98db56e616b5b3eec498a8c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 9 Dec 2020 17:11:49 +0500 Subject: [PATCH 1/8] init Iam Api Server --- weed/command/command.go | 1 + weed/command/iam.go | 97 +++++++++++++++++++++++ weed/iamapi/iamapi_handlers.go | 81 +++++++++++++++++++ weed/iamapi/iamapi_management_handlers.go | 69 ++++++++++++++++ weed/iamapi/iamapi_server.go | 72 +++++++++++++++++ 5 files changed, 320 insertions(+) create mode 100644 weed/command/iam.go create mode 100644 weed/iamapi/iamapi_handlers.go create mode 100644 weed/iamapi/iamapi_management_handlers.go create mode 100644 weed/iamapi/iamapi_server.go diff --git a/weed/command/command.go b/weed/command/command.go index bbc2e0423..fea1dd9d3 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -23,6 +23,7 @@ var Commands = []*Command{ cmdMaster, cmdMount, cmdS3, + cmdIam, cmdMsgBroker, cmdScaffold, cmdServer, diff --git a/weed/command/iam.go b/weed/command/iam.go new file mode 100644 index 000000000..ddcddbec9 --- /dev/null +++ b/weed/command/iam.go @@ -0,0 +1,97 @@ +package command + +import ( + "context" + "fmt" + "net/http" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/iamapi" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/gorilla/mux" + "time" +) + +var ( + iamStandaloneOptions IamOptions +) + +type IamOptions struct { + filer *string + masters *string + port *int +} + +func init() { + cmdIam.Run = runIam // break init cycle + iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "filer server address") + iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers") + iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port") +} + +var cmdIam = &Command{ + UsageLine: "iam [-port=8111] [-filer=] [-masters=,]", + Short: "start a iam API compatible server", + Long: "start a iam API compatible server.", +} + +func runIam(cmd *Command, args []string) bool { + return iamStandaloneOptions.startIamServer() +} + +func (iamopt *IamOptions) startIamServer() bool { + filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*iamopt.filer) + if err != nil { + glog.Fatal(err) + return false + } + + grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + for { + err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) + } + glog.V(0).Infof("IAM read filer configuration: %s", resp) + return nil + }) + if err != nil { + glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress) + time.Sleep(time.Second) + } else { + glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress) + break + } + } + + router := mux.NewRouter().SkipClean(true) + _, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{ + Filer: *iamopt.filer, + Port: *iamopt.port, + FilerGrpcAddress: filerGrpcAddress, + GrpcDialOption: grpcDialOption, + }) + glog.V(0).Info("NewIamApiServer created") + if iamApiServer_err != nil { + glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err) + } + + httpS := &http.Server{Handler: router} + + listenAddress := fmt.Sprintf(":%d", *iamopt.port) + iamApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second) + if err != nil { + glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err) + } + + glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port) + if err = httpS.Serve(iamApiListener); err != nil { + glog.Fatalf("IAM API Server Fail to serve: %v", err) + } + + return true +} diff --git a/weed/iamapi/iamapi_handlers.go b/weed/iamapi/iamapi_handlers.go new file mode 100644 index 000000000..c436ba998 --- /dev/null +++ b/weed/iamapi/iamapi_handlers.go @@ -0,0 +1,81 @@ +package iamapi + +import ( + "bytes" + "encoding/xml" + "fmt" + "strconv" + + "net/http" + "net/url" + "time" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" +) + +type mimeType string + +const ( + mimeNone mimeType = "" + mimeXML mimeType = "application/xml" +) + +func setCommonHeaders(w http.ResponseWriter) { + w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano())) + w.Header().Set("Accept-Ranges", "bytes") +} + +// Encodes the response headers into XML format. +func encodeResponse(response interface{}) []byte { + var bytesBuffer bytes.Buffer + bytesBuffer.WriteString(xml.Header) + e := xml.NewEncoder(&bytesBuffer) + e.Encode(response) + return bytesBuffer.Bytes() +} + +// If none of the http routes match respond with MethodNotAllowed +func notFoundHandler(w http.ResponseWriter, r *http.Request) { + glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI) + writeErrorResponse(w, s3err.ErrMethodNotAllowed, r.URL) +} + +func writeErrorResponse(w http.ResponseWriter, errorCode s3err.ErrorCode, reqURL *url.URL) { + apiError := s3err.GetAPIError(errorCode) + errorResponse := getRESTErrorResponse(apiError, reqURL.Path) + encodedErrorResponse := encodeResponse(errorResponse) + writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML) +} + +func getRESTErrorResponse(err s3err.APIError, resource string) s3err.RESTErrorResponse { + return s3err.RESTErrorResponse{ + Code: err.Code, + Message: err.Description, + Resource: resource, + RequestID: fmt.Sprintf("%d", time.Now().UnixNano()), + } +} + +func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { + setCommonHeaders(w) + if response != nil { + w.Header().Set("Content-Length", strconv.Itoa(len(response))) + } + if mType != mimeNone { + w.Header().Set("Content-Type", string(mType)) + } + w.WriteHeader(statusCode) + if response != nil { + glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response)) + _, err := w.Write(response) + if err != nil { + glog.V(0).Infof("write err: %v", err) + } + w.(http.Flusher).Flush() + } +} + +func writeSuccessResponseXML(w http.ResponseWriter, response []byte) { + writeResponse(w, http.StatusOK, response, mimeXML) +} diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go new file mode 100644 index 000000000..a46a506f1 --- /dev/null +++ b/weed/iamapi/iamapi_management_handlers.go @@ -0,0 +1,69 @@ +package iamapi + +import ( + "encoding/xml" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "net/http" + "net/url" + + // "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" +) + +const ( + version = "2010-05-08" +) + +type ListUsersResponse struct { + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListUsersResponse"` + ListUsersResult struct { + Users []*iam.User `xml:"Users>member"` + IsTruncated bool `xml:"IsTruncated"` + } `xml:"ListUsersResult"` + ResponseMetadata struct { + RequestId string `xml:"RequestId"` + } `xml:"ResponseMetadata"` +} + +// {'Action': 'CreateUser', 'Version': '2010-05-08', 'UserName': 'Bob'} +// {'Action': 'ListUsers', 'Version': '2010-05-08'} +func (iama *IamApiServer) ListUsers(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) ListUsersResponse { + glog.Info("Do ListUsers") + resp := ListUsersResponse{} + for _, ident := range s3cfg.Identities { + resp.ListUsersResult.Users = append(resp.ListUsersResult.Users, &iam.User{UserName: &ident.Name}) + } + return resp +} + +func (iama *IamApiServer) ListAccessKeys(values url.Values) ListUsersResponse { + return ListUsersResponse{} +} + +func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) + return + } + values := r.PostForm + s3cfg := &iam_pb.S3ApiConfiguration{} + if err := iama.GetS3ApiConfiguration(s3cfg); err != nil { + writeErrorResponse(w, s3err.ErrInternalError, r.URL) + return + } + + glog.Info("values ", values) + var response interface{} + switch r.Form.Get("Action") { + case "ListUsers": + response = iama.ListUsers(s3cfg, values) + case "ListAccessKeys": + response = iama.ListAccessKeys(values) + default: + writeErrorResponse(w, s3err.ErrNotImplemented, r.URL) + return + } + writeSuccessResponseXML(w, encodeResponse(response)) +} diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go new file mode 100644 index 000000000..00c4a69a2 --- /dev/null +++ b/weed/iamapi/iamapi_server.go @@ -0,0 +1,72 @@ +package iamapi + +// https://docs.aws.amazon.com/cli/latest/reference/iam/list-roles.html +// https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateRole.html + +import ( + "bytes" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "github.com/gorilla/mux" + "google.golang.org/grpc" + "net/http" + "strings" +) + +type IamServerOption struct { + Masters string + Filer string + Port int + FilerGrpcAddress string + GrpcDialOption grpc.DialOption +} + +type IamApiServer struct { + option *IamServerOption + masterClient *wdclient.MasterClient + filerclient *filer_pb.SeaweedFilerClient +} + +func NewIamApiServer(router *mux.Router, option *IamServerOption) (iamApiServer *IamApiServer, err error) { + iamApiServer = &IamApiServer{ + option: option, + masterClient: wdclient.NewMasterClient(option.GrpcDialOption, pb.AdminShellClient, "", 0, "", strings.Split(option.Masters, ",")), + } + + iamApiServer.registerRouter(router) + + return iamApiServer, nil +} + +func (iama *IamApiServer) registerRouter(router *mux.Router) { + // API Router + apiRouter := router.PathPrefix("/").Subrouter() + // ListBuckets + + // apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST")) + apiRouter.Path("/").Methods("POST").HandlerFunc(iama.DoActions) + // NotFound + apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler) +} + +func (iama *IamApiServer) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) { + var buf bytes.Buffer + err = pb.WithGrpcFilerClient(iama.option.FilerGrpcAddress, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + if err = filer.ReadEntry(iama.masterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf); err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + if buf.Len() > 0 { + if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), s3cfg); err != nil { + return err + } + } + return nil +} From d7719d0542f58d9f12448b8bfe43c4ebc0374a17 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 10 Dec 2020 00:03:14 +0500 Subject: [PATCH 2/8] base handlers --- weed/iamapi/iamapi_management_handlers.go | 167 ++++++++++++++++++++-- 1 file changed, 155 insertions(+), 12 deletions(-) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index a46a506f1..2347c9a80 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -2,44 +2,179 @@ package iamapi import ( "encoding/xml" + "fmt" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "math/rand" "net/http" "net/url" + "time" // "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" ) const ( - version = "2010-05-08" + version = "2010-05-08" + charsetUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + charset = charsetUpper + "abcdefghijklmnopqrstuvwxyz/" ) +var seededRand *rand.Rand = rand.New( + rand.NewSource(time.Now().UnixNano())) + +type Response interface { + SetRequestId() +} + +type CommonResponse struct { + ResponseMetadata struct { + RequestId string `xml:"RequestId"` + } `xml:"ResponseMetadata"` +} + type ListUsersResponse struct { + CommonResponse XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListUsersResponse"` ListUsersResult struct { Users []*iam.User `xml:"Users>member"` IsTruncated bool `xml:"IsTruncated"` } `xml:"ListUsersResult"` - ResponseMetadata struct { - RequestId string `xml:"RequestId"` - } `xml:"ResponseMetadata"` } -// {'Action': 'CreateUser', 'Version': '2010-05-08', 'UserName': 'Bob'} -// {'Action': 'ListUsers', 'Version': '2010-05-08'} -func (iama *IamApiServer) ListUsers(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) ListUsersResponse { - glog.Info("Do ListUsers") - resp := ListUsersResponse{} +type ListAccessKeysResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAccessKeysResponse"` + ListAccessKeysResult struct { + AccessKeyMetadata []*iam.AccessKeyMetadata `xml:"AccessKeyMetadata>member"` + IsTruncated bool `xml:"IsTruncated"` + } `xml:"ListAccessKeysResult"` +} + +type DeleteUserResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserResponse"` +} + +type DeleteAccessKeyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteAccessKeyResponse"` +} + +type CreateUserResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateUserResponse"` + CreateUserResult struct { + User iam.User `xml:"User"` + } `xml:"CreateUserResult"` +} + +type CreateAccessKeyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"` + CreateAccessKeyResult struct { + AccessKey iam.AccessKey `xml:"AccessKey"` + } `xml:"CreateAccessKeyResult"` +} + +func (r *CommonResponse) SetRequestId() { + r.ResponseMetadata.RequestId = fmt.Sprintf("%d", time.Now().UnixNano()) +} + +func StringWithCharset(length int, charset string) string { + b := make([]byte, length) + for i := range b { + b[i] = charset[seededRand.Intn(len(charset))] + } + return string(b) +} + +func (iama *IamApiServer) ListUsers(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListUsersResponse) { for _, ident := range s3cfg.Identities { resp.ListUsersResult.Users = append(resp.ListUsersResult.Users, &iam.User{UserName: &ident.Name}) } return resp } -func (iama *IamApiServer) ListAccessKeys(values url.Values) ListUsersResponse { - return ListUsersResponse{} +func (iama *IamApiServer) ListAccessKeys(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListAccessKeysResponse) { + status := iam.StatusTypeActive + for _, ident := range s3cfg.Identities { + for _, cred := range ident.Credentials { + resp.ListAccessKeysResult.AccessKeyMetadata = append(resp.ListAccessKeysResult.AccessKeyMetadata, + &iam.AccessKeyMetadata{UserName: &ident.Name, AccessKeyId: &cred.AccessKey, Status: &status}, + ) + } + } + return resp +} + +func (iama *IamApiServer) CreateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateUserResponse) { + userName := values.Get("UserName") + resp.CreateUserResult.User.UserName = &userName + s3cfg.Identities = append(s3cfg.Identities, &iam_pb.Identity{Name: userName}) + return resp +} + +func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteUserResponse) { + userName := values.Get("UserName") + for i, ident := range s3cfg.Identities { + if userName == ident.Name { + ident.Credentials = append(ident.Credentials[:i], ident.Credentials[i+1:]...) + break + } + } + return resp +} + +func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateAccessKeyResponse) { + userName := values.Get("UserName") + status := iam.StatusTypeActive + accessKeyId := StringWithCharset(21, charsetUpper) + secretAccessKey := StringWithCharset(42, charset) + resp.CreateAccessKeyResult.AccessKey.AccessKeyId = &accessKeyId + resp.CreateAccessKeyResult.AccessKey.SecretAccessKey = &secretAccessKey + resp.CreateAccessKeyResult.AccessKey.UserName = &userName + resp.CreateAccessKeyResult.AccessKey.Status = &status + changed := false + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + ident.Credentials = append(ident.Credentials, + &iam_pb.Credential{AccessKey: accessKeyId, SecretKey: secretAccessKey}) + changed = true + break + } + } + if !changed { + s3cfg.Identities = append(s3cfg.Identities, + &iam_pb.Identity{Name: userName, + Credentials: []*iam_pb.Credential{ + { + AccessKey: accessKeyId, + SecretKey: secretAccessKey, + }, + }, + }, + ) + } + return resp +} + +func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteAccessKeyResponse) { + userName := values.Get("UserName") + accessKeyId := values.Get("AccessKeyId") + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + for i, cred := range ident.Credentials { + if cred.AccessKey == accessKeyId { + ident.Credentials = append(ident.Credentials[:i], ident.Credentials[i+1:]...) + break + } + } + break + } + } + return resp } func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { @@ -60,7 +195,15 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { case "ListUsers": response = iama.ListUsers(s3cfg, values) case "ListAccessKeys": - response = iama.ListAccessKeys(values) + response = iama.ListAccessKeys(s3cfg, values) + case "CreateUser": + response = iama.CreateUser(s3cfg, values) + case "DeleteUser": + response = iama.DeleteUser(s3cfg, values) + case "CreateAccessKey": + response = iama.CreateAccessKey(s3cfg, values) + case "DeleteAccessKey": + response = iama.DeleteAccessKey(s3cfg, values) default: writeErrorResponse(w, s3err.ErrNotImplemented, r.URL) return From 82b0463fac817d434fd9272d2745bd4c2fe75329 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 10 Dec 2020 15:36:04 +0500 Subject: [PATCH 3/8] handler PutUserPolicy https://docs.aws.amazon.com/IAM/latest/APIReference/API_PutUserPolicy.html --- weed/iamapi/iamapi_management_handlers.go | 143 +++++++++++++++++++++- 1 file changed, 139 insertions(+), 4 deletions(-) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 2347c9a80..22bc8748a 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -1,14 +1,18 @@ package iamapi import ( + "crypto/sha1" + "encoding/json" "encoding/xml" "fmt" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "math/rand" "net/http" "net/url" + "strings" "time" // "github.com/aws/aws-sdk-go/aws" @@ -21,11 +25,19 @@ const ( charset = charsetUpper + "abcdefghijklmnopqrstuvwxyz/" ) -var seededRand *rand.Rand = rand.New( - rand.NewSource(time.Now().UnixNano())) +var ( + seededRand *rand.Rand = rand.New( + rand.NewSource(time.Now().UnixNano())) + policyDocuments = map[string]*PolicyDocument{} +) -type Response interface { - SetRequestId() +type PolicyDocument struct { + Version string `json:"Version"` + Statement []struct { + Effect string `json:"Effect"` + Action []string `json:"Action"` + Resource []string `json:"Resource"` + } `json:"Statement"` } type CommonResponse struct { @@ -62,6 +74,14 @@ type DeleteAccessKeyResponse struct { XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteAccessKeyResponse"` } +type CreatePolicyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreatePolicyResponse"` + CreatePolicyResult struct { + Policy iam.Policy `xml:"Policy"` + } `xml:"CreatePolicyResult"` +} + type CreateUserResponse struct { CommonResponse XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateUserResponse"` @@ -78,10 +98,21 @@ type CreateAccessKeyResponse struct { } `xml:"CreateAccessKeyResult"` } +type PutUserPolicyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutUserPolicyResponse"` +} + func (r *CommonResponse) SetRequestId() { r.ResponseMetadata.RequestId = fmt.Sprintf("%d", time.Now().UnixNano()) } +func Hash(s *string) string { + h := sha1.New() + h.Write([]byte(*s)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + func StringWithCharset(length int, charset string) string { b := make([]byte, length) for i := range b { @@ -115,6 +146,96 @@ func (iama *IamApiServer) CreateUser(s3cfg *iam_pb.S3ApiConfiguration, values ur s3cfg.Identities = append(s3cfg.Identities, &iam_pb.Identity{Name: userName}) return resp } +func GetPolicyDocument(policy *string) (policyDocument PolicyDocument, err error) { + if err = json.Unmarshal([]byte(*policy), &policyDocument); err != nil { + return PolicyDocument{}, err + } + return policyDocument, err +} + +func (iama *IamApiServer) CreatePolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreatePolicyResponse, err error) { + policyName := values.Get("PolicyName") + policyDocumentString := values.Get("PolicyDocument") + policyDocument, err := GetPolicyDocument(&policyDocumentString) + if err != nil { + return CreatePolicyResponse{}, err + } + policyId := Hash(&policyDocumentString) + arn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName) + resp.CreatePolicyResult.Policy.PolicyName = &policyName + resp.CreatePolicyResult.Policy.Arn = &arn + resp.CreatePolicyResult.Policy.PolicyId = &policyId + policyDocuments[policyName] = &policyDocument + return resp, nil +} + +func (iama *IamApiServer) PutUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp PutUserPolicyResponse, err error) { + userName := values.Get("UserName") + policyName := values.Get("PolicyName") + policyDocumentString := values.Get("PolicyDocument") + policyDocument, err := GetPolicyDocument(&policyDocumentString) + if err != nil { + return PutUserPolicyResponse{}, err + } + policyDocuments[policyName] = &policyDocument + actions := GetActions(&policyDocument) + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + for _, action := range actions { + ident.Actions = append(ident.Actions, action) + } + break + } + } + return resp, nil +} + +func MapAction(action string) string { + switch action { + case "*": + return s3_constants.ACTION_ADMIN + case "Put*": + return s3_constants.ACTION_WRITE + case "Get*": + return s3_constants.ACTION_READ + case "List*": + return s3_constants.ACTION_LIST + default: + return s3_constants.ACTION_TAGGING + } +} + +func GetActions(policy *PolicyDocument) (actions []string) { + for _, statement := range policy.Statement { + if statement.Effect != "Allow" { + continue + } + for _, resource := range statement.Resource { + // Parse "arn:aws:s3:::my-bucket/shared/*" + res := strings.Split(resource, ":") + if len(res) != 6 || res[0] != "arn:" || res[1] != "aws" || res[2] != "s3" { + continue + } + for _, action := range statement.Action { + // Parse "s3:Get*" + act := strings.Split(action, ":") + if len(act) != 2 || act[0] != "s3" { + continue + } + if res[5] == "*" { + actions = append(actions, MapAction(act[1])) + continue + } + // Parse my-bucket/shared/* + path := strings.Split(res[5], "/") + if len(path) != 2 || path[1] != "*" { + actions = append(actions, fmt.Sprintf("%s:%s", MapAction(act[1]), path[0])) + } + } + } + } + return actions +} func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteUserResponse) { userName := values.Get("UserName") @@ -204,6 +325,20 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { response = iama.CreateAccessKey(s3cfg, values) case "DeleteAccessKey": response = iama.DeleteAccessKey(s3cfg, values) + case "CreatePolicy": + var err error + response, err = iama.CreatePolicy(s3cfg, values) + if err != nil { + writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) + return + } + case "PutUserPolicy": + var err error + response, err = iama.PutUserPolicy(s3cfg, values) + if err != nil { + writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) + return + } default: writeErrorResponse(w, s3err.ErrNotImplemented, r.URL) return From 9f26f2815c756d2125bf35032058ddd0d0efba1d Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 10 Dec 2020 17:03:55 +0500 Subject: [PATCH 4/8] SaveAs S3 Configuration --- weed/filer/read_write.go | 8 ++++-- weed/iamapi/iamapi_management_handlers.go | 33 +++++++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/weed/filer/read_write.go b/weed/filer/read_write.go index 7a6da3beb..bf9159b41 100644 --- a/weed/filer/read_write.go +++ b/weed/filer/read_write.go @@ -41,8 +41,12 @@ func ReadContent(filerAddress string, dir, name string) ([]byte, error) { } func SaveAs(host string, port int, dir, name string, contentType string, byteBuffer *bytes.Buffer) error { - - target := fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name) + var target string + if port == 0 { + target = fmt.Sprintf("http://%s%s/%s", host, dir, name) + } else { + target = fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name) + } // set the HTTP method, url, and request body req, err := http.NewRequest(http.MethodPut, target, byteBuffer) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 22bc8748a..4316f0fd5 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -1,10 +1,12 @@ package iamapi import ( + "bytes" "crypto/sha1" "encoding/json" "encoding/xml" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" @@ -213,13 +215,15 @@ func GetActions(policy *PolicyDocument) (actions []string) { for _, resource := range statement.Resource { // Parse "arn:aws:s3:::my-bucket/shared/*" res := strings.Split(resource, ":") - if len(res) != 6 || res[0] != "arn:" || res[1] != "aws" || res[2] != "s3" { + if len(res) != 6 || res[0] != "arn" || res[1] != "aws" || res[2] != "s3" { + glog.Infof("not math resource: %s", res) continue } for _, action := range statement.Action { // Parse "s3:Get*" act := strings.Split(action, ":") if len(act) != 2 || act[0] != "s3" { + glog.Infof("not match action: %s", act) continue } if res[5] == "*" { @@ -229,8 +233,11 @@ func GetActions(policy *PolicyDocument) (actions []string) { // Parse my-bucket/shared/* path := strings.Split(res[5], "/") if len(path) != 2 || path[1] != "*" { - actions = append(actions, fmt.Sprintf("%s:%s", MapAction(act[1]), path[0])) + glog.Infof("not match bucket: %s", path) + continue } + actions = append(actions, fmt.Sprintf("%s:%s", MapAction(act[1]), path[0])) + } } } @@ -312,11 +319,14 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { glog.Info("values ", values) var response interface{} + changed := true switch r.Form.Get("Action") { case "ListUsers": response = iama.ListUsers(s3cfg, values) + changed = false case "ListAccessKeys": response = iama.ListAccessKeys(s3cfg, values) + changed = false case "CreateUser": response = iama.CreateUser(s3cfg, values) case "DeleteUser": @@ -343,5 +353,24 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { writeErrorResponse(w, s3err.ErrNotImplemented, r.URL) return } + if changed { + buf := bytes.Buffer{} + if err := filer.S3ConfigurationToText(&buf, s3cfg); err != nil { + glog.Error("S3ConfigurationToText: ", err) + writeErrorResponse(w, s3err.ErrInternalError, r.URL) + return + } + if err := filer.SaveAs( + iama.option.Filer, + 0, + filer.IamConfigDirecotry, + filer.IamIdentityFile, + "text/plain; charset=utf-8", + &buf); err != nil { + glog.Error("SaveAs: ", err) + writeErrorResponse(w, s3err.ErrInternalError, r.URL) + return + } + } writeSuccessResponseXML(w, encodeResponse(response)) } From 40938d6a47136a47f612d7a4c93862ff5e17ec45 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 10 Dec 2020 21:03:25 +0500 Subject: [PATCH 5/8] SaveInsideFiler S3 Configuration --- weed/iamapi/iamapi_management_handlers.go | 24 +++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 4316f0fd5..322c16d73 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -8,6 +8,8 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" @@ -216,7 +218,7 @@ func GetActions(policy *PolicyDocument) (actions []string) { // Parse "arn:aws:s3:::my-bucket/shared/*" res := strings.Split(resource, ":") if len(res) != 6 || res[0] != "arn" || res[1] != "aws" || res[2] != "s3" { - glog.Infof("not math resource: %s", res) + glog.Infof("not match resource: %s", res) continue } for _, action := range statement.Action { @@ -237,7 +239,6 @@ func GetActions(policy *PolicyDocument) (actions []string) { continue } actions = append(actions, fmt.Sprintf("%s:%s", MapAction(act[1]), path[0])) - } } } @@ -360,14 +361,17 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { writeErrorResponse(w, s3err.ErrInternalError, r.URL) return } - if err := filer.SaveAs( - iama.option.Filer, - 0, - filer.IamConfigDirecotry, - filer.IamIdentityFile, - "text/plain; charset=utf-8", - &buf); err != nil { - glog.Error("SaveAs: ", err) + err := pb.WithGrpcFilerClient( + iama.option.FilerGrpcAddress, + iama.option.GrpcDialOption, + func(client filer_pb.SeaweedFilerClient) error { + if err := filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile, buf.Bytes()); err != nil { + return err + } + return nil + }, + ) + if err != nil { writeErrorResponse(w, s3err.ErrInternalError, r.URL) return } From 8a95f9c10c3d4c9e1f6761f5620da3e5253398f5 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Mon, 29 Mar 2021 12:01:44 +0500 Subject: [PATCH 6/8] iam GetUser --- weed/command/filer.go | 25 ++++- weed/iamapi/iamapi_handlers.go | 18 ++++ weed/iamapi/iamapi_management_handlers.go | 119 +++++++--------------- weed/iamapi/iamapi_response.go | 93 +++++++++++++++++ 4 files changed, 167 insertions(+), 88 deletions(-) create mode 100644 weed/iamapi/iamapi_response.go diff --git a/weed/command/filer.go b/weed/command/filer.go index 534bd9e04..b256e385f 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -25,6 +25,8 @@ var ( filerS3Options S3Options filerStartWebDav *bool filerWebDavOptions WebDavOption + filerStartIam *bool + filerIamOptions IamOptions ) type FilerOptions struct { @@ -89,6 +91,10 @@ func init() { filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB") + + // start iam on filer + filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service") + filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port") } var cmdFiler = &Command{ @@ -119,22 +125,33 @@ func runFiler(cmd *Command, args []string) bool { go stats_collect.StartMetricsServer(*f.metricsHttpPort) + filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port) + startDelay := time.Duration(2) if *filerStartS3 { - filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port) filerS3Options.filer = &filerAddress go func() { - time.Sleep(2 * time.Second) + time.Sleep(startDelay * time.Second) filerS3Options.startS3Server() }() + startDelay++ } if *filerStartWebDav { - filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port) filerWebDavOptions.filer = &filerAddress go func() { - time.Sleep(2 * time.Second) + time.Sleep(startDelay * time.Second) filerWebDavOptions.startWebDav() }() + startDelay++ + } + + if *filerStartIam { + filerIamOptions.filer = &filerAddress + filerIamOptions.masters = f.masters + go func() { + time.Sleep(startDelay * time.Second) + filerIamOptions.startIamServer() + }() } f.startFiler() diff --git a/weed/iamapi/iamapi_handlers.go b/weed/iamapi/iamapi_handlers.go index c436ba998..962717ad9 100644 --- a/weed/iamapi/iamapi_handlers.go +++ b/weed/iamapi/iamapi_handlers.go @@ -12,6 +12,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + + "github.com/aws/aws-sdk-go/service/iam" ) type mimeType string @@ -48,6 +50,22 @@ func writeErrorResponse(w http.ResponseWriter, errorCode s3err.ErrorCode, reqURL writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML) } +func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string) { + errCode := err.Error() + errorResp := ErrorResponse{} + errorResp.Error.Type = "Sender" + errorResp.Error.Code = &errCode + switch errCode { + case iam.ErrCodeNoSuchEntityException: + msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value) + errorResp.Error.Message = &msg + writeResponse(w, http.StatusNotFound, encodeResponse(errorResp), mimeXML) + default: + writeResponse(w, http.StatusInternalServerError, encodeResponse(errorResp), mimeXML) + + } +} + func getRESTErrorResponse(err s3err.APIError, resource string) s3err.RESTErrorResponse { return s3err.RESTErrorResponse{ Code: err.Code, diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 322c16d73..e4daa081f 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/sha1" "encoding/json" - "encoding/xml" "fmt" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" @@ -19,12 +18,10 @@ import ( "strings" "time" - // "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" ) const ( - version = "2010-05-08" charsetUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" charset = charsetUpper + "abcdefghijklmnopqrstuvwxyz/" ) @@ -44,73 +41,6 @@ type PolicyDocument struct { } `json:"Statement"` } -type CommonResponse struct { - ResponseMetadata struct { - RequestId string `xml:"RequestId"` - } `xml:"ResponseMetadata"` -} - -type ListUsersResponse struct { - CommonResponse - XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListUsersResponse"` - ListUsersResult struct { - Users []*iam.User `xml:"Users>member"` - IsTruncated bool `xml:"IsTruncated"` - } `xml:"ListUsersResult"` -} - -type ListAccessKeysResponse struct { - CommonResponse - XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAccessKeysResponse"` - ListAccessKeysResult struct { - AccessKeyMetadata []*iam.AccessKeyMetadata `xml:"AccessKeyMetadata>member"` - IsTruncated bool `xml:"IsTruncated"` - } `xml:"ListAccessKeysResult"` -} - -type DeleteUserResponse struct { - CommonResponse - XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserResponse"` -} - -type DeleteAccessKeyResponse struct { - CommonResponse - XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteAccessKeyResponse"` -} - -type CreatePolicyResponse struct { - CommonResponse - XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreatePolicyResponse"` - CreatePolicyResult struct { - Policy iam.Policy `xml:"Policy"` - } `xml:"CreatePolicyResult"` -} - -type CreateUserResponse struct { - CommonResponse - XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateUserResponse"` - CreateUserResult struct { - User iam.User `xml:"User"` - } `xml:"CreateUserResult"` -} - -type CreateAccessKeyResponse struct { - CommonResponse - XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"` - CreateAccessKeyResult struct { - AccessKey iam.AccessKey `xml:"AccessKey"` - } `xml:"CreateAccessKeyResult"` -} - -type PutUserPolicyResponse struct { - CommonResponse - XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutUserPolicyResponse"` -} - -func (r *CommonResponse) SetRequestId() { - r.ResponseMetadata.RequestId = fmt.Sprintf("%d", time.Now().UnixNano()) -} - func Hash(s *string) string { h := sha1.New() h.Write([]byte(*s)) @@ -150,6 +80,27 @@ func (iama *IamApiServer) CreateUser(s3cfg *iam_pb.S3ApiConfiguration, values ur s3cfg.Identities = append(s3cfg.Identities, &iam_pb.Identity{Name: userName}) return resp } + +func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp DeleteUserResponse, err error) { + for i, ident := range s3cfg.Identities { + if userName == ident.Name { + ident.Credentials = append(ident.Credentials[:i], ident.Credentials[i+1:]...) + return resp, nil + } + } + return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException) +} + +func (iama *IamApiServer) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (resp GetUserResponse, err error) { + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + resp.GetUserResult.User = iam.User{UserName: &ident.Name} + return resp, nil + } + } + return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException) +} + func GetPolicyDocument(policy *string) (policyDocument PolicyDocument, err error) { if err = json.Unmarshal([]byte(*policy), &policyDocument); err != nil { return PolicyDocument{}, err @@ -245,17 +196,6 @@ func GetActions(policy *PolicyDocument) (actions []string) { return actions } -func (iama *IamApiServer) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteUserResponse) { - userName := values.Get("UserName") - for i, ident := range s3cfg.Identities { - if userName == ident.Name { - ident.Credentials = append(ident.Credentials[:i], ident.Credentials[i+1:]...) - break - } - } - return resp -} - func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateAccessKeyResponse) { userName := values.Get("UserName") status := iam.StatusTypeActive @@ -320,6 +260,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { glog.Info("values ", values) var response interface{} + var err error changed := true switch r.Form.Get("Action") { case "ListUsers": @@ -330,21 +271,31 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { changed = false case "CreateUser": response = iama.CreateUser(s3cfg, values) + case "GetUser": + userName := values.Get("UserName") + response, err = iama.GetUser(s3cfg, userName) + if err != nil { + writeIamErrorResponse(w, err, "user", userName) + return + } case "DeleteUser": - response = iama.DeleteUser(s3cfg, values) + userName := values.Get("UserName") + response, err = iama.DeleteUser(s3cfg, userName) + if err != nil { + writeIamErrorResponse(w, err, "user", userName) + return + } case "CreateAccessKey": response = iama.CreateAccessKey(s3cfg, values) case "DeleteAccessKey": response = iama.DeleteAccessKey(s3cfg, values) case "CreatePolicy": - var err error response, err = iama.CreatePolicy(s3cfg, values) if err != nil { writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) return } case "PutUserPolicy": - var err error response, err = iama.PutUserPolicy(s3cfg, values) if err != nil { writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) diff --git a/weed/iamapi/iamapi_response.go b/weed/iamapi/iamapi_response.go new file mode 100644 index 000000000..26dd0f263 --- /dev/null +++ b/weed/iamapi/iamapi_response.go @@ -0,0 +1,93 @@ +package iamapi + +import ( + "encoding/xml" + "fmt" + "time" + + "github.com/aws/aws-sdk-go/service/iam" +) + +type CommonResponse struct { + ResponseMetadata struct { + RequestId string `xml:"RequestId"` + } `xml:"ResponseMetadata"` +} + +type ListUsersResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListUsersResponse"` + ListUsersResult struct { + Users []*iam.User `xml:"Users>member"` + IsTruncated bool `xml:"IsTruncated"` + } `xml:"ListUsersResult"` +} + +type ListAccessKeysResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAccessKeysResponse"` + ListAccessKeysResult struct { + AccessKeyMetadata []*iam.AccessKeyMetadata `xml:"AccessKeyMetadata>member"` + IsTruncated bool `xml:"IsTruncated"` + } `xml:"ListAccessKeysResult"` +} + +type DeleteAccessKeyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteAccessKeyResponse"` +} + +type CreatePolicyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreatePolicyResponse"` + CreatePolicyResult struct { + Policy iam.Policy `xml:"Policy"` + } `xml:"CreatePolicyResult"` +} + +type CreateUserResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateUserResponse"` + CreateUserResult struct { + User iam.User `xml:"User"` + } `xml:"CreateUserResult"` +} + +type DeleteUserResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserResponse"` +} + +type GetUserResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserResponse"` + GetUserResult struct { + User iam.User `xml:"User"` + } `xml:"GetUserResult"` +} + +type CreateAccessKeyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"` + CreateAccessKeyResult struct { + AccessKey iam.AccessKey `xml:"AccessKey"` + } `xml:"CreateAccessKeyResult"` +} + +type PutUserPolicyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutUserPolicyResponse"` +} + +type ErrorResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ErrorResponse"` + Error struct { + iam.ErrorDetails + Type string `xml:"Type"` + } `xml:"Error"` +} + +func (r *CommonResponse) SetRequestId() { + r.ResponseMetadata.RequestId = fmt.Sprintf("%d", time.Now().UnixNano()) +} From ed79baa30fe5687a35a9a61e2dcf3b4750064d36 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Tue, 6 Apr 2021 13:43:08 +0500 Subject: [PATCH 7/8] add tests --- weed/iamapi/iamapi_handlers.go | 1 + weed/iamapi/iamapi_management_handlers.go | 47 +++---- weed/iamapi/iamapi_server.go | 48 +++++-- weed/iamapi/iamapi_test.go | 157 ++++++++++++++++++++++ 4 files changed, 217 insertions(+), 36 deletions(-) create mode 100644 weed/iamapi/iamapi_test.go diff --git a/weed/iamapi/iamapi_handlers.go b/weed/iamapi/iamapi_handlers.go index 962717ad9..fdaf4dd69 100644 --- a/weed/iamapi/iamapi_handlers.go +++ b/weed/iamapi/iamapi_handlers.go @@ -55,6 +55,7 @@ func writeIamErrorResponse(w http.ResponseWriter, err error, object string, valu errorResp := ErrorResponse{} errorResp.Error.Type = "Sender" errorResp.Error.Code = &errCode + glog.Errorf("Response %+v", err) switch errCode { case iam.ErrCodeNoSuchEntityException: msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index e4daa081f..470731064 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -1,14 +1,10 @@ package iamapi import ( - "bytes" "crypto/sha1" "encoding/json" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" @@ -16,6 +12,7 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/aws/aws-sdk-go/service/iam" @@ -32,13 +29,15 @@ var ( policyDocuments = map[string]*PolicyDocument{} ) +type Statement struct { + Effect string `json:"Effect"` + Action []string `json:"Action"` + Resource []string `json:"Resource"` +} + type PolicyDocument struct { - Version string `json:"Version"` - Statement []struct { - Effect string `json:"Effect"` - Action []string `json:"Action"` - Resource []string `json:"Resource"` - } `json:"Statement"` + Version string `json:"Version"` + Statement []*Statement `json:"Statement"` } func Hash(s *string) string { @@ -252,13 +251,16 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { return } values := r.PostForm + var s3cfgLock sync.RWMutex + s3cfgLock.RLock() s3cfg := &iam_pb.S3ApiConfiguration{} - if err := iama.GetS3ApiConfiguration(s3cfg); err != nil { + if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil { writeErrorResponse(w, s3err.ErrInternalError, r.URL) return } + s3cfgLock.RUnlock() - glog.Info("values ", values) + glog.V(4).Infof("DoActions: %+v", values) var response interface{} var err error changed := true @@ -292,12 +294,14 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { case "CreatePolicy": response, err = iama.CreatePolicy(s3cfg, values) if err != nil { + glog.Errorf("CreatePolicy: %+v", err) writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) return } case "PutUserPolicy": response, err = iama.PutUserPolicy(s3cfg, values) if err != nil { + glog.Errorf("PutUserPolicy: %+v", err) writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL) return } @@ -306,22 +310,9 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { return } if changed { - buf := bytes.Buffer{} - if err := filer.S3ConfigurationToText(&buf, s3cfg); err != nil { - glog.Error("S3ConfigurationToText: ", err) - writeErrorResponse(w, s3err.ErrInternalError, r.URL) - return - } - err := pb.WithGrpcFilerClient( - iama.option.FilerGrpcAddress, - iama.option.GrpcDialOption, - func(client filer_pb.SeaweedFilerClient) error { - if err := filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile, buf.Bytes()); err != nil { - return err - } - return nil - }, - ) + s3cfgLock.Lock() + err := iama.s3ApiConfig.PutS3ApiConfiguration(s3cfg) + s3cfgLock.Unlock() if err != nil { writeErrorResponse(w, s3err.ErrInternalError, r.URL) return diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go index 00c4a69a2..7698fab71 100644 --- a/weed/iamapi/iamapi_server.go +++ b/weed/iamapi/iamapi_server.go @@ -1,10 +1,10 @@ package iamapi // https://docs.aws.amazon.com/cli/latest/reference/iam/list-roles.html -// https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateRole.html import ( "bytes" + "fmt" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -16,6 +16,16 @@ import ( "strings" ) +type IamS3ApiConfig interface { + GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) + PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) +} + +type IamS3ApiConfigure struct { + option *IamServerOption + masterClient *wdclient.MasterClient +} + type IamServerOption struct { Masters string Filer string @@ -25,17 +35,22 @@ type IamServerOption struct { } type IamApiServer struct { - option *IamServerOption - masterClient *wdclient.MasterClient - filerclient *filer_pb.SeaweedFilerClient + s3ApiConfig IamS3ApiConfig + filerclient *filer_pb.SeaweedFilerClient } +var s3ApiConfigure IamS3ApiConfig + func NewIamApiServer(router *mux.Router, option *IamServerOption) (iamApiServer *IamApiServer, err error) { - iamApiServer = &IamApiServer{ + s3ApiConfigure = IamS3ApiConfigure{ option: option, masterClient: wdclient.NewMasterClient(option.GrpcDialOption, pb.AdminShellClient, "", 0, "", strings.Split(option.Masters, ",")), } + iamApiServer = &IamApiServer{ + s3ApiConfig: s3ApiConfigure, + } + iamApiServer.registerRouter(router) return iamApiServer, nil @@ -52,10 +67,10 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) { apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler) } -func (iama *IamApiServer) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) { +func (iam IamS3ApiConfigure) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) { var buf bytes.Buffer - err = pb.WithGrpcFilerClient(iama.option.FilerGrpcAddress, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - if err = filer.ReadEntry(iama.masterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf); err != nil { + err = pb.WithGrpcFilerClient(iam.option.FilerGrpcAddress, iam.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + if err = filer.ReadEntry(iam.masterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf); err != nil { return err } return nil @@ -70,3 +85,20 @@ func (iama *IamApiServer) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration } return nil } + +func (iam IamS3ApiConfigure) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) { + buf := bytes.Buffer{} + if err := filer.S3ConfigurationToText(&buf, s3cfg); err != nil { + return fmt.Errorf("S3ConfigurationToText: %s", err) + } + return pb.WithGrpcFilerClient( + iam.option.FilerGrpcAddress, + iam.option.GrpcDialOption, + func(client filer_pb.SeaweedFilerClient) error { + if err := filer.SaveInsideFiler(client, filer.IamConfigDirecotry, filer.IamIdentityFile, buf.Bytes()); err != nil { + return err + } + return nil + }, + ) +} diff --git a/weed/iamapi/iamapi_test.go b/weed/iamapi/iamapi_test.go new file mode 100644 index 000000000..f989626e6 --- /dev/null +++ b/weed/iamapi/iamapi_test.go @@ -0,0 +1,157 @@ +package iamapi + +import ( + "encoding/xml" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" +) + +var S3config iam_pb.S3ApiConfiguration +var GetS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error) +var PutS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error) + +type iamS3ApiConfigureMock struct{} + +func (iam iamS3ApiConfigureMock) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) { + s3cfg = &S3config + return nil +} + +func (iam iamS3ApiConfigureMock) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) { + S3config = *s3cfg + return nil +} + +var a = IamApiServer{} + +func TestCreateUser(t *testing.T) { + userName := aws.String("Test") + params := &iam.CreateUserInput{UserName: userName} + req, _ := iam.New(session.New()).CreateUserRequest(params) + _ = req.Build() + out := CreateUserResponse{} + response, err := executeRequest(req.HTTPRequest, out) + assert.Equal(t, nil, err) + assert.Equal(t, http.StatusOK, response.Code) + //assert.Equal(t, out.XMLName, "lol") +} + +func TestListUsers(t *testing.T) { + params := &iam.ListUsersInput{} + req, _ := iam.New(session.New()).ListUsersRequest(params) + _ = req.Build() + out := ListUsersResponse{} + response, err := executeRequest(req.HTTPRequest, out) + assert.Equal(t, nil, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +func TestListAccessKeys(t *testing.T) { + svc := iam.New(session.New()) + params := &iam.ListAccessKeysInput{} + req, _ := svc.ListAccessKeysRequest(params) + _ = req.Build() + out := ListAccessKeysResponse{} + response, err := executeRequest(req.HTTPRequest, out) + assert.Equal(t, nil, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +func TestDeleteUser(t *testing.T) { + userName := aws.String("Test") + params := &iam.DeleteUserInput{UserName: userName} + req, _ := iam.New(session.New()).DeleteUserRequest(params) + _ = req.Build() + out := DeleteUserResponse{} + response, err := executeRequest(req.HTTPRequest, out) + assert.Equal(t, nil, err) + assert.Equal(t, http.StatusNotFound, response.Code) +} + +func TestGetUser(t *testing.T) { + userName := aws.String("Test") + params := &iam.GetUserInput{UserName: userName} + req, _ := iam.New(session.New()).GetUserRequest(params) + _ = req.Build() + out := GetUserResponse{} + response, err := executeRequest(req.HTTPRequest, out) + assert.Equal(t, nil, err) + assert.Equal(t, http.StatusNotFound, response.Code) +} + +// Todo flat statement +func TestCreatePolicy(t *testing.T) { + params := &iam.CreatePolicyInput{ + PolicyName: aws.String("S3-read-only-example-bucket"), + PolicyDocument: aws.String(` + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:List*" + ], + "Resource": [ + "arn:aws:s3:::EXAMPLE-BUCKET", + "arn:aws:s3:::EXAMPLE-BUCKET/*" + ] + } + ] + }`), + } + req, _ := iam.New(session.New()).CreatePolicyRequest(params) + _ = req.Build() + out := CreatePolicyResponse{} + response, err := executeRequest(req.HTTPRequest, out) + assert.Equal(t, nil, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +func TestPutUserPolicy(t *testing.T) { + userName := aws.String("Test") + params := &iam.PutUserPolicyInput{ + UserName: userName, + PolicyName: aws.String("S3-read-only-example-bucket"), + PolicyDocument: aws.String( + `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:List*" + ], + "Resource": [ + "arn:aws:s3:::EXAMPLE-BUCKET", + "arn:aws:s3:::EXAMPLE-BUCKET/*" + ] + } + ] + }`), + } + req, _ := iam.New(session.New()).PutUserPolicyRequest(params) + _ = req.Build() + out := PutUserPolicyResponse{} + response, err := executeRequest(req.HTTPRequest, out) + assert.Equal(t, nil, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +func executeRequest(req *http.Request, v interface{}) (*httptest.ResponseRecorder, error) { + rr := httptest.NewRecorder() + apiRouter := mux.NewRouter().SkipClean(true) + a.s3ApiConfig = iamS3ApiConfigureMock{} + apiRouter.Path("/").Methods("POST").HandlerFunc(a.DoActions) + apiRouter.ServeHTTP(rr, req) + return rr, xml.Unmarshal(rr.Body.Bytes(), &v) +} From f5f8eec8e2f75045d7cc8685dc5fb86508700d2b Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Tue, 6 Apr 2021 13:53:56 +0500 Subject: [PATCH 8/8] fix get filerGrpcAddress --- weed/command/iam.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/iam.go b/weed/command/iam.go index ddcddbec9..17d0832cb 100644 --- a/weed/command/iam.go +++ b/weed/command/iam.go @@ -43,7 +43,7 @@ func runIam(cmd *Command, args []string) bool { } func (iamopt *IamOptions) startIamServer() bool { - filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*iamopt.filer) + filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*iamopt.filer) if err != nil { glog.Fatal(err) return false