From 6206737df2ee5b6a999c9869fc48d9583c3f8e31 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 25 Nov 2020 21:02:31 +0500 Subject: [PATCH] s3 configure --- weed/s3api/filer_util.go | 87 +++++++++++----- weed/shell/command_s3_configure.go | 160 +++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+), 26 deletions(-) create mode 100644 weed/shell/command_s3_configure.go diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index e4d7eb04f..7e61aa46d 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -3,16 +3,20 @@ package s3api import ( "context" "fmt" - "strings" - + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "strings" + proto "github.com/golang/protobuf/proto" "google.golang.org/grpc" - "github.com/chrislusf/seaweedfs/weed/util" ) +const S3ConfName = "s3.conf" + func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error { return filer_pb.Mkdir(s3a, parentDirectoryPath, dirName, fn) @@ -84,37 +88,68 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry * return filer_pb.GetEntry(s3a, fullPath) } +func LoadS3configFromEntryExtended(extended *map[string][]byte, identities *[]*Identity) (err error) { + for name, ident := range *extended { + t := &Identity{ + Name: name, + Credentials: nil, + Actions: nil, + } + identity := &iam_pb.Identity{} + if err := proto.Unmarshal(ident, identity); err != nil { + return err + } + for _, action := range identity.Actions { + t.Actions = append(t.Actions, Action(action)) + } + for _, cred := range identity.Credentials { + t.Credentials = append(t.Credentials, &Credential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + }) + } + *identities = append(*identities, t) + } + return nil +} + +func SaveS3configToEntryExtended(extended *map[string][]byte, identities *[]*Identity) (err error) { + for _, identity := range *identities { + i := &iam_pb.Identity{ + Name: identity.Name, + Credentials: []*iam_pb.Credential{}, + Actions: []string{}, + } + for _, cred := range identity.Credentials { + i.Credentials = append(i.Credentials, &iam_pb.Credential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + }) + } + for _, action := range identity.Actions { + i.Actions = append(i.Actions, string(action)) + } + ident, err := proto.Marshal(i) + if err != nil { + return err + } + (*extended)[identity.Name] = ident + } + return nil +} + func loadS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) error { return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ - Directory: "/.configs", - Name: "s3identities", + Directory: filer.DirectoryEtc, + Name: S3ConfName, }) if err != nil { return err } - for name, ident := range resp.Entry.Extended { - t := &Identity{ - Name: name, - Credentials: nil, - Actions: nil, - } - identity := &iam_pb.Identity{} - if err := proto.Unmarshal(ident, identity); err != nil { - return err - } - for _, action := range identity.Actions { - t.Actions = append(t.Actions, Action(action)) - } - for _, cred := range identity.Credentials { - t.Credentials = append(t.Credentials, &Credential{ - AccessKey: cred.AccessKey, - SecretKey: cred.SecretKey, - }) - glog.V(0).Infof("AccessKey %s, SecretKey: %s", cred.AccessKey, cred.SecretKey) - } - iam.identities = append(iam.identities, t) + if err = LoadS3configFromEntryExtended(&resp.Entry.Extended, &iam.identities); err != nil { + return err } return nil }, option.FilerGrpcAddress, option.GrpcDialOption) diff --git a/weed/shell/command_s3_configure.go b/weed/shell/command_s3_configure.go new file mode 100644 index 000000000..a4d45a4f9 --- /dev/null +++ b/weed/shell/command_s3_configure.go @@ -0,0 +1,160 @@ +package shell + +import ( + "flag" + "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api" + "io" + "sort" + "strings" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +func init() { + Commands = append(Commands, &commandS3Configure{}) +} + +type commandS3Configure struct { +} + +func (c *commandS3Configure) Name() string { + return "s3.configure" +} + +func (c *commandS3Configure) Help() string { + return `configure and apply s3 options for each bucket + # see the current configuration file content + s3.configure + ` +} + +func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + s3ConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + actions := s3ConfigureCommand.String("actions", "", "actions names") + user := s3ConfigureCommand.String("user", "", "user name") + buckets := s3ConfigureCommand.String("buckets", "", "bucket name") + accessKey := s3ConfigureCommand.String("access_key", "", "specify the access key") + secretKey := s3ConfigureCommand.String("secret_key", "", "specify the secret key") + isDelete := s3ConfigureCommand.Bool("delete", false, "delete users, actions or access keys") + apply := s3ConfigureCommand.Bool("apply", false, "update and apply s3 configuration") + + if err = s3ConfigureCommand.Parse(args); err != nil { + return nil + } + + var identities []*s3api.Identity + if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + request := &filer_pb.LookupDirectoryEntryRequest{ + Directory: filer.DirectoryEtc, + Name: s3api.S3ConfName, + } + respLookupEntry, err := filer_pb.LookupEntry(client, request) + if err != nil { + return err + } + if err = s3api.LoadS3configFromEntryExtended(&respLookupEntry.Entry.Extended, &identities); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + idx := 0 + changed := false + if *user != "" && *buckets != "" { + for i, identity := range identities { + if *user == identity.Name { + idx = i + changed = true + break + } + } + } + cmdActions := []s3api.Action{} + for _, bucket := range strings.Split(*buckets, ",") { + for _, action := range strings.Split(*actions, ",") { + cmdActions = append(cmdActions, s3api.Action(fmt.Sprintf("%s:%s", action, bucket))) + } + } + cmdCredential := &s3api.Credential{ + AccessKey: *accessKey, + SecretKey: *secretKey, + } + + if changed { + if *isDelete { + exists := []int{} + for _, cmdAction := range cmdActions { + for i, currentAction := range identities[idx].Actions { + if cmdAction == currentAction { + exists = append(exists, i) + } + } + } + sort.Sort(sort.Reverse(sort.IntSlice(exists))) + for _, i := range exists { + identities[idx].Actions = append(identities[idx].Actions[:i], identities[idx].Actions[i+1:]...) + } + if *accessKey != "" { + exists = []int{} + for i, credential := range identities[idx].Credentials { + if credential.AccessKey == *accessKey { + exists = append(exists, i) + } + } + sort.Sort(sort.Reverse(sort.IntSlice(exists))) + for _, i := range exists { + identities[idx].Credentials = append(identities[idx].Credentials[:i], identities[idx].Credentials[:i+1]...) + } + + } + if *actions == "" && *accessKey == "" { + identities = append(identities[:idx], identities[idx+1:]...) + } + } else { + identities[idx].Actions = append(identities[idx].Actions, cmdActions...) + identities[idx].Credentials = append(identities[idx].Credentials, &s3api.Credential{ + AccessKey: *accessKey, + SecretKey: *secretKey, + }) + } + } else { + identity := s3api.Identity{ + Name: *user, + Actions: cmdActions, + } + identity.Credentials = append(identity.Credentials, &s3api.Credential{ + AccessKey: *accessKey, + SecretKey: *secretKey, + }) + identities = append(identities, &identity) + } + + fmt.Fprintf(writer, fmt.Sprintf("%+v\n", identities)) + fmt.Fprintln(writer) + + if !*apply { + return nil + } + + if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + request := &filer_pb.LookupDirectoryEntryRequest{ + Directory: filer.DirectoryEtc, + Name: s3api.S3ConfName, + } + respLookupEntry, err := filer_pb.LookupEntry(client, request) + if err != nil { + return err + } + if err = s3api.SaveS3configToEntryExtended(&respLookupEntry.Entry.Extended, &identities); err != nil { + return err + } + return nil + }); err != nil { + return err + } + return nil +}