From e1190b3224638616cf4e1318ddcba0b1575f2130 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Tue, 3 Nov 2020 21:45:56 +0500 Subject: [PATCH] load S3 config from filer https://github.com/chrislusf/seaweedfs/issues/1500 --- weed/s3api/auth_credentials.go | 15 ++-- weed/s3api/auto_signature_v4_test.go | 6 +- weed/s3api/filer_util.go | 105 ++++++++++++++++++++++++++- weed/s3api/s3api_server.go | 2 +- 4 files changed, 116 insertions(+), 12 deletions(-) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 2b7666345..55cc9f727 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -43,15 +43,17 @@ type Credential struct { SecretKey string } -func NewIdentityAccessManagement(fileName string, domain string) *IdentityAccessManagement { +func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement { iam := &IdentityAccessManagement{ - domain: domain, + domain: option.DomainName, } - if fileName == "" { - return iam + if err := loadS3config(iam, option); err != nil { + glog.Warningf("fail to load config %v", err) } - if err := iam.loadS3ApiConfiguration(fileName); err != nil { - glog.Fatalf("fail to load config file %s: %v", fileName, err) + if len(iam.identities) == 0 && option.Config != "" { + if err := iam.loadS3ApiConfiguration(option.Config); err != nil { + glog.Fatalf("fail to load config file %s: %v", option.Config, err) + } } return iam } @@ -59,7 +61,6 @@ func NewIdentityAccessManagement(fileName string, domain string) *IdentityAccess func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) error { s3ApiConfiguration := &iam_pb.S3ApiConfiguration{} - rawData, readErr := ioutil.ReadFile(fileName) if readErr != nil { glog.Warningf("fail to read %s : %v", fileName, readErr) diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index 8f1c9b470..4c8255768 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -57,7 +57,8 @@ func TestIsRequestPresignedSignatureV4(t *testing.T) { // Tests is requested authenticated function, tests replies for s3 errors. func TestIsReqAuthenticated(t *testing.T) { - iam := NewIdentityAccessManagement("", "") + option := S3ApiServerOption{} + iam := NewIdentityAccessManagement(&option) iam.identities = []*Identity{ { Name: "someone", @@ -92,7 +93,8 @@ func TestIsReqAuthenticated(t *testing.T) { } func TestCheckAdminRequestAuthType(t *testing.T) { - iam := NewIdentityAccessManagement("", "") + option := S3ApiServerOption{} + iam := NewIdentityAccessManagement(&option) iam.identities = []*Identity{ { Name: "someone", diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index ebdbe8245..f6cfed785 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -3,10 +3,13 @@ package s3api import ( "context" "fmt" - "strings" - "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + proto "github.com/golang/protobuf/proto" + "google.golang.org/grpc" + "strings" ) func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error { @@ -75,6 +78,104 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD } +func loadS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) error { + return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + client := filer_pb.NewSeaweedFilerClient(grpcConnection) + resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: "/.configs", + Name: "s3identities", + }) + if err != nil { + return err + } + for name, ident := range resp.Entry.Extended { + t := &Identity{ + Name: name, + Credentials: nil, + Actions: nil, + } + identity := &iam_pb.Identity{} + if err := proto.Unmarshal(ident, identity); err != nil { + return err + } + for _, action := range identity.Actions { + t.Actions = append(t.Actions, Action(action)) + } + for _, cred := range identity.Credentials { + t.Credentials = append(t.Credentials, &Credential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + }) + glog.V(0).Infof("AccessKey %s, SecretKey: %s", cred.AccessKey, cred.SecretKey) + } + iam.identities = append(iam.identities, t) + } + return nil + }, option.FilerGrpcAddress, option.GrpcDialOption) +} + +/* testing save +func saveS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) (error) { + return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { + client := filer_pb.NewSeaweedFilerClient(grpcConnection) + entry := &filer_pb.Entry{ + Name: "s3identities", + IsDirectory: false, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(0644), + Collection: "", + Replication: "", + }, + Extended: make(map[string][]byte), + } + for _, identity := range iam.identities { + glog.V(0).Infof("get iam identities %s", identity.Name) + i := &iam_pb.Identity{ + Name: identity.Name, + Credentials: []*iam_pb.Credential{}, + Actions: []string{}, + } + for _, cred := range identity.Credentials { + i.Credentials = append(i.Credentials, &iam_pb.Credential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + }) + } + for _, action := range identity.Actions { + i.Actions = append(i.Actions, string(action)) + } + ident, err := proto.Marshal(i) + if err != nil { + return err + } + entry.Extended[identity.Name] = ident + } + _, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: "/.configs", + Name: "s3identities", + }) + if err == filer_pb.ErrNotFound { + err = filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{ + Directory: "/.configs", + Entry: entry, + IsFromOtherCluster: false, + Signatures: nil, + }) + } else { + err = filer_pb.UpdateEntry(client, &filer_pb.UpdateEntryRequest{ + Directory: "/.configs", + Entry: entry, + IsFromOtherCluster: false, + Signatures: nil, + }) + } + return err + },option.FilerGrpcAddress, option.GrpcDialOption) +} +*/ + func objectKey(key *string) *string { if strings.HasPrefix(*key, "/") { t := (*key)[1:] diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index b1e1cfe80..850a02171 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -27,7 +27,7 @@ type S3ApiServer struct { func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) { s3ApiServer = &S3ApiServer{ option: option, - iam: NewIdentityAccessManagement(option.Config, option.DomainName), + iam: NewIdentityAccessManagement(option), } s3ApiServer.registerRouter(router)