Browse Source
Implement IAM propagation to S3 servers (#8130)
Implement IAM propagation to S3 servers (#8130)
* Implement IAM propagation to S3 servers - Add PropagatingCredentialStore to propagate IAM changes to S3 servers via gRPC - Add Policy management RPCs to S3 proto and S3ApiServer - Update CredentialManager to use PropagatingCredentialStore when MasterClient is available - Wire FilerServer to enable propagation * Implement parallel IAM propagation and fix S3 cluster registration - Parallelized IAM change propagation with 10s timeout. - Refined context usage in PropagatingCredentialStore. - Added S3Type support to cluster node management. - Enabled S3 servers to register with gRPC address to the master. - Ensured IAM configuration reload after policy updates via gRPC. * Optimize IAM propagation with direct in-memory cache updates * Secure IAM propagation: Use metadata to skip persistence only on propagation * pb: refactor IAM and S3 services for unidirectional IAM propagation - Move SeaweedS3IamCache service from iam.proto to s3.proto. - Remove legacy IAM management RPCs and empty SeaweedS3 service from s3.proto. - Enforce that S3 servers only use the synchronization interface. * pb: regenerate Go code for IAM and S3 services Updated generated code following the proto refactoring of IAM synchronization services. * s3api: implement read-only mode for Embedded IAM API - Add readOnly flag to EmbeddedIamApi to reject write operations via HTTP. - Enable read-only mode by default in S3ApiServer. - Handle AccessDenied error in writeIamErrorResponse. - Embed SeaweedS3IamCacheServer in S3ApiServer. * credential: refactor PropagatingCredentialStore for unidirectional IAM flow - Update to use s3_pb.SeaweedS3IamCacheClient for propagation to S3 servers. - Propagate full Identity object via PutIdentity for consistency. - Remove redundant propagation of specific user/account/policy management RPCs. - Add timeout context for propagation calls. * s3api: implement SeaweedS3IamCacheServer for unidirectional sync - Update S3ApiServer to implement the cache synchronization gRPC interface. - Methods (PutIdentity, RemoveIdentity, etc.) now perform direct in-memory cache updates. - Register SeaweedS3IamCacheServer in command/s3.go. - Remove registration for the legacy and now empty SeaweedS3 service. * s3api: update tests for read-only IAM and propagation - Added TestEmbeddedIamReadOnly to verify rejection of write operations in read-only mode. - Update test setup to pass readOnly=false to NewEmbeddedIamApi in routing tests. - Updated EmbeddedIamApiForTest helper with read-only checks matching production behavior. * s3api: add back temporary debug logs for IAM updates Log IAM updates received via: - gRPC propagation (PutIdentity, PutPolicy, etc.) - Metadata configuration reloads (LoadS3ApiConfigurationFromCredentialManager) - Core identity management (UpsertIdentity, RemoveIdentity) * IAM: finalize propagation fix with reduced logging and clarified architecture * Allow configuring IAM read-only mode for S3 server integration tests * s3api: add defensive validation to UpsertIdentity * s3api: fix log message to reference correct IAM read-only flag * test/s3/iam: ensure WaitForS3Service checks for IAM write permissions * test: enable writable IAM in Makefile for integration tests * IAM: add GetPolicy/ListPolicies RPCs to s3.proto * S3: add GetBucketPolicy and ListBucketPolicies helpers * S3: support storing generic IAM policies in IdentityAccessManagement * S3: implement IAM policy RPCs using IdentityAccessManagement * IAM: fix stale user identity on rename propagationpull/8135/head
committed by
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 1116 additions and 1021 deletions
-
1test/s3/iam/Makefile
-
1test/s3/iam/run_tests.sh
-
35test/s3/iam/s3_iam_framework.go
-
35test/s3/normal/s3_integration_test.go
-
10weed/cluster/cluster.go
-
5weed/command/mini.go
-
23weed/command/s3.go
-
1weed/command/server.go
-
7weed/credential/credential_manager.go
-
297weed/credential/propagating_store.go
-
19weed/pb/iam.proto
-
267weed/pb/iam_pb/iam.pb.go
-
31weed/pb/s3.proto
-
140weed/pb/s3_pb/s3.pb.go
-
618weed/pb/s3_pb/s3_grpc.pb.go
-
121weed/s3api/auth_credentials.go
-
48weed/s3api/auth_credentials_test.go
-
10weed/s3api/s3api_bucket_policy_engine.go
-
30weed/s3api/s3api_embedded_iam.go
-
46weed/s3api/s3api_embedded_iam_test.go
-
21weed/s3api/s3api_server.go
-
356weed/s3api/s3api_server_grpc.go
-
2weed/s3api/s3api_server_routing_test.go
-
4weed/server/filer_server.go
-
8weed/server/filer_server_handlers_iam_grpc.go
-
1weed/server/master_grpc_server.go
@ -0,0 +1,297 @@ |
|||
package credential |
|||
|
|||
import ( |
|||
"context" |
|||
"encoding/json" |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/cluster" |
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" |
|||
"github.com/seaweedfs/seaweedfs/weed/wdclient" |
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
var _ CredentialStore = &PropagatingCredentialStore{} |
|||
var _ PolicyManager = &PropagatingCredentialStore{} |
|||
|
|||
type PropagatingCredentialStore struct { |
|||
CredentialStore |
|||
masterClient *wdclient.MasterClient |
|||
grpcDialOption grpc.DialOption |
|||
} |
|||
|
|||
func NewPropagatingCredentialStore(upstream CredentialStore, masterClient *wdclient.MasterClient, grpcDialOption grpc.DialOption) *PropagatingCredentialStore { |
|||
return &PropagatingCredentialStore{ |
|||
CredentialStore: upstream, |
|||
masterClient: masterClient, |
|||
grpcDialOption: grpcDialOption, |
|||
} |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) propagateChange(ctx context.Context, fn func(context.Context, s3_pb.SeaweedS3IamCacheClient) error) { |
|||
if s.masterClient == nil { |
|||
return |
|||
} |
|||
|
|||
// List S3 servers
|
|||
var s3Servers []string |
|||
err := s.masterClient.WithClient(false, func(client master_pb.SeaweedClient) error { |
|||
glog.V(4).Infof("IAM: listing S3 servers (FilerGroup: '%s')", s.masterClient.FilerGroup) |
|||
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ |
|||
ClientType: cluster.S3Type, |
|||
FilerGroup: s.masterClient.FilerGroup, |
|||
}) |
|||
if err != nil { |
|||
glog.V(1).Infof("failed to list S3 servers: %v", err) |
|||
return err |
|||
} |
|||
for _, node := range resp.ClusterNodes { |
|||
s3Servers = append(s3Servers, node.Address) |
|||
} |
|||
|
|||
return nil |
|||
}) |
|||
if err != nil { |
|||
glog.V(1).Infof("failed to list s3 servers via master client: %v", err) |
|||
return |
|||
} |
|||
glog.V(1).Infof("IAM: propagating change to %d S3 servers: %v", len(s3Servers), s3Servers) |
|||
|
|||
// Create context with timeout for the propagation process
|
|||
propagateCtx, cancel := context.WithTimeout(ctx, 10*time.Second) |
|||
defer cancel() |
|||
|
|||
var wg sync.WaitGroup |
|||
for _, server := range s3Servers { |
|||
wg.Add(1) |
|||
go func(server string) { |
|||
defer wg.Done() |
|||
err := pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error { |
|||
glog.V(4).Infof("IAM: successfully connected to S3 server %s for propagation", server) |
|||
client := s3_pb.NewSeaweedS3IamCacheClient(conn) |
|||
return fn(propagateCtx, client) |
|||
}, server, false, s.grpcDialOption) |
|||
if err != nil { |
|||
glog.V(1).Infof("failed to propagate change to s3 server %s: %v", server, err) |
|||
} |
|||
}(server) |
|||
} |
|||
wg.Wait() |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) CreateUser(ctx context.Context, identity *iam_pb.Identity) error { |
|||
glog.V(4).Infof("IAM: PropagatingCredentialStore.CreateUser %s", identity.Name) |
|||
if err := s.CredentialStore.CreateUser(ctx, identity); err != nil { |
|||
return err |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
_, err := client.PutIdentity(tx, &iam_pb.PutIdentityRequest{Identity: identity}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error { |
|||
glog.V(4).Infof("IAM: PropagatingCredentialStore.UpdateUser %s", username) |
|||
if err := s.CredentialStore.UpdateUser(ctx, username, identity); err != nil { |
|||
return err |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
if _, err := client.PutIdentity(tx, &iam_pb.PutIdentityRequest{Identity: identity}); err != nil { |
|||
return err |
|||
} |
|||
if username != identity.Name { |
|||
if _, err := client.RemoveIdentity(tx, &iam_pb.RemoveIdentityRequest{Username: username}); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
return nil |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) DeleteUser(ctx context.Context, username string) error { |
|||
glog.V(4).Infof("IAM: PropagatingCredentialStore.DeleteUser %s", username) |
|||
if err := s.CredentialStore.DeleteUser(ctx, username); err != nil { |
|||
return err |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
_, err := client.RemoveIdentity(tx, &iam_pb.RemoveIdentityRequest{Username: username}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) CreateAccessKey(ctx context.Context, username string, credential *iam_pb.Credential) error { |
|||
if err := s.CredentialStore.CreateAccessKey(ctx, username, credential); err != nil { |
|||
return err |
|||
} |
|||
// Fetch updated identity to propagate
|
|||
identity, err := s.CredentialStore.GetUser(ctx, username) |
|||
if err != nil { |
|||
glog.Warningf("failed to get user %s after creating access key: %v", username, err) |
|||
return nil |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
_, err := client.PutIdentity(tx, &iam_pb.PutIdentityRequest{Identity: identity}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error { |
|||
if err := s.CredentialStore.DeleteAccessKey(ctx, username, accessKey); err != nil { |
|||
return err |
|||
} |
|||
// Fetch updated identity to propagate
|
|||
identity, err := s.CredentialStore.GetUser(ctx, username) |
|||
if err != nil { |
|||
glog.Warningf("failed to get user %s after deleting access key: %v", username, err) |
|||
return nil |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
_, err := client.PutIdentity(tx, &iam_pb.PutIdentityRequest{Identity: identity}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) PutPolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error { |
|||
glog.V(4).Infof("IAM: PropagatingCredentialStore.PutPolicy %s", name) |
|||
if err := s.CredentialStore.PutPolicy(ctx, name, document); err != nil { |
|||
return err |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
content, err := json.Marshal(document) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
_, err = client.PutPolicy(tx, &iam_pb.PutPolicyRequest{Name: name, Content: string(content)}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) DeletePolicy(ctx context.Context, name string) error { |
|||
glog.V(4).Infof("IAM: PropagatingCredentialStore.DeletePolicy %s", name) |
|||
if err := s.CredentialStore.DeletePolicy(ctx, name); err != nil { |
|||
return err |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
_, err := client.DeletePolicy(tx, &iam_pb.DeletePolicyRequest{Name: name}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) CreatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error { |
|||
if pm, ok := s.CredentialStore.(PolicyManager); ok { |
|||
if err := pm.CreatePolicy(ctx, name, document); err != nil { |
|||
return err |
|||
} |
|||
} else { |
|||
if err := s.CredentialStore.PutPolicy(ctx, name, document); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
content, err := json.Marshal(document) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
_, err = client.PutPolicy(tx, &iam_pb.PutPolicyRequest{Name: name, Content: string(content)}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) UpdatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error { |
|||
if pm, ok := s.CredentialStore.(PolicyManager); ok { |
|||
if err := pm.UpdatePolicy(ctx, name, document); err != nil { |
|||
return err |
|||
} |
|||
} else { |
|||
if err := s.CredentialStore.PutPolicy(ctx, name, document); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
content, err := json.Marshal(document) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
_, err = client.PutPolicy(tx, &iam_pb.PutPolicyRequest{Name: name, Content: string(content)}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) CreateServiceAccount(ctx context.Context, sa *iam_pb.ServiceAccount) error { |
|||
glog.V(4).Infof("IAM: PropagatingCredentialStore.CreateServiceAccount %s (parent: %s)", sa.Id, sa.ParentUser) |
|||
if err := s.CredentialStore.CreateServiceAccount(ctx, sa); err != nil { |
|||
return err |
|||
} |
|||
// Fetch parent identity to propagate
|
|||
identity, err := s.CredentialStore.GetUser(ctx, sa.ParentUser) |
|||
if err != nil { |
|||
glog.Warningf("failed to get parent user %s after creating service account: %v", sa.ParentUser, err) |
|||
return nil |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
_, err := client.PutIdentity(tx, &iam_pb.PutIdentityRequest{Identity: identity}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) UpdateServiceAccount(ctx context.Context, id string, sa *iam_pb.ServiceAccount) error { |
|||
if err := s.CredentialStore.UpdateServiceAccount(ctx, id, sa); err != nil { |
|||
return err |
|||
} |
|||
// Fetch parent identity to propagate
|
|||
identity, err := s.CredentialStore.GetUser(ctx, sa.ParentUser) |
|||
if err != nil { |
|||
glog.Warningf("failed to get parent user %s after updating service account: %v", sa.ParentUser, err) |
|||
return nil |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
_, err := client.PutIdentity(tx, &iam_pb.PutIdentityRequest{Identity: identity}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
func (s *PropagatingCredentialStore) DeleteServiceAccount(ctx context.Context, id string) error { |
|||
// Retrieve SA first to get ParentUser
|
|||
sa, err := s.CredentialStore.GetServiceAccount(ctx, id) |
|||
if err != nil { |
|||
// If accessing non-existent SA, just proceed to delete (idempotency)
|
|||
// But we can't propagate to parent...
|
|||
if err := s.CredentialStore.DeleteServiceAccount(ctx, id); err != nil { |
|||
return err |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
if err := s.CredentialStore.DeleteServiceAccount(ctx, id); err != nil { |
|||
return err |
|||
} |
|||
|
|||
// Fetch parent identity to propagate
|
|||
identity, err := s.CredentialStore.GetUser(ctx, sa.ParentUser) |
|||
if err != nil { |
|||
glog.Warningf("failed to get parent user %s after deleting service account: %v", sa.ParentUser, err) |
|||
return nil |
|||
} |
|||
s.propagateChange(ctx, func(tx context.Context, client s3_pb.SeaweedS3IamCacheClient) error { |
|||
_, err := client.PutIdentity(tx, &iam_pb.PutIdentityRequest{Identity: identity}) |
|||
return err |
|||
}) |
|||
return nil |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue