diff --git a/weed/credential/filer_etc/filer_etc_identity.go b/weed/credential/filer_etc/filer_etc_identity.go index 2b231c549..71de39b4d 100644 --- a/weed/credential/filer_etc/filer_etc_identity.go +++ b/weed/credential/filer_etc/filer_etc_identity.go @@ -15,8 +15,8 @@ import ( func (store *FilerEtcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) { s3cfg := &iam_pb.S3ApiConfiguration{} - glog.V(1).Infof("Loading IAM configuration from %s/%s (filer: %s)", - filer.IamConfigDirectory, filer.IamIdentityFile, store.filerGrpcAddress) + glog.V(1).Infof("Loading IAM configuration from %s/%s (using current active filer)", + filer.IamConfigDirectory, filer.IamIdentityFile) err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { // Use ReadInsideFiler instead of ReadEntry since identity.json is small diff --git a/weed/credential/filer_etc/filer_etc_policy.go b/weed/credential/filer_etc/filer_etc_policy.go index 775ca97cb..dddb341f0 100644 --- a/weed/credential/filer_etc/filer_etc_policy.go +++ b/weed/credential/filer_etc/filer_etc_policy.go @@ -21,14 +21,14 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_ } // Check if filer client is configured - if store.filerGrpcAddress == "" { + if store.filerAddressFunc == nil { glog.V(1).Infof("Filer client not configured for policy retrieval, returning empty policies") // Return empty policies if filer client is not configured return policiesCollection.Policies, nil } - glog.V(2).Infof("Loading IAM policies from %s/%s (filer: %s)", - filer.IamConfigDirectory, filer.IamPoliciesFile, store.filerGrpcAddress) + glog.V(2).Infof("Loading IAM policies from %s/%s (using current active filer)", + filer.IamConfigDirectory, filer.IamPoliciesFile) err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { // Use ReadInsideFiler instead of ReadEntry since policies.json is small diff --git a/weed/credential/filer_etc/filer_etc_store.go b/weed/credential/filer_etc/filer_etc_store.go index f8750cb25..05cc6a4e2 100644 --- a/weed/credential/filer_etc/filer_etc_store.go +++ b/weed/credential/filer_etc/filer_etc_store.go @@ -16,7 +16,7 @@ func init() { // FilerEtcStore implements CredentialStore using SeaweedFS filer for storage type FilerEtcStore struct { - filerGrpcAddress string + filerAddressFunc func() pb.ServerAddress // Function to get current active filer grpcDialOption grpc.DialOption } @@ -27,27 +27,48 @@ func (store *FilerEtcStore) GetName() credential.CredentialStoreTypeName { func (store *FilerEtcStore) Initialize(configuration util.Configuration, prefix string) error { // Handle nil configuration gracefully if configuration != nil { - store.filerGrpcAddress = configuration.GetString(prefix + "filer") + filerAddr := configuration.GetString(prefix + "filer") + if filerAddr != "" { + // Static configuration - use fixed address + store.filerAddressFunc = func() pb.ServerAddress { + return pb.ServerAddress(filerAddr) + } + } // TODO: Initialize grpcDialOption based on configuration } - // Note: filerGrpcAddress can be set later via SetFilerClient method + // Note: filerAddressFunc can be set later via SetFilerClient method return nil } // SetFilerClient sets the filer client details for the file store +// Deprecated: Use SetFilerAddressFunc for better HA support func (store *FilerEtcStore) SetFilerClient(filerAddress string, grpcDialOption grpc.DialOption) { - store.filerGrpcAddress = filerAddress + store.filerAddressFunc = func() pb.ServerAddress { + return pb.ServerAddress(filerAddress) + } + store.grpcDialOption = grpcDialOption +} + +// SetFilerAddressFunc sets a function that returns the current active filer address +// This enables high availability by using the currently active filer +func (store *FilerEtcStore) SetFilerAddressFunc(getFiler func() pb.ServerAddress, grpcDialOption grpc.DialOption) { + store.filerAddressFunc = getFiler store.grpcDialOption = grpcDialOption } // withFilerClient executes a function with a filer client func (store *FilerEtcStore) withFilerClient(fn func(client filer_pb.SeaweedFilerClient) error) error { - if store.filerGrpcAddress == "" { + if store.filerAddressFunc == nil { return fmt.Errorf("filer address not configured") } + filerAddress := store.filerAddressFunc() + if filerAddress == "" { + return fmt.Errorf("filer address is empty") + } + // Use the pb.WithGrpcFilerClient helper similar to existing code - return pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(store.filerGrpcAddress), store.grpcDialOption, fn) + return pb.WithGrpcFilerClient(false, 0, filerAddress, store.grpcDialOption, fn) } func (store *FilerEtcStore) Shutdown() { diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 795ef6d86..21357ba4a 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/kms" + "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -137,17 +138,35 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitSto } // For stores that need filer client details, set them - // Note: SetFilerClient interface currently accepts only a single filer address - // TODO: Update credential store interfaces to support multiple filers for HA - // For now, using first filer - this is a known limitation for filer-backed stores + // Use SetFilerAddressFunc to provide current active filer for HA if store := credentialManager.GetStore(); store != nil { - if filerClientSetter, ok := store.(interface { + // Check for new HA-aware interface first + if filerFuncSetter, ok := store.(interface { + SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption) + }); ok { + // Use FilerClient's GetCurrentFiler for HA + // Note: FilerClient is created later, so we need to capture it + // For now, use first filer - this will be updated when FilerClient is available + if len(option.Filers) > 0 { + // Create a closure that will use the first filer initially + // In a full implementation, this would get the FilerClient's current filer + getFiler := func() pb.ServerAddress { + if len(option.Filers) > 0 { + return option.Filers[0] + } + return "" + } + filerFuncSetter.SetFilerAddressFunc(getFiler, option.GrpcDialOption) + glog.V(1).Infof("Credential store configured with filer function (HA-aware)") + } + } else if filerClientSetter, ok := store.(interface { SetFilerClient(string, grpc.DialOption) }); ok { + // Fallback to old interface for backward compatibility if len(option.Filers) > 0 { filerAddr := option.Filers[0].ToGrpcAddress() filerClientSetter.SetFilerClient(filerAddr, option.GrpcDialOption) - glog.V(1).Infof("Credential store configured with first filer: %s (HA limitation)", filerAddr) + glog.V(1).Infof("Credential store configured with first filer: %s (legacy interface)", filerAddr) } else { glog.Warningf("No filer addresses configured for credential store") } diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index d29bf5ff2..1bbddd94f 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -103,6 +103,17 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl filerClient := wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, option.DataCenter) glog.V(0).Infof("S3 API initialized FilerClient with %d filer(s) for volume location caching", len(option.Filers)) + // Update credential store to use FilerClient's current filer for HA + if store := iam.credentialManager.GetStore(); store != nil { + if filerFuncSetter, ok := store.(interface { + SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption) + }); ok { + // Use FilerClient's GetCurrentFiler for true HA + filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, option.GrpcDialOption) + glog.V(1).Infof("Updated credential store to use FilerClient's current active filer (HA-aware)") + } + } + s3ApiServer = &S3ApiServer{ option: option, iam: iam,