diff --git a/weed/cluster/cluster.go b/weed/cluster/cluster.go index 638553b04..a52aa2721 100644 --- a/weed/cluster/cluster.go +++ b/weed/cluster/cluster.go @@ -13,6 +13,7 @@ const ( VolumeServerType = "volumeServer" FilerType = "filer" BrokerType = "broker" + S3Type = "s3" ) type FilerGroupName string diff --git a/weed/command/iam.go b/weed/command/iam.go index 8f4ac878d..8fae7ec96 100644 --- a/weed/command/iam.go +++ b/weed/command/iam.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/seaweedfs/seaweedfs/weed/util/grace" // Import credential stores to register them _ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc" @@ -35,16 +36,18 @@ type IamOptions struct { func init() { cmdIam.Run = runIam // break init cycle - iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "filer server address") + iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "comma-separated filer server addresses for high availability") iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers") iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address") iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port") } var cmdIam = &Command{ - UsageLine: "iam [-port=8111] [-filer=] [-master=,]", + UsageLine: "iam [-port=8111] [-filer=[,]...] [-master=,]", Short: "start a iam API compatible server", - Long: "start a iam API compatible server.", + Long: `start a iam API compatible server. + + Multiple filer addresses can be specified for high availability, separated by commas.`, } func runIam(cmd *Command, args []string) bool { @@ -52,24 +55,24 @@ func runIam(cmd *Command, args []string) bool { } func (iamopt *IamOptions) startIamServer() bool { - filerAddress := pb.ServerAddress(*iamopt.filer) + filerAddresses := pb.ServerAddresses(*iamopt.filer).ToAddresses() util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") for { - err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err := pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { - return fmt.Errorf("get filer %s configuration: %v", filerAddress, err) + return fmt.Errorf("get filer configuration: %v", err) } glog.V(0).Infof("IAM read filer configuration: %s", resp) return nil }) if err != nil { - glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress()) + glog.V(0).Infof("wait to connect to filers %v", filerAddresses) time.Sleep(time.Second) } else { - glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress()) + glog.V(0).Infof("connected to filers %v", filerAddresses) break } } @@ -78,7 +81,7 @@ func (iamopt *IamOptions) startIamServer() bool { router := mux.NewRouter().SkipClean(true) iamApiServer, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{ Masters: masters, - Filer: filerAddress, + Filers: filerAddresses, Port: *iamopt.port, GrpcDialOption: grpcDialOption, }) @@ -87,8 +90,10 @@ func (iamopt *IamOptions) startIamServer() bool { glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err) } - // Ensure cleanup on shutdown - defer iamApiServer.Shutdown() + // Register shutdown handler to prevent goroutine leak + grace.OnInterrupt(func() { + iamApiServer.Shutdown() + }) listenAddress := fmt.Sprintf(":%d", *iamopt.port) iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(*iamopt.ip, *iamopt.port, time.Duration(10)*time.Second) diff --git a/weed/command/s3.go b/weed/command/s3.go index fa575b3db..995d15f8a 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -61,7 +61,7 @@ type S3Options struct { func init() { cmdS3.Run = runS3 // break init cycle - s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address") + s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "comma-separated filer server addresses for high availability") s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.") s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port") s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port") @@ -86,9 +86,12 @@ func init() { } var cmdS3 = &Command{ - UsageLine: "s3 [-port=8333] [-filer=] [-config=]", - Short: "start a s3 API compatible server that is backed by a filer", - Long: `start a s3 API compatible server that is backed by a filer. + UsageLine: "s3 [-port=8333] [-filer=[,]...] [-config=]", + Short: "start a s3 API compatible server that is backed by filer(s)", + Long: `start a s3 API compatible server that is backed by filer(s). + + Multiple filer addresses can be specified for high availability, separated by commas. + The S3 server will automatically failover between filers if one becomes unavailable. By default, you can use any access key and secret key to access the S3 APIs. To enable credential based access, create a config.json file similar to this: @@ -200,10 +203,11 @@ func (s3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Cer func (s3opt *S3Options) startS3Server() bool { - filerAddress := pb.ServerAddress(*s3opt.filer) + filerAddresses := pb.ServerAddresses(*s3opt.filer).ToAddresses() filerBucketsPath := "/buckets" filerGroup := "" + var masterAddresses []pb.ServerAddress grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") @@ -212,22 +216,27 @@ func (s3opt *S3Options) startS3Server() bool { var metricsIntervalSec int for { - err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err := pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { - return fmt.Errorf("get filer %s configuration: %v", filerAddress, err) + return fmt.Errorf("get filer configuration: %v", err) } filerBucketsPath = resp.DirBuckets filerGroup = resp.FilerGroup + // Get master addresses for filer discovery + masterAddresses = pb.ServerAddresses(strings.Join(resp.Masters, ",")).ToAddresses() metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec) glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath) + if len(masterAddresses) > 0 { + glog.V(0).Infof("S3 read master addresses for discovery: %v", masterAddresses) + } return nil }) if err != nil { - glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress()) + glog.V(0).Infof("wait to connect to filers %v grpc address", filerAddresses) time.Sleep(time.Second) } else { - glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress()) + glog.V(0).Infof("connected to filers %v", filerAddresses) break } } @@ -252,7 +261,8 @@ func (s3opt *S3Options) startS3Server() bool { } s3ApiServer, s3ApiServer_err = s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{ - Filer: filerAddress, + Filers: filerAddresses, + Masters: masterAddresses, Port: *s3opt.port, Config: *s3opt.config, DomainName: *s3opt.domainName, diff --git a/weed/credential/filer_etc/filer_etc_identity.go b/weed/credential/filer_etc/filer_etc_identity.go index 2b231c549..71de39b4d 100644 --- a/weed/credential/filer_etc/filer_etc_identity.go +++ b/weed/credential/filer_etc/filer_etc_identity.go @@ -15,8 +15,8 @@ import ( func (store *FilerEtcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) { s3cfg := &iam_pb.S3ApiConfiguration{} - glog.V(1).Infof("Loading IAM configuration from %s/%s (filer: %s)", - filer.IamConfigDirectory, filer.IamIdentityFile, store.filerGrpcAddress) + glog.V(1).Infof("Loading IAM configuration from %s/%s (using current active filer)", + filer.IamConfigDirectory, filer.IamIdentityFile) err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { // Use ReadInsideFiler instead of ReadEntry since identity.json is small diff --git a/weed/credential/filer_etc/filer_etc_policy.go b/weed/credential/filer_etc/filer_etc_policy.go index 775ca97cb..99931bae8 100644 --- a/weed/credential/filer_etc/filer_etc_policy.go +++ b/weed/credential/filer_etc/filer_etc_policy.go @@ -20,15 +20,19 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_ Policies: make(map[string]policy_engine.PolicyDocument), } - // Check if filer client is configured - if store.filerGrpcAddress == "" { + // Check if filer client is configured (with mutex protection) + store.mu.RLock() + configured := store.filerAddressFunc != nil + store.mu.RUnlock() + + if !configured { glog.V(1).Infof("Filer client not configured for policy retrieval, returning empty policies") // Return empty policies if filer client is not configured return policiesCollection.Policies, nil } - glog.V(2).Infof("Loading IAM policies from %s/%s (filer: %s)", - filer.IamConfigDirectory, filer.IamPoliciesFile, store.filerGrpcAddress) + glog.V(2).Infof("Loading IAM policies from %s/%s (using current active filer)", + filer.IamConfigDirectory, filer.IamPoliciesFile) err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { // Use ReadInsideFiler instead of ReadEntry since policies.json is small diff --git a/weed/credential/filer_etc/filer_etc_store.go b/weed/credential/filer_etc/filer_etc_store.go index f8750cb25..b181a55f0 100644 --- a/weed/credential/filer_etc/filer_etc_store.go +++ b/weed/credential/filer_etc/filer_etc_store.go @@ -2,6 +2,7 @@ package filer_etc import ( "fmt" + "sync" "github.com/seaweedfs/seaweedfs/weed/credential" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -16,8 +17,9 @@ func init() { // FilerEtcStore implements CredentialStore using SeaweedFS filer for storage type FilerEtcStore struct { - filerGrpcAddress string + filerAddressFunc func() pb.ServerAddress // Function to get current active filer grpcDialOption grpc.DialOption + mu sync.RWMutex // Protects filerAddressFunc and grpcDialOption } func (store *FilerEtcStore) GetName() credential.CredentialStoreTypeName { @@ -27,27 +29,48 @@ func (store *FilerEtcStore) GetName() credential.CredentialStoreTypeName { func (store *FilerEtcStore) Initialize(configuration util.Configuration, prefix string) error { // Handle nil configuration gracefully if configuration != nil { - store.filerGrpcAddress = configuration.GetString(prefix + "filer") + filerAddr := configuration.GetString(prefix + "filer") + if filerAddr != "" { + // Static configuration - use fixed address + store.mu.Lock() + store.filerAddressFunc = func() pb.ServerAddress { + return pb.ServerAddress(filerAddr) + } + store.mu.Unlock() + } // TODO: Initialize grpcDialOption based on configuration } - // Note: filerGrpcAddress can be set later via SetFilerClient method + // Note: filerAddressFunc can be set later via SetFilerAddressFunc method return nil } -// SetFilerClient sets the filer client details for the file store -func (store *FilerEtcStore) SetFilerClient(filerAddress string, grpcDialOption grpc.DialOption) { - store.filerGrpcAddress = filerAddress +// SetFilerAddressFunc sets a function that returns the current active filer address +// This enables high availability by using the currently active filer +func (store *FilerEtcStore) SetFilerAddressFunc(getFiler func() pb.ServerAddress, grpcDialOption grpc.DialOption) { + store.mu.Lock() + defer store.mu.Unlock() + store.filerAddressFunc = getFiler store.grpcDialOption = grpcDialOption } // withFilerClient executes a function with a filer client func (store *FilerEtcStore) withFilerClient(fn func(client filer_pb.SeaweedFilerClient) error) error { - if store.filerGrpcAddress == "" { - return fmt.Errorf("filer address not configured") + store.mu.RLock() + if store.filerAddressFunc == nil { + store.mu.RUnlock() + return fmt.Errorf("filer_etc: filer address function not configured") + } + + filerAddress := store.filerAddressFunc() + dialOption := store.grpcDialOption + store.mu.RUnlock() + + if filerAddress == "" { + return fmt.Errorf("filer_etc: filer address is empty") } // Use the pb.WithGrpcFilerClient helper similar to existing code - return pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(store.filerGrpcAddress), store.grpcDialOption, fn) + return pb.WithGrpcFilerClient(false, 0, filerAddress, dialOption, fn) } func (store *FilerEtcStore) Shutdown() { diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index e93279fba..869b3b93d 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -32,22 +32,34 @@ type FilerConf struct { } func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) { - var buf bytes.Buffer - if err := pb.WithGrpcFilerClient(false, 0, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + return ReadFilerConfFromFilers([]pb.ServerAddress{filerGrpcAddress}, grpcDialOption, masterClient) +} + +// ReadFilerConfFromFilers reads filer configuration with multi-filer failover support +func ReadFilerConfFromFilers(filerGrpcAddresses []pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) { + var data []byte + if err := pb.WithOneOfGrpcFilerClients(false, filerGrpcAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { if masterClient != nil { - return ReadEntry(masterClient, client, DirectoryEtcSeaweedFS, FilerConfName, &buf) - } else { - content, err := ReadInsideFiler(client, DirectoryEtcSeaweedFS, FilerConfName) - buf = *bytes.NewBuffer(content) + var buf bytes.Buffer + if err := ReadEntry(masterClient, client, DirectoryEtcSeaweedFS, FilerConfName, &buf); err != nil { + return err + } + data = buf.Bytes() + return nil + } + content, err := ReadInsideFiler(client, DirectoryEtcSeaweedFS, FilerConfName) + if err != nil { return err } + data = content + return nil }); err != nil && err != filer_pb.ErrNotFound { return nil, fmt.Errorf("read %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err) } fc := NewFilerConf() - if buf.Len() > 0 { - if err := fc.LoadFromBytes(buf.Bytes()); err != nil { + if len(data) > 0 { + if err := fc.LoadFromBytes(data); err != nil { return nil, fmt.Errorf("parse %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err) } } diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go index 361d9bec9..e3979e416 100644 --- a/weed/iamapi/iamapi_server.go +++ b/weed/iamapi/iamapi_server.go @@ -40,7 +40,7 @@ type IamS3ApiConfigure struct { type IamServerOption struct { Masters map[string]pb.ServerAddress - Filer pb.ServerAddress + Filers []pb.ServerAddress Port int GrpcDialOption grpc.DialOption } @@ -60,6 +60,10 @@ func NewIamApiServer(router *mux.Router, option *IamServerOption) (iamApiServer } func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, explicitStore string) (iamApiServer *IamApiServer, err error) { + if len(option.Filers) == 0 { + return nil, fmt.Errorf("at least one filer address is required") + } + masterClient := wdclient.NewMasterClient(option.GrpcDialOption, "", "iam", "", "", "", *pb.NewServiceDiscoveryFromMap(option.Masters)) // Create a cancellable context for the master client connection @@ -80,7 +84,7 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli s3ApiConfigure = configure s3Option := s3api.S3ApiServerOption{ - Filer: option.Filer, + Filers: option.Filers, GrpcDialOption: option.GrpcDialOption, } @@ -149,7 +153,7 @@ func (iama *IamS3ApiConfigure) PutS3ApiConfigurationToCredentialManager(s3cfg *i func (iama *IamS3ApiConfigure) GetS3ApiConfigurationFromFiler(s3cfg *iam_pb.S3ApiConfiguration) (err error) { var buf bytes.Buffer - err = pb.WithGrpcFilerClient(false, 0, iama.option.Filer, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = pb.WithOneOfGrpcFilerClients(false, iama.option.Filers, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { if err = filer.ReadEntry(iama.masterClient, client, filer.IamConfigDirectory, filer.IamIdentityFile, &buf); err != nil { return err } @@ -171,7 +175,7 @@ func (iama *IamS3ApiConfigure) PutS3ApiConfigurationToFiler(s3cfg *iam_pb.S3ApiC if err := filer.ProtoToText(&buf, s3cfg); err != nil { return fmt.Errorf("ProtoToText: %s", err) } - return pb.WithGrpcFilerClient(false, 0, iama.option.Filer, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + return pb.WithOneOfGrpcFilerClients(false, iama.option.Filers, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { err = util.Retry("saveIamIdentity", func() error { return filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamIdentityFile, buf.Bytes()) }) @@ -184,7 +188,7 @@ func (iama *IamS3ApiConfigure) PutS3ApiConfigurationToFiler(s3cfg *iam_pb.S3ApiC func (iama *IamS3ApiConfigure) GetPolicies(policies *Policies) (err error) { var buf bytes.Buffer - err = pb.WithGrpcFilerClient(false, 0, iama.option.Filer, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + err = pb.WithOneOfGrpcFilerClients(false, iama.option.Filers, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { if err = filer.ReadEntry(iama.masterClient, client, filer.IamConfigDirectory, filer.IamPoliciesFile, &buf); err != nil { return err } @@ -208,7 +212,7 @@ func (iama *IamS3ApiConfigure) PutPolicies(policies *Policies) (err error) { if b, err = json.Marshal(policies); err != nil { return err } - return pb.WithGrpcFilerClient(false, 0, iama.option.Filer, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + return pb.WithOneOfGrpcFilerClients(false, iama.option.Filers, iama.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { if err := filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile, b); err != nil { return err } diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index cebcd17f5..148839d3e 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/kms" + "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -136,12 +137,24 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitSto glog.Fatalf("failed to initialize credential manager: %v", err) } - // For stores that need filer client details, set them + // For stores that need filer client details, set them temporarily + // This will be updated to use FilerClient's GetCurrentFiler after FilerClient is created if store := credentialManager.GetStore(); store != nil { - if filerClientSetter, ok := store.(interface { - SetFilerClient(string, grpc.DialOption) + if filerFuncSetter, ok := store.(interface { + SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption) }); ok { - filerClientSetter.SetFilerClient(string(option.Filer), option.GrpcDialOption) + // Temporary setup: use first filer until FilerClient is available + // See s3api_server.go where this is updated to FilerClient.GetCurrentFiler + if len(option.Filers) > 0 { + getFiler := func() pb.ServerAddress { + if len(option.Filers) > 0 { + return option.Filers[0] + } + return "" + } + filerFuncSetter.SetFilerAddressFunc(getFiler, option.GrpcDialOption) + glog.V(1).Infof("Credential store configured with temporary filer function (will be updated after FilerClient creation)") + } } } diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 4b8fbaa62..1e4635ead 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "math" + "net/url" "path/filepath" "slices" "sort" @@ -42,6 +43,20 @@ type InitiateMultipartUploadResult struct { s3.CreateMultipartUploadOutput } +// getRequestScheme determines the URL scheme (http or https) from the request +// Checks X-Forwarded-Proto header first (for proxies), then TLS state +func getRequestScheme(r *http.Request) string { + // Check X-Forwarded-Proto header for proxied requests + if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" { + return proto + } + // Check if connection is TLS + if r.TLS != nil { + return "https" + } + return "http" +} + func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code s3err.ErrorCode) { glog.V(2).Infof("createMultipartUpload input %v", input) @@ -183,8 +198,10 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl entryName, dirName := s3a.getEntryNameAndDir(input) if entry, _ := s3a.getEntry(dirName, entryName); entry != nil && entry.Extended != nil { if uploadId, ok := entry.Extended[s3_constants.SeaweedFSUploadId]; ok && *input.UploadId == string(uploadId) { + // Location uses the S3 endpoint that the client connected to + // Format: scheme://s3-endpoint/bucket/object (following AWS S3 API) return &CompleteMultipartUploadResult{ - Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))), + Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))), Bucket: input.Bucket, ETag: aws.String("\"" + filer.ETagChunks(entry.GetChunks()) + "\""), Key: objectKey(input.Key), @@ -401,7 +418,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl // The latest version information is tracked in the .versions directory metadata output = &CompleteMultipartUploadResult{ - Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))), + Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))), Bucket: input.Bucket, ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""), Key: objectKey(input.Key), @@ -454,7 +471,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl // Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec output = &CompleteMultipartUploadResult{ - Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))), + Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))), Bucket: input.Bucket, ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""), Key: objectKey(input.Key), @@ -511,7 +528,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl // For non-versioned buckets, return response without VersionId output = &CompleteMultipartUploadResult{ - Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))), + Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))), Bucket: input.Bucket, ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""), Key: objectKey(input.Key), diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index d73fabd2f..f0704fe23 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -877,7 +877,8 @@ func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWr s3err.WriteErrorResponse(w, r, err) return } - fc, err := filer.ReadFilerConf(s3a.option.Filer, s3a.option.GrpcDialOption, nil) + // ReadFilerConfFromFilers provides multi-filer failover + fc, err := filer.ReadFilerConfFromFilers(s3a.option.Filers, s3a.option.GrpcDialOption, nil) if err != nil { glog.Errorf("GetBucketLifecycleConfigurationHandler: %s", err) s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) @@ -938,7 +939,7 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr return } - fc, err := filer.ReadFilerConf(s3a.option.Filer, s3a.option.GrpcDialOption, nil) + fc, err := filer.ReadFilerConfFromFilers(s3a.option.Filers, s3a.option.GrpcDialOption, nil) if err != nil { glog.Errorf("PutBucketLifecycleConfigurationHandler read filer config: %s", err) s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) @@ -1020,7 +1021,7 @@ func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *h return } - fc, err := filer.ReadFilerConf(s3a.option.Filer, s3a.option.GrpcDialOption, nil) + fc, err := filer.ReadFilerConfFromFilers(s3a.option.Filers, s3a.option.GrpcDialOption, nil) if err != nil { glog.Errorf("DeleteBucketLifecycleHandler read filer config: %s", err) s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) diff --git a/weed/s3api/s3api_circuit_breaker.go b/weed/s3api/s3api_circuit_breaker.go index 47efa728a..2f5e1f580 100644 --- a/weed/s3api/s3api_circuit_breaker.go +++ b/weed/s3api/s3api_circuit_breaker.go @@ -29,7 +29,8 @@ func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker { limitations: make(map[string]int64), } - err := pb.WithFilerClient(false, 0, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + // Use WithOneOfGrpcFilerClients to support multiple filers with failover + err := pb.WithOneOfGrpcFilerClients(false, option.Filers, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { content, err := filer.ReadInsideFiler(client, s3_constants.CircuitBreakerConfigDir, s3_constants.CircuitBreakerConfigFile) if errors.Is(err, filer_pb.ErrNotFound) { return nil @@ -41,6 +42,7 @@ func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker { }) if err != nil { + glog.Warningf("S3 circuit breaker disabled; failed to load config from any filer: %v", err) } return cb diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index c146a8b15..6c47e8256 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "google.golang.org/grpc" @@ -15,12 +16,75 @@ import ( var _ = filer_pb.FilerClient(&S3ApiServer{}) func (s3a *S3ApiServer) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - + // Use filerClient for proper connection management and failover + if s3a.filerClient != nil { + return s3a.withFilerClientFailover(streamingMode, fn) + } + + // Fallback to direct connection if filerClient not initialized + // This should only happen during initialization or testing return pb.WithGrpcClient(streamingMode, s3a.randomClientId, func(grpcConnection *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(grpcConnection) return fn(client) - }, s3a.option.Filer.ToGrpcAddress(), false, s3a.option.GrpcDialOption) + }, s3a.getFilerAddress().ToGrpcAddress(), false, s3a.option.GrpcDialOption) + +} +// withFilerClientFailover attempts to execute fn with automatic failover to other filers +func (s3a *S3ApiServer) withFilerClientFailover(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { + // Get current filer as starting point + currentFiler := s3a.filerClient.GetCurrentFiler() + + // Try current filer first (fast path) + err := pb.WithGrpcClient(streamingMode, s3a.randomClientId, func(grpcConnection *grpc.ClientConn) error { + client := filer_pb.NewSeaweedFilerClient(grpcConnection) + return fn(client) + }, currentFiler.ToGrpcAddress(), false, s3a.option.GrpcDialOption) + + if err == nil { + s3a.filerClient.RecordFilerSuccess(currentFiler) + return nil + } + + // Record failure for current filer + s3a.filerClient.RecordFilerFailure(currentFiler) + + // Current filer failed - try all other filers with health-aware selection + filers := s3a.filerClient.GetAllFilers() + var lastErr error = err + + for _, filer := range filers { + if filer == currentFiler { + continue // Already tried this one + } + + // Skip filers known to be unhealthy (circuit breaker pattern) + if s3a.filerClient.ShouldSkipUnhealthyFiler(filer) { + glog.V(2).Infof("WithFilerClient: skipping unhealthy filer %s", filer) + continue + } + + err = pb.WithGrpcClient(streamingMode, s3a.randomClientId, func(grpcConnection *grpc.ClientConn) error { + client := filer_pb.NewSeaweedFilerClient(grpcConnection) + return fn(client) + }, filer.ToGrpcAddress(), false, s3a.option.GrpcDialOption) + + if err == nil { + // Success! Record success and update current filer for future requests + s3a.filerClient.RecordFilerSuccess(filer) + s3a.filerClient.SetCurrentFiler(filer) + glog.V(1).Infof("WithFilerClient: failover from %s to %s succeeded", currentFiler, filer) + return nil + } + + // Record failure for health tracking + s3a.filerClient.RecordFilerFailure(filer) + glog.V(2).Infof("WithFilerClient: failover to %s failed: %v", filer, err) + lastErr = err + } + + // All filers failed + return fmt.Errorf("all filers failed, last error: %w", lastErr) } func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string { diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index cd0e82421..b1446c3e7 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -404,11 +404,11 @@ func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bu return listEntry } -func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string { - object = urlPathEscape(removeDuplicateSlashes(object)) - destUrl := fmt.Sprintf("http://%s%s/%s%s", - s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, object) - return destUrl +func (s3a *S3ApiServer) toFilerPath(bucket, object string) string { + // Returns the raw file path - no URL escaping needed + // The path is used directly, not embedded in a URL + object = removeDuplicateSlashes(object) + return fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) } // hasConditionalHeaders checks if the request has any conditional headers diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index 2d9f8e620..becbd9bf9 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -404,7 +404,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ } } - uploadUrl := s3a.genPartUploadUrl(bucket, uploadID, partID) + filePath := s3a.genPartUploadPath(bucket, uploadID, partID) if partID == 1 && r.Header.Get("Content-Type") == "" { dataReader = mimeDetect(r, dataReader) @@ -413,7 +413,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ glog.V(2).Infof("PutObjectPart: bucket=%s, object=%s, uploadId=%s, partNumber=%d, size=%d", bucket, object, uploadID, partID, r.ContentLength) - etag, errCode, sseMetadata := s3a.putToFiler(r, uploadUrl, dataReader, bucket, partID) + etag, errCode, sseMetadata := s3a.putToFiler(r, filePath, dataReader, bucket, partID) if errCode != s3err.ErrNone { glog.Errorf("PutObjectPart: putToFiler failed with error code %v for bucket=%s, object=%s, partNumber=%d", errCode, bucket, object, partID) @@ -437,9 +437,11 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string { return fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, s3_constants.MultipartUploadsFolder) } -func (s3a *S3ApiServer) genPartUploadUrl(bucket, uploadID string, partID int) string { - return fmt.Sprintf("http://%s%s/%s/%04d_%s.part", - s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, uuid.NewString()) +func (s3a *S3ApiServer) genPartUploadPath(bucket, uploadID string, partID int) string { + // Returns just the file path - no filer address needed + // Upload traffic goes directly to volume servers, not through filer + return fmt.Sprintf("%s/%s/%04d_%s.part", + s3a.genUploadsFolder(bucket), uploadID, partID, uuid.NewString()) } // Generate uploadID hash string from object diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index ecb2ac8d1..3ec6147f5 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -114,7 +114,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R } } - uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlEscapeObject(object)) + filePath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) // Get ContentType from post formData // Otherwise from formFile ContentType @@ -136,7 +136,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R } } - etag, errCode, sseMetadata := s3a.putToFiler(r, uploadUrl, fileBody, bucket, 1) + etag, errCode, sseMetadata := s3a.putToFiler(r, filePath, fileBody, bucket, 1) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 540a1e512..100796b2e 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "net/http" - "net/url" "path/filepath" "strconv" "strings" @@ -223,12 +222,12 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) s3a.setSSEResponseHeaders(w, r, sseMetadata) default: // Handle regular PUT (never configured versioning) - uploadUrl := s3a.toFilerUrl(bucket, object) + filePath := s3a.toFilerPath(bucket, object) if objectContentType == "" { dataReader = mimeDetect(r, dataReader) } - etag, errCode, sseMetadata := s3a.putToFiler(r, uploadUrl, dataReader, bucket, 1) + etag, errCode, sseMetadata := s3a.putToFiler(r, filePath, dataReader, bucket, 1) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -248,9 +247,10 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) writeSuccessResponseEmpty(w, r) } -func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader, bucket string, partNumber int) (etag string, code s3err.ErrorCode, sseMetadata SSEResponseMetadata) { +func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader io.Reader, bucket string, partNumber int) (etag string, code s3err.ErrorCode, sseMetadata SSEResponseMetadata) { // NEW OPTIMIZATION: Write directly to volume servers, bypassing filer proxy // This eliminates the filer proxy overhead for PUT operations + // Note: filePath is now passed directly instead of URL (no parsing needed) // For SSE, encrypt with offset=0 for all parts // Each part is encrypted independently, then decrypted using metadata during GET @@ -311,20 +311,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader glog.V(4).Infof("putToFiler: explicit encryption already applied, skipping bucket default encryption") } - // Parse the upload URL to extract the file path - // uploadUrl format: http://filer:8888/path/to/bucket/object (or https://, IPv6, etc.) - // Use proper URL parsing instead of string manipulation for robustness - parsedUrl, parseErr := url.Parse(uploadUrl) - if parseErr != nil { - glog.Errorf("putToFiler: failed to parse uploadUrl %q: %v", uploadUrl, parseErr) - return "", s3err.ErrInternalError, SSEResponseMetadata{} - } - - // Use parsedUrl.Path directly - it's already decoded by url.Parse() - // Per Go documentation: "Path is stored in decoded form: /%47%6f%2f becomes /Go/" - // Calling PathUnescape again would double-decode and fail on keys like "b%ar" - filePath := parsedUrl.Path - + // filePath is already provided directly - no URL parsing needed // Step 1 & 2: Use auto-chunking to handle large files without OOM // This splits large uploads into 8MB chunks, preventing memory issues on both S3 API and volume servers const chunkSize = 8 * 1024 * 1024 // 8MB chunks (S3 standard) @@ -743,7 +730,7 @@ func (s3a *S3ApiServer) setObjectOwnerFromRequest(r *http.Request, entry *filer_ // For suspended versioning, objects are stored as regular files (version ID "null") in the bucket directory, // while existing versions from when versioning was enabled remain preserved in the .versions subdirectory. func (s3a *S3ApiServer) putSuspendedVersioningObject(r *http.Request, bucket, object string, dataReader io.Reader, objectContentType string) (etag string, errCode s3err.ErrorCode, sseMetadata SSEResponseMetadata) { - // Normalize object path to ensure consistency with toFilerUrl behavior + // Normalize object path to ensure consistency with toFilerPath behavior normalizedObject := removeDuplicateSlashes(object) glog.V(3).Infof("putSuspendedVersioningObject: START bucket=%s, object=%s, normalized=%s", @@ -783,7 +770,7 @@ func (s3a *S3ApiServer) putSuspendedVersioningObject(r *http.Request, bucket, ob glog.V(3).Infof("putSuspendedVersioningObject: no .versions directory for %s/%s", bucket, object) } - uploadUrl := s3a.toFilerUrl(bucket, normalizedObject) + filePath := s3a.toFilerPath(bucket, normalizedObject) body := dataReader if objectContentType == "" { @@ -846,7 +833,7 @@ func (s3a *S3ApiServer) putSuspendedVersioningObject(r *http.Request, bucket, ob } // Upload the file using putToFiler - this will create the file with version metadata - etag, errCode, sseMetadata = s3a.putToFiler(r, uploadUrl, body, bucket, 1) + etag, errCode, sseMetadata = s3a.putToFiler(r, filePath, body, bucket, 1) if errCode != s3err.ErrNone { glog.Errorf("putSuspendedVersioningObject: failed to upload object: %v", errCode) return "", errCode, SSEResponseMetadata{} @@ -937,7 +924,7 @@ func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object strin // Generate version ID versionId = generateVersionId() - // Normalize object path to ensure consistency with toFilerUrl behavior + // Normalize object path to ensure consistency with toFilerPath behavior normalizedObject := removeDuplicateSlashes(object) glog.V(2).Infof("putVersionedObject: creating version %s for %s/%s (normalized: %s)", versionId, bucket, object, normalizedObject) @@ -948,7 +935,7 @@ func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object strin // Upload directly to the versions directory // We need to construct the object path relative to the bucket versionObjectPath := normalizedObject + s3_constants.VersionsFolder + "/" + versionFileName - versionUploadUrl := s3a.toFilerUrl(bucket, versionObjectPath) + versionFilePath := s3a.toFilerPath(bucket, versionObjectPath) // Ensure the .versions directory exists before uploading bucketDir := s3a.option.BucketsPath + "/" + bucket @@ -966,9 +953,9 @@ func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object strin body = mimeDetect(r, body) } - glog.V(2).Infof("putVersionedObject: uploading %s/%s version %s to %s", bucket, object, versionId, versionUploadUrl) + glog.V(2).Infof("putVersionedObject: uploading %s/%s version %s to %s", bucket, object, versionId, versionFilePath) - etag, errCode, sseMetadata = s3a.putToFiler(r, versionUploadUrl, body, bucket, 1) + etag, errCode, sseMetadata = s3a.putToFiler(r, versionFilePath, body, bucket, 1) if errCode != s3err.ErrNone { glog.Errorf("putVersionedObject: failed to upload version: %v", errCode) return "", "", errCode, SSEResponseMetadata{} diff --git a/weed/s3api/s3api_object_handlers_test.go b/weed/s3api/s3api_object_handlers_test.go index cf650a36e..a6592ad4b 100644 --- a/weed/s3api/s3api_object_handlers_test.go +++ b/weed/s3api/s3api_object_handlers_test.go @@ -114,7 +114,7 @@ func TestRemoveDuplicateSlashes(t *testing.T) { } } -func TestS3ApiServer_toFilerUrl(t *testing.T) { +func TestS3ApiServer_toFilerPath(t *testing.T) { tests := []struct { name string args string diff --git a/weed/s3api/s3api_object_versioning.go b/weed/s3api/s3api_object_versioning.go index 1c1dbee03..bbc43f205 100644 --- a/weed/s3api/s3api_object_versioning.go +++ b/weed/s3api/s3api_object_versioning.go @@ -607,7 +607,7 @@ func (s3a *S3ApiServer) calculateETagFromChunks(chunks []*filer_pb.FileChunk) st // getSpecificObjectVersion retrieves a specific version of an object func (s3a *S3ApiServer) getSpecificObjectVersion(bucket, object, versionId string) (*filer_pb.Entry, error) { - // Normalize object path to ensure consistency with toFilerUrl behavior + // Normalize object path to ensure consistency with toFilerPath behavior normalizedObject := removeDuplicateSlashes(object) if versionId == "" { @@ -639,7 +639,7 @@ func (s3a *S3ApiServer) getSpecificObjectVersion(bucket, object, versionId strin // deleteSpecificObjectVersion deletes a specific version of an object func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId string) error { - // Normalize object path to ensure consistency with toFilerUrl behavior + // Normalize object path to ensure consistency with toFilerPath behavior normalizedObject := removeDuplicateSlashes(object) if versionId == "" { @@ -843,7 +843,7 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http // getLatestObjectVersion finds the latest version of an object by reading .versions directory metadata func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb.Entry, error) { - // Normalize object path to ensure consistency with toFilerUrl behavior + // Normalize object path to ensure consistency with toFilerPath behavior normalizedObject := removeDuplicateSlashes(object) bucketDir := s3a.option.BucketsPath + "/" + bucket diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 992027fda..dcf3a55f2 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -11,29 +11,31 @@ import ( "strings" "time" + "github.com/gorilla/mux" + "google.golang.org/grpc" + + "github.com/seaweedfs/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/credential" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/iam/integration" "github.com/seaweedfs/seaweedfs/weed/iam/policy" "github.com/seaweedfs/seaweedfs/weed/iam/sts" - "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" - "github.com/seaweedfs/seaweedfs/weed/util/grace" - "github.com/seaweedfs/seaweedfs/weed/wdclient" - - "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/seaweedfs/seaweedfs/weed/util/grace" util_http "github.com/seaweedfs/seaweedfs/weed/util/http" util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client" - "google.golang.org/grpc" + "github.com/seaweedfs/seaweedfs/weed/wdclient" ) type S3ApiServerOption struct { - Filer pb.ServerAddress + Filers []pb.ServerAddress + Masters []pb.ServerAddress // For filer discovery Port int Config string DomainName string @@ -69,6 +71,10 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer } func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, explicitStore string) (s3ApiServer *S3ApiServer, err error) { + if len(option.Filers) == 0 { + return nil, fmt.Errorf("at least one filer address is required") + } + startTsNs := time.Now().UnixNano() v := util.GetViper() @@ -95,9 +101,38 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl // Initialize FilerClient for volume location caching // Uses the battle-tested vidMap with filer-based lookups - // S3 API typically connects to a single filer, but wrap in slice for consistency - filerClient := wdclient.NewFilerClient([]pb.ServerAddress{option.Filer}, option.GrpcDialOption, option.DataCenter) - glog.V(0).Infof("S3 API initialized FilerClient for volume location caching") + // Supports multiple filer addresses with automatic failover for high availability + var filerClient *wdclient.FilerClient + if len(option.Masters) > 0 && option.FilerGroup != "" { + // Enable filer discovery via master + masterMap := make(map[string]pb.ServerAddress) + for i, addr := range option.Masters { + masterMap[fmt.Sprintf("master%d", i)] = addr + } + masterClient := wdclient.NewMasterClient(option.GrpcDialOption, option.FilerGroup, cluster.S3Type, "", "", "", *pb.NewServiceDiscoveryFromMap(masterMap)) + + filerClient = wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, option.DataCenter, &wdclient.FilerClientOption{ + MasterClient: masterClient, + FilerGroup: option.FilerGroup, + DiscoveryInterval: 5 * time.Minute, + }) + glog.V(0).Infof("S3 API initialized FilerClient with %d filer(s) and discovery enabled (group: %s, masters: %v)", + len(option.Filers), option.FilerGroup, option.Masters) + } else { + filerClient = wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, option.DataCenter) + glog.V(0).Infof("S3 API initialized FilerClient with %d filer(s) (no discovery)", len(option.Filers)) + } + + // Update credential store to use FilerClient's current filer for HA + if store := iam.credentialManager.GetStore(); store != nil { + if filerFuncSetter, ok := store.(interface { + SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption) + }); ok { + // Use FilerClient's GetCurrentFiler for true HA + filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, option.GrpcDialOption) + glog.V(1).Infof("Updated credential store to use FilerClient's current active filer (HA-aware)") + } + } s3ApiServer = &S3ApiServer{ option: option, @@ -119,14 +154,16 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl if option.IamConfig != "" { glog.V(1).Infof("Loading advanced IAM configuration from: %s", option.IamConfig) + // Use FilerClient's GetCurrentFiler for HA-aware filer selection iamManager, err := loadIAMManagerFromConfig(option.IamConfig, func() string { - return string(option.Filer) + return string(filerClient.GetCurrentFiler()) }) if err != nil { glog.Errorf("Failed to load IAM configuration: %v", err) } else { // Create S3 IAM integration with the loaded IAM manager - s3iam := NewS3IAMIntegration(iamManager, string(option.Filer)) + // filerAddress not actually used, just for backward compatibility + s3iam := NewS3IAMIntegration(iamManager, "") // Set IAM integration in server s3ApiServer.iamIntegration = s3iam @@ -134,7 +171,7 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl // Set the integration in the traditional IAM for compatibility iam.SetIAMIntegration(s3iam) - glog.V(1).Infof("Advanced IAM system initialized successfully") + glog.V(1).Infof("Advanced IAM system initialized successfully with HA filer support") } } @@ -173,6 +210,21 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl return s3ApiServer, nil } +// getFilerAddress returns the current active filer address +// Uses FilerClient's tracked current filer which is updated on successful operations +// This provides better availability than always using the first filer +func (s3a *S3ApiServer) getFilerAddress() pb.ServerAddress { + if s3a.filerClient != nil { + return s3a.filerClient.GetCurrentFiler() + } + // Fallback to first filer if filerClient not initialized + if len(s3a.option.Filers) > 0 { + return s3a.option.Filers[0] + } + glog.Warningf("getFilerAddress: no filer addresses available") + return "" +} + // syncBucketPolicyToEngine syncs a bucket policy to the policy engine // This helper method centralizes the logic for loading bucket policies into the engine // to avoid duplication and ensure consistent error handling diff --git a/weed/wdclient/filer_client.go b/weed/wdclient/filer_client.go index f0dd5f2e6..2222575d6 100644 --- a/weed/wdclient/filer_client.go +++ b/weed/wdclient/filer_client.go @@ -5,6 +5,7 @@ import ( "fmt" "math/rand" "strings" + "sync" "sync/atomic" "time" @@ -12,6 +13,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/seaweedfs/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -35,9 +37,11 @@ type filerHealth struct { // It uses the shared vidMap cache for efficient lookups // Supports multiple filer addresses with automatic failover for high availability // Tracks filer health to avoid repeatedly trying known-unhealthy filers +// Can discover additional filers from master server when configured with filer group type FilerClient struct { *vidMapClient filerAddresses []pb.ServerAddress + filerAddressesMu sync.RWMutex // Protects filerAddresses and filerHealth filerIndex int32 // atomic: current filer index for round-robin filerHealth []*filerHealth // health status per filer (same order as filerAddresses) grpcDialOption grpc.DialOption @@ -50,6 +54,13 @@ type FilerClient struct { maxRetries int // Retry: maximum retry attempts for transient failures initialRetryWait time.Duration // Retry: initial wait time before first retry retryBackoffFactor float64 // Retry: backoff multiplier for wait time + + // Filer discovery fields + masterClient *MasterClient // Optional: for discovering filers in the same group + filerGroup string // Optional: filer group for discovery + discoveryInterval time.Duration // How often to refresh filer list from master + stopDiscovery chan struct{} // Signal to stop discovery goroutine + closeDiscoveryOnce sync.Once // Ensures discovery channel is closed at most once } // filerVolumeProvider implements VolumeLocationProvider by querying filer @@ -68,6 +79,11 @@ type FilerClientOption struct { MaxRetries int // Retry: maximum retry attempts for transient failures (0 = use default of 3) InitialRetryWait time.Duration // Retry: initial wait time before first retry (0 = use default of 1s) RetryBackoffFactor float64 // Retry: backoff multiplier for wait time (0 = use default of 1.5) + + // Filer discovery options + MasterClient *MasterClient // Optional: enables filer discovery from master + FilerGroup string // Optional: filer group name for discovery (required if MasterClient is set) + DiscoveryInterval time.Duration // Optional: how often to refresh filer list (0 = use default of 5 minutes) } // NewFilerClient creates a new client that queries filer(s) for volume locations @@ -87,6 +103,9 @@ func NewFilerClient(filerAddresses []pb.ServerAddress, grpcDialOption grpc.DialO maxRetries := 3 // Default: 3 retry attempts for transient failures initialRetryWait := time.Second // Default: 1 second initial retry wait retryBackoffFactor := 1.5 // Default: 1.5x backoff multiplier + var masterClient *MasterClient + var filerGroup string + discoveryInterval := 5 * time.Minute // Default: refresh every 5 minutes // Override with provided options if len(opts) > 0 && opts[0] != nil { @@ -115,6 +134,13 @@ func NewFilerClient(filerAddresses []pb.ServerAddress, grpcDialOption grpc.DialO if opt.RetryBackoffFactor > 0 { retryBackoffFactor = opt.RetryBackoffFactor } + if opt.MasterClient != nil { + masterClient = opt.MasterClient + filerGroup = opt.FilerGroup + if opt.DiscoveryInterval > 0 { + discoveryInterval = opt.DiscoveryInterval + } + } } // Initialize health tracking for each filer @@ -137,6 +163,17 @@ func NewFilerClient(filerAddresses []pb.ServerAddress, grpcDialOption grpc.DialO maxRetries: maxRetries, initialRetryWait: initialRetryWait, retryBackoffFactor: retryBackoffFactor, + masterClient: masterClient, + filerGroup: filerGroup, + discoveryInterval: discoveryInterval, + } + + // Start filer discovery if master client is configured + // Empty filerGroup is valid (represents default group) + if masterClient != nil { + fc.stopDiscovery = make(chan struct{}) + go fc.discoverFilers() + glog.V(0).Infof("FilerClient: started filer discovery for group '%s' (refresh interval: %v)", filerGroup, discoveryInterval) } // Create provider that references this FilerClient for failover support @@ -149,6 +186,204 @@ func NewFilerClient(filerAddresses []pb.ServerAddress, grpcDialOption grpc.DialO return fc } +// GetCurrentFiler returns the currently active filer address +// This is the filer that was last successfully used or the one indicated by round-robin +// Returns empty string if no filers are configured +func (fc *FilerClient) GetCurrentFiler() pb.ServerAddress { + fc.filerAddressesMu.RLock() + defer fc.filerAddressesMu.RUnlock() + + if len(fc.filerAddresses) == 0 { + return "" + } + + // Get current index (atomically updated on successful operations) + index := atomic.LoadInt32(&fc.filerIndex) + if index >= int32(len(fc.filerAddresses)) { + index = 0 + } + + return fc.filerAddresses[index] +} + +// GetAllFilers returns a snapshot of all filer addresses +// Returns a copy to avoid concurrent modification issues +func (fc *FilerClient) GetAllFilers() []pb.ServerAddress { + fc.filerAddressesMu.RLock() + defer fc.filerAddressesMu.RUnlock() + + // Return a copy to avoid concurrent modification + filers := make([]pb.ServerAddress, len(fc.filerAddresses)) + copy(filers, fc.filerAddresses) + return filers +} + +// SetCurrentFiler updates the current filer index to the specified address +// This is useful after successful failover to prefer the healthy filer for future requests +func (fc *FilerClient) SetCurrentFiler(addr pb.ServerAddress) { + fc.filerAddressesMu.RLock() + defer fc.filerAddressesMu.RUnlock() + + // Find the index of the specified filer address + for i, filer := range fc.filerAddresses { + if filer == addr { + atomic.StoreInt32(&fc.filerIndex, int32(i)) + return + } + } + // If address not found, leave index unchanged +} + +// ShouldSkipUnhealthyFiler checks if a filer address should be skipped based on health tracking +// Returns true if the filer has exceeded failure threshold and reset timeout hasn't elapsed +func (fc *FilerClient) ShouldSkipUnhealthyFiler(addr pb.ServerAddress) bool { + fc.filerAddressesMu.RLock() + defer fc.filerAddressesMu.RUnlock() + + // Find the health for this filer address + for i, filer := range fc.filerAddresses { + if filer == addr { + if i < len(fc.filerHealth) { + return fc.shouldSkipUnhealthyFilerWithHealth(fc.filerHealth[i]) + } + return false + } + } + // If address not found, don't skip it + return false +} + +// RecordFilerSuccess resets failure tracking for a successful filer +func (fc *FilerClient) RecordFilerSuccess(addr pb.ServerAddress) { + fc.filerAddressesMu.RLock() + defer fc.filerAddressesMu.RUnlock() + + // Find the health for this filer address + for i, filer := range fc.filerAddresses { + if filer == addr { + if i < len(fc.filerHealth) { + fc.recordFilerSuccessWithHealth(fc.filerHealth[i]) + } + return + } + } +} + +// RecordFilerFailure increments failure count for an unhealthy filer +func (fc *FilerClient) RecordFilerFailure(addr pb.ServerAddress) { + fc.filerAddressesMu.RLock() + defer fc.filerAddressesMu.RUnlock() + + // Find the health for this filer address + for i, filer := range fc.filerAddresses { + if filer == addr { + if i < len(fc.filerHealth) { + fc.recordFilerFailureWithHealth(fc.filerHealth[i]) + } + return + } + } +} + +// Close stops the filer discovery goroutine if running +// Safe to call multiple times (idempotent) +func (fc *FilerClient) Close() { + if fc.stopDiscovery != nil { + fc.closeDiscoveryOnce.Do(func() { + close(fc.stopDiscovery) + }) + } +} + +// discoverFilers periodically queries the master to discover filers in the same group +// and updates the filer list. This runs in a background goroutine. +func (fc *FilerClient) discoverFilers() { + defer func() { + if r := recover(); r != nil { + glog.Errorf("FilerClient: panic in filer discovery goroutine for group '%s': %v", fc.filerGroup, r) + } + }() + + // Do an initial discovery + fc.refreshFilerList() + + ticker := time.NewTicker(fc.discoveryInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + fc.refreshFilerList() + case <-fc.stopDiscovery: + glog.V(0).Infof("FilerClient: stopping filer discovery for group '%s'", fc.filerGroup) + return + } + } +} + +// refreshFilerList queries the master for the current list of filers and updates the local list +func (fc *FilerClient) refreshFilerList() { + if fc.masterClient == nil { + return + } + + // Get current master address + currentMaster := fc.masterClient.GetMaster(context.Background()) + if currentMaster == "" { + glog.V(1).Infof("FilerClient: no master available for filer discovery") + return + } + + // Query master for filers in our group + updates := cluster.ListExistingPeerUpdates(currentMaster, fc.grpcDialOption, fc.filerGroup, cluster.FilerType) + + if len(updates) == 0 { + glog.V(2).Infof("FilerClient: no filers found in group '%s'", fc.filerGroup) + return + } + + // Build new filer address list + discoveredFilers := make(map[pb.ServerAddress]bool) + for _, update := range updates { + if update.Address != "" { + discoveredFilers[pb.ServerAddress(update.Address)] = true + } + } + + // Thread-safe update of filer list + fc.filerAddressesMu.Lock() + defer fc.filerAddressesMu.Unlock() + + // Build a map of existing filers for efficient O(1) lookup + existingFilers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses)) + for _, f := range fc.filerAddresses { + existingFilers[f] = struct{}{} + } + + // Find new filers - O(N+M) instead of O(N*M) + var newFilers []pb.ServerAddress + for addr := range discoveredFilers { + if _, found := existingFilers[addr]; !found { + newFilers = append(newFilers, addr) + } + } + + // Add new filers + if len(newFilers) > 0 { + glog.V(0).Infof("FilerClient: discovered %d new filer(s) in group '%s': %v", len(newFilers), fc.filerGroup, newFilers) + fc.filerAddresses = append(fc.filerAddresses, newFilers...) + + // Initialize health tracking for new filers + for range newFilers { + fc.filerHealth = append(fc.filerHealth, &filerHealth{}) + } + } + + // Optionally, remove filers that are no longer in the cluster + // For now, we keep all filers and rely on health checks to avoid dead ones + // This prevents removing filers that might be temporarily unavailable +} + // GetLookupFileIdFunction returns a lookup function with URL preference handling func (fc *FilerClient) GetLookupFileIdFunction() LookupFileIdFunctionType { if fc.urlPreference == PreferUrl { @@ -245,8 +480,9 @@ func isRetryableGrpcError(err error) bool { // shouldSkipUnhealthyFiler checks if we should skip a filer based on recent failures // Circuit breaker pattern: skip filers with multiple recent consecutive failures -func (fc *FilerClient) shouldSkipUnhealthyFiler(index int32) bool { - health := fc.filerHealth[index] +// shouldSkipUnhealthyFilerWithHealth checks if a filer should be skipped based on health +// Uses atomic operations only - safe to call without locks +func (fc *FilerClient) shouldSkipUnhealthyFilerWithHealth(health *filerHealth) bool { failureCount := atomic.LoadInt32(&health.failureCount) // Check if failure count exceeds threshold @@ -267,17 +503,53 @@ func (fc *FilerClient) shouldSkipUnhealthyFiler(index int32) bool { return true // Skip this unhealthy filer } +// Deprecated: Use shouldSkipUnhealthyFilerWithHealth instead +// This function is kept for backward compatibility but requires array access +// Note: This function is now thread-safe. +func (fc *FilerClient) shouldSkipUnhealthyFiler(index int32) bool { + fc.filerAddressesMu.RLock() + if index >= int32(len(fc.filerHealth)) { + fc.filerAddressesMu.RUnlock() + return true // Invalid index - skip + } + health := fc.filerHealth[index] + fc.filerAddressesMu.RUnlock() + return fc.shouldSkipUnhealthyFilerWithHealth(health) +} + +// recordFilerSuccessWithHealth resets failure tracking for a successful filer +func (fc *FilerClient) recordFilerSuccessWithHealth(health *filerHealth) { + atomic.StoreInt32(&health.failureCount, 0) +} + // recordFilerSuccess resets failure tracking for a successful filer func (fc *FilerClient) recordFilerSuccess(index int32) { + fc.filerAddressesMu.RLock() + if index >= int32(len(fc.filerHealth)) { + fc.filerAddressesMu.RUnlock() + return // Invalid index + } health := fc.filerHealth[index] - atomic.StoreInt32(&health.failureCount, 0) + fc.filerAddressesMu.RUnlock() + fc.recordFilerSuccessWithHealth(health) +} + +// recordFilerFailureWithHealth increments failure count for an unhealthy filer +func (fc *FilerClient) recordFilerFailureWithHealth(health *filerHealth) { + atomic.AddInt32(&health.failureCount, 1) + atomic.StoreInt64(&health.lastFailureTimeNs, time.Now().UnixNano()) } // recordFilerFailure increments failure count for an unhealthy filer func (fc *FilerClient) recordFilerFailure(index int32) { + fc.filerAddressesMu.RLock() + if index >= int32(len(fc.filerHealth)) { + fc.filerAddressesMu.RUnlock() + return // Invalid index + } health := fc.filerHealth[index] - atomic.AddInt32(&health.failureCount, 1) - atomic.StoreInt64(&health.lastFailureTimeNs, time.Now().UnixNano()) + fc.filerAddressesMu.RUnlock() + fc.recordFilerFailureWithHealth(health) } // LookupVolumeIds queries the filer for volume locations with automatic failover @@ -299,13 +571,34 @@ func (p *filerVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []s // Try all filer addresses with round-robin starting from current index // Skip known-unhealthy filers (circuit breaker pattern) i := atomic.LoadInt32(&fc.filerIndex) + + // Get filer count with read lock + fc.filerAddressesMu.RLock() n := int32(len(fc.filerAddresses)) + fc.filerAddressesMu.RUnlock() for x := int32(0); x < n; x++ { - // Circuit breaker: skip unhealthy filers - if fc.shouldSkipUnhealthyFiler(i) { + // Get current filer address and health with read lock + fc.filerAddressesMu.RLock() + if len(fc.filerAddresses) == 0 { + fc.filerAddressesMu.RUnlock() + lastErr = fmt.Errorf("no filers available") + break + } + if i >= int32(len(fc.filerAddresses)) { + // Filer list changed, reset index + i = 0 + } + + // Get health pointer while holding lock + health := fc.filerHealth[i] + filerAddress := fc.filerAddresses[i] + fc.filerAddressesMu.RUnlock() + + // Circuit breaker: skip unhealthy filers (no lock needed - uses atomics) + if fc.shouldSkipUnhealthyFilerWithHealth(health) { glog.V(2).Infof("FilerClient: skipping unhealthy filer %s (consecutive failures: %d)", - fc.filerAddresses[i], atomic.LoadInt32(&fc.filerHealth[i].failureCount)) + filerAddress, atomic.LoadInt32(&health.failureCount)) i++ if i >= n { i = 0 @@ -313,8 +606,6 @@ func (p *filerVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []s continue } - filerAddress := fc.filerAddresses[i] - // Use anonymous function to ensure defer cancel() is called per iteration, not accumulated err := func() error { // Create a fresh timeout context for each filer attempt @@ -367,7 +658,7 @@ func (p *filerVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []s if err != nil { glog.V(1).Infof("FilerClient: filer %s lookup failed (attempt %d/%d, retry %d/%d): %v", filerAddress, x+1, n, retry+1, maxRetries, err) - fc.recordFilerFailure(i) + fc.recordFilerFailureWithHealth(health) lastErr = err i++ if i >= n { @@ -378,7 +669,7 @@ func (p *filerVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []s // Success - update the preferred filer index and reset health tracking atomic.StoreInt32(&fc.filerIndex, i) - fc.recordFilerSuccess(i) + fc.recordFilerSuccessWithHealth(health) glog.V(3).Infof("FilerClient: looked up %d volumes on %s, found %d", len(volumeIds), filerAddress, len(result)) return result, nil } @@ -400,5 +691,8 @@ func (p *filerVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []s } // All retries exhausted - return nil, fmt.Errorf("all %d filer(s) failed after %d attempts, last error: %w", len(fc.filerAddresses), maxRetries, lastErr) + fc.filerAddressesMu.RLock() + totalFilers := len(fc.filerAddresses) + fc.filerAddressesMu.RUnlock() + return nil, fmt.Errorf("all %d filer(s) failed after %d attempts, last error: %w", totalFilers, maxRetries, lastErr) }