Browse Source

Enable filer discovery: pass master addresses to FilerClient

Fix two critical issues:

1. **Filer Discovery Not Working**: Master client was not being passed to
   FilerClient, so peer discovery couldn't work

2. **Credential Store Design**: Already uses FilerClient via GetCurrentFiler
   function - this is the correct design for HA

Changes:

**Command (s3.go):**
- Read master addresses from GetFilerConfiguration response
- Pass masterAddresses to S3ApiServerOption
- Log master addresses for visibility

**S3ApiServerOption:**
- Add Masters []pb.ServerAddress field for discovery

**S3ApiServer:**
- Create MasterClient from Masters when available
- Pass MasterClient + FilerGroup to FilerClient via options
- Enable discovery with 5-minute refresh interval
- Log whether discovery is enabled or disabled

**Credential Store:**
- Already correctly uses filerClient.GetCurrentFiler via function
- This provides HA without tight coupling to FilerClient struct
- Function-based design is clean and thread-safe

Discovery Flow:
1. S3 command reads filer config → gets masters + filer group
2. S3ApiServer creates MasterClient from masters
3. FilerClient uses MasterClient to query for peer filers
4. Background goroutine refreshes peer list every 5 minutes
5. Credential store uses GetCurrentFiler to get active filer

Now filer discovery actually works! ��
pull/7550/head
Chris Lu 3 days ago
parent
commit
af3a0770ff
  1. 7
      weed/command/s3.go
  2. 23
      weed/s3api/s3api_server.go

7
weed/command/s3.go

@ -207,6 +207,7 @@ func (s3opt *S3Options) startS3Server() bool {
filerBucketsPath := "/buckets"
filerGroup := ""
var masterAddresses []pb.ServerAddress
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
@ -222,8 +223,13 @@ func (s3opt *S3Options) startS3Server() bool {
}
filerBucketsPath = resp.DirBuckets
filerGroup = resp.FilerGroup
// Get master addresses for filer discovery
masterAddresses = pb.ServerAddresses(strings.Join(resp.Masters, ",")).ToAddresses()
metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
if len(masterAddresses) > 0 {
glog.V(0).Infof("S3 read master addresses for discovery: %v", masterAddresses)
}
return nil
})
if err != nil {
@ -256,6 +262,7 @@ func (s3opt *S3Options) startS3Server() bool {
s3ApiServer, s3ApiServer_err = s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
Filers: filerAddresses,
Masters: masterAddresses,
Port: *s3opt.port,
Config: *s3opt.config,
DomainName: *s3opt.domainName,

23
weed/s3api/s3api_server.go

@ -34,6 +34,7 @@ import (
type S3ApiServerOption struct {
Filers []pb.ServerAddress
Masters []pb.ServerAddress // For filer discovery
Port int
Config string
DomainName string
@ -100,8 +101,26 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
// Initialize FilerClient for volume location caching
// Uses the battle-tested vidMap with filer-based lookups
// Supports multiple filer addresses with automatic failover for high availability
filerClient := wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, option.DataCenter)
glog.V(0).Infof("S3 API initialized FilerClient with %d filer(s) for volume location caching", len(option.Filers))
var filerClient *wdclient.FilerClient
if len(option.Masters) > 0 && option.FilerGroup != "" {
// Enable filer discovery via master
masterMap := make(map[string]pb.ServerAddress)
for i, addr := range option.Masters {
masterMap[fmt.Sprintf("master%d", i)] = addr
}
masterClient := wdclient.NewMasterClient(option.GrpcDialOption, option.FilerGroup, "s3", "", "", "", *pb.NewServiceDiscoveryFromMap(masterMap))
filerClient = wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, option.DataCenter, &wdclient.FilerClientOption{
MasterClient: masterClient,
FilerGroup: option.FilerGroup,
DiscoveryInterval: 5 * time.Minute,
})
glog.V(0).Infof("S3 API initialized FilerClient with %d filer(s) and discovery enabled (group: %s, masters: %v)",
len(option.Filers), option.FilerGroup, option.Masters)
} else {
filerClient = wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, option.DataCenter)
glog.V(0).Infof("S3 API initialized FilerClient with %d filer(s) (no discovery)", len(option.Filers))
}
// Update credential store to use FilerClient's current filer for HA
if store := iam.credentialManager.GetStore(); store != nil {

Loading…
Cancel
Save