volume: get metrics configuration from master

fix https://github.com/chrislusf/seaweedfs/issues/1354

weed/command/master.go

@@ -57,7 +57,7 @@ func init() {
 	m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	m.whiteList = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
 	m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
-	m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address")
+	m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address <host>:<port>")
 	m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 }
 

weed/command/s3.go

@@ -152,6 +152,8 @@ func (s3opt *S3Options) startS3Server() bool {
 			break
 		}
 	}
+
+	glog.V(0).Infof("s3 server sends metrics to %s every %d seconds", metricsAddress, metricsIntervalSec)
 	if metricsAddress != "" && metricsIntervalSec > 0 {
 		go stats_collect.LoopPushingMetric("s3", stats_collect.SourceName(uint32(*s3opt.port)), stats_collect.S3Gather, metricsAddress, metricsIntervalSec)
 	}

weed/server/filer_server.go

@@ -156,6 +156,9 @@ func (fs *FilerServer) maybeStartMetrics() {
 			}
 		}
 	}
+
+	glog.V(0).Infof("filer sends metrics to %s every %d seconds", fs.metricsAddress, fs.metricsIntervalSec)
+
 	if fs.metricsAddress == "" && fs.metricsIntervalSec <= 0 {
 		return
 	}

weed/server/master_grpc_server_volume.go

@@ -3,6 +3,7 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 
 	"github.com/chrislusf/raft"
@@ -182,6 +183,8 @@ func (ms *MasterServer) LookupEcVolume(ctx context.Context, req *master_pb.Looku
 
 func (ms *MasterServer) GetMasterConfiguration(ctx context.Context, req *master_pb.GetMasterConfigurationRequest) (*master_pb.GetMasterConfigurationResponse, error) {
 
+	glog.V(0).Infof("master sends metrics to %s every %d seconds", ms.option.MetricsAddress, ms.option.MetricsIntervalSec)
+
 	resp := &master_pb.GetMasterConfigurationResponse{
 		MetricsAddress:         ms.option.MetricsAddress,
 		MetricsIntervalSeconds: uint32(ms.option.MetricsIntervalSec),

weed/server/volume_grpc_client_to_master.go

@@ -24,21 +24,25 @@ func (vs *VolumeServer) GetMaster() string {
 }
 
 func (vs *VolumeServer) checkWithMaster() (err error) {
-	for _, master := range vs.SeedMasterNodes {
-		err = operation.WithMasterServerClient(master, vs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
-			resp, err := masterClient.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
-			if err != nil {
-				return fmt.Errorf("get master %s configuration: %v", master, err)
+	isConnected := false
+	for !isConnected {
+		for _, master := range vs.SeedMasterNodes {
+			err = operation.WithMasterServerClient(master, vs.grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+				resp, err := masterClient.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
+				if err != nil {
+					return fmt.Errorf("get master %s configuration: %v", master, err)
+				}
+				vs.metricsAddress, vs.metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
+				backend.LoadFromPbStorageBackends(resp.StorageBackends)
+				return nil
+			})
+			if err == nil {
+				return
+			} else {
+				glog.V(0).Infof("checkWithMaster %s: %v", master, err)
 			}
-			vs.MetricsAddress, vs.MetricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
-			backend.LoadFromPbStorageBackends(resp.StorageBackends)
-			return nil
-		})
-		if err == nil {
-			return
-		} else {
-			glog.V(0).Infof("checkWithMaster %s: %v", master, err)
 		}
+		time.Sleep(1790 * time.Millisecond)
 	}
 	return
 }

weed/server/volume_server.go

@@ -28,8 +28,8 @@ type VolumeServer struct {
 	FixJpgOrientation       bool
 	ReadRedirect            bool
 	compactionBytePerSecond int64
-	MetricsAddress          string
-	MetricsIntervalSec      int
+	metricsAddress          string
+	metricsIntervalSec      int
 	fileSizeLimitBytes      int64
 	isHeartbeating          bool
 	stopChan                chan bool
@@ -97,8 +97,9 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	}
 
 	go vs.heartbeat()
+	glog.V(0).Infof("volume server sends metrics to %s every %d seconds", vs.metricsAddress, vs.metricsIntervalSec)
 	hostAddress := fmt.Sprintf("%s:%d", ip, port)
-	go stats.LoopPushingMetric("volumeServer", hostAddress, stats.VolumeServerGather, vs.MetricsAddress, vs.MetricsIntervalSec)
+	go stats.LoopPushingMetric("volumeServer", hostAddress, stats.VolumeServerGather, vs.metricsAddress, vs.metricsIntervalSec)
 
 	return vs
 }