From 5f6c9825f8032ae34c64b3459f9ff2aff0c5efcd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 14 Jun 2019 00:54:56 -0700 Subject: [PATCH] volume server: adds basic metrics --- weed/command/server.go | 2 ++ weed/command/volume.go | 6 ++++ weed/server/filer_server.go | 10 +++---- weed/server/metrics.go | 31 +++++++++++++++++++-- weed/server/volume_server.go | 5 ++++ weed/server/volume_server_handlers_read.go | 5 ++++ weed/server/volume_server_handlers_write.go | 10 +++++++ 7 files changed, 61 insertions(+), 8 deletions(-) diff --git a/weed/command/server.go b/weed/command/server.go index 30618c452..f6c2b5036 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -149,6 +149,8 @@ func runServer(cmd *Command, args []string) bool { filerOptions.metricsAddress = serverOptions.metricsAddress filerOptions.metricsIntervalSec = serverOptions.metricsIntervalSec + serverOptions.v.metricsAddress = serverOptions.metricsAddress + serverOptions.v.metricsIntervalSec = serverOptions.metricsIntervalSec filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port) s3Options.filer = &filerAddress diff --git a/weed/command/volume.go b/weed/command/volume.go index c775ac5cf..5726528d5 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -45,6 +45,8 @@ type VolumeServerOptions struct { cpuProfile *string memProfile *string compactionMBPerSecond *int + metricsAddress *string + metricsIntervalSec *int } func init() { @@ -66,6 +68,8 @@ func init() { v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file") v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file") v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second") + v.metricsAddress = cmdVolume.Flag.String("metrics.address", "", "Prometheus gateway address") + v.metricsIntervalSec = cmdVolume.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds") } var cmdVolume = &Command{ @@ -161,6 +165,8 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v v.whiteList, *v.fixJpgOrientation, *v.readRedirect, *v.compactionMBPerSecond, + *v.metricsAddress, + *v.metricsIntervalSec, ) listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 0046035b0..9cd425981 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -88,23 +88,23 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) readonlyMux.HandleFunc("/", fs.readonlyFilerHandler) } - startPushingMetric(option.MetricsAddress, option.MetricsIntervalSec) + startPushingMetric("filer", filerGather, option.MetricsAddress, option.MetricsIntervalSec) return fs, nil } -func startPushingMetric(addr string, intervalSeconds int) { +func startPushingMetric(name string, gatherer *prometheus.Registry, addr string, intervalSeconds int) { if intervalSeconds == 0 || addr == "" { glog.V(0).Info("disable metrics reporting") return } glog.V(0).Infof("push metrics to %s every %d seconds", addr, intervalSeconds) - go loopPushMetrics(addr, intervalSeconds) + go loopPushMetrics(name, gatherer, addr, intervalSeconds) } -func loopPushMetrics(addr string, intervalSeconds int) { +func loopPushMetrics(name string, gatherer *prometheus.Registry, addr string, intervalSeconds int) { - pusher := push.New(addr, "filer").Gatherer(prometheus.DefaultGatherer) + pusher := push.New(addr, name).Gatherer(gatherer) for { err := pusher.Push() diff --git a/weed/server/metrics.go b/weed/server/metrics.go index 89a7d2a96..0bee27d22 100644 --- a/weed/server/metrics.go +++ b/weed/server/metrics.go @@ -3,6 +3,9 @@ package weed_server import "github.com/prometheus/client_golang/prometheus" var ( + filerGather = prometheus.NewRegistry() + volumeServerGather = prometheus.NewRegistry() + filerRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "SeaweedFS", @@ -17,11 +20,33 @@ var ( Subsystem: "filer", Name: "request_seconds", Help: "Bucketed histogram of filer request processing time.", - Buckets: prometheus.ExponentialBuckets(0.0005, 2, 18), + Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), + }, []string{"type"}) + + volumeServerRequestCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "SeaweedFS", + Subsystem: "volumeServer", + Name: "request_total", + Help: "Counter of filer requests.", + }, []string{"type"}) + + volumeServerHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "SeaweedFS", + Subsystem: "volumeServer", + Name: "request_seconds", + Help: "Bucketed histogram of filer request processing time.", + Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), }, []string{"type"}) ) func init() { - prometheus.MustRegister(filerRequestCounter) - prometheus.MustRegister(filerRequestHistogram) + + filerGather.MustRegister(filerRequestCounter) + filerGather.MustRegister(filerRequestHistogram) + + volumeServerGather.MustRegister(volumeServerRequestCounter) + volumeServerGather.MustRegister(volumeServerHistogram) + } diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index f0da20323..0bb2f0f6f 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -24,6 +24,8 @@ type VolumeServer struct { FixJpgOrientation bool ReadRedirect bool compactionBytePerSecond int64 + MetricsAddress string + MetricsIntervalSec int } func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, @@ -36,6 +38,8 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, fixJpgOrientation bool, readRedirect bool, compactionMBPerSecond int, + metricsAddress string, + metricsIntervalSec int, ) *VolumeServer { v := viper.GetViper() @@ -80,6 +84,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, } go vs.heartbeat() + startPushingMetric("volumeServer", volumeServerGather, metricsAddress, metricsIntervalSec) return vs } diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index d2aaa4e24..5e8bc7469 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -26,6 +26,11 @@ import ( var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { + + volumeServerRequestCounter.WithLabelValues("get").Inc() + start := time.Now() + defer func() { volumeServerHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }() + n := new(needle.Needle) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go index 7c0e08554..b53defd26 100644 --- a/weed/server/volume_server_handlers_write.go +++ b/weed/server/volume_server_handlers_write.go @@ -15,6 +15,11 @@ import ( ) func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { + + volumeServerRequestCounter.WithLabelValues("post").Inc() + start := time.Now() + defer func() { volumeServerHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) }() + if e := r.ParseForm(); e != nil { glog.V(0).Infoln("form parse error:", e) writeJsonError(w, r, http.StatusBadRequest, e) @@ -60,6 +65,11 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { } func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { + + volumeServerRequestCounter.WithLabelValues("delete").Inc() + start := time.Now() + defer func() { volumeServerHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) }() + n := new(needle.Needle) vid, fid, _, _, _ := parseURLPath(r.URL.Path) volumeId, _ := needle.NewVolumeId(vid)