From ab50f21ad92c994ebe514017493e70c051ef175a Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 1 Aug 2022 21:16:57 +0500 Subject: [PATCH 1/5] Detailed metrics VolumeServerRequestHistogram for writing to disk and replication --- weed/topology/store_replicate.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index e426e15de..578cf758d 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -9,6 +9,7 @@ import ( "net/url" "strconv" "strings" + "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/operation" @@ -43,7 +44,9 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt } if s.GetVolume(volumeId) != nil { + start := time.Now() isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync) + stats.VolumeServerRequestHistogram.WithLabelValues("write").Observe(time.Since(start).Seconds()) if err != nil { stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc() err = fmt.Errorf("failed to write to local disk: %v", err) @@ -53,7 +56,8 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt } if len(remoteLocations) > 0 { //send to other replica locations - if err = DistributedOperation(remoteLocations, func(location operation.Location) error { + start := time.Now() + err = DistributedOperation(remoteLocations, func(location operation.Location) error { u := url.URL{ Scheme: "http", Host: location.Url, @@ -97,7 +101,9 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt } _, err := operation.UploadData(n.Data, uploadOption) return err - }); err != nil { + }) + stats.VolumeServerRequestHistogram.WithLabelValues("replicate").Observe(time.Since(start).Seconds()) + if err != nil { stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc() err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) glog.V(0).Infoln(err) From 0a5d3c3cf65516737f07ca6802bd6a2d2b327446 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 1 Aug 2022 23:09:09 +0500 Subject: [PATCH 2/5] rename to localWrite and replicatedWrite --- weed/topology/store_replicate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index 578cf758d..1a26becbb 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -46,7 +46,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt if s.GetVolume(volumeId) != nil { start := time.Now() isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync) - stats.VolumeServerRequestHistogram.WithLabelValues("write").Observe(time.Since(start).Seconds()) + stats.VolumeServerRequestHistogram.WithLabelValues("localWrite").Observe(time.Since(start).Seconds()) if err != nil { stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc() err = fmt.Errorf("failed to write to local disk: %v", err) @@ -102,7 +102,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt _, err := operation.UploadData(n.Data, uploadOption) return err }) - stats.VolumeServerRequestHistogram.WithLabelValues("replicate").Observe(time.Since(start).Seconds()) + stats.VolumeServerRequestHistogram.WithLabelValues("replicatedWrite").Observe(time.Since(start).Seconds()) if err != nil { stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc() err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) From df9cc31a0a403daf2694048dbb3593e14ba72b6a Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 2 Aug 2022 01:18:26 +0500 Subject: [PATCH 3/5] refactor --- weed/server/volume_server_handlers.go | 6 +++++- weed/server/volume_server_handlers_read.go | 6 ------ weed/server/volume_server_handlers_write.go | 15 --------------- weed/stats/metrics_names.go | 2 ++ weed/topology/store_replicate.go | 4 ++-- 5 files changed, 9 insertions(+), 24 deletions(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index e95ecc8da..fd043ad51 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -36,6 +36,11 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Credentials", "true") } + stats.VolumeServerRequestCounter.WithLabelValues(r.Method).Inc() + start := time.Now() + defer func() { + stats.VolumeServerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds()) + }() switch r.Method { case "GET", "HEAD": stats.ReadRequest() @@ -56,7 +61,6 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque stats.DeleteRequest() vs.guard.WhiteList(vs.DeleteHandler)(w, r) case "PUT", "POST": - contentLength := getContentLength(r) // exclude the replication from the concurrentUploadLimitMB if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 { diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index eaebc1630..4208dd4fc 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -20,7 +20,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/images" "github.com/seaweedfs/seaweedfs/weed/operation" - "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/util" @@ -29,11 +28,6 @@ import ( var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { - - stats.VolumeServerRequestCounter.WithLabelValues("get").Inc() - start := time.Now() - defer func() { stats.VolumeServerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }() - n := new(needle.Needle) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go index c25c40174..009980f56 100644 --- a/weed/server/volume_server_handlers_write.go +++ b/weed/server/volume_server_handlers_write.go @@ -11,19 +11,11 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/operation" - "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/topology" ) func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { - - stats.VolumeServerRequestCounter.WithLabelValues("post").Inc() - start := time.Now() - defer func() { - stats.VolumeServerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds()) - }() - if e := r.ParseForm(); e != nil { glog.V(0).Infoln("form parse error:", e) writeJsonError(w, r, http.StatusBadRequest, e) @@ -79,13 +71,6 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { } func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { - - stats.VolumeServerRequestCounter.WithLabelValues("delete").Inc() - start := time.Now() - defer func() { - stats.VolumeServerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) - }() - n := new(needle.Needle) vid, fid, _, _, _ := parseURLPath(r.URL.Path) volumeId, _ := needle.NewVolumeId(vid) diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index 59ae022aa..d4f9c23c9 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -4,6 +4,8 @@ package stats // The naming convention is ErrorSomeThing = "error.some.thing" const ( // volume server + LocalWrite = "localWrite" + ReplicatedWrite = "replicatedWrite" ErrorSizeMismatchOffsetSize = "errorSizeMismatchOffsetSize" ErrorSizeMismatch = "errorSizeMismatch" ErrorCRC = "errorCRC" diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index 1a26becbb..0fff29ba6 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -46,7 +46,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt if s.GetVolume(volumeId) != nil { start := time.Now() isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync) - stats.VolumeServerRequestHistogram.WithLabelValues("localWrite").Observe(time.Since(start).Seconds()) + stats.VolumeServerRequestHistogram.WithLabelValues(stats.LocalWrite).Observe(time.Since(start).Seconds()) if err != nil { stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc() err = fmt.Errorf("failed to write to local disk: %v", err) @@ -102,7 +102,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt _, err := operation.UploadData(n.Data, uploadOption) return err }) - stats.VolumeServerRequestHistogram.WithLabelValues("replicatedWrite").Observe(time.Since(start).Seconds()) + stats.VolumeServerRequestHistogram.WithLabelValues(stats.ReplicatedWrite).Observe(time.Since(start).Seconds()) if err != nil { stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc() err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) From 2a5629dd28410658129513dbb08048e06761e502 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 2 Aug 2022 01:29:44 +0500 Subject: [PATCH 4/5] align --- weed/stats/metrics_names.go | 3 ++- weed/topology/store_replicate.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index d4f9c23c9..722f4ad94 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -4,7 +4,8 @@ package stats // The naming convention is ErrorSomeThing = "error.some.thing" const ( // volume server - LocalWrite = "localWrite" + WriteToLocalDisk = "writeToLocalDisk" + WriteToReplicas = "writeToReplicas" ReplicatedWrite = "replicatedWrite" ErrorSizeMismatchOffsetSize = "errorSizeMismatchOffsetSize" ErrorSizeMismatch = "errorSizeMismatch" diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index 0fff29ba6..816f1fbe4 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -46,7 +46,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt if s.GetVolume(volumeId) != nil { start := time.Now() isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync) - stats.VolumeServerRequestHistogram.WithLabelValues(stats.LocalWrite).Observe(time.Since(start).Seconds()) + stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToLocalDisk).Observe(time.Since(start).Seconds()) if err != nil { stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc() err = fmt.Errorf("failed to write to local disk: %v", err) @@ -102,7 +102,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt _, err := operation.UploadData(n.Data, uploadOption) return err }) - stats.VolumeServerRequestHistogram.WithLabelValues(stats.ReplicatedWrite).Observe(time.Since(start).Seconds()) + stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToReplicas).Observe(time.Since(start).Seconds()) if err != nil { stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc() err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) From 3ffa4ba91dfdeba985ab6f7933b4d789ab702442 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 2 Aug 2022 01:30:38 +0500 Subject: [PATCH 5/5] clean --- weed/stats/metrics_names.go | 1 - 1 file changed, 1 deletion(-) diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index 722f4ad94..3c62dd48e 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -6,7 +6,6 @@ const ( // volume server WriteToLocalDisk = "writeToLocalDisk" WriteToReplicas = "writeToReplicas" - ReplicatedWrite = "replicatedWrite" ErrorSizeMismatchOffsetSize = "errorSizeMismatchOffsetSize" ErrorSizeMismatch = "errorSizeMismatch" ErrorCRC = "errorCRC"