fix: record and delete bucket metrics after inactive (#6349)

weed/s3api/stats.go

@@ -1,12 +1,13 @@
 package s3api
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
-	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"net/http"
 	"strconv"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 func track(f http.HandlerFunc, action string) http.HandlerFunc {
@@ -25,10 +26,12 @@ func track(f http.HandlerFunc, action string) http.HandlerFunc {
 		}
 		stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds())
 		stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc()
+		stats_collect.RecordBucketActiveTime(bucket)
 	}
 }
 
 func TimeToFirstByte(action string, start time.Time, r *http.Request) {
 	bucket, _ := s3_constants.GetBucketAndObject(r)
 	stats_collect.S3TimeToFirstByteHistogram.WithLabelValues(action, bucket).Observe(float64(time.Since(start).Milliseconds()))
+	stats_collect.RecordBucketActiveTime(bucket)
 }

weed/stats/metrics.go

@@ -23,9 +23,11 @@ const (
 	NoWriteOrDelete  = "noWriteOrDelete"
 	NoWriteCanDelete = "noWriteCanDelete"
 	IsDiskSpaceLow   = "isDiskSpaceLow"
+	bucketAtiveTTL   = 10 * time.Minute
 )
 
 var readOnlyVolumeTypes = [4]string{IsReadOnly, NoWriteOrDelete, NoWriteCanDelete, IsDiskSpaceLow}
+var bucketLastActiveTsNs map[string]int64 = map[string]int64{}
 
 var (
 	Gather = prometheus.NewRegistry()
@@ -281,6 +283,7 @@ var (
 			Name:      "request_total",
 			Help:      "Counter of s3 requests.",
 		}, []string{"type", "code", "bucket"})
+
 	S3HandlerCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: Namespace,
@@ -288,6 +291,7 @@ var (
 			Name:      "handler_total",
 			Help:      "Counter of s3 server handlers.",
 		}, []string{"type"})
+
 	S3RequestHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: Namespace,
@@ -296,6 +300,7 @@ var (
 			Help:      "Bucketed histogram of s3 request processing time.",
 			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 24),
 		}, []string{"type", "bucket"})
+
 	S3TimeToFirstByteHistogram = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: Namespace,
@@ -354,6 +359,8 @@ func init() {
 	Gather.MustRegister(S3RequestHistogram)
 	Gather.MustRegister(S3InFlightRequestsGauge)
 	Gather.MustRegister(S3TimeToFirstByteHistogram)
+
+	go bucketMetricTTLControl()
 }
 
 func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {
@@ -401,11 +408,40 @@ func SourceName(port uint32) string {
 	return net.JoinHostPort(hostname, strconv.Itoa(int(port)))
 }
 
-// todo - can be changed to DeletePartialMatch when https://github.com/prometheus/client_golang/pull/1013 gets released
+func RecordBucketActiveTime(bucket string) {
+	bucketLastActiveTsNs[bucket] = time.Now().UnixNano()
+}
+
 func DeleteCollectionMetrics(collection string) {
-	VolumeServerDiskSizeGauge.DeleteLabelValues(collection, "normal")
-	for _, volume_type := range readOnlyVolumeTypes {
-		VolumeServerReadOnlyVolumeGauge.DeleteLabelValues(collection, volume_type)
+	labels := prometheus.Labels{"collection": collection}
+	c := MasterReplicaPlacementMismatch.DeletePartialMatch(labels)
+	c += MasterVolumeLayoutWritable.DeletePartialMatch(labels)
+	c += MasterVolumeLayoutCrowded.DeletePartialMatch(labels)
+	c += VolumeServerDiskSizeGauge.DeletePartialMatch(labels)
+	c += VolumeServerVolumeGauge.DeletePartialMatch(labels)
+	c += VolumeServerReadOnlyVolumeGauge.DeletePartialMatch(labels)
+
+	glog.V(0).Infof("delete collection metrics, %s: %d", collection, c)
+}
+
+func bucketMetricTTLControl() {
+	ttlNs := bucketAtiveTTL.Nanoseconds()
+	for {
+		now := time.Now().UnixNano()
+
+		for bucket, ts := range bucketLastActiveTsNs {
+			if (now - ts) > ttlNs {
+				delete(bucketLastActiveTsNs, bucket)
+
+				labels := prometheus.Labels{"bucket": bucket}
+				c := S3RequestCounter.DeletePartialMatch(labels)
+				c += S3RequestHistogram.DeletePartialMatch(labels)
+				c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels)
+				glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c)
+			}
 		}
-	VolumeServerVolumeGauge.DeleteLabelValues(collection, "volume")
+
+		time.Sleep(bucketAtiveTTL)
+	}
+
 }