diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index c4cfd1bd9..249933035 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -20,6 +20,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + stats_collect "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" @@ -396,8 +397,9 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque return } - // Clean up bucket-related caches and locks after successful deletion + // Clean up bucket-related caches, locks, and metrics after successful deletion s3a.invalidateBucketConfigCache(bucket) + stats_collect.DeleteBucketMetrics(bucket) s3err.WriteEmptyResponse(w, r, http.StatusNoContent) } diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 9c3b902a3..d07b53d76 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -573,6 +573,26 @@ func RecordBucketActiveTime(bucket string) { bucketLastActiveLock.Unlock() } +func DeleteBucketMetrics(bucket string) { + bucketLastActiveLock.Lock() + delete(bucketLastActiveTsNs, bucket) + bucketLastActiveLock.Unlock() + + labels := prometheus.Labels{"bucket": bucket} + c := S3RequestCounter.DeletePartialMatch(labels) + c += S3RequestHistogram.DeletePartialMatch(labels) + c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels) + c += S3BucketTrafficReceivedBytesCounter.DeletePartialMatch(labels) + c += S3BucketTrafficSentBytesCounter.DeletePartialMatch(labels) + c += S3DeletedObjectsCounter.DeletePartialMatch(labels) + c += S3UploadedObjectsCounter.DeletePartialMatch(labels) + c += S3BucketSizeBytesGauge.DeletePartialMatch(labels) + c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels) + c += S3BucketObjectCountGauge.DeletePartialMatch(labels) + + glog.V(0).Infof("delete bucket metrics, %s: %d", bucket, c) +} + func DeleteCollectionMetrics(collection string) { labels := prometheus.Labels{"collection": collection} c := MasterReplicaPlacementMismatch.DeletePartialMatch(labels) @@ -605,13 +625,11 @@ func bucketMetricTTLControl() { for _, bucket := range expiredBuckets { labels := prometheus.Labels{"bucket": bucket} - c := S3RequestCounter.DeletePartialMatch(labels) - c += S3RequestHistogram.DeletePartialMatch(labels) + // Only delete gauges and histograms, which represent current state. + // Counters (traffic, requests, objects) must persist for the process + // lifetime so that Prometheus rate()/increase() queries work correctly. + c := S3RequestHistogram.DeletePartialMatch(labels) c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels) - c += S3BucketTrafficReceivedBytesCounter.DeletePartialMatch(labels) - c += S3BucketTrafficSentBytesCounter.DeletePartialMatch(labels) - c += S3DeletedObjectsCounter.DeletePartialMatch(labels) - c += S3UploadedObjectsCounter.DeletePartialMatch(labels) c += S3BucketSizeBytesGauge.DeletePartialMatch(labels) c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels) c += S3BucketObjectCountGauge.DeletePartialMatch(labels)