Browse Source
feat: add S3 bucket size and object count metrics (#7776)
feat: add S3 bucket size and object count metrics (#7776)
* feat: add S3 bucket size and object count metrics Adds periodic collection of bucket size metrics: - SeaweedFS_s3_bucket_size_bytes: logical size (deduplicated across replicas) - SeaweedFS_s3_bucket_physical_size_bytes: physical size (including replicas) - SeaweedFS_s3_bucket_object_count: object count (deduplicated) Collection runs every 1 minute via background goroutine that queries filer Statistics RPC for each bucket's collection. Also adds Grafana dashboard panels for: - S3 Bucket Size (logical vs physical) - S3 Bucket Object Count * address PR comments: fix bucket size metrics collection 1. Fix collectCollectionInfoFromMaster to use master VolumeList API - Now properly queries master for topology info - Uses WithMasterClient to get volume list from master - Correctly calculates logical vs physical size based on replication 2. Return error when filerClient is nil to trigger fallback - Changed from 'return nil, nil' to 'return nil, error' - Ensures fallback to filer stats is properly triggered 3. Implement pagination in listBucketNames - Added listBucketPageSize constant (1000) - Uses StartFromFileName for pagination - Continues fetching until fewer entries than limit returned 4. Handle NewReplicaPlacementFromByte error and prevent division by zero - Check error return from NewReplicaPlacementFromByte - Default to 1 copy if error occurs - Add explicit check for copyCount == 0 * simplify bucket size metrics: remove filer fallback, align with quota enforcement - Remove fallback to filer Statistics RPC - Use only master topology for collection info (same as s3.bucket.quota.enforce) - Updated comments to clarify this runs the same collection logic as quota enforcement - Simplified code by removing collectBucketSizeFromFilerStats * use s3a.option.Masters directly instead of querying filer * address PR comments: fix dashboard overlaps and improve metrics collection Grafana dashboard fixes: - Fix overlapping panels 55 and 59 in grafana_seaweedfs.json (moved 59 to y=30) - Fix grid collision in k8s dashboard (moved panel 72 to y=48) - Aggregate bucket metrics with max() by (bucket) for multi-instance S3 gateways Go code improvements: - Add graceful shutdown support via context cancellation - Use ticker instead of time.Sleep for better shutdown responsiveness - Distinguish EOF from actual errors in stream handling * improve bucket size metrics: multi-master failover and proper error handling - Initial delay now respects context cancellation using select with time.After - Use WithOneOfGrpcMasterClients for multi-master failover instead of hardcoding Masters[0] - Properly propagate stream errors instead of just logging them (EOF vs real errors) * improve bucket size metrics: distributed lock and volume ID deduplication - Add distributed lock (LiveLock) so only one S3 instance collects metrics at a time - Add IsLocked() method to LiveLock for checking lock status - Fix deduplication: use volume ID tracking instead of dividing by copyCount - Previous approach gave wrong results if replicas were missing - Now tracks seen volume IDs and counts each volume only once - Physical size still includes all replicas for accurate disk usage reporting * rename lock to s3.leader * simplify: remove StartBucketSizeMetricsCollection wrapper function * fix data race: use atomic operations for LiveLock.isLocked field - Change isLocked from bool to int32 - Use atomic.LoadInt32/StoreInt32 for all reads/writes - Sync shared isLocked field in StartLongLivedLock goroutine * add nil check for topology info to prevent panic * fix bucket metrics: use Ticker for consistent intervals, fix pagination logic - Use time.Ticker instead of time.After for consistent interval execution - Fix pagination: count all entries (not just directories) for proper termination - Update lastFileName for all entries to prevent pagination issues * address PR comments: remove redundant atomic store, propagate context - Remove redundant atomic.StoreInt32 in StartLongLivedLock (AttemptToLock already sets it) - Propagate context through metrics collection for proper cancellation on shutdown - collectAndUpdateBucketSizeMetrics now accepts ctx - collectCollectionInfoFromMaster uses ctx for VolumeList RPC - listBucketNames uses ctx for ListEntries RPCpull/7782/head
Chris Lu
3 days ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 1100 additions and 9 deletions
-
203k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json
-
194other/metrics/grafana_seaweedfs.json
-
196other/metrics/grafana_seaweedfs_heartbeat.json
-
203other/metrics/grafana_seaweedfs_k8s.json
-
2test/s3/iam/test_config.json
-
18weed/cluster/lock_client.go
-
242weed/s3api/bucket_size_metrics.go
-
4weed/s3api/s3api_server.go
-
41weed/stats/metrics.go
@ -0,0 +1,242 @@ |
|||||
|
package s3api |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"io" |
||||
|
"strings" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/cluster" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/glog" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/stats" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
bucketSizeMetricsInterval = 1 * time.Minute |
||||
|
listBucketPageSize = 1000 // Page size for paginated bucket listing
|
||||
|
s3MetricsLockName = "s3.leader" |
||||
|
) |
||||
|
|
||||
|
// CollectionInfo holds collection statistics
|
||||
|
// Used for both metrics collection and quota enforcement
|
||||
|
type CollectionInfo struct { |
||||
|
FileCount float64 |
||||
|
DeleteCount float64 |
||||
|
DeletedByteCount float64 |
||||
|
Size float64 // Logical size (deduplicated by volume ID)
|
||||
|
PhysicalSize float64 // Physical size (including all replicas)
|
||||
|
VolumeCount int // Logical volume count (deduplicated by volume ID)
|
||||
|
} |
||||
|
|
||||
|
// volumeKey uniquely identifies a volume for deduplication
|
||||
|
type volumeKey struct { |
||||
|
collection string |
||||
|
volumeId uint32 |
||||
|
} |
||||
|
|
||||
|
// startBucketSizeMetricsLoop periodically collects bucket size metrics and updates Prometheus gauges.
|
||||
|
// Uses a distributed lock to ensure only one S3 instance collects metrics at a time.
|
||||
|
// Should be called as a goroutine; stops when the provided context is cancelled.
|
||||
|
func (s3a *S3ApiServer) startBucketSizeMetricsLoop(ctx context.Context) { |
||||
|
// Initial delay to let the system stabilize
|
||||
|
select { |
||||
|
case <-time.After(10 * time.Second): |
||||
|
case <-ctx.Done(): |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// Create lock client for distributed lock
|
||||
|
if len(s3a.option.Filers) == 0 { |
||||
|
glog.V(1).Infof("No filers configured, skipping bucket size metrics collection") |
||||
|
return |
||||
|
} |
||||
|
filer := s3a.option.Filers[0] |
||||
|
lockClient := cluster.NewLockClient(s3a.option.GrpcDialOption, filer) |
||||
|
owner := string(filer) + "-s3-metrics" |
||||
|
|
||||
|
// Start long-lived lock - this S3 instance will only collect metrics when it holds the lock
|
||||
|
lock := lockClient.StartLongLivedLock(s3MetricsLockName, owner, func(newLockOwner string) { |
||||
|
glog.V(1).Infof("S3 bucket size metrics lock owner changed to: %s", newLockOwner) |
||||
|
}) |
||||
|
defer lock.Stop() |
||||
|
|
||||
|
ticker := time.NewTicker(bucketSizeMetricsInterval) |
||||
|
defer ticker.Stop() |
||||
|
|
||||
|
for { |
||||
|
select { |
||||
|
case <-ctx.Done(): |
||||
|
glog.V(1).Infof("Stopping bucket size metrics collection") |
||||
|
return |
||||
|
case <-ticker.C: |
||||
|
// Only collect metrics if we hold the lock
|
||||
|
if lock.IsLocked() { |
||||
|
s3a.collectAndUpdateBucketSizeMetrics(ctx) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// collectAndUpdateBucketSizeMetrics collects bucket sizes from master topology
|
||||
|
// and updates Prometheus metrics. Uses the same approach as quota enforcement.
|
||||
|
func (s3a *S3ApiServer) collectAndUpdateBucketSizeMetrics(ctx context.Context) { |
||||
|
// Collect collection info from master topology (same as quota enforcement)
|
||||
|
collectionInfos, err := s3a.collectCollectionInfoFromMaster(ctx) |
||||
|
if err != nil { |
||||
|
glog.V(2).Infof("Failed to collect collection info from master: %v", err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// Get list of buckets
|
||||
|
buckets, err := s3a.listBucketNames(ctx) |
||||
|
if err != nil { |
||||
|
glog.V(2).Infof("Failed to list buckets for size metrics: %v", err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// Map collections to buckets and update metrics
|
||||
|
for _, bucket := range buckets { |
||||
|
collection := s3a.getCollectionName(bucket) |
||||
|
if info, found := collectionInfos[collection]; found { |
||||
|
stats.UpdateBucketSizeMetrics(bucket, info.Size, info.PhysicalSize, info.FileCount) |
||||
|
glog.V(3).Infof("Updated bucket size metrics: bucket=%s, logicalSize=%.0f, physicalSize=%.0f, objects=%.0f", |
||||
|
bucket, info.Size, info.PhysicalSize, info.FileCount) |
||||
|
} else { |
||||
|
// Bucket exists but no collection data (empty bucket)
|
||||
|
stats.UpdateBucketSizeMetrics(bucket, 0, 0, 0) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// collectCollectionInfoFromMaster queries the master for topology info and extracts collection sizes.
|
||||
|
// This is the same approach used by shell command s3.bucket.quota.enforce.
|
||||
|
func (s3a *S3ApiServer) collectCollectionInfoFromMaster(ctx context.Context) (map[string]*CollectionInfo, error) { |
||||
|
if len(s3a.option.Masters) == 0 { |
||||
|
return nil, fmt.Errorf("no masters configured") |
||||
|
} |
||||
|
|
||||
|
// Convert masters slice to map for WithOneOfGrpcMasterClients
|
||||
|
masterMap := make(map[string]pb.ServerAddress) |
||||
|
for _, master := range s3a.option.Masters { |
||||
|
masterMap[string(master)] = master |
||||
|
} |
||||
|
|
||||
|
// Connect to any available master and get volume list with topology
|
||||
|
collectionInfos := make(map[string]*CollectionInfo) |
||||
|
|
||||
|
err := pb.WithOneOfGrpcMasterClients(false, masterMap, s3a.option.GrpcDialOption, func(client master_pb.SeaweedClient) error { |
||||
|
resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{}) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("failed to get volume list: %w", err) |
||||
|
} |
||||
|
if resp == nil || resp.TopologyInfo == nil { |
||||
|
return fmt.Errorf("empty topology info from master") |
||||
|
} |
||||
|
collectCollectionInfoFromTopology(resp.TopologyInfo, collectionInfos) |
||||
|
return nil |
||||
|
}) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
|
||||
|
return collectionInfos, nil |
||||
|
} |
||||
|
|
||||
|
// listBucketNames returns a list of all bucket names using pagination
|
||||
|
func (s3a *S3ApiServer) listBucketNames(ctx context.Context) ([]string, error) { |
||||
|
var buckets []string |
||||
|
|
||||
|
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { |
||||
|
lastFileName := "" |
||||
|
for { |
||||
|
request := &filer_pb.ListEntriesRequest{ |
||||
|
Directory: s3a.option.BucketsPath, |
||||
|
StartFromFileName: lastFileName, |
||||
|
Limit: listBucketPageSize, |
||||
|
InclusiveStartFrom: lastFileName == "", |
||||
|
} |
||||
|
|
||||
|
stream, err := client.ListEntries(ctx, request) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
entriesReceived := 0 |
||||
|
for { |
||||
|
resp, err := stream.Recv() |
||||
|
if err != nil { |
||||
|
if err == io.EOF { |
||||
|
break |
||||
|
} |
||||
|
return fmt.Errorf("error receiving bucket list entries: %w", err) |
||||
|
} |
||||
|
entriesReceived++ |
||||
|
if resp.Entry != nil { |
||||
|
lastFileName = resp.Entry.Name |
||||
|
if resp.Entry.IsDirectory { |
||||
|
// Skip .uploads and other hidden directories
|
||||
|
if !strings.HasPrefix(resp.Entry.Name, ".") { |
||||
|
buckets = append(buckets, resp.Entry.Name) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// If we got fewer entries than the limit, we're done
|
||||
|
if entriesReceived < listBucketPageSize { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
}) |
||||
|
|
||||
|
return buckets, err |
||||
|
} |
||||
|
|
||||
|
// collectCollectionInfoFromTopology extracts collection info from topology.
|
||||
|
// Deduplicates by volume ID to correctly handle missing replicas.
|
||||
|
// Unlike dividing by copyCount (which would give wrong results if replicas are missing),
|
||||
|
// we track seen volume IDs and only count each volume once for logical size/count.
|
||||
|
func collectCollectionInfoFromTopology(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) { |
||||
|
// Track which volumes we've already seen to deduplicate by volume ID
|
||||
|
seenVolumes := make(map[volumeKey]bool) |
||||
|
|
||||
|
for _, dc := range t.DataCenterInfos { |
||||
|
for _, r := range dc.RackInfos { |
||||
|
for _, dn := range r.DataNodeInfos { |
||||
|
for _, diskInfo := range dn.DiskInfos { |
||||
|
for _, vi := range diskInfo.VolumeInfos { |
||||
|
c := vi.Collection |
||||
|
cif, found := collectionInfos[c] |
||||
|
if !found { |
||||
|
cif = &CollectionInfo{} |
||||
|
collectionInfos[c] = cif |
||||
|
} |
||||
|
|
||||
|
// Always add to physical size (all replicas)
|
||||
|
cif.PhysicalSize += float64(vi.Size) |
||||
|
|
||||
|
// Check if we've already counted this volume for logical stats
|
||||
|
key := volumeKey{collection: c, volumeId: vi.Id} |
||||
|
if seenVolumes[key] { |
||||
|
// Already counted this volume, skip logical stats
|
||||
|
continue |
||||
|
} |
||||
|
seenVolumes[key] = true |
||||
|
|
||||
|
// First time seeing this volume - add to logical stats
|
||||
|
cif.Size += float64(vi.Size) |
||||
|
cif.FileCount += float64(vi.FileCount) |
||||
|
cif.DeleteCount += float64(vi.DeleteCount) |
||||
|
cif.DeletedByteCount += float64(vi.DeletedByteCount) |
||||
|
cif.VolumeCount++ |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue