diff --git a/docker/admin_integration/create_vacuum_test_data.go b/docker/admin_integration/create_vacuum_test_data.go index 5f43a9744..0d7490394 100644 --- a/docker/admin_integration/create_vacuum_test_data.go +++ b/docker/admin_integration/create_vacuum_test_data.go @@ -472,9 +472,14 @@ type VolumeInfo struct { } type VolumeStatus struct { - IsLeader bool `json:"IsLeader"` - Leader string `json:"Leader"` - Volumes []VolumeInfo `json:"Volumes"` + Version string `json:"Version"` + Volumes VolumeLayout `json:"Volumes"` +} + +type VolumeLayout struct { + DataCenters map[string]map[string]map[string][]VolumeInfo `json:"DataCenters"` + Free int `json:"Free"` + Max int `json:"Max"` } func getVolumeStatusForDeletion() []VolumeInfo { @@ -492,7 +497,20 @@ func getVolumeStatusForDeletion() []VolumeInfo { return nil } - return volumeStatus.Volumes + // Extract all volumes from the nested structure + var allVolumes []VolumeInfo + for dcName, dataCenter := range volumeStatus.Volumes.DataCenters { + log.Printf("Processing data center: %s", dcName) + for rackName, rack := range dataCenter { + log.Printf("Processing rack: %s", rackName) + for serverName, volumes := range rack { + log.Printf("Found %d volumes on server %s", len(volumes), serverName) + allVolumes = append(allVolumes, volumes...) + } + } + } + + return allVolumes } type StoredFilePaths struct { diff --git a/docker/admin_integration/ec_test_files.json b/docker/admin_integration/ec_test_files.json index c1cbc0513..fb9181f24 100644 --- a/docker/admin_integration/ec_test_files.json +++ b/docker/admin_integration/ec_test_files.json @@ -1,37 +1,32 @@ { "file_paths": [ - "/ec_test/large_file_1754816105_0.dat", - "/ec_test/large_file_1754816105_1.dat", - "/ec_test/large_file_1754816105_2.dat", - "/ec_test/large_file_1754816105_3.dat", - "/ec_test/large_file_1754816105_4.dat", - "/ec_test/large_file_1754816105_5.dat", - "/ec_test/large_file_1754816105_6.dat", - "/ec_test/large_file_1754816105_7.dat", - "/ec_test/large_file_1754816105_8.dat", - "/ec_test/large_file_1754816105_9.dat", - "/ec_test/large_file_1754816105_10.dat", - "/ec_test/large_file_1754816105_11.dat", - "/ec_test/large_file_1754816105_12.dat", - "/ec_test/large_file_1754816106_13.dat", - "/ec_test/large_file_1754816106_14.dat", - "/ec_test/large_file_1754816106_15.dat", - "/ec_test/large_file_1754816106_16.dat", - "/ec_test/large_file_1754816106_17.dat", - "/ec_test/large_file_1754816106_18.dat", - "/ec_test/large_file_1754816106_19.dat", - "/ec_test/large_file_1754816106_20.dat", - "/ec_test/large_file_1754816106_21.dat", - "/ec_test/large_file_1754816106_22.dat", - "/ec_test/large_file_1754816106_23.dat", - "/ec_test/large_file_1754816106_24.dat", - "/ec_test/large_file_1754816106_25.dat", - "/ec_test/large_file_1754816106_26.dat", - "/ec_test/large_file_1754816106_27.dat", - "/ec_test/large_file_1754816106_28.dat", - "/ec_test/large_file_1754816106_29.dat" + "/ec_test/large_file_1754854356_0.dat", + "/ec_test/large_file_1754854356_1.dat", + "/ec_test/large_file_1754854356_2.dat", + "/ec_test/large_file_1754854356_3.dat", + "/ec_test/large_file_1754854356_4.dat", + "/ec_test/large_file_1754854356_5.dat", + "/ec_test/large_file_1754854356_6.dat", + "/ec_test/large_file_1754854356_7.dat", + "/ec_test/large_file_1754854356_8.dat", + "/ec_test/large_file_1754854356_9.dat", + "/ec_test/large_file_1754854356_10.dat", + "/ec_test/large_file_1754854356_11.dat", + "/ec_test/large_file_1754854356_12.dat", + "/ec_test/large_file_1754854356_13.dat", + "/ec_test/large_file_1754854356_14.dat", + "/ec_test/large_file_1754854356_15.dat", + "/ec_test/large_file_1754854356_16.dat", + "/ec_test/large_file_1754854356_17.dat", + "/ec_test/large_file_1754854357_18.dat", + "/ec_test/large_file_1754854357_19.dat", + "/ec_test/large_file_1754854357_20.dat", + "/ec_test/large_file_1754854357_21.dat", + "/ec_test/large_file_1754854357_22.dat", + "/ec_test/large_file_1754854357_23.dat", + "/ec_test/large_file_1754854357_24.dat" ], - "timestamp": "2025-08-10T08:55:06.363144049Z", - "file_count": 30, + "timestamp": "2025-08-10T19:32:37.127812169Z", + "file_count": 25, "file_size_kb": 3000 } \ No newline at end of file diff --git a/weed/admin/dash/ec_shard_management.go b/weed/admin/dash/ec_shard_management.go index 34574ecdb..556b970bc 100644 --- a/weed/admin/dash/ec_shard_management.go +++ b/weed/admin/dash/ec_shard_management.go @@ -727,6 +727,20 @@ func (s *AdminServer) GetEcVolumeDetails(volumeID uint32, sortBy string, sortOrd serverList = append(serverList, server) } + // Get EC volume health metrics (deletion information) + volumeHealth, err := s.getEcVolumeHealthMetrics(volumeID) + if err != nil { + glog.V(1).Infof("Failed to get EC volume health metrics for volume %d: %v", volumeID, err) + // Don't fail the request, just use default values + volumeHealth = &EcVolumeHealthInfo{ + TotalSize: 0, + DeletedByteCount: 0, + FileCount: 0, + DeleteCount: 0, + GarbageRatio: 0.0, + } + } + data := &EcVolumeDetailsData{ VolumeID: volumeID, Collection: collection, @@ -737,9 +751,159 @@ func (s *AdminServer) GetEcVolumeDetails(volumeID uint32, sortBy string, sortOrd DataCenters: dcList, Servers: serverList, LastUpdated: time.Now(), - SortBy: sortBy, - SortOrder: sortOrder, + + // Volume health metrics (for EC vacuum) + TotalSize: volumeHealth.TotalSize, + DeletedByteCount: volumeHealth.DeletedByteCount, + FileCount: volumeHealth.FileCount, + DeleteCount: volumeHealth.DeleteCount, + GarbageRatio: volumeHealth.GarbageRatio, + + SortBy: sortBy, + SortOrder: sortOrder, } return data, nil } + +// getEcVolumeHealthMetrics retrieves health metrics for an EC volume +func (s *AdminServer) getEcVolumeHealthMetrics(volumeID uint32) (*EcVolumeHealthInfo, error) { + // Get list of servers that have shards for this EC volume + var servers []string + + err := s.WithMasterClient(func(client master_pb.SeaweedClient) error { + resp, err := client.VolumeList(context.Background(), &master_pb.VolumeListRequest{}) + if err != nil { + return err + } + + if resp.TopologyInfo != nil { + for _, dc := range resp.TopologyInfo.DataCenterInfos { + for _, rack := range dc.RackInfos { + for _, node := range rack.DataNodeInfos { + for _, diskInfo := range node.DiskInfos { + // Check if this node has EC shards for our volume + for _, ecShardInfo := range diskInfo.EcShardInfos { + if ecShardInfo.Id == volumeID { + servers = append(servers, node.Id) + goto nextNode // Found shards on this node, move to next node + } + } + } + } + nextNode: + } + } + } + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to get topology info: %v", err) + } + + if len(servers) == 0 { + return nil, fmt.Errorf("no servers found with EC shards for volume %d", volumeID) + } + + // Try to get volume file status from servers that have EC shards + // The volume health metrics should be stored with the EC volume metadata + for _, server := range servers { + healthInfo, err := s.getVolumeHealthFromServer(server, volumeID) + if err != nil { + glog.V(2).Infof("Failed to get volume health from server %s for volume %d: %v", server, volumeID, err) + continue // Try next server + } + if healthInfo != nil { + return healthInfo, nil + } + } + + // If we can't get the original metrics, try to calculate from EC shards + return s.calculateHealthFromEcShards(volumeID, servers) +} + +// getVolumeHealthFromServer gets volume health information from a specific server +func (s *AdminServer) getVolumeHealthFromServer(server string, volumeID uint32) (*EcVolumeHealthInfo, error) { + var healthInfo *EcVolumeHealthInfo + + err := s.WithVolumeServerClient(pb.ServerAddress(server), func(client volume_server_pb.VolumeServerClient) error { + // Try to get volume file status (which may include original volume metrics) + resp, err := client.ReadVolumeFileStatus(context.Background(), &volume_server_pb.ReadVolumeFileStatusRequest{ + VolumeId: volumeID, + }) + if err != nil { + return err + } + + // Extract health metrics from volume info + if resp.VolumeInfo != nil { + totalSize := uint64(resp.VolumeInfo.DatFileSize) + if totalSize > 0 { + healthInfo = &EcVolumeHealthInfo{ + TotalSize: totalSize, + DeletedByteCount: 0, // EC volumes don't track deletions in VolumeInfo + FileCount: resp.FileCount, + DeleteCount: 0, // Not available in current API + GarbageRatio: 0.0, + } + } + } + + return nil + }) + + return healthInfo, err +} + +// calculateHealthFromEcShards attempts to calculate health metrics from EC shard information +func (s *AdminServer) calculateHealthFromEcShards(volumeID uint32, servers []string) (*EcVolumeHealthInfo, error) { + var totalShardSize uint64 + shardCount := 0 + + // Get shard sizes from all servers + for _, server := range servers { + err := s.WithVolumeServerClient(pb.ServerAddress(server), func(client volume_server_pb.VolumeServerClient) error { + resp, err := client.VolumeEcShardsInfo(context.Background(), &volume_server_pb.VolumeEcShardsInfoRequest{ + VolumeId: volumeID, + }) + if err != nil { + return err + } + + for _, shardInfo := range resp.EcShardInfos { + totalShardSize += uint64(shardInfo.Size) + shardCount++ + } + + return nil + }) + if err != nil { + glog.V(2).Infof("Failed to get EC shard info from server %s: %v", server, err) + } + } + + if shardCount == 0 { + return nil, fmt.Errorf("no EC shard information found for volume %d", volumeID) + } + + // For EC volumes, we can estimate the original size from the data shards + // EC uses 10 data shards + 4 parity shards = 14 total + // The original volume size is approximately the sum of the 10 data shards + dataShardCount := 10 // erasure_coding.DataShardsCount + estimatedOriginalSize := totalShardSize + + if shardCount >= dataShardCount { + // If we have info from data shards, estimate better + avgShardSize := totalShardSize / uint64(shardCount) + estimatedOriginalSize = avgShardSize * uint64(dataShardCount) + } + + return &EcVolumeHealthInfo{ + TotalSize: estimatedOriginalSize, + DeletedByteCount: 0, // Cannot determine from EC shards alone + FileCount: 0, // Cannot determine from EC shards alone + DeleteCount: 0, // Cannot determine from EC shards alone + GarbageRatio: 0.0, + }, nil +} diff --git a/weed/admin/dash/types.go b/weed/admin/dash/types.go index 18c46a48d..676d51e89 100644 --- a/weed/admin/dash/types.go +++ b/weed/admin/dash/types.go @@ -224,6 +224,13 @@ type EcVolumeDetailsData struct { Servers []string `json:"servers"` LastUpdated time.Time `json:"last_updated"` + // Volume health metrics (for EC vacuum) + TotalSize uint64 `json:"total_size"` // Total volume size before EC + DeletedByteCount uint64 `json:"deleted_byte_count"` // Deleted bytes count + FileCount uint64 `json:"file_count"` // Total file count + DeleteCount uint64 `json:"delete_count"` // Deleted file count + GarbageRatio float64 `json:"garbage_ratio"` // Deletion ratio (0.0-1.0) + // Sorting SortBy string `json:"sort_by"` SortOrder string `json:"sort_order"` @@ -237,6 +244,15 @@ type VolumeDetailsData struct { LastUpdated time.Time `json:"last_updated"` } +// EcVolumeHealthInfo represents health metrics for an EC volume +type EcVolumeHealthInfo struct { + TotalSize uint64 `json:"total_size"` // Original volume size before EC + DeletedByteCount uint64 `json:"deleted_byte_count"` // Deleted bytes count + FileCount uint64 `json:"file_count"` // Total file count + DeleteCount uint64 `json:"delete_count"` // Deleted file count + GarbageRatio float64 `json:"garbage_ratio"` // Deletion ratio (0.0-1.0) +} + // Collection management structures type CollectionInfo struct { Name string `json:"name"` diff --git a/weed/admin/view/app/ec_volume_details.templ b/weed/admin/view/app/ec_volume_details.templ index caf506d0f..1c69b489d 100644 --- a/weed/admin/view/app/ec_volume_details.templ +++ b/weed/admin/view/app/ec_volume_details.templ @@ -33,7 +33,7 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) {
| Volume ID: | ")
+ templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "Volume Information
Shard Distribution") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, " |
This volume may not be EC encoded yet.
This volume may not be EC encoded yet.