You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
248 lines
14 KiB
248 lines
14 KiB
package shell
|
|
|
|
import (
|
|
"regexp"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestSelectVolumeIdsFromTopology(t *testing.T) {
|
|
// Topology volumeSizeLimit:1000 MB hdd(volume:28/30 active:28 free:2 remote:0)
|
|
// DataCenter DefaultDataCenter hdd(volume:28/30 active:28 free:2 remote:0)
|
|
// Rack DefaultRack hdd(volume:28/30 active:28 free:2 remote:0)
|
|
// DataNode seaweedfs-volume-0.seaweedfs-volume.sea:8080 hdd(volume:14/15 active:14 free:1 remote:0)
|
|
// Disk hdd(volume:14/15 active:14 free:1 remote:0) id:0
|
|
// volume Id:1, Size:23520, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:3, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366011
|
|
// volume Id:2, Size:28536, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:1, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365291
|
|
// volume Id:3, Size:100752, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364931
|
|
// volume Id:4, Size:3112, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300850
|
|
// volume Id:5, Size:145208, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365652
|
|
// volume Id:6, Size:146456, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364812
|
|
// volume Id:7, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300824
|
|
// volume Id:8, Size:808, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300796
|
|
// volume Id:9, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300814
|
|
// volume Id:10, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364737
|
|
// volume Id:11, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364796
|
|
// volume Id:12, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364829
|
|
// volume Id:13, Size:570428616, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:68, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366062
|
|
// volume Id:14, Size:478153400, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:57, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366020
|
|
// Disk hdd {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
|
|
// DataNode seaweedfs-volume-0.seaweedfs-volume.sea:8080 {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
|
|
// DataCenter DefaultDataCenter hdd(volume:28/30 active:28 free:2 remote:0)
|
|
// Rack DefaultRack hdd(volume:28/30 active:28 free:2 remote:0)
|
|
// DataNode seaweedfs-volume-1.seaweedfs-volume.sea:8080 hdd(volume:14/15 active:14 free:1 remote:0)
|
|
// Disk hdd(volume:14/15 active:14 free:1 remote:0) id:0
|
|
// volume Id:1, Size:23520, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:3, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366012
|
|
// volume Id:2, Size:28536, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:1, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365292
|
|
// volume Id:3, Size:100752, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364931
|
|
// volume Id:4, Size:3112, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300850
|
|
// volume Id:5, Size:145208, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365653
|
|
// volume Id:6, Size:146456, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364812
|
|
// volume Id:7, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300824
|
|
// volume Id:8, Size:808, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300796
|
|
// volume Id:9, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300814
|
|
// volume Id:10, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364737
|
|
// volume Id:11, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364796
|
|
// volume Id:12, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364828
|
|
// volume Id:13, Size:570428616, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:68, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366062
|
|
// volume Id:14, Size:478153400, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:57, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366020
|
|
// Disk hdd {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
|
|
// DataNode seaweedfs-volume-1.seaweedfs-volume.sea:8080 {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
|
|
// Rack DefaultRack {Size:8389552896 FileCount:1036 DeletedFileCount:0 DeletedBytes:0}
|
|
// DataCenter DefaultDataCenter {Size:8389552896 FileCount:1036 DeletedFileCount:0 DeletedBytes:0}
|
|
// total size:8389552896 file_count:1036
|
|
|
|
topologyInfo := &master_pb.TopologyInfo{
|
|
Id: "DefaultDataCenter",
|
|
DataCenterInfos: []*master_pb.DataCenterInfo{
|
|
{
|
|
Id: "DefaultDataCenter",
|
|
RackInfos: []*master_pb.RackInfo{
|
|
{
|
|
Id: "DefaultRack",
|
|
DataNodeInfos: []*master_pb.DataNodeInfo{
|
|
{
|
|
Id: "seaweedfs-volume-0.seaweedfs-volume.sea:8080",
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"hdd": {
|
|
Type: "hdd",
|
|
FreeVolumeCount: 1,
|
|
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
|
{Id: 1, Size: 23520, Collection: "", ModifiedAtSecond: 1770366011},
|
|
{Id: 2, Size: 28536, Collection: "", ModifiedAtSecond: 1770365291},
|
|
{Id: 3, Size: 100752, Collection: "", ModifiedAtSecond: 1770364931},
|
|
{Id: 4, Size: 3112, Collection: "", ModifiedAtSecond: 1770300850},
|
|
{Id: 5, Size: 145208, Collection: "", ModifiedAtSecond: 1770365652},
|
|
{Id: 6, Size: 146456, Collection: "", ModifiedAtSecond: 1770364812},
|
|
{Id: 7, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300824},
|
|
{Id: 8, Size: 808, Collection: "encrypt-data", ModifiedAtSecond: 1770300796},
|
|
{Id: 9, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300814},
|
|
{Id: 10, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364737},
|
|
{Id: 11, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364796},
|
|
{Id: 12, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364829},
|
|
{Id: 13, Size: 570428616, Collection: "test", ModifiedAtSecond: 1770366062},
|
|
{Id: 14, Size: 478153400, Collection: "test", ModifiedAtSecond: 1770366020},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Id: "seaweedfs-volume-1.seaweedfs-volume.sea:8080",
|
|
DiskInfos: map[string]*master_pb.DiskInfo{
|
|
"hdd": {
|
|
Type: "hdd",
|
|
FreeVolumeCount: 1,
|
|
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
|
{Id: 1, Size: 23520, Collection: "", ModifiedAtSecond: 1770366012},
|
|
{Id: 2, Size: 28536, Collection: "", ModifiedAtSecond: 1770365292},
|
|
{Id: 3, Size: 100752, Collection: "", ModifiedAtSecond: 1770364931},
|
|
{Id: 4, Size: 3112, Collection: "", ModifiedAtSecond: 1770300850},
|
|
{Id: 5, Size: 145208, Collection: "", ModifiedAtSecond: 1770365653},
|
|
{Id: 6, Size: 146456, Collection: "", ModifiedAtSecond: 1770364812},
|
|
{Id: 7, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300824},
|
|
{Id: 8, Size: 808, Collection: "encrypt-data", ModifiedAtSecond: 1770300796},
|
|
{Id: 9, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300814},
|
|
{Id: 10, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364737},
|
|
{Id: 11, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364796},
|
|
{Id: 12, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364828},
|
|
{Id: 13, Size: 570428616, Collection: "test", ModifiedAtSecond: 1770366062},
|
|
{Id: 14, Size: 478153400, Collection: "test", ModifiedAtSecond: 1770366020},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
volumeSizeLimitMb := uint64(1000)
|
|
collectionPattern := ".*"
|
|
collectionRegex, _ := regexp.Compile(collectionPattern)
|
|
|
|
// ec.encode -force
|
|
// force means we ignore the check for 4+ volume servers.
|
|
// But in this unit test we are testing selectVolumeIdsFromTopology, which selects volumes.
|
|
// The -force flag is handled in the caller of this function.
|
|
|
|
// Default values
|
|
quietPeriod := time.Hour
|
|
fullPercentage := 95.0
|
|
verbose := true
|
|
|
|
// Mock time to be slightly after the latest modification time in the test data
|
|
// The latest ModifiedAtSecond in the test data is 1770366062 (Volume 13)
|
|
// Let's set now to be 1770366062 + quietPeriod.Seconds() + 100
|
|
nowUnixSeconds := int64(1770366062) + int64(quietPeriod.Seconds()) + 100
|
|
quietSeconds := int64(quietPeriod.Seconds())
|
|
|
|
// Test case 1: Select all volumes with sufficient size
|
|
// Note: The logic requires (FreeVolumeCount >= 2) unless the volume is already selected?
|
|
// Wait, distinct volume IDs.
|
|
// Logic:
|
|
// if diskInfo.FreeVolumeCount < 2 { skip }
|
|
// In the test data, FreeVolumeCount is 1. So it should skip all volumes basically?
|
|
// Let's check the code:
|
|
/*
|
|
// check free disk space
|
|
if good, found := vidMap[v.Id]; found {
|
|
if good {
|
|
if diskInfo.FreeVolumeCount < 2 {
|
|
// ...
|
|
vidMap[v.Id] = false
|
|
noFreeDisk++
|
|
}
|
|
}
|
|
} else {
|
|
if diskInfo.FreeVolumeCount < 2 {
|
|
// ...
|
|
vidMap[v.Id] = false
|
|
noFreeDisk++
|
|
} else {
|
|
// ...
|
|
vidMap[v.Id] = true
|
|
}
|
|
}
|
|
*/
|
|
// Yes, if FreeVolumeCount < 2, it marks the volume as bad (false in vidMap).
|
|
// In the provided topology, FreeVolumeCount is 1 ("free:1"), so it is < 2.
|
|
// So ALL volumes should be skipped due to insufficient free disk space.
|
|
|
|
vids, _ := selectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
|
|
|
|
assert.Equal(t, 0, len(vids), "Should select 0 volumes because FreeVolumeCount is 1 (less than 2)")
|
|
|
|
// Test case 2: If we had enough free space
|
|
topologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos["hdd"].FreeVolumeCount = 2
|
|
topologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[1].DiskInfos["hdd"].FreeVolumeCount = 2
|
|
|
|
// Update expected selection based on size and other filters
|
|
// Size threshold: 95% of 1000MB = 950MB = 996147200 bytes
|
|
// Volumes >= 950MB:
|
|
// Vol 10: 1048582008 > 950MB
|
|
// Vol 11: 1048582008 > 950MB
|
|
// Vol 12: 1048582008 > 950MB
|
|
// Vol 13: 570428616 < 950MB
|
|
// Vol 14: 478153400 < 950MB
|
|
// ... others are small
|
|
|
|
// So expected volumes: 10, 11, 12.
|
|
|
|
vids, _ = selectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
|
|
|
|
expectedVids := []needle.VolumeId{10, 11, 12}
|
|
assert.Equal(t, len(expectedVids), len(vids), "Should select 3 volumes")
|
|
|
|
// Check content
|
|
vidMap := make(map[needle.VolumeId]bool)
|
|
for _, vid := range vids {
|
|
vidMap[vid] = true
|
|
}
|
|
for _, vid := range expectedVids {
|
|
assert.True(t, vidMap[vid], "Volume %d should be selected", vid)
|
|
}
|
|
}
|
|
|
|
func TestEcEncodeNodeCountCheck(t *testing.T) {
|
|
topologyInfo := &master_pb.TopologyInfo{
|
|
Id: "DefaultDataCenter",
|
|
DataCenterInfos: []*master_pb.DataCenterInfo{
|
|
{
|
|
Id: "DefaultDataCenter",
|
|
RackInfos: []*master_pb.RackInfo{
|
|
{
|
|
Id: "DefaultRack",
|
|
DataNodeInfos: []*master_pb.DataNodeInfo{
|
|
{Id: "node1"},
|
|
{Id: "node2"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
nodeCount := 0
|
|
eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
|
|
nodeCount++
|
|
})
|
|
|
|
// Default parity shards count is 4
|
|
minNodeCount := 4
|
|
|
|
// Case 1: Without -force
|
|
forceChanges := false
|
|
willProceed := forceChanges || nodeCount >= minNodeCount
|
|
assert.False(t, willProceed, "Should NOT proceed with %d nodes (min %d) without force", nodeCount, minNodeCount)
|
|
|
|
// Case 2: With -force
|
|
forceChanges = true
|
|
willProceed = forceChanges || nodeCount >= minNodeCount
|
|
assert.True(t, willProceed, "Should proceed with -force even with %d nodes", nodeCount)
|
|
}
|