package topology import ( "testing" "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestActiveTopologyBasicOperations tests basic topology management func TestActiveTopologyBasicOperations(t *testing.T) { topology := NewActiveTopology(10) assert.NotNil(t, topology) assert.Equal(t, 10, topology.recentTaskWindowSeconds) // Test empty topology assert.Equal(t, 0, len(topology.nodes)) assert.Equal(t, 0, len(topology.disks)) assert.Equal(t, 0, len(topology.pendingTasks)) } // TestActiveTopologyUpdate tests topology updates from master func TestActiveTopologyUpdate(t *testing.T) { topology := NewActiveTopology(10) // Create sample topology info topologyInfo := createSampleTopology() err := topology.UpdateTopology(topologyInfo) require.NoError(t, err) // Verify topology structure assert.Equal(t, 2, len(topology.nodes)) // 2 nodes assert.Equal(t, 4, len(topology.disks)) // 4 disks total (2 per node) // Verify node structure node1, exists := topology.nodes["10.0.0.1:8080"] require.True(t, exists) assert.Equal(t, "dc1", node1.dataCenter) assert.Equal(t, "rack1", node1.rack) assert.Equal(t, 2, len(node1.disks)) // Verify disk structure disk1, exists := topology.disks["10.0.0.1:8080:0"] require.True(t, exists) assert.Equal(t, uint32(0), disk1.DiskID) assert.Equal(t, "hdd", disk1.DiskType) assert.Equal(t, "dc1", disk1.DataCenter) } // TestTaskLifecycle tests the complete task lifecycle func TestTaskLifecycle(t *testing.T) { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) taskID := "balance-001" // 1. Add pending task topology.AddPendingTask(taskID, TaskTypeBalance, 1001, "10.0.0.1:8080", 0, "10.0.0.2:8080", 1) // Verify pending state assert.Equal(t, 1, len(topology.pendingTasks)) assert.Equal(t, 0, len(topology.assignedTasks)) assert.Equal(t, 0, len(topology.recentTasks)) task := topology.pendingTasks[taskID] assert.Equal(t, TaskStatusPending, task.Status) assert.Equal(t, uint32(1001), task.VolumeID) // Verify task assigned to disks sourceDisk := topology.disks["10.0.0.1:8080:0"] targetDisk := topology.disks["10.0.0.2:8080:1"] assert.Equal(t, 1, len(sourceDisk.pendingTasks)) assert.Equal(t, 1, len(targetDisk.pendingTasks)) // 2. Assign task err := topology.AssignTask(taskID) require.NoError(t, err) // Verify assigned state assert.Equal(t, 0, len(topology.pendingTasks)) assert.Equal(t, 1, len(topology.assignedTasks)) assert.Equal(t, 0, len(topology.recentTasks)) task = topology.assignedTasks[taskID] assert.Equal(t, TaskStatusInProgress, task.Status) // Verify task moved to assigned on disks assert.Equal(t, 0, len(sourceDisk.pendingTasks)) assert.Equal(t, 1, len(sourceDisk.assignedTasks)) assert.Equal(t, 0, len(targetDisk.pendingTasks)) assert.Equal(t, 1, len(targetDisk.assignedTasks)) // 3. Complete task err = topology.CompleteTask(taskID) require.NoError(t, err) // Verify completed state assert.Equal(t, 0, len(topology.pendingTasks)) assert.Equal(t, 0, len(topology.assignedTasks)) assert.Equal(t, 1, len(topology.recentTasks)) task = topology.recentTasks[taskID] assert.Equal(t, TaskStatusCompleted, task.Status) assert.False(t, task.CompletedAt.IsZero()) } // TestTaskDetectionScenarios tests various task detection scenarios func TestTaskDetectionScenarios(t *testing.T) { tests := []struct { name string scenario func() *ActiveTopology expectedTasks map[string]bool // taskType -> shouldDetect }{ { name: "Empty cluster - no tasks needed", scenario: func() *ActiveTopology { topology := NewActiveTopology(10) topology.UpdateTopology(createEmptyTopology()) return topology }, expectedTasks: map[string]bool{ "balance": false, "vacuum": false, "ec": false, }, }, { name: "Unbalanced cluster - balance task needed", scenario: func() *ActiveTopology { topology := NewActiveTopology(10) topology.UpdateTopology(createUnbalancedTopology()) return topology }, expectedTasks: map[string]bool{ "balance": true, "vacuum": false, "ec": false, }, }, { name: "High garbage ratio - vacuum task needed", scenario: func() *ActiveTopology { topology := NewActiveTopology(10) topology.UpdateTopology(createHighGarbageTopology()) return topology }, expectedTasks: map[string]bool{ "balance": false, "vacuum": true, "ec": false, }, }, { name: "Large volumes - EC task needed", scenario: func() *ActiveTopology { topology := NewActiveTopology(10) topology.UpdateTopology(createLargeVolumeTopology()) return topology }, expectedTasks: map[string]bool{ "balance": false, "vacuum": false, "ec": true, }, }, { name: "Recent tasks - no immediate re-detection", scenario: func() *ActiveTopology { topology := NewActiveTopology(10) topology.UpdateTopology(createUnbalancedTopology()) // Add recent balance task topology.recentTasks["recent-balance"] = &taskState{ VolumeID: 1001, TaskType: TaskTypeBalance, Status: TaskStatusCompleted, CompletedAt: time.Now().Add(-5 * time.Second), // 5 seconds ago } return topology }, expectedTasks: map[string]bool{ "balance": false, // Should not detect due to recent task "vacuum": false, "ec": false, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { topology := tt.scenario() // Test balance task detection shouldDetectBalance := tt.expectedTasks["balance"] actualDetectBalance := !topology.HasRecentTaskForVolume(1001, TaskTypeBalance) if shouldDetectBalance { assert.True(t, actualDetectBalance, "Should detect balance task") } else { // Note: In real implementation, task detection would be more sophisticated // This is a simplified test of the recent task prevention mechanism } // Test that recent tasks prevent re-detection if len(topology.recentTasks) > 0 { for _, task := range topology.recentTasks { hasRecent := topology.HasRecentTaskForVolume(task.VolumeID, task.TaskType) assert.True(t, hasRecent, "Should find recent task for volume %d", task.VolumeID) } } }) } } // TestTargetSelectionScenarios tests target selection for different task types func TestTargetSelectionScenarios(t *testing.T) { tests := []struct { name string topology *ActiveTopology taskType TaskType excludeNode string expectedTargets int expectedBestTarget string }{ { name: "Balance task - find least loaded disk", topology: createTopologyWithLoad(), taskType: TaskTypeBalance, excludeNode: "10.0.0.1:8080", // Exclude source node expectedTargets: 2, // 2 disks on other node }, { name: "EC task - find multiple available disks", topology: createTopologyForEC(), taskType: TaskTypeErasureCoding, excludeNode: "", // Don't exclude any nodes expectedTargets: 4, // All 4 disks available }, { name: "Vacuum task - avoid conflicting disks", topology: createTopologyWithConflicts(), taskType: TaskTypeVacuum, excludeNode: "", expectedTargets: 1, // Only 1 disk without conflicts (conflicts exclude more disks) }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { availableDisks := tt.topology.GetAvailableDisks(tt.taskType, tt.excludeNode) assert.Equal(t, tt.expectedTargets, len(availableDisks), "Expected %d available disks, got %d", tt.expectedTargets, len(availableDisks)) // Verify disks are actually available for _, disk := range availableDisks { assert.NotEqual(t, tt.excludeNode, disk.NodeID, "Available disk should not be on excluded node") load := tt.topology.GetDiskLoad(disk.NodeID, disk.DiskID) assert.Less(t, load, 2, "Disk load should be less than 2") } }) } } // TestDiskLoadCalculation tests disk load calculation func TestDiskLoadCalculation(t *testing.T) { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) // Initially no load load := topology.GetDiskLoad("10.0.0.1:8080", 0) assert.Equal(t, 0, load) // Add pending task topology.AddPendingTask("task1", TaskTypeBalance, 1001, "10.0.0.1:8080", 0, "10.0.0.2:8080", 1) // Check load increased load = topology.GetDiskLoad("10.0.0.1:8080", 0) assert.Equal(t, 1, load) // Add another task to same disk topology.AddPendingTask("task2", TaskTypeVacuum, 1002, "10.0.0.1:8080", 0, "", 0) load = topology.GetDiskLoad("10.0.0.1:8080", 0) assert.Equal(t, 2, load) // Move one task to assigned topology.AssignTask("task1") // Load should still be 2 (1 pending + 1 assigned) load = topology.GetDiskLoad("10.0.0.1:8080", 0) assert.Equal(t, 2, load) // Complete one task topology.CompleteTask("task1") // Load should decrease to 1 load = topology.GetDiskLoad("10.0.0.1:8080", 0) assert.Equal(t, 1, load) } // TestTaskConflictDetection tests task conflict detection func TestTaskConflictDetection(t *testing.T) { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) // Add a balance task topology.AddPendingTask("balance1", TaskTypeBalance, 1001, "10.0.0.1:8080", 0, "10.0.0.2:8080", 1) topology.AssignTask("balance1") // Try to get available disks for vacuum (conflicts with balance) availableDisks := topology.GetAvailableDisks(TaskTypeVacuum, "") // Source disk should not be available due to conflict sourceDiskAvailable := false for _, disk := range availableDisks { if disk.NodeID == "10.0.0.1:8080" && disk.DiskID == 0 { sourceDiskAvailable = true break } } assert.False(t, sourceDiskAvailable, "Source disk should not be available due to task conflict") } // TestPublicInterfaces tests the public interface methods func TestPublicInterfaces(t *testing.T) { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) // Test GetAllNodes nodes := topology.GetAllNodes() assert.Equal(t, 2, len(nodes)) assert.Contains(t, nodes, "10.0.0.1:8080") assert.Contains(t, nodes, "10.0.0.2:8080") // Test GetNodeDisks disks := topology.GetNodeDisks("10.0.0.1:8080") assert.Equal(t, 2, len(disks)) // Test with non-existent node disks = topology.GetNodeDisks("non-existent") assert.Nil(t, disks) } // Helper functions to create test topologies func createSampleTopology() *master_pb.TopologyInfo { return &master_pb.TopologyInfo{ DataCenterInfos: []*master_pb.DataCenterInfo{ { Id: "dc1", RackInfos: []*master_pb.RackInfo{ { Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ { Id: "10.0.0.1:8080", DiskInfos: map[string]*master_pb.DiskInfo{ "hdd": {DiskId: 0, VolumeCount: 10, MaxVolumeCount: 100}, "ssd": {DiskId: 1, VolumeCount: 5, MaxVolumeCount: 50}, }, }, { Id: "10.0.0.2:8080", DiskInfos: map[string]*master_pb.DiskInfo{ "hdd": {DiskId: 0, VolumeCount: 8, MaxVolumeCount: 100}, "ssd": {DiskId: 1, VolumeCount: 3, MaxVolumeCount: 50}, }, }, }, }, }, }, }, } } func createEmptyTopology() *master_pb.TopologyInfo { return &master_pb.TopologyInfo{ DataCenterInfos: []*master_pb.DataCenterInfo{ { Id: "dc1", RackInfos: []*master_pb.RackInfo{ { Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ { Id: "10.0.0.1:8080", DiskInfos: map[string]*master_pb.DiskInfo{ "hdd": {DiskId: 0, VolumeCount: 0, MaxVolumeCount: 100}, }, }, }, }, }, }, }, } } func createUnbalancedTopology() *master_pb.TopologyInfo { return &master_pb.TopologyInfo{ DataCenterInfos: []*master_pb.DataCenterInfo{ { Id: "dc1", RackInfos: []*master_pb.RackInfo{ { Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ { Id: "10.0.0.1:8080", DiskInfos: map[string]*master_pb.DiskInfo{ "hdd": {DiskId: 0, VolumeCount: 90, MaxVolumeCount: 100}, // Very loaded }, }, { Id: "10.0.0.2:8080", DiskInfos: map[string]*master_pb.DiskInfo{ "hdd": {DiskId: 0, VolumeCount: 10, MaxVolumeCount: 100}, // Lightly loaded }, }, }, }, }, }, }, } } func createHighGarbageTopology() *master_pb.TopologyInfo { // In a real implementation, this would include volume-level garbage metrics return createSampleTopology() } func createLargeVolumeTopology() *master_pb.TopologyInfo { // In a real implementation, this would include volume-level size metrics return createSampleTopology() } func createTopologyWithLoad() *ActiveTopology { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) // Add some existing tasks to create load topology.AddPendingTask("existing1", TaskTypeVacuum, 2001, "10.0.0.1:8080", 0, "", 0) topology.AssignTask("existing1") return topology } func createTopologyForEC() *ActiveTopology { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) return topology } func createTopologyWithConflicts() *ActiveTopology { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) // Add conflicting tasks topology.AddPendingTask("balance1", TaskTypeBalance, 3001, "10.0.0.1:8080", 0, "10.0.0.2:8080", 0) topology.AssignTask("balance1") topology.AddPendingTask("ec1", TaskTypeErasureCoding, 3002, "10.0.0.1:8080", 1, "", 0) topology.AssignTask("ec1") return topology } // TestDestinationPlanning tests destination planning functionality func TestDestinationPlanning(t *testing.T) { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) // Test balance destination planning t.Run("Balance destination planning", func(t *testing.T) { plan, err := topology.PlanBalanceDestination(1001, "10.0.0.1:8080", "rack1", "dc1", 1024*1024) // 1MB require.NoError(t, err) require.NotNil(t, plan) // Should not target the source node assert.NotEqual(t, "10.0.0.1:8080", plan.TargetNode) assert.Equal(t, "10.0.0.2:8080", plan.TargetNode) assert.NotEmpty(t, plan.TargetRack) assert.NotEmpty(t, plan.TargetDC) assert.Greater(t, plan.PlacementScore, 0.0) }) // Test EC destination planning t.Run("EC destination planning", func(t *testing.T) { multiPlan, err := topology.PlanECDestinations(1002, "10.0.0.1:8080", "rack1", "dc1", 3) // Ask for 3 shards - source node can be included require.NoError(t, err) require.NotNil(t, multiPlan) assert.Greater(t, len(multiPlan.Plans), 0) assert.LessOrEqual(t, len(multiPlan.Plans), 3) // Should get at most 3 shards assert.Equal(t, len(multiPlan.Plans), multiPlan.TotalShards) // Check that all plans have valid target nodes for _, plan := range multiPlan.Plans { assert.NotEmpty(t, plan.TargetNode) assert.NotEmpty(t, plan.TargetRack) assert.NotEmpty(t, plan.TargetDC) assert.GreaterOrEqual(t, plan.PlacementScore, 0.0) } // Check diversity metrics assert.GreaterOrEqual(t, multiPlan.SuccessfulRack, 1) assert.GreaterOrEqual(t, multiPlan.SuccessfulDCs, 1) }) // Test destination planning with load t.Run("Destination planning considers load", func(t *testing.T) { // Add load to one disk topology.AddPendingTask("task1", TaskTypeBalance, 2001, "10.0.0.2:8080", 0, "", 0) plan, err := topology.PlanBalanceDestination(1003, "10.0.0.1:8080", "rack1", "dc1", 1024*1024) require.NoError(t, err) require.NotNil(t, plan) // Should prefer less loaded disk (disk 1 over disk 0 on node2) assert.Equal(t, "10.0.0.2:8080", plan.TargetNode) assert.Equal(t, uint32(1), plan.TargetDisk) // Should prefer SSD (disk 1) which has no load }) // Test insufficient destinations t.Run("Handle insufficient destinations", func(t *testing.T) { // Try to plan for more EC shards than available disks multiPlan, err := topology.PlanECDestinations(1004, "10.0.0.1:8080", "rack1", "dc1", 100) // Should get an error for insufficient disks assert.Error(t, err) assert.Nil(t, multiPlan) }) } // TestDestinationPlanningWithActiveTopology tests the integration between task detection and destination planning func TestDestinationPlanningWithActiveTopology(t *testing.T) { topology := NewActiveTopology(10) topology.UpdateTopology(createUnbalancedTopology()) // Test that tasks are created with destinations t.Run("Balance task with destination", func(t *testing.T) { // Simulate what the balance detector would create sourceNode := "10.0.0.1:8080" // Overloaded node volumeID := uint32(1001) plan, err := topology.PlanBalanceDestination(volumeID, sourceNode, "rack1", "dc1", 1024*1024) require.NoError(t, err) require.NotNil(t, plan) // Verify the destination is different from source assert.NotEqual(t, sourceNode, plan.TargetNode) assert.Equal(t, "10.0.0.2:8080", plan.TargetNode) // Should be the lightly loaded node // Verify placement quality assert.Greater(t, plan.PlacementScore, 0.0) assert.LessOrEqual(t, plan.PlacementScore, 1.0) }) // Test task state integration t.Run("Task state affects future planning", func(t *testing.T) { volumeID := uint32(1002) sourceNode := "10.0.0.1:8080" targetNode := "10.0.0.2:8080" // Plan first destination plan1, err := topology.PlanBalanceDestination(volumeID, sourceNode, "rack1", "dc1", 1024*1024) require.NoError(t, err) require.NotNil(t, plan1) // Add a pending task to the target topology.AddPendingTask("task1", TaskTypeBalance, volumeID, sourceNode, 0, targetNode, 0) // Plan another destination - should consider the pending task load plan2, err := topology.PlanBalanceDestination(1003, sourceNode, "rack1", "dc1", 1024*1024) require.NoError(t, err) require.NotNil(t, plan2) // The placement score should reflect the increased load // (This test might need adjustment based on the actual scoring algorithm) glog.V(1).Infof("Plan1 score: %.3f, Plan2 score: %.3f", plan1.PlacementScore, plan2.PlacementScore) }) } // TestECDestinationPlanningDetailed tests the EC destination planning with multiple shards func TestECDestinationPlanningDetailed(t *testing.T) { topology := NewActiveTopology(10) topology.UpdateTopology(createSampleTopology()) t.Run("EC multiple destinations", func(t *testing.T) { // Plan for 3 EC shards (now including source node, we have 4 disks total) multiPlan, err := topology.PlanECDestinations(1005, "10.0.0.1:8080", "rack1", "dc1", 3) require.NoError(t, err) require.NotNil(t, multiPlan) // Should get 3 destinations (can include source node's disks) assert.Equal(t, 3, len(multiPlan.Plans)) assert.Equal(t, 3, multiPlan.TotalShards) // Count node distribution - source node can now be included nodeCount := make(map[string]int) for _, plan := range multiPlan.Plans { nodeCount[plan.TargetNode]++ } // Should distribute across available nodes (both nodes can be used) assert.GreaterOrEqual(t, len(nodeCount), 1, "Should use at least 1 node") assert.LessOrEqual(t, len(nodeCount), 2, "Should use at most 2 nodes") glog.V(1).Infof("EC destinations node distribution: %v", nodeCount) glog.V(1).Infof("EC destinations: %d plans across %d racks, %d DCs", multiPlan.TotalShards, multiPlan.SuccessfulRack, multiPlan.SuccessfulDCs) }) t.Run("EC destination planning with task conflicts", func(t *testing.T) { // Create a fresh topology for this test to avoid conflicts from previous test freshTopology := NewActiveTopology(10) freshTopology.UpdateTopology(createSampleTopology()) // Add tasks to create conflicts on some disks freshTopology.AddPendingTask("conflict1", TaskTypeVacuum, 2001, "10.0.0.2:8080", 0, "", 0) freshTopology.AddPendingTask("conflict2", TaskTypeBalance, 2002, "10.0.0.1:8080", 0, "", 0) freshTopology.AssignTask("conflict1") freshTopology.AssignTask("conflict2") // Plan EC destinations - should still succeed using available disks multiPlan, err := freshTopology.PlanECDestinations(1006, "10.0.0.1:8080", "rack1", "dc1", 2) require.NoError(t, err) require.NotNil(t, multiPlan) // Should get destinations (using disks that don't have conflicts) assert.GreaterOrEqual(t, len(multiPlan.Plans), 1) assert.LessOrEqual(t, len(multiPlan.Plans), 2) // Available disks should be: node1/disk1 and node2/disk1 (since disk0 on both nodes have conflicts) for _, plan := range multiPlan.Plans { assert.Equal(t, uint32(1), plan.TargetDisk, "Should prefer disk 1 which has no conflicts") } glog.V(1).Infof("EC destination planning with conflicts: found %d destinations", len(multiPlan.Plans)) }) }