5 changed files with 548 additions and 46 deletions
-
119weed/worker/tasks/ec_vacuum/ec_vacuum_task.go
-
422weed/worker/tasks/ec_vacuum/execution_validation_test.go
-
2weed/worker/tasks/ec_vacuum/register.go
-
37weed/worker/tasks/ec_vacuum/safety_checks.go
-
14weed/worker/tasks/ec_vacuum/safety_checks_test.go
@ -0,0 +1,422 @@ |
|||
package ec_vacuum |
|||
|
|||
import ( |
|||
"testing" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
) |
|||
|
|||
// TestExecutionPlanValidation validates that the execution properly follows the vacuum plan
|
|||
func TestExecutionPlanValidation(t *testing.T) { |
|||
tests := []struct { |
|||
name string |
|||
params *worker_pb.TaskParams |
|||
expectedSourceGen uint32 |
|||
expectedTargetGen uint32 |
|||
expectedCleanupGens []uint32 |
|||
expectedExecutionSteps []string |
|||
validateExecution func(*testing.T, *EcVacuumTask, *VacuumPlan) |
|||
}{ |
|||
{ |
|||
name: "single_generation_execution", |
|||
params: &worker_pb.TaskParams{ |
|||
VolumeId: 100, |
|||
Collection: "test", |
|||
Sources: []*worker_pb.TaskSource{ |
|||
{ |
|||
Node: "node1:8080", |
|||
Generation: 1, |
|||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, |
|||
}, |
|||
}, |
|||
}, |
|||
expectedSourceGen: 1, |
|||
expectedTargetGen: 2, |
|||
expectedCleanupGens: []uint32{1}, |
|||
expectedExecutionSteps: []string{ |
|||
"create_plan", |
|||
"validate_plan", |
|||
"collect_shards_from_generation_1", |
|||
"decode_and_vacuum", |
|||
"encode_to_generation_2", |
|||
"distribute_generation_2", |
|||
"activate_generation_2", |
|||
"cleanup_generation_1", |
|||
}, |
|||
validateExecution: func(t *testing.T, task *EcVacuumTask, plan *VacuumPlan) { |
|||
// Validate plan reflects multi-generation logic
|
|||
if plan.CurrentGeneration != 1 { |
|||
t.Errorf("expected source generation 1, got %d", plan.CurrentGeneration) |
|||
} |
|||
if plan.TargetGeneration != 2 { |
|||
t.Errorf("expected target generation 2, got %d", plan.TargetGeneration) |
|||
} |
|||
if len(plan.GenerationsToCleanup) != 1 || plan.GenerationsToCleanup[0] != 1 { |
|||
t.Errorf("expected cleanup generations [1], got %v", plan.GenerationsToCleanup) |
|||
} |
|||
|
|||
// Validate task uses plan values
|
|||
if task.sourceGeneration != plan.CurrentGeneration { |
|||
t.Errorf("task source generation %d != plan current generation %d", |
|||
task.sourceGeneration, plan.CurrentGeneration) |
|||
} |
|||
if task.targetGeneration != plan.TargetGeneration { |
|||
t.Errorf("task target generation %d != plan target generation %d", |
|||
task.targetGeneration, plan.TargetGeneration) |
|||
} |
|||
}, |
|||
}, |
|||
{ |
|||
name: "multi_generation_cleanup_execution", |
|||
params: &worker_pb.TaskParams{ |
|||
VolumeId: 200, |
|||
Collection: "data", |
|||
Sources: []*worker_pb.TaskSource{ |
|||
{ |
|||
Node: "node1:8080", |
|||
Generation: 0, |
|||
ShardIds: []uint32{0, 1, 2}, // Incomplete - should not be selected
|
|||
}, |
|||
{ |
|||
Node: "node2:8080", |
|||
Generation: 1, |
|||
ShardIds: []uint32{0, 1, 2, 3, 4}, // Incomplete - should not be selected
|
|||
}, |
|||
{ |
|||
Node: "node3:8080", |
|||
Generation: 2, |
|||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, // Complete - should be selected
|
|||
}, |
|||
}, |
|||
}, |
|||
expectedSourceGen: 2, // Should pick generation 2 (most complete)
|
|||
expectedTargetGen: 3, // max(0,1,2) + 1 = 3
|
|||
expectedCleanupGens: []uint32{0, 1, 2}, // Should cleanup ALL old generations
|
|||
expectedExecutionSteps: []string{ |
|||
"create_plan", |
|||
"validate_plan", |
|||
"collect_shards_from_generation_2", // Use most complete generation
|
|||
"decode_and_vacuum", |
|||
"encode_to_generation_3", |
|||
"distribute_generation_3", |
|||
"activate_generation_3", |
|||
"cleanup_generation_0", // Cleanup ALL old generations
|
|||
"cleanup_generation_1", |
|||
"cleanup_generation_2", |
|||
}, |
|||
validateExecution: func(t *testing.T, task *EcVacuumTask, plan *VacuumPlan) { |
|||
// Validate plan correctly identifies most complete generation
|
|||
if plan.CurrentGeneration != 2 { |
|||
t.Errorf("expected source generation 2 (most complete), got %d", plan.CurrentGeneration) |
|||
} |
|||
if plan.TargetGeneration != 3 { |
|||
t.Errorf("expected target generation 3, got %d", plan.TargetGeneration) |
|||
} |
|||
|
|||
// Validate cleanup includes ALL old generations
|
|||
expectedCleanup := map[uint32]bool{0: true, 1: true, 2: true} |
|||
for _, gen := range plan.GenerationsToCleanup { |
|||
if !expectedCleanup[gen] { |
|||
t.Errorf("unexpected generation %d in cleanup list", gen) |
|||
} |
|||
delete(expectedCleanup, gen) |
|||
} |
|||
for gen := range expectedCleanup { |
|||
t.Errorf("missing generation %d in cleanup list", gen) |
|||
} |
|||
|
|||
// Validate source nodes only include nodes from selected generation
|
|||
expectedNodeCount := 1 // Only node3 has generation 2 shards
|
|||
if len(plan.SourceDistribution.Nodes) != expectedNodeCount { |
|||
t.Errorf("expected %d source nodes (generation 2 only), got %d", |
|||
expectedNodeCount, len(plan.SourceDistribution.Nodes)) |
|||
} |
|||
|
|||
// Validate the selected node has all shards
|
|||
for _, shardBits := range plan.SourceDistribution.Nodes { |
|||
if shardBits.ShardIdCount() != 14 { |
|||
t.Errorf("expected 14 shards from selected generation, got %d", shardBits.ShardIdCount()) |
|||
} |
|||
} |
|||
}, |
|||
}, |
|||
} |
|||
|
|||
logic := NewEcVacuumLogic() |
|||
|
|||
for _, tt := range tests { |
|||
t.Run(tt.name, func(t *testing.T) { |
|||
// Step 1: Create vacuum plan
|
|||
plan, err := logic.CreateVacuumPlan(tt.params.VolumeId, tt.params.Collection, tt.params) |
|||
if err != nil { |
|||
t.Fatalf("failed to create vacuum plan: %v", err) |
|||
} |
|||
|
|||
// Step 2: Create task (simulating the execution setup)
|
|||
sourceNodes, err := logic.ParseSourceNodes(tt.params, plan.CurrentGeneration) |
|||
if err != nil { |
|||
t.Fatalf("failed to parse source nodes: %v", err) |
|||
} |
|||
|
|||
task := NewEcVacuumTask("test-execution", tt.params.VolumeId, tt.params.Collection, sourceNodes) |
|||
task.plan = plan |
|||
task.sourceGeneration = plan.CurrentGeneration |
|||
task.targetGeneration = plan.TargetGeneration |
|||
|
|||
// Step 3: Validate plan matches expectations
|
|||
if plan.CurrentGeneration != tt.expectedSourceGen { |
|||
t.Errorf("source generation: expected %d, got %d", tt.expectedSourceGen, plan.CurrentGeneration) |
|||
} |
|||
if plan.TargetGeneration != tt.expectedTargetGen { |
|||
t.Errorf("target generation: expected %d, got %d", tt.expectedTargetGen, plan.TargetGeneration) |
|||
} |
|||
|
|||
// Step 4: Validate cleanup generations
|
|||
if !equalUint32Slices(plan.GenerationsToCleanup, tt.expectedCleanupGens) { |
|||
t.Errorf("cleanup generations: expected %v, got %v", tt.expectedCleanupGens, plan.GenerationsToCleanup) |
|||
} |
|||
|
|||
// Step 5: Run custom validation
|
|||
if tt.validateExecution != nil { |
|||
tt.validateExecution(t, task, plan) |
|||
} |
|||
|
|||
// Step 6: Validate execution readiness
|
|||
err = logic.ValidateShardDistribution(plan.SourceDistribution) |
|||
if err != nil { |
|||
t.Errorf("plan validation failed: %v", err) |
|||
} |
|||
|
|||
t.Logf("✅ Execution plan validation passed:") |
|||
t.Logf(" Volume: %d (%s)", plan.VolumeID, plan.Collection) |
|||
t.Logf(" Source generation: %d (most complete)", plan.CurrentGeneration) |
|||
t.Logf(" Target generation: %d", plan.TargetGeneration) |
|||
t.Logf(" Generations to cleanup: %v", plan.GenerationsToCleanup) |
|||
t.Logf(" Source nodes: %d", len(plan.SourceDistribution.Nodes)) |
|||
t.Logf(" Safety checks: %d", len(plan.SafetyChecks)) |
|||
}) |
|||
} |
|||
} |
|||
|
|||
// TestExecutionStepValidation validates individual execution steps
|
|||
func TestExecutionStepValidation(t *testing.T) { |
|||
// Create a realistic multi-generation scenario
|
|||
params := &worker_pb.TaskParams{ |
|||
VolumeId: 300, |
|||
Collection: "test", |
|||
Sources: []*worker_pb.TaskSource{ |
|||
{ |
|||
Node: "node1:8080", |
|||
Generation: 0, |
|||
ShardIds: []uint32{0, 1, 2, 3}, // Incomplete old generation
|
|||
}, |
|||
{ |
|||
Node: "node2:8080", |
|||
Generation: 1, |
|||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, // Complete generation (should be selected)
|
|||
}, |
|||
{ |
|||
Node: "node3:8080", |
|||
Generation: 1, |
|||
ShardIds: []uint32{10, 11, 12, 13}, // Additional shards for generation 1
|
|||
}, |
|||
}, |
|||
} |
|||
|
|||
logic := NewEcVacuumLogic() |
|||
|
|||
// Create plan
|
|||
plan, err := logic.CreateVacuumPlan(params.VolumeId, params.Collection, params) |
|||
if err != nil { |
|||
t.Fatalf("failed to create plan: %v", err) |
|||
} |
|||
|
|||
// Validate Step 1: Plan Creation
|
|||
t.Run("step_1_plan_creation", func(t *testing.T) { |
|||
if plan.CurrentGeneration != 1 { |
|||
t.Errorf("plan should select generation 1 (complete), got %d", plan.CurrentGeneration) |
|||
} |
|||
if plan.TargetGeneration != 2 { |
|||
t.Errorf("plan should target generation 2, got %d", plan.TargetGeneration) |
|||
} |
|||
if len(plan.GenerationsToCleanup) != 2 { |
|||
t.Errorf("plan should cleanup 2 generations (0,1), got %d", len(plan.GenerationsToCleanup)) |
|||
} |
|||
}) |
|||
|
|||
// Validate Step 2: Source Node Selection
|
|||
t.Run("step_2_source_node_selection", func(t *testing.T) { |
|||
sourceNodes, err := logic.ParseSourceNodes(params, plan.CurrentGeneration) |
|||
if err != nil { |
|||
t.Fatalf("failed to parse source nodes: %v", err) |
|||
} |
|||
|
|||
// Should only include nodes from generation 1
|
|||
expectedNodes := 2 // node2 and node3 have generation 1 shards
|
|||
if len(sourceNodes) != expectedNodes { |
|||
t.Errorf("expected %d source nodes (generation 1 only), got %d", expectedNodes, len(sourceNodes)) |
|||
} |
|||
|
|||
// Verify node2 has the right shards (0-9)
|
|||
node2Addr := pb.ServerAddress("node2:8080") |
|||
if shardBits, exists := sourceNodes[node2Addr]; exists { |
|||
if shardBits.ShardIdCount() != 10 { |
|||
t.Errorf("node2 should have 10 shards, got %d", shardBits.ShardIdCount()) |
|||
} |
|||
} else { |
|||
t.Errorf("node2 should be in source nodes") |
|||
} |
|||
|
|||
// Verify node3 has the right shards (10-13)
|
|||
node3Addr := pb.ServerAddress("node3:8080") |
|||
if shardBits, exists := sourceNodes[node3Addr]; exists { |
|||
if shardBits.ShardIdCount() != 4 { |
|||
t.Errorf("node3 should have 4 shards, got %d", shardBits.ShardIdCount()) |
|||
} |
|||
} else { |
|||
t.Errorf("node3 should be in source nodes") |
|||
} |
|||
}) |
|||
|
|||
// Validate Step 3: Cleanup Planning
|
|||
t.Run("step_3_cleanup_planning", func(t *testing.T) { |
|||
// Should cleanup both generation 0 and 1, but not generation 2
|
|||
cleanupMap := make(map[uint32]bool) |
|||
for _, gen := range plan.GenerationsToCleanup { |
|||
cleanupMap[gen] = true |
|||
} |
|||
|
|||
expectedCleanup := []uint32{0, 1} |
|||
for _, expectedGen := range expectedCleanup { |
|||
if !cleanupMap[expectedGen] { |
|||
t.Errorf("generation %d should be in cleanup list", expectedGen) |
|||
} |
|||
} |
|||
|
|||
// Should NOT cleanup target generation
|
|||
if cleanupMap[plan.TargetGeneration] { |
|||
t.Errorf("target generation %d should NOT be in cleanup list", plan.TargetGeneration) |
|||
} |
|||
}) |
|||
|
|||
// Validate Step 4: Safety Checks
|
|||
t.Run("step_4_safety_checks", func(t *testing.T) { |
|||
if len(plan.SafetyChecks) == 0 { |
|||
t.Errorf("plan should include safety checks") |
|||
} |
|||
|
|||
// Verify shard distribution is sufficient
|
|||
err := logic.ValidateShardDistribution(plan.SourceDistribution) |
|||
if err != nil { |
|||
t.Errorf("shard distribution validation failed: %v", err) |
|||
} |
|||
}) |
|||
|
|||
t.Logf("✅ All execution step validations passed") |
|||
} |
|||
|
|||
// TestExecutionErrorHandling tests error scenarios in execution
|
|||
func TestExecutionErrorHandling(t *testing.T) { |
|||
logic := NewEcVacuumLogic() |
|||
|
|||
tests := []struct { |
|||
name string |
|||
params *worker_pb.TaskParams |
|||
expectError bool |
|||
errorMsg string |
|||
}{ |
|||
{ |
|||
name: "no_sufficient_generations", |
|||
params: &worker_pb.TaskParams{ |
|||
VolumeId: 400, |
|||
Collection: "test", |
|||
Sources: []*worker_pb.TaskSource{ |
|||
{ |
|||
Node: "node1:8080", |
|||
Generation: 0, |
|||
ShardIds: []uint32{0, 1, 2}, // Only 3 shards - insufficient
|
|||
}, |
|||
{ |
|||
Node: "node2:8080", |
|||
Generation: 1, |
|||
ShardIds: []uint32{3, 4, 5}, // Only 6 total shards - insufficient
|
|||
}, |
|||
}, |
|||
}, |
|||
expectError: true, |
|||
errorMsg: "no generation has sufficient shards", |
|||
}, |
|||
{ |
|||
name: "empty_sources", |
|||
params: &worker_pb.TaskParams{ |
|||
VolumeId: 500, |
|||
Collection: "test", |
|||
Sources: []*worker_pb.TaskSource{}, |
|||
}, |
|||
expectError: false, // Should fall back to defaults
|
|||
errorMsg: "", |
|||
}, |
|||
{ |
|||
name: "mixed_valid_invalid_generations", |
|||
params: &worker_pb.TaskParams{ |
|||
VolumeId: 600, |
|||
Collection: "test", |
|||
Sources: []*worker_pb.TaskSource{ |
|||
{ |
|||
Node: "node1:8080", |
|||
Generation: 0, |
|||
ShardIds: []uint32{0, 1}, // Insufficient
|
|||
}, |
|||
{ |
|||
Node: "node2:8080", |
|||
Generation: 1, |
|||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, // Complete - should be selected
|
|||
}, |
|||
}, |
|||
}, |
|||
expectError: false, // Should use generation 1
|
|||
errorMsg: "", |
|||
}, |
|||
} |
|||
|
|||
for _, tt := range tests { |
|||
t.Run(tt.name, func(t *testing.T) { |
|||
plan, err := logic.CreateVacuumPlan(tt.params.VolumeId, tt.params.Collection, tt.params) |
|||
|
|||
if tt.expectError { |
|||
if err == nil { |
|||
t.Errorf("expected error but got none") |
|||
} else if tt.errorMsg != "" && !contains(err.Error(), tt.errorMsg) { |
|||
t.Errorf("expected error containing '%s', got '%s'", tt.errorMsg, err.Error()) |
|||
} |
|||
} else { |
|||
if err != nil { |
|||
t.Errorf("unexpected error: %v", err) |
|||
} else { |
|||
// Validate the plan is reasonable
|
|||
if plan.TargetGeneration <= plan.CurrentGeneration { |
|||
t.Errorf("target generation %d should be > current generation %d", |
|||
plan.TargetGeneration, plan.CurrentGeneration) |
|||
} |
|||
} |
|||
} |
|||
}) |
|||
} |
|||
} |
|||
|
|||
// Helper function to check if string contains substring
|
|||
func contains(s, substr string) bool { |
|||
return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && |
|||
(s[:len(substr)] == substr || s[len(s)-len(substr):] == substr || |
|||
len(s) > len(substr) && someContains(s, substr))) |
|||
} |
|||
|
|||
func someContains(s, substr string) bool { |
|||
for i := 0; i <= len(s)-len(substr); i++ { |
|||
if s[i:i+len(substr)] == substr { |
|||
return true |
|||
} |
|||
} |
|||
return false |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue