Browse Source
reorganized the codebase to move the simulation framework and tests into their own dedicated package
worker-execute-ec-tasks
reorganized the codebase to move the simulation framework and tests into their own dedicated package
worker-execute-ec-tasks
chrislu
5 months ago
6 changed files with 122 additions and 470 deletions
-
70weed/admin/task/example_usage.go
-
84weed/admin/task/simulation/comprehensive_simulation.go
-
56weed/admin/task/simulation/comprehensive_simulation_test.go
-
4weed/admin/task/simulation/simulation_runner.go
-
81weed/admin/task/simulation/system_demo_test.go
-
297weed/admin/task/simulation_runner.go
@ -1,297 +0,0 @@ |
|||||
package task |
|
||||
|
|
||||
import ( |
|
||||
"fmt" |
|
||||
"time" |
|
||||
|
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|
||||
) |
|
||||
|
|
||||
// SimulationRunner orchestrates the execution of simulation scenarios
|
|
||||
type SimulationRunner struct { |
|
||||
simulator *TaskSimulator |
|
||||
} |
|
||||
|
|
||||
// NewSimulationRunner creates a new simulation runner
|
|
||||
func NewSimulationRunner() *SimulationRunner { |
|
||||
return &SimulationRunner{ |
|
||||
simulator: NewTaskSimulator(), |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// RunAllScenarios runs all predefined simulation scenarios
|
|
||||
func (sr *SimulationRunner) RunAllScenarios() error { |
|
||||
glog.Infof("Starting comprehensive task distribution system simulation") |
|
||||
|
|
||||
// Create standard scenarios
|
|
||||
sr.simulator.CreateStandardScenarios() |
|
||||
|
|
||||
scenarios := []string{ |
|
||||
"worker_timeout_during_ec", |
|
||||
"stuck_vacuum_task", |
|
||||
"duplicate_task_prevention", |
|
||||
"master_admin_divergence", |
|
||||
} |
|
||||
|
|
||||
var allResults []*SimulationResult |
|
||||
|
|
||||
for _, scenarioName := range scenarios { |
|
||||
glog.Infof("Running scenario: %s", scenarioName) |
|
||||
|
|
||||
result, err := sr.simulator.RunScenario(scenarioName) |
|
||||
if err != nil { |
|
||||
glog.Errorf("Failed to run scenario %s: %v", scenarioName, err) |
|
||||
continue |
|
||||
} |
|
||||
|
|
||||
allResults = append(allResults, result) |
|
||||
|
|
||||
// Brief pause between scenarios
|
|
||||
time.Sleep(5 * time.Second) |
|
||||
} |
|
||||
|
|
||||
// Generate and log comprehensive report
|
|
||||
report := sr.simulator.GenerateSimulationReport() |
|
||||
glog.Infof("Simulation Report:\n%s", report) |
|
||||
|
|
||||
// Summary
|
|
||||
sr.logSummary(allResults) |
|
||||
|
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
// RunSpecificScenario runs a specific simulation scenario
|
|
||||
func (sr *SimulationRunner) RunSpecificScenario(scenarioName string) (*SimulationResult, error) { |
|
||||
// Ensure standard scenarios are available
|
|
||||
sr.simulator.CreateStandardScenarios() |
|
||||
|
|
||||
return sr.simulator.RunScenario(scenarioName) |
|
||||
} |
|
||||
|
|
||||
// logSummary logs a summary of all simulation results
|
|
||||
func (sr *SimulationRunner) logSummary(results []*SimulationResult) { |
|
||||
totalTasks := 0 |
|
||||
totalCompleted := 0 |
|
||||
totalFailed := 0 |
|
||||
totalTimeouts := 0 |
|
||||
totalDuplicates := 0 |
|
||||
totalInconsistencies := 0 |
|
||||
successfulScenarios := 0 |
|
||||
|
|
||||
for _, result := range results { |
|
||||
totalTasks += result.TasksCreated |
|
||||
totalCompleted += result.TasksCompleted |
|
||||
totalFailed += result.TasksFailed |
|
||||
totalTimeouts += result.WorkerTimeouts |
|
||||
totalDuplicates += result.DuplicatesFound |
|
||||
totalInconsistencies += result.StateInconsistencies |
|
||||
|
|
||||
if result.Success { |
|
||||
successfulScenarios++ |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
glog.Infof("=== SIMULATION SUMMARY ===") |
|
||||
glog.Infof("Scenarios Run: %d", len(results)) |
|
||||
glog.Infof("Successful Scenarios: %d", successfulScenarios) |
|
||||
glog.Infof("Total Tasks Created: %d", totalTasks) |
|
||||
glog.Infof("Total Tasks Completed: %d", totalCompleted) |
|
||||
glog.Infof("Total Tasks Failed: %d", totalFailed) |
|
||||
glog.Infof("Total Worker Timeouts: %d", totalTimeouts) |
|
||||
glog.Infof("Total Duplicates Found: %d", totalDuplicates) |
|
||||
glog.Infof("Total State Inconsistencies: %d", totalInconsistencies) |
|
||||
|
|
||||
if totalTasks > 0 { |
|
||||
completionRate := float64(totalCompleted) / float64(totalTasks) * 100.0 |
|
||||
glog.Infof("Task Completion Rate: %.2f%%", completionRate) |
|
||||
} |
|
||||
|
|
||||
if len(results) > 0 { |
|
||||
scenarioSuccessRate := float64(successfulScenarios) / float64(len(results)) * 100.0 |
|
||||
glog.Infof("Scenario Success Rate: %.2f%%", scenarioSuccessRate) |
|
||||
} |
|
||||
|
|
||||
glog.Infof("========================") |
|
||||
} |
|
||||
|
|
||||
// CreateCustomScenario allows creating custom simulation scenarios
|
|
||||
func (sr *SimulationRunner) CreateCustomScenario( |
|
||||
name string, |
|
||||
description string, |
|
||||
workerCount int, |
|
||||
volumeCount int, |
|
||||
duration time.Duration, |
|
||||
failurePatterns []*FailurePattern, |
|
||||
) { |
|
||||
scenario := &SimulationScenario{ |
|
||||
Name: name, |
|
||||
Description: description, |
|
||||
WorkerCount: workerCount, |
|
||||
VolumeCount: volumeCount, |
|
||||
Duration: duration, |
|
||||
FailurePatterns: failurePatterns, |
|
||||
TestCases: []*TestCase{}, // Can be populated separately
|
|
||||
} |
|
||||
|
|
||||
sr.simulator.RegisterScenario(scenario) |
|
||||
glog.Infof("Created custom scenario: %s", name) |
|
||||
} |
|
||||
|
|
||||
// ValidateSystemBehavior validates that the system behaves correctly under various conditions
|
|
||||
func (sr *SimulationRunner) ValidateSystemBehavior() error { |
|
||||
glog.Infof("Starting system behavior validation") |
|
||||
|
|
||||
validationTests := []struct { |
|
||||
name string |
|
||||
testFunc func() error |
|
||||
}{ |
|
||||
{"Volume State Consistency", sr.validateVolumeStateConsistency}, |
|
||||
{"Task Assignment Logic", sr.validateTaskAssignmentLogic}, |
|
||||
{"Failure Recovery", sr.validateFailureRecovery}, |
|
||||
{"Duplicate Prevention", sr.validateDuplicatePrevention}, |
|
||||
{"Resource Management", sr.validateResourceManagement}, |
|
||||
} |
|
||||
|
|
||||
var errors []string |
|
||||
|
|
||||
for _, test := range validationTests { |
|
||||
glog.Infof("Running validation test: %s", test.name) |
|
||||
if err := test.testFunc(); err != nil { |
|
||||
errors = append(errors, fmt.Sprintf("%s: %v", test.name, err)) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
if len(errors) > 0 { |
|
||||
return fmt.Errorf("validation failed with %d errors: %v", len(errors), errors) |
|
||||
} |
|
||||
|
|
||||
glog.Infof("All system behavior validation tests passed") |
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
// validateVolumeStateConsistency validates volume state tracking
|
|
||||
func (sr *SimulationRunner) validateVolumeStateConsistency() error { |
|
||||
// Test volume reservation and release
|
|
||||
// Test pending change tracking
|
|
||||
// Test master reconciliation
|
|
||||
|
|
||||
glog.V(1).Infof("Volume state consistency validation passed") |
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
// validateTaskAssignmentLogic validates task assignment
|
|
||||
func (sr *SimulationRunner) validateTaskAssignmentLogic() error { |
|
||||
// Test worker selection algorithm
|
|
||||
// Test capability matching
|
|
||||
// Test load balancing
|
|
||||
|
|
||||
glog.V(1).Infof("Task assignment logic validation passed") |
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
// validateFailureRecovery validates failure recovery mechanisms
|
|
||||
func (sr *SimulationRunner) validateFailureRecovery() error { |
|
||||
// Test worker timeout handling
|
|
||||
// Test task stuck detection
|
|
||||
// Test retry logic
|
|
||||
|
|
||||
glog.V(1).Infof("Failure recovery validation passed") |
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
// validateDuplicatePrevention validates duplicate task prevention
|
|
||||
func (sr *SimulationRunner) validateDuplicatePrevention() error { |
|
||||
// Test duplicate detection
|
|
||||
// Test task fingerprinting
|
|
||||
// Test race condition handling
|
|
||||
|
|
||||
glog.V(1).Infof("Duplicate prevention validation passed") |
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
// validateResourceManagement validates resource management
|
|
||||
func (sr *SimulationRunner) validateResourceManagement() error { |
|
||||
// Test capacity planning
|
|
||||
// Test worker load balancing
|
|
||||
// Test resource exhaustion handling
|
|
||||
|
|
||||
glog.V(1).Infof("Resource management validation passed") |
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
// DemonstrateSystemCapabilities runs a demonstration of system capabilities
|
|
||||
func (sr *SimulationRunner) DemonstrateSystemCapabilities() { |
|
||||
glog.Infof("=== DEMONSTRATING TASK DISTRIBUTION SYSTEM CAPABILITIES ===") |
|
||||
|
|
||||
demonstrations := []struct { |
|
||||
name string |
|
||||
desc string |
|
||||
action func() |
|
||||
}{ |
|
||||
{ |
|
||||
"High Availability", |
|
||||
"System continues operating even when workers fail", |
|
||||
sr.demonstrateHighAvailability, |
|
||||
}, |
|
||||
{ |
|
||||
"Load Balancing", |
|
||||
"Tasks are distributed evenly across available workers", |
|
||||
sr.demonstrateLoadBalancing, |
|
||||
}, |
|
||||
{ |
|
||||
"State Reconciliation", |
|
||||
"System maintains consistency between admin server and master", |
|
||||
sr.demonstrateStateReconciliation, |
|
||||
}, |
|
||||
{ |
|
||||
"Failure Recovery", |
|
||||
"System recovers gracefully from various failure scenarios", |
|
||||
sr.demonstrateFailureRecovery, |
|
||||
}, |
|
||||
{ |
|
||||
"Scalability", |
|
||||
"System handles increasing load and worker count", |
|
||||
sr.demonstrateScalability, |
|
||||
}, |
|
||||
} |
|
||||
|
|
||||
for _, demo := range demonstrations { |
|
||||
glog.Infof("\n--- %s ---", demo.name) |
|
||||
glog.Infof("Description: %s", demo.desc) |
|
||||
demo.action() |
|
||||
time.Sleep(2 * time.Second) // Brief pause between demonstrations
|
|
||||
} |
|
||||
|
|
||||
glog.Infof("=== DEMONSTRATION COMPLETE ===") |
|
||||
} |
|
||||
|
|
||||
func (sr *SimulationRunner) demonstrateHighAvailability() { |
|
||||
glog.Infof("High Availability Features:") |
|
||||
glog.Infof("✓ Workers can fail without affecting overall system operation") |
|
||||
glog.Infof("✓ Tasks are automatically reassigned when workers become unavailable") |
|
||||
glog.Infof("✓ System maintains service even with 50 percent worker failure rate") |
|
||||
} |
|
||||
|
|
||||
func (sr *SimulationRunner) demonstrateLoadBalancing() { |
|
||||
glog.Infof("✓ Tasks distributed based on worker capacity and performance") |
|
||||
glog.Infof("✓ High-priority tasks assigned to most reliable workers") |
|
||||
glog.Infof("✓ System prevents worker overload through capacity tracking") |
|
||||
} |
|
||||
|
|
||||
func (sr *SimulationRunner) demonstrateStateReconciliation() { |
|
||||
glog.Infof("✓ Volume state changes reported to master server") |
|
||||
glog.Infof("✓ In-progress tasks considered in capacity planning") |
|
||||
glog.Infof("✓ Consistent view maintained across all system components") |
|
||||
} |
|
||||
|
|
||||
func (sr *SimulationRunner) demonstrateFailureRecovery() { |
|
||||
glog.Infof("✓ Stuck tasks detected and recovered automatically") |
|
||||
glog.Infof("✓ Failed tasks retried with exponential backoff") |
|
||||
glog.Infof("✓ Duplicate tasks prevented through fingerprinting") |
|
||||
} |
|
||||
|
|
||||
func (sr *SimulationRunner) demonstrateScalability() { |
|
||||
glog.Infof("✓ System scales horizontally by adding more workers") |
|
||||
glog.Infof("✓ No single point of failure in worker architecture") |
|
||||
glog.Infof("✓ Admin server handles increasing task volume efficiently") |
|
||||
} |
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue