Browse Source
reorganized the codebase to move the simulation framework and tests into their own dedicated package
worker-execute-ec-tasks
reorganized the codebase to move the simulation framework and tests into their own dedicated package
worker-execute-ec-tasks
chrislu
4 months ago
6 changed files with 122 additions and 470 deletions
-
70weed/admin/task/example_usage.go
-
84weed/admin/task/simulation/comprehensive_simulation.go
-
56weed/admin/task/simulation/comprehensive_simulation_test.go
-
4weed/admin/task/simulation/simulation_runner.go
-
81weed/admin/task/simulation/system_demo_test.go
-
297weed/admin/task/simulation_runner.go
@ -1,297 +0,0 @@ |
|||
package task |
|||
|
|||
import ( |
|||
"fmt" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
) |
|||
|
|||
// SimulationRunner orchestrates the execution of simulation scenarios
|
|||
type SimulationRunner struct { |
|||
simulator *TaskSimulator |
|||
} |
|||
|
|||
// NewSimulationRunner creates a new simulation runner
|
|||
func NewSimulationRunner() *SimulationRunner { |
|||
return &SimulationRunner{ |
|||
simulator: NewTaskSimulator(), |
|||
} |
|||
} |
|||
|
|||
// RunAllScenarios runs all predefined simulation scenarios
|
|||
func (sr *SimulationRunner) RunAllScenarios() error { |
|||
glog.Infof("Starting comprehensive task distribution system simulation") |
|||
|
|||
// Create standard scenarios
|
|||
sr.simulator.CreateStandardScenarios() |
|||
|
|||
scenarios := []string{ |
|||
"worker_timeout_during_ec", |
|||
"stuck_vacuum_task", |
|||
"duplicate_task_prevention", |
|||
"master_admin_divergence", |
|||
} |
|||
|
|||
var allResults []*SimulationResult |
|||
|
|||
for _, scenarioName := range scenarios { |
|||
glog.Infof("Running scenario: %s", scenarioName) |
|||
|
|||
result, err := sr.simulator.RunScenario(scenarioName) |
|||
if err != nil { |
|||
glog.Errorf("Failed to run scenario %s: %v", scenarioName, err) |
|||
continue |
|||
} |
|||
|
|||
allResults = append(allResults, result) |
|||
|
|||
// Brief pause between scenarios
|
|||
time.Sleep(5 * time.Second) |
|||
} |
|||
|
|||
// Generate and log comprehensive report
|
|||
report := sr.simulator.GenerateSimulationReport() |
|||
glog.Infof("Simulation Report:\n%s", report) |
|||
|
|||
// Summary
|
|||
sr.logSummary(allResults) |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// RunSpecificScenario runs a specific simulation scenario
|
|||
func (sr *SimulationRunner) RunSpecificScenario(scenarioName string) (*SimulationResult, error) { |
|||
// Ensure standard scenarios are available
|
|||
sr.simulator.CreateStandardScenarios() |
|||
|
|||
return sr.simulator.RunScenario(scenarioName) |
|||
} |
|||
|
|||
// logSummary logs a summary of all simulation results
|
|||
func (sr *SimulationRunner) logSummary(results []*SimulationResult) { |
|||
totalTasks := 0 |
|||
totalCompleted := 0 |
|||
totalFailed := 0 |
|||
totalTimeouts := 0 |
|||
totalDuplicates := 0 |
|||
totalInconsistencies := 0 |
|||
successfulScenarios := 0 |
|||
|
|||
for _, result := range results { |
|||
totalTasks += result.TasksCreated |
|||
totalCompleted += result.TasksCompleted |
|||
totalFailed += result.TasksFailed |
|||
totalTimeouts += result.WorkerTimeouts |
|||
totalDuplicates += result.DuplicatesFound |
|||
totalInconsistencies += result.StateInconsistencies |
|||
|
|||
if result.Success { |
|||
successfulScenarios++ |
|||
} |
|||
} |
|||
|
|||
glog.Infof("=== SIMULATION SUMMARY ===") |
|||
glog.Infof("Scenarios Run: %d", len(results)) |
|||
glog.Infof("Successful Scenarios: %d", successfulScenarios) |
|||
glog.Infof("Total Tasks Created: %d", totalTasks) |
|||
glog.Infof("Total Tasks Completed: %d", totalCompleted) |
|||
glog.Infof("Total Tasks Failed: %d", totalFailed) |
|||
glog.Infof("Total Worker Timeouts: %d", totalTimeouts) |
|||
glog.Infof("Total Duplicates Found: %d", totalDuplicates) |
|||
glog.Infof("Total State Inconsistencies: %d", totalInconsistencies) |
|||
|
|||
if totalTasks > 0 { |
|||
completionRate := float64(totalCompleted) / float64(totalTasks) * 100.0 |
|||
glog.Infof("Task Completion Rate: %.2f%%", completionRate) |
|||
} |
|||
|
|||
if len(results) > 0 { |
|||
scenarioSuccessRate := float64(successfulScenarios) / float64(len(results)) * 100.0 |
|||
glog.Infof("Scenario Success Rate: %.2f%%", scenarioSuccessRate) |
|||
} |
|||
|
|||
glog.Infof("========================") |
|||
} |
|||
|
|||
// CreateCustomScenario allows creating custom simulation scenarios
|
|||
func (sr *SimulationRunner) CreateCustomScenario( |
|||
name string, |
|||
description string, |
|||
workerCount int, |
|||
volumeCount int, |
|||
duration time.Duration, |
|||
failurePatterns []*FailurePattern, |
|||
) { |
|||
scenario := &SimulationScenario{ |
|||
Name: name, |
|||
Description: description, |
|||
WorkerCount: workerCount, |
|||
VolumeCount: volumeCount, |
|||
Duration: duration, |
|||
FailurePatterns: failurePatterns, |
|||
TestCases: []*TestCase{}, // Can be populated separately
|
|||
} |
|||
|
|||
sr.simulator.RegisterScenario(scenario) |
|||
glog.Infof("Created custom scenario: %s", name) |
|||
} |
|||
|
|||
// ValidateSystemBehavior validates that the system behaves correctly under various conditions
|
|||
func (sr *SimulationRunner) ValidateSystemBehavior() error { |
|||
glog.Infof("Starting system behavior validation") |
|||
|
|||
validationTests := []struct { |
|||
name string |
|||
testFunc func() error |
|||
}{ |
|||
{"Volume State Consistency", sr.validateVolumeStateConsistency}, |
|||
{"Task Assignment Logic", sr.validateTaskAssignmentLogic}, |
|||
{"Failure Recovery", sr.validateFailureRecovery}, |
|||
{"Duplicate Prevention", sr.validateDuplicatePrevention}, |
|||
{"Resource Management", sr.validateResourceManagement}, |
|||
} |
|||
|
|||
var errors []string |
|||
|
|||
for _, test := range validationTests { |
|||
glog.Infof("Running validation test: %s", test.name) |
|||
if err := test.testFunc(); err != nil { |
|||
errors = append(errors, fmt.Sprintf("%s: %v", test.name, err)) |
|||
} |
|||
} |
|||
|
|||
if len(errors) > 0 { |
|||
return fmt.Errorf("validation failed with %d errors: %v", len(errors), errors) |
|||
} |
|||
|
|||
glog.Infof("All system behavior validation tests passed") |
|||
return nil |
|||
} |
|||
|
|||
// validateVolumeStateConsistency validates volume state tracking
|
|||
func (sr *SimulationRunner) validateVolumeStateConsistency() error { |
|||
// Test volume reservation and release
|
|||
// Test pending change tracking
|
|||
// Test master reconciliation
|
|||
|
|||
glog.V(1).Infof("Volume state consistency validation passed") |
|||
return nil |
|||
} |
|||
|
|||
// validateTaskAssignmentLogic validates task assignment
|
|||
func (sr *SimulationRunner) validateTaskAssignmentLogic() error { |
|||
// Test worker selection algorithm
|
|||
// Test capability matching
|
|||
// Test load balancing
|
|||
|
|||
glog.V(1).Infof("Task assignment logic validation passed") |
|||
return nil |
|||
} |
|||
|
|||
// validateFailureRecovery validates failure recovery mechanisms
|
|||
func (sr *SimulationRunner) validateFailureRecovery() error { |
|||
// Test worker timeout handling
|
|||
// Test task stuck detection
|
|||
// Test retry logic
|
|||
|
|||
glog.V(1).Infof("Failure recovery validation passed") |
|||
return nil |
|||
} |
|||
|
|||
// validateDuplicatePrevention validates duplicate task prevention
|
|||
func (sr *SimulationRunner) validateDuplicatePrevention() error { |
|||
// Test duplicate detection
|
|||
// Test task fingerprinting
|
|||
// Test race condition handling
|
|||
|
|||
glog.V(1).Infof("Duplicate prevention validation passed") |
|||
return nil |
|||
} |
|||
|
|||
// validateResourceManagement validates resource management
|
|||
func (sr *SimulationRunner) validateResourceManagement() error { |
|||
// Test capacity planning
|
|||
// Test worker load balancing
|
|||
// Test resource exhaustion handling
|
|||
|
|||
glog.V(1).Infof("Resource management validation passed") |
|||
return nil |
|||
} |
|||
|
|||
// DemonstrateSystemCapabilities runs a demonstration of system capabilities
|
|||
func (sr *SimulationRunner) DemonstrateSystemCapabilities() { |
|||
glog.Infof("=== DEMONSTRATING TASK DISTRIBUTION SYSTEM CAPABILITIES ===") |
|||
|
|||
demonstrations := []struct { |
|||
name string |
|||
desc string |
|||
action func() |
|||
}{ |
|||
{ |
|||
"High Availability", |
|||
"System continues operating even when workers fail", |
|||
sr.demonstrateHighAvailability, |
|||
}, |
|||
{ |
|||
"Load Balancing", |
|||
"Tasks are distributed evenly across available workers", |
|||
sr.demonstrateLoadBalancing, |
|||
}, |
|||
{ |
|||
"State Reconciliation", |
|||
"System maintains consistency between admin server and master", |
|||
sr.demonstrateStateReconciliation, |
|||
}, |
|||
{ |
|||
"Failure Recovery", |
|||
"System recovers gracefully from various failure scenarios", |
|||
sr.demonstrateFailureRecovery, |
|||
}, |
|||
{ |
|||
"Scalability", |
|||
"System handles increasing load and worker count", |
|||
sr.demonstrateScalability, |
|||
}, |
|||
} |
|||
|
|||
for _, demo := range demonstrations { |
|||
glog.Infof("\n--- %s ---", demo.name) |
|||
glog.Infof("Description: %s", demo.desc) |
|||
demo.action() |
|||
time.Sleep(2 * time.Second) // Brief pause between demonstrations
|
|||
} |
|||
|
|||
glog.Infof("=== DEMONSTRATION COMPLETE ===") |
|||
} |
|||
|
|||
func (sr *SimulationRunner) demonstrateHighAvailability() { |
|||
glog.Infof("High Availability Features:") |
|||
glog.Infof("✓ Workers can fail without affecting overall system operation") |
|||
glog.Infof("✓ Tasks are automatically reassigned when workers become unavailable") |
|||
glog.Infof("✓ System maintains service even with 50 percent worker failure rate") |
|||
} |
|||
|
|||
func (sr *SimulationRunner) demonstrateLoadBalancing() { |
|||
glog.Infof("✓ Tasks distributed based on worker capacity and performance") |
|||
glog.Infof("✓ High-priority tasks assigned to most reliable workers") |
|||
glog.Infof("✓ System prevents worker overload through capacity tracking") |
|||
} |
|||
|
|||
func (sr *SimulationRunner) demonstrateStateReconciliation() { |
|||
glog.Infof("✓ Volume state changes reported to master server") |
|||
glog.Infof("✓ In-progress tasks considered in capacity planning") |
|||
glog.Infof("✓ Consistent view maintained across all system components") |
|||
} |
|||
|
|||
func (sr *SimulationRunner) demonstrateFailureRecovery() { |
|||
glog.Infof("✓ Stuck tasks detected and recovered automatically") |
|||
glog.Infof("✓ Failed tasks retried with exponential backoff") |
|||
glog.Infof("✓ Duplicate tasks prevented through fingerprinting") |
|||
} |
|||
|
|||
func (sr *SimulationRunner) demonstrateScalability() { |
|||
glog.Infof("✓ System scales horizontally by adding more workers") |
|||
glog.Infof("✓ No single point of failure in worker architecture") |
|||
glog.Infof("✓ Admin server handles increasing task volume efficiently") |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue