package task import ( "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/worker/types" ) // ExampleUsage demonstrates how to use the task distribution system func ExampleUsage() { glog.Infof("=== SeaweedFS Task Distribution System Example ===") // Example 1: Setting up the Admin Server setupAdminServerExample() // Example 2: Simulating Workers simulateWorkersExample() // Example 3: Running Simulations runSimulationsExample() // Example 4: Demonstrating Features demonstrateFeaturesExample() } // setupAdminServerExample shows how to set up the admin server func setupAdminServerExample() { glog.Infof("\n--- Example 1: Setting up Admin Server ---") // Create master client (in real usage, this would connect to actual master) masterClient := &wdclient.MasterClient{} // Simplified for example // Create admin server configuration config := &AdminConfig{ ScanInterval: 30 * time.Minute, WorkerTimeout: 5 * time.Minute, TaskTimeout: 10 * time.Minute, MaxRetries: 3, ReconcileInterval: 5 * time.Minute, EnableFailureRecovery: true, MaxConcurrentTasks: 10, } // Create admin server adminServer := NewAdminServer(config, masterClient) // Start the admin server if err := adminServer.Start(); err != nil { glog.Errorf("Failed to start admin server: %v", err) return } glog.Infof("✓ Admin server started with configuration:") glog.Infof(" - Scan Interval: %v", config.ScanInterval) glog.Infof(" - Worker Timeout: %v", config.WorkerTimeout) glog.Infof(" - Max Concurrent Tasks: %d", config.MaxConcurrentTasks) // Simulate some operations time.Sleep(2 * time.Second) // Stop the admin server adminServer.Stop() glog.Infof("✓ Admin server stopped gracefully") } // simulateWorkersExample shows how workers would register and operate func simulateWorkersExample() { glog.Infof("\n--- Example 2: Worker Registration and Operation ---") // Create mock workers workers := []*types.Worker{ { ID: "worker-ec-01", Address: "192.168.1.100:8080", Capabilities: []types.TaskType{types.TaskTypeErasureCoding}, MaxConcurrent: 2, Status: "active", CurrentLoad: 0, }, { ID: "worker-vacuum-01", Address: "192.168.1.101:8080", Capabilities: []types.TaskType{types.TaskTypeVacuum}, MaxConcurrent: 3, Status: "active", CurrentLoad: 0, }, { ID: "worker-multi-01", Address: "192.168.1.102:8080", Capabilities: []types.TaskType{types.TaskTypeErasureCoding, types.TaskTypeVacuum}, MaxConcurrent: 2, Status: "active", CurrentLoad: 0, }, } // Create worker registry registry := NewWorkerRegistry() // Register workers for _, worker := range workers { if err := registry.RegisterWorker(worker); err != nil { glog.Errorf("Failed to register worker %s: %v", worker.ID, err) } else { glog.Infof("✓ Registered worker %s with capabilities: %v", worker.ID, worker.Capabilities) } } // Demonstrate worker selection bestECWorker := registry.GetBestWorkerForTask(types.TaskTypeErasureCoding) if bestECWorker != nil { glog.Infof("✓ Best worker for EC tasks: %s", bestECWorker.ID) } bestVacuumWorker := registry.GetBestWorkerForTask(types.TaskTypeVacuum) if bestVacuumWorker != nil { glog.Infof("✓ Best worker for vacuum tasks: %s", bestVacuumWorker.ID) } // Show registry statistics stats := registry.GetRegistryStats() glog.Infof("✓ Registry statistics: %+v", stats) } // runSimulationsExample shows how to run simulation scenarios func runSimulationsExample() { glog.Infof("\n--- Example 3: Running Simulation Scenarios ---") // Create simulation runner runner := NewSimulationRunner() // Demonstrate system capabilities runner.DemonstrateSystemCapabilities() // Create a custom scenario runner.CreateCustomScenario( "custom_test", "Custom test scenario for demonstration", 3, // 3 workers 10, // 10 volumes 60*time.Second, // 60 second duration []*FailurePattern{ { Type: FailureWorkerTimeout, Probability: 0.2, // 20% chance Timing: &TimingSpec{ MinProgress: 30.0, MaxProgress: 70.0, }, }, }, ) // Run specific scenario result, err := runner.RunSpecificScenario("custom_test") if err != nil { glog.Errorf("Failed to run scenario: %v", err) } else { glog.Infof("✓ Custom scenario completed:") glog.Infof(" - Tasks Created: %d", result.TasksCreated) glog.Infof(" - Tasks Completed: %d", result.TasksCompleted) glog.Infof(" - Duration: %v", result.Duration) glog.Infof(" - Success: %v", result.Success) } // Validate system behavior if err := runner.ValidateSystemBehavior(); err != nil { glog.Errorf("System validation failed: %v", err) } else { glog.Infof("✓ All system validation tests passed") } } // demonstrateFeaturesExample shows key system features func demonstrateFeaturesExample() { glog.Infof("\n--- Example 4: Key System Features ---") // Feature 1: Task Discovery demonstrateTaskDiscovery() // Feature 2: Volume State Tracking demonstrateVolumeStateTracking() // Feature 3: Failure Handling demonstrateFailureHandling() // Feature 4: Task Scheduling demonstrateTaskScheduling() } // demonstrateTaskDiscovery shows how task discovery works func demonstrateTaskDiscovery() { glog.Infof("\n Feature 1: Task Discovery") // Create mock volumes volumes := []*VolumeInfo{ { ID: 1, Size: 28 * 1024 * 1024 * 1024, // 28GB (93% of 30GB) Collection: "photos", DeletedByteCount: 0, ReadOnly: false, ModifiedAtSecond: time.Now().Add(-2 * time.Hour).Unix(), // 2 hours old }, { ID: 2, Size: 20 * 1024 * 1024 * 1024, // 20GB Collection: "documents", DeletedByteCount: 8 * 1024 * 1024 * 1024, // 8GB garbage (40%) ReadOnly: false, ModifiedAtSecond: time.Now().Add(-1 * time.Hour).Unix(), // 1 hour old }, } // Create detectors ecDetector := NewECDetector() vacuumDetector := NewVacuumDetector() // Test EC detection ecCandidates, _ := ecDetector.DetectECCandidates(volumes) glog.Infof(" ✓ EC detector found %d candidates", len(ecCandidates)) for _, candidate := range ecCandidates { glog.Infof(" - Volume %d: %s (priority: %d)", candidate.VolumeID, candidate.Reason, candidate.Priority) } // Test vacuum detection vacuumCandidates, _ := vacuumDetector.DetectVacuumCandidates(volumes) glog.Infof(" ✓ Vacuum detector found %d candidates", len(vacuumCandidates)) for _, candidate := range vacuumCandidates { glog.Infof(" - Volume %d: %s (priority: %d)", candidate.VolumeID, candidate.Reason, candidate.Priority) } } // demonstrateVolumeStateTracking shows volume state management func demonstrateVolumeStateTracking() { glog.Infof("\n Feature 2: Volume State Tracking") // Create volume state tracker tracker := NewVolumeStateTracker(nil, 5*time.Minute) // Reserve volumes for tasks tracker.ReserveVolume(1, "task-ec-001") tracker.ReserveVolume(2, "task-vacuum-001") glog.Infof(" ✓ Reserved volumes for tasks") // Check reservations if tracker.IsVolumeReserved(1) { glog.Infof(" ✓ Volume 1 is correctly reserved") } // Record volume changes tracker.RecordVolumeChange(1, types.TaskTypeErasureCoding, "task-ec-001") glog.Infof(" ✓ Recorded volume change for EC completion") // Get pending changes if change := tracker.GetPendingChange(1); change != nil { glog.Infof(" ✓ Pending change found: %s for volume %d", change.ChangeType, change.VolumeID) } // Release reservation tracker.ReleaseVolume(2, "task-vacuum-001") glog.Infof(" ✓ Released volume reservation") // Show statistics stats := tracker.GetStats() glog.Infof(" ✓ Tracker statistics: %+v", stats) } // demonstrateFailureHandling shows failure recovery mechanisms func demonstrateFailureHandling() { glog.Infof("\n Feature 3: Failure Handling") // Create failure handler config := DefaultAdminConfig() handler := NewFailureHandler(config) // Create mock task task := &InProgressTask{ Task: &types.Task{ ID: "test-task-001", Type: types.TaskTypeErasureCoding, VolumeID: 1, RetryCount: 0, }, WorkerID: "worker-01", StartedAt: time.Now(), LastUpdate: time.Now().Add(-30 * time.Minute), // 30 minutes ago Progress: 45.0, } // Demonstrate different failure scenarios glog.Infof(" ✓ Simulating worker timeout scenario") handler.HandleWorkerTimeout("worker-01", []*InProgressTask{task}) glog.Infof(" ✓ Simulating stuck task scenario") handler.HandleTaskStuck(task) glog.Infof(" ✓ Simulating duplicate task detection") handler.HandleDuplicateTask("existing-task", "duplicate-task", 1) // Show failure statistics stats := handler.GetFailureStats() glog.Infof(" ✓ Failure handler statistics: %+v", stats) } // demonstrateTaskScheduling shows task scheduling logic func demonstrateTaskScheduling() { glog.Infof("\n Feature 4: Task Scheduling") // Create worker registry and task queue registry := NewWorkerRegistry() queue := NewPriorityTaskQueue() scheduler := NewTaskScheduler(registry, queue) // Add mock worker worker := &types.Worker{ ID: "scheduler-worker-01", Capabilities: []types.TaskType{types.TaskTypeErasureCoding, types.TaskTypeVacuum}, MaxConcurrent: 2, Status: "active", CurrentLoad: 0, } registry.RegisterWorker(worker) // Create mock tasks with different priorities highPriorityTask := &types.Task{ ID: "high-priority-task", Type: types.TaskTypeErasureCoding, Priority: types.TaskPriorityHigh, VolumeID: 1, } normalPriorityTask := &types.Task{ ID: "normal-priority-task", Type: types.TaskTypeVacuum, Priority: types.TaskPriorityNormal, VolumeID: 2, } // Add tasks to queue queue.Push(normalPriorityTask) queue.Push(highPriorityTask) // Should be prioritized glog.Infof(" ✓ Added tasks to priority queue (size: %d)", queue.Size()) // Test worker selection selectedWorker := scheduler.SelectWorker(highPriorityTask, []*types.Worker{worker}) if selectedWorker != nil { glog.Infof(" ✓ Selected worker %s for high-priority task", selectedWorker.ID) } // Test task retrieval nextTask := scheduler.GetNextTask("scheduler-worker-01", []types.TaskType{types.TaskTypeErasureCoding, types.TaskTypeVacuum}) if nextTask != nil { glog.Infof(" ✓ Next task for worker: %s (priority: %d)", nextTask.ID, nextTask.Priority) } glog.Infof(" ✓ Task scheduling demonstration complete") } // RunComprehensiveDemo runs a full demonstration of the system func RunComprehensiveDemo() { glog.Infof("Starting comprehensive task distribution system demonstration...") // Run the main example ExampleUsage() // Run all simulation scenarios runner := NewSimulationRunner() if err := runner.RunAllScenarios(); err != nil { glog.Errorf("Failed to run all scenarios: %v", err) } glog.Infof("=== Comprehensive demonstration complete ===") glog.Infof("The task distribution system is ready for production use!") glog.Infof("Key benefits demonstrated:") glog.Infof(" ✓ Automatic task discovery and assignment") glog.Infof(" ✓ Robust failure handling and recovery") glog.Infof(" ✓ Volume state consistency and reconciliation") glog.Infof(" ✓ Worker load balancing and performance tracking") glog.Infof(" ✓ Comprehensive simulation and validation framework") }