seaweedfs/weed/admin/task/comprehensive_simulation_test.go

package task

import (
	"fmt"
	"testing"
	"time"
)

func TestComprehensiveSimulation_VolumeCreationDuringTask(t *testing.T) {
	simulator := NewComprehensiveSimulator()

	scenario := &StateTestScenario{
		Name:        "volume_creation_during_task",
		Description: "Tests state consistency when master reports new volume while task is creating it",
		InitialState: &ClusterState{
			Volumes:  make(map[uint32]*VolumeInfo),
			ECShards: make(map[uint32]map[int]*ShardInfo),
		},
		EventSequence: []*SimulationEvent{
			{Type: EventTaskStarted, VolumeID: 1, TaskID: "create_task_1", Parameters: map[string]interface{}{"type": "create"}},
			{Type: EventVolumeCreated, VolumeID: 1, Parameters: map[string]interface{}{"size": int64(1024 * 1024 * 1024)}},
			{Type: EventMasterSync},
			{Type: EventTaskCompleted, TaskID: "create_task_1"},
		},
		InconsistencyChecks: []*InconsistencyCheck{
			{Name: "No unexpected volumes", Type: InconsistencyVolumeUnexpected, MaxAllowedCount: 0},
		},
		Duration: 30 * time.Second,
	}

	err := simulator.runScenario(scenario)
	if err != nil {
		t.Errorf("Volume creation during task scenario failed: %v", err)
	}

	t.Log("✅ Volume creation during task test passed")
}

func TestComprehensiveSimulation_VolumeDeletionDuringTask(t *testing.T) {
	simulator := NewComprehensiveSimulator()

	scenario := &StateTestScenario{
		Name:        "volume_deletion_during_task",
		Description: "Tests handling when volume is deleted while task is working on it",
		InitialState: &ClusterState{
			Volumes: map[uint32]*VolumeInfo{
				1: {ID: 1, Size: 1024 * 1024 * 1024},
			},
		},
		EventSequence: []*SimulationEvent{
			{Type: EventTaskStarted, VolumeID: 1, TaskID: "vacuum_task_1", Parameters: map[string]interface{}{"type": "vacuum"}},
			{Type: EventVolumeDeleted, VolumeID: 1},
			{Type: EventMasterSync},
			{Type: EventTaskFailed, TaskID: "vacuum_task_1", Parameters: map[string]interface{}{"reason": "volume_deleted"}},
		},
		InconsistencyChecks: []*InconsistencyCheck{
			{Name: "Missing volume detected", Type: InconsistencyVolumeMissing, ExpectedCount: 1, MaxAllowedCount: 1},
		},
		Duration: 30 * time.Second,
	}

	err := simulator.runScenario(scenario)
	if err != nil {
		t.Errorf("Volume deletion during task scenario failed: %v", err)
	}

	t.Log("✅ Volume deletion during task test passed")
}

func TestComprehensiveSimulation_ShardCreationRaceCondition(t *testing.T) {
	simulator := NewComprehensiveSimulator()

	scenario := &StateTestScenario{
		Name:        "shard_creation_race_condition",
		Description: "Tests race condition between EC task creating shards and master sync",
		InitialState: &ClusterState{
			Volumes: map[uint32]*VolumeInfo{
				1: {ID: 1, Size: 28 * 1024 * 1024 * 1024}, // Large volume ready for EC
			},
		},
		EventSequence: []*SimulationEvent{
			{Type: EventTaskStarted, VolumeID: 1, TaskID: "ec_task_1", Parameters: map[string]interface{}{"type": "ec_encode"}},
			// Simulate shards being created one by one
			{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(0), Server: "server1"},
			{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(1), Server: "server1"},
			{Type: EventMasterSync}, // Master sync happens while shards are being created
			{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(2), Server: "server2"},
			{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(3), Server: "server2"},
			{Type: EventTaskCompleted, TaskID: "ec_task_1"},
			{Type: EventMasterSync},
		},
		InconsistencyChecks: []*InconsistencyCheck{
			{Name: "All shards accounted for", Type: InconsistencyShardMissing, MaxAllowedCount: 0},
		},
		Duration: 45 * time.Second,
	}

	err := simulator.runScenario(scenario)
	if err != nil {
		t.Errorf("Shard creation race condition scenario failed: %v", err)
	}

	t.Log("✅ Shard creation race condition test passed")
}

func TestComprehensiveSimulation_NetworkPartitionRecovery(t *testing.T) {
	simulator := NewComprehensiveSimulator()

	scenario := &StateTestScenario{
		Name:        "network_partition_recovery",
		Description: "Tests state consistency during and after network partitions",
		EventSequence: []*SimulationEvent{
			{Type: EventTaskStarted, VolumeID: 1, TaskID: "partition_task_1"},
			{Type: EventNetworkPartition, Parameters: map[string]interface{}{"duration": "5s"}}, // Shorter for test
			{Type: EventVolumeCreated, VolumeID: 2},                                             // Created during partition
			{Type: EventNetworkHealed},
			{Type: EventMasterReconnected},
			{Type: EventMasterSync},
			{Type: EventTaskCompleted, TaskID: "partition_task_1"},
		},
		InconsistencyChecks: []*InconsistencyCheck{
			{Name: "State reconciled after partition", Type: InconsistencyVolumeUnexpected, MaxAllowedCount: 1},
		},
		Duration: 30 * time.Second,
	}

	err := simulator.runScenario(scenario)
	if err != nil {
		t.Errorf("Network partition recovery scenario failed: %v", err)
	}

	t.Log("✅ Network partition recovery test passed")
}

func TestComprehensiveSimulation_ConcurrentTasksCapacityTracking(t *testing.T) {
	simulator := NewComprehensiveSimulator()

	scenario := &StateTestScenario{
		Name:        "concurrent_tasks_capacity_tracking",
		Description: "Tests capacity tracking with multiple concurrent tasks",
		EventSequence: []*SimulationEvent{
			{Type: EventTaskStarted, VolumeID: 1, TaskID: "ec_task_1"},
			{Type: EventTaskStarted, VolumeID: 2, TaskID: "vacuum_task_1"},
			{Type: EventTaskStarted, VolumeID: 3, TaskID: "ec_task_2"},
			{Type: EventMasterSync},
			{Type: EventTaskCompleted, TaskID: "vacuum_task_1"},
			{Type: EventTaskCompleted, TaskID: "ec_task_1"},
			{Type: EventTaskCompleted, TaskID: "ec_task_2"},
			{Type: EventMasterSync},
		},
		InconsistencyChecks: []*InconsistencyCheck{
			{Name: "Capacity tracking accurate", Type: InconsistencyCapacityMismatch, MaxAllowedCount: 0},
		},
		Duration: 60 * time.Second,
	}

	err := simulator.runScenario(scenario)
	if err != nil {
		t.Errorf("Concurrent tasks capacity tracking scenario failed: %v", err)
	}

	t.Log("✅ Concurrent tasks capacity tracking test passed")
}

func TestComprehensiveSimulation_ComplexECOperation(t *testing.T) {
	simulator := NewComprehensiveSimulator()

	scenario := &StateTestScenario{
		Name:        "complex_ec_operation",
		Description: "Tests complex EC operations with shard movements and rebuilds",
		EventSequence: []*SimulationEvent{
			{Type: EventTaskStarted, VolumeID: 1, TaskID: "ec_encode_1"},
			// Create some shards
			{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(0), Server: "server1"},
			{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(1), Server: "server1"},
			{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(2), Server: "server2"},
			{Type: EventTaskCompleted, TaskID: "ec_encode_1"},
			{Type: EventShardCorrupted, VolumeID: 1, ShardID: intPtr(2)},
			{Type: EventTaskStarted, VolumeID: 1, TaskID: "ec_rebuild_1"},
			{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(2), Server: "server3"}, // Rebuilt
			{Type: EventTaskCompleted, TaskID: "ec_rebuild_1"},
			{Type: EventMasterSync},
		},
		Duration: 60 * time.Second,
	}

	err := simulator.runScenario(scenario)
	if err != nil {
		t.Errorf("Complex EC operation scenario failed: %v", err)
	}

	t.Log("✅ Complex EC operation test passed")
}

func TestComprehensiveSimulation_HighLoadStressTest(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping high load stress test in short mode")
	}

	simulator := NewComprehensiveSimulator()

	events := []*SimulationEvent{}

	// Create 50 concurrent tasks (reduced from 100 for faster test)
	for i := 0; i < 50; i++ {
		events = append(events, &SimulationEvent{
			Type:     EventTaskStarted,
			VolumeID: uint32(i + 1),
			TaskID:   fmt.Sprintf("stress_task_%d", i),
		})
	}

	// Add master syncs throughout
	for i := 0; i < 5; i++ {
		events = append(events, &SimulationEvent{
			Type: EventMasterSync,
		})
	}

	// Complete all tasks
	for i := 0; i < 50; i++ {
		events = append(events, &SimulationEvent{
			Type:   EventTaskCompleted,
			TaskID: fmt.Sprintf("stress_task_%d", i),
		})
	}

	scenario := &StateTestScenario{
		Name:          "high_load_stress_test",
		Description:   "Tests system under high load with many concurrent operations",
		EventSequence: events,
		Duration:      2 * time.Minute, // Reduced for faster test
	}

	err := simulator.runScenario(scenario)
	if err != nil {
		t.Errorf("High load stress test scenario failed: %v", err)
	}

	t.Log("✅ High load stress test passed")
}

func TestComprehensiveSimulation_AllScenarios(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping comprehensive simulation in short mode")
	}

	simulator := NewComprehensiveSimulator()
	simulator.CreateComprehensiveScenarios()

	// Run a subset of scenarios for testing (full suite would be too slow)
	testScenarios := []string{
		"volume_creation_during_task",
		"volume_deletion_during_task",
		"shard_creation_race_condition",
		"network_partition_recovery",
		"concurrent_tasks_capacity_tracking",
	}

	passedScenarios := 0
	totalScenarios := len(testScenarios)

	for _, scenarioName := range testScenarios {
		t.Run(scenarioName, func(t *testing.T) {
			// Find the scenario
			var scenario *StateTestScenario
			for _, s := range simulator.scenarios {
				if s.Name == scenarioName {
					scenario = s
					break
				}
			}

			if scenario == nil {
				t.Errorf("Scenario %s not found", scenarioName)
				return
			}

			// Reduce duration for faster testing
			scenario.Duration = 15 * time.Second

			err := simulator.runScenario(scenario)
			if err != nil {
				t.Errorf("Scenario %s failed: %v", scenarioName, err)
			} else {
				passedScenarios++
				t.Logf("✅ Scenario %s passed", scenarioName)
			}
		})
	}

	successRate := float64(passedScenarios) / float64(totalScenarios) * 100.0
	t.Logf("=== COMPREHENSIVE SIMULATION TEST RESULTS ===")
	t.Logf("Scenarios Passed: %d/%d (%.1f%%)", passedScenarios, totalScenarios, successRate)

	if successRate < 100.0 {
		t.Errorf("Some scenarios failed. Success rate: %.1f%%", successRate)
	} else {
		t.Log("🎉 All comprehensive simulation scenarios passed!")
	}
}

func TestComprehensiveSimulation_SimulationFramework(t *testing.T) {
	// Test the simulation framework itself
	simulator := NewComprehensiveSimulator()

	// Test event execution
	event := &SimulationEvent{
		Type:     EventTaskStarted,
		VolumeID: 1,
		TaskID:   "test_task",
		Parameters: map[string]interface{}{
			"type": "vacuum",
		},
	}

	err := simulator.executeEvent(event)
	if err != nil {
		t.Errorf("Event execution failed: %v", err)
	}

	// Verify task was registered
	if simulator.results.TasksExecuted != 1 {
		t.Errorf("Expected 1 task executed, got %d", simulator.results.TasksExecuted)
	}

	// Test event logging
	simulator.logEvent(event)
	if len(simulator.eventLog) != 1 {
		t.Errorf("Expected 1 logged event, got %d", len(simulator.eventLog))
	}

	// Test mock master
	simulator.mockMaster.CreateVolume(1, 1024*1024*1024)
	if len(simulator.mockMaster.volumes) != 1 {
		t.Errorf("Expected 1 volume in mock master, got %d", len(simulator.mockMaster.volumes))
	}

	t.Log("✅ Simulation framework test passed")
}

// Integration test that validates the complete state management flow
func TestComprehensiveSimulation_StateManagementIntegration(t *testing.T) {
	// This test validates the core requirement: accurate volume/shard state tracking
	simulator := NewComprehensiveSimulator()

	// Use mock master client instead of nil to avoid nil pointer errors
	simulator.stateManager.masterClient = nil // Skip master client calls for test

	// Setup realistic initial state
	initialState := &ClusterState{
		Volumes: map[uint32]*VolumeInfo{
			1: {ID: 1, Size: 28 * 1024 * 1024 * 1024, Server: "server1"},                                           // Ready for EC
			2: {ID: 2, Size: 20 * 1024 * 1024 * 1024, Server: "server2", DeletedByteCount: 8 * 1024 * 1024 * 1024}, // Needs vacuum
		},
		ServerCapacity: map[string]*CapacityInfo{
			"server1": {Server: "server1", TotalCapacity: 100 * 1024 * 1024 * 1024, UsedCapacity: 30 * 1024 * 1024 * 1024},
			"server2": {Server: "server2", TotalCapacity: 100 * 1024 * 1024 * 1024, UsedCapacity: 25 * 1024 * 1024 * 1024},
		},
	}

	// Complex event sequence that tests state consistency (excluding master sync for test)
	eventSequence := []*SimulationEvent{
		// Start EC task on volume 1
		{Type: EventTaskStarted, VolumeID: 1, TaskID: "ec_task_1", Parameters: map[string]interface{}{"type": "ec_encode"}},

		// Start vacuum task on volume 2
		{Type: EventTaskStarted, VolumeID: 2, TaskID: "vacuum_task_1", Parameters: map[string]interface{}{"type": "vacuum"}},

		// EC task creates shards
		{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(0), Server: "server1"},
		{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(1), Server: "server1"},
		{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(2), Server: "server2"},

		// Vacuum task completes (volume 2 size reduces)
		{Type: EventTaskCompleted, TaskID: "vacuum_task_1"},
		{Type: EventVolumeSizeChanged, VolumeID: 2, Parameters: map[string]interface{}{"new_size": int64(12 * 1024 * 1024 * 1024)}},

		// EC task completes
		{Type: EventTaskCompleted, TaskID: "ec_task_1"},
		{Type: EventVolumeReadOnly, VolumeID: 1}, // Volume becomes read-only after EC
	}

	scenario := &StateTestScenario{
		Name:          "state_management_integration",
		Description:   "Complete state management integration test",
		InitialState:  initialState,
		EventSequence: eventSequence,
		Duration:      30 * time.Second, // Reduced for faster test
		InconsistencyChecks: []*InconsistencyCheck{
			{Name: "No state inconsistencies", Type: InconsistencyVolumeUnexpected, MaxAllowedCount: 0},
			{Name: "No capacity mismatches", Type: InconsistencyCapacityMismatch, MaxAllowedCount: 0},
			{Name: "No orphaned tasks", Type: InconsistencyTaskOrphaned, MaxAllowedCount: 0},
		},
	}

	err := simulator.runScenario(scenario)
	if err != nil {
		t.Errorf("State management integration test failed: %v", err)
	}

	// Verify final state
	if simulator.results.TasksExecuted != 2 {
		t.Errorf("Expected 2 tasks executed, got %d", simulator.results.TasksExecuted)
	}

	if simulator.results.TasksSucceeded != 2 {
		t.Errorf("Expected 2 tasks succeeded, got %d", simulator.results.TasksSucceeded)
	}

	t.Log("✅ State management integration test passed")
	t.Log("✅ System accurately tracked volume/shard states throughout complex operation sequence")
}

// Performance test for simulation framework
func BenchmarkComprehensiveSimulation_EventExecution(b *testing.B) {
	simulator := NewComprehensiveSimulator()

	events := []*SimulationEvent{
		{Type: EventTaskStarted, VolumeID: 1, TaskID: "task_1"},
		{Type: EventVolumeCreated, VolumeID: 2},
		{Type: EventShardCreated, VolumeID: 1, ShardID: intPtr(0), Server: "server1"},
		{Type: EventMasterSync},
		{Type: EventTaskCompleted, TaskID: "task_1"},
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		for _, event := range events {
			simulator.executeEvent(event)
		}
	}
}

// Helper functions for tests
func createTestVolumeInfo(id uint32, size uint64) *VolumeInfo {
	return &VolumeInfo{
		ID:   id,
		Size: size,
	}
}