diff --git a/weed/storage/disk_location_ec_test.go b/weed/storage/disk_location_ec_test.go new file mode 100644 index 000000000..6cbbe6879 --- /dev/null +++ b/weed/storage/disk_location_ec_test.go @@ -0,0 +1,461 @@ +package storage + +import ( + "os" + "path/filepath" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// TestIncompleteEcEncodingCleanup tests the cleanup logic for incomplete EC encoding scenarios +func TestIncompleteEcEncodingCleanup(t *testing.T) { + // Create temporary test directory + tempDir, err := os.MkdirTemp("", "ec_cleanup_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + + tests := []struct { + name string + volumeId needle.VolumeId + collection string + createDatFile bool + createEcxFile bool + createEcjFile bool + numShards int + expectCleanup bool + expectLoadSuccess bool + }{ + { + name: "Incomplete EC: shards without .ecx, .dat exists - should cleanup", + volumeId: 100, + collection: "", + createDatFile: true, + createEcxFile: false, + createEcjFile: false, + numShards: 14, // All shards but no .ecx + expectCleanup: true, + expectLoadSuccess: false, + }, + { + name: "Distributed EC: shards without .ecx, .dat deleted - should NOT cleanup", + volumeId: 101, + collection: "", + createDatFile: false, + createEcxFile: false, + createEcjFile: false, + numShards: 5, // Partial shards, distributed + expectCleanup: false, + expectLoadSuccess: false, + }, + { + name: "Incomplete EC: shards with .ecx but < 10 shards, .dat exists - should cleanup", + volumeId: 102, + collection: "", + createDatFile: true, + createEcxFile: true, + createEcjFile: false, + numShards: 7, // Less than DataShardsCount (10) + expectCleanup: true, + expectLoadSuccess: false, + }, + { + name: "Valid local EC: shards with .ecx, >= 10 shards, .dat exists - should load", + volumeId: 103, + collection: "", + createDatFile: true, + createEcxFile: true, + createEcjFile: false, + numShards: 14, // All shards + expectCleanup: false, + expectLoadSuccess: true, // Would succeed if .ecx was valid + }, + { + name: "Distributed EC: shards with .ecx, .dat deleted - should load", + volumeId: 104, + collection: "", + createDatFile: false, + createEcxFile: true, + createEcjFile: false, + numShards: 10, // Enough shards + expectCleanup: false, + expectLoadSuccess: true, // Would succeed if .ecx was valid + }, + { + name: "Incomplete EC with collection: shards without .ecx, .dat exists - should cleanup", + volumeId: 105, + collection: "test_collection", + createDatFile: true, + createEcxFile: false, + createEcjFile: false, + numShards: 14, + expectCleanup: true, + expectLoadSuccess: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create DiskLocation + minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} + diskLocation := &DiskLocation{ + Directory: tempDir, + DirectoryUuid: "test-uuid", + IdxDirectory: tempDir, + DiskType: types.HddType, + MaxVolumeCount: 100, + OriginalMaxVolumeCount: 100, + MinFreeSpace: minFreeSpace, + } + diskLocation.volumes = make(map[needle.VolumeId]*Volume) + diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) + + // Setup test files + baseFileName := erasure_coding.EcShardFileName(tt.collection, tempDir, int(tt.volumeId)) + + // Create .dat file if needed + if tt.createDatFile { + datFile, err := os.Create(baseFileName + ".dat") + if err != nil { + t.Fatalf("Failed to create .dat file: %v", err) + } + datFile.WriteString("dummy data") + datFile.Close() + } + + // Create EC shard files + for i := 0; i < tt.numShards; i++ { + shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i)) + if err != nil { + t.Fatalf("Failed to create shard file: %v", err) + } + shardFile.WriteString("dummy shard data") + shardFile.Close() + } + + // Create .ecx file if needed + if tt.createEcxFile { + ecxFile, err := os.Create(baseFileName + ".ecx") + if err != nil { + t.Fatalf("Failed to create .ecx file: %v", err) + } + ecxFile.WriteString("dummy ecx data") + ecxFile.Close() + } + + // Create .ecj file if needed + if tt.createEcjFile { + ecjFile, err := os.Create(baseFileName + ".ecj") + if err != nil { + t.Fatalf("Failed to create .ecj file: %v", err) + } + ecjFile.WriteString("dummy ecj data") + ecjFile.Close() + } + + // Run loadAllEcShards + err = diskLocation.loadAllEcShards() + if err != nil { + t.Logf("loadAllEcShards returned error (expected in some cases): %v", err) + } + + // Verify cleanup expectations + if tt.expectCleanup { + // Check that files were cleaned up + if util.FileExists(baseFileName + ".ecx") { + t.Errorf("Expected .ecx to be cleaned up but it still exists") + } + if util.FileExists(baseFileName + ".ecj") { + t.Errorf("Expected .ecj to be cleaned up but it still exists") + } + for i := 0; i < erasure_coding.TotalShardsCount; i++ { + shardFile := baseFileName + erasure_coding.ToExt(i) + if util.FileExists(shardFile) { + t.Errorf("Expected shard %d to be cleaned up but it still exists", i) + } + } + // .dat file should still exist (not cleaned up) + if tt.createDatFile && !util.FileExists(baseFileName+".dat") { + t.Errorf("Expected .dat file to remain but it was deleted") + } + } else { + // Check that files were NOT cleaned up + for i := 0; i < tt.numShards; i++ { + shardFile := baseFileName + erasure_coding.ToExt(i) + if !util.FileExists(shardFile) { + t.Errorf("Expected shard %d to remain but it was cleaned up", i) + } + } + if tt.createEcxFile && !util.FileExists(baseFileName+".ecx") { + t.Errorf("Expected .ecx to remain but it was cleaned up") + } + } + + // Cleanup test files for next iteration + os.Remove(baseFileName + ".dat") + os.Remove(baseFileName + ".ecx") + os.Remove(baseFileName + ".ecj") + for i := 0; i < erasure_coding.TotalShardsCount; i++ { + os.Remove(baseFileName + erasure_coding.ToExt(i)) + } + }) + } +} + +// TestValidateEcVolume tests the validateEcVolume function +func TestValidateEcVolume(t *testing.T) { + tempDir, err := os.MkdirTemp("", "ec_validate_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + + minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} + diskLocation := &DiskLocation{ + Directory: tempDir, + DirectoryUuid: "test-uuid", + IdxDirectory: tempDir, + DiskType: types.HddType, + MinFreeSpace: minFreeSpace, + } + + tests := []struct { + name string + volumeId needle.VolumeId + collection string + createDatFile bool + numShards int + expectValid bool + }{ + { + name: "Valid: .dat exists with 10+ shards", + volumeId: 200, + collection: "", + createDatFile: true, + numShards: 10, + expectValid: true, + }, + { + name: "Invalid: .dat exists with < 10 shards", + volumeId: 201, + collection: "", + createDatFile: true, + numShards: 9, + expectValid: false, + }, + { + name: "Valid: .dat deleted (distributed EC) with any shards", + volumeId: 202, + collection: "", + createDatFile: false, + numShards: 5, + expectValid: true, + }, + { + name: "Valid: .dat deleted (distributed EC) with no shards", + volumeId: 203, + collection: "", + createDatFile: false, + numShards: 0, + expectValid: true, + }, + { + name: "Invalid: zero-byte shard files should not count", + volumeId: 204, + collection: "", + createDatFile: true, + numShards: 0, // Will create 10 zero-byte files below + expectValid: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + baseFileName := erasure_coding.EcShardFileName(tt.collection, tempDir, int(tt.volumeId)) + + // Create .dat file if needed + if tt.createDatFile { + datFile, err := os.Create(baseFileName + ".dat") + if err != nil { + t.Fatalf("Failed to create .dat file: %v", err) + } + datFile.WriteString("dummy data") + datFile.Close() + } + + // Create EC shard files + for i := 0; i < tt.numShards; i++ { + shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i)) + if err != nil { + t.Fatalf("Failed to create shard file: %v", err) + } + shardFile.WriteString("dummy shard data") + shardFile.Close() + } + + // For zero-byte test case, create 10 empty files + if tt.volumeId == 204 { + for i := 0; i < 10; i++ { + shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i)) + if err != nil { + t.Fatalf("Failed to create empty shard file: %v", err) + } + // Don't write anything - leave as zero-byte + shardFile.Close() + } + } + + // Test validation + isValid := diskLocation.validateEcVolume(tt.collection, tt.volumeId) + if isValid != tt.expectValid { + t.Errorf("Expected validation result %v but got %v", tt.expectValid, isValid) + } + + // Cleanup + os.Remove(baseFileName + ".dat") + for i := 0; i < erasure_coding.TotalShardsCount; i++ { + os.Remove(baseFileName + erasure_coding.ToExt(i)) + } + }) + } +} + +// TestRemoveEcVolumeFiles tests the removeEcVolumeFiles function +func TestRemoveEcVolumeFiles(t *testing.T) { + tempDir, err := os.MkdirTemp("", "ec_remove_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + + minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} + diskLocation := &DiskLocation{ + Directory: tempDir, + DirectoryUuid: "test-uuid", + IdxDirectory: tempDir, + DiskType: types.HddType, + MinFreeSpace: minFreeSpace, + } + + volumeId := needle.VolumeId(300) + collection := "" + baseFileName := erasure_coding.EcShardFileName(collection, tempDir, int(volumeId)) + + // Create all EC files + for i := 0; i < erasure_coding.TotalShardsCount; i++ { + shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i)) + if err != nil { + t.Fatalf("Failed to create shard file: %v", err) + } + shardFile.WriteString("dummy shard data") + shardFile.Close() + } + + ecxFile, _ := os.Create(baseFileName + ".ecx") + ecxFile.WriteString("dummy ecx data") + ecxFile.Close() + + ecjFile, _ := os.Create(baseFileName + ".ecj") + ecjFile.WriteString("dummy ecj data") + ecjFile.Close() + + // Create .dat file that should NOT be removed + datFile, _ := os.Create(baseFileName + ".dat") + datFile.WriteString("dummy dat data") + datFile.Close() + + // Call removeEcVolumeFiles + diskLocation.removeEcVolumeFiles(collection, volumeId) + + // Verify all EC files are removed + for i := 0; i < erasure_coding.TotalShardsCount; i++ { + shardFile := baseFileName + erasure_coding.ToExt(i) + if util.FileExists(shardFile) { + t.Errorf("Shard file %d should be removed but still exists", i) + } + } + + if util.FileExists(baseFileName + ".ecx") { + t.Errorf(".ecx file should be removed but still exists") + } + + if util.FileExists(baseFileName + ".ecj") { + t.Errorf(".ecj file should be removed but still exists") + } + + // Verify .dat file is NOT removed + if !util.FileExists(baseFileName + ".dat") { + t.Errorf(".dat file should NOT be removed but was deleted") + } + + // Cleanup + os.Remove(baseFileName + ".dat") +} + +// TestEcCleanupWithSeparateIdxDirectory tests EC cleanup when idx directory is different +func TestEcCleanupWithSeparateIdxDirectory(t *testing.T) { + tempDir, err := os.MkdirTemp("", "ec_cleanup_idx_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + + idxDir := filepath.Join(tempDir, "idx") + dataDir := filepath.Join(tempDir, "data") + os.MkdirAll(idxDir, 0755) + os.MkdirAll(dataDir, 0755) + + minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} + diskLocation := &DiskLocation{ + Directory: dataDir, + DirectoryUuid: "test-uuid", + IdxDirectory: idxDir, + DiskType: types.HddType, + MinFreeSpace: minFreeSpace, + } + diskLocation.volumes = make(map[needle.VolumeId]*Volume) + diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) + + volumeId := needle.VolumeId(400) + collection := "" + + // Create shards in data directory + dataBaseFileName := erasure_coding.EcShardFileName(collection, dataDir, int(volumeId)) + for i := 0; i < 14; i++ { + shardFile, _ := os.Create(dataBaseFileName + erasure_coding.ToExt(i)) + shardFile.WriteString("dummy shard data") + shardFile.Close() + } + + // Create .dat in data directory + datFile, _ := os.Create(dataBaseFileName + ".dat") + datFile.WriteString("dummy data") + datFile.Close() + + // Create .ecx and .ecj in idx directory (but no .ecx to trigger cleanup) + // Don't create .ecx to test orphaned shards cleanup + + // Run loadAllEcShards + err = diskLocation.loadAllEcShards() + if err != nil { + t.Logf("loadAllEcShards error: %v", err) + } + + // Verify cleanup occurred + for i := 0; i < erasure_coding.TotalShardsCount; i++ { + shardFile := dataBaseFileName + erasure_coding.ToExt(i) + if util.FileExists(shardFile) { + t.Errorf("Shard file %d should be cleaned up but still exists", i) + } + } + + // .dat should still exist + if !util.FileExists(dataBaseFileName + ".dat") { + t.Errorf(".dat file should remain but was deleted") + } +}