chrislu
1 month ago
1 changed files with 461 additions and 0 deletions
@ -0,0 +1,461 @@ |
|||||
|
package storage |
||||
|
|
||||
|
import ( |
||||
|
"os" |
||||
|
"path/filepath" |
||||
|
"testing" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/storage/types" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
// TestIncompleteEcEncodingCleanup tests the cleanup logic for incomplete EC encoding scenarios
|
||||
|
func TestIncompleteEcEncodingCleanup(t *testing.T) { |
||||
|
// Create temporary test directory
|
||||
|
tempDir, err := os.MkdirTemp("", "ec_cleanup_test") |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create temp dir: %v", err) |
||||
|
} |
||||
|
defer os.RemoveAll(tempDir) |
||||
|
|
||||
|
tests := []struct { |
||||
|
name string |
||||
|
volumeId needle.VolumeId |
||||
|
collection string |
||||
|
createDatFile bool |
||||
|
createEcxFile bool |
||||
|
createEcjFile bool |
||||
|
numShards int |
||||
|
expectCleanup bool |
||||
|
expectLoadSuccess bool |
||||
|
}{ |
||||
|
{ |
||||
|
name: "Incomplete EC: shards without .ecx, .dat exists - should cleanup", |
||||
|
volumeId: 100, |
||||
|
collection: "", |
||||
|
createDatFile: true, |
||||
|
createEcxFile: false, |
||||
|
createEcjFile: false, |
||||
|
numShards: 14, // All shards but no .ecx
|
||||
|
expectCleanup: true, |
||||
|
expectLoadSuccess: false, |
||||
|
}, |
||||
|
{ |
||||
|
name: "Distributed EC: shards without .ecx, .dat deleted - should NOT cleanup", |
||||
|
volumeId: 101, |
||||
|
collection: "", |
||||
|
createDatFile: false, |
||||
|
createEcxFile: false, |
||||
|
createEcjFile: false, |
||||
|
numShards: 5, // Partial shards, distributed
|
||||
|
expectCleanup: false, |
||||
|
expectLoadSuccess: false, |
||||
|
}, |
||||
|
{ |
||||
|
name: "Incomplete EC: shards with .ecx but < 10 shards, .dat exists - should cleanup", |
||||
|
volumeId: 102, |
||||
|
collection: "", |
||||
|
createDatFile: true, |
||||
|
createEcxFile: true, |
||||
|
createEcjFile: false, |
||||
|
numShards: 7, // Less than DataShardsCount (10)
|
||||
|
expectCleanup: true, |
||||
|
expectLoadSuccess: false, |
||||
|
}, |
||||
|
{ |
||||
|
name: "Valid local EC: shards with .ecx, >= 10 shards, .dat exists - should load", |
||||
|
volumeId: 103, |
||||
|
collection: "", |
||||
|
createDatFile: true, |
||||
|
createEcxFile: true, |
||||
|
createEcjFile: false, |
||||
|
numShards: 14, // All shards
|
||||
|
expectCleanup: false, |
||||
|
expectLoadSuccess: true, // Would succeed if .ecx was valid
|
||||
|
}, |
||||
|
{ |
||||
|
name: "Distributed EC: shards with .ecx, .dat deleted - should load", |
||||
|
volumeId: 104, |
||||
|
collection: "", |
||||
|
createDatFile: false, |
||||
|
createEcxFile: true, |
||||
|
createEcjFile: false, |
||||
|
numShards: 10, // Enough shards
|
||||
|
expectCleanup: false, |
||||
|
expectLoadSuccess: true, // Would succeed if .ecx was valid
|
||||
|
}, |
||||
|
{ |
||||
|
name: "Incomplete EC with collection: shards without .ecx, .dat exists - should cleanup", |
||||
|
volumeId: 105, |
||||
|
collection: "test_collection", |
||||
|
createDatFile: true, |
||||
|
createEcxFile: false, |
||||
|
createEcjFile: false, |
||||
|
numShards: 14, |
||||
|
expectCleanup: true, |
||||
|
expectLoadSuccess: false, |
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
for _, tt := range tests { |
||||
|
t.Run(tt.name, func(t *testing.T) { |
||||
|
// Create DiskLocation
|
||||
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} |
||||
|
diskLocation := &DiskLocation{ |
||||
|
Directory: tempDir, |
||||
|
DirectoryUuid: "test-uuid", |
||||
|
IdxDirectory: tempDir, |
||||
|
DiskType: types.HddType, |
||||
|
MaxVolumeCount: 100, |
||||
|
OriginalMaxVolumeCount: 100, |
||||
|
MinFreeSpace: minFreeSpace, |
||||
|
} |
||||
|
diskLocation.volumes = make(map[needle.VolumeId]*Volume) |
||||
|
diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) |
||||
|
|
||||
|
// Setup test files
|
||||
|
baseFileName := erasure_coding.EcShardFileName(tt.collection, tempDir, int(tt.volumeId)) |
||||
|
|
||||
|
// Create .dat file if needed
|
||||
|
if tt.createDatFile { |
||||
|
datFile, err := os.Create(baseFileName + ".dat") |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create .dat file: %v", err) |
||||
|
} |
||||
|
datFile.WriteString("dummy data") |
||||
|
datFile.Close() |
||||
|
} |
||||
|
|
||||
|
// Create EC shard files
|
||||
|
for i := 0; i < tt.numShards; i++ { |
||||
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i)) |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create shard file: %v", err) |
||||
|
} |
||||
|
shardFile.WriteString("dummy shard data") |
||||
|
shardFile.Close() |
||||
|
} |
||||
|
|
||||
|
// Create .ecx file if needed
|
||||
|
if tt.createEcxFile { |
||||
|
ecxFile, err := os.Create(baseFileName + ".ecx") |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create .ecx file: %v", err) |
||||
|
} |
||||
|
ecxFile.WriteString("dummy ecx data") |
||||
|
ecxFile.Close() |
||||
|
} |
||||
|
|
||||
|
// Create .ecj file if needed
|
||||
|
if tt.createEcjFile { |
||||
|
ecjFile, err := os.Create(baseFileName + ".ecj") |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create .ecj file: %v", err) |
||||
|
} |
||||
|
ecjFile.WriteString("dummy ecj data") |
||||
|
ecjFile.Close() |
||||
|
} |
||||
|
|
||||
|
// Run loadAllEcShards
|
||||
|
err = diskLocation.loadAllEcShards() |
||||
|
if err != nil { |
||||
|
t.Logf("loadAllEcShards returned error (expected in some cases): %v", err) |
||||
|
} |
||||
|
|
||||
|
// Verify cleanup expectations
|
||||
|
if tt.expectCleanup { |
||||
|
// Check that files were cleaned up
|
||||
|
if util.FileExists(baseFileName + ".ecx") { |
||||
|
t.Errorf("Expected .ecx to be cleaned up but it still exists") |
||||
|
} |
||||
|
if util.FileExists(baseFileName + ".ecj") { |
||||
|
t.Errorf("Expected .ecj to be cleaned up but it still exists") |
||||
|
} |
||||
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
||||
|
shardFile := baseFileName + erasure_coding.ToExt(i) |
||||
|
if util.FileExists(shardFile) { |
||||
|
t.Errorf("Expected shard %d to be cleaned up but it still exists", i) |
||||
|
} |
||||
|
} |
||||
|
// .dat file should still exist (not cleaned up)
|
||||
|
if tt.createDatFile && !util.FileExists(baseFileName+".dat") { |
||||
|
t.Errorf("Expected .dat file to remain but it was deleted") |
||||
|
} |
||||
|
} else { |
||||
|
// Check that files were NOT cleaned up
|
||||
|
for i := 0; i < tt.numShards; i++ { |
||||
|
shardFile := baseFileName + erasure_coding.ToExt(i) |
||||
|
if !util.FileExists(shardFile) { |
||||
|
t.Errorf("Expected shard %d to remain but it was cleaned up", i) |
||||
|
} |
||||
|
} |
||||
|
if tt.createEcxFile && !util.FileExists(baseFileName+".ecx") { |
||||
|
t.Errorf("Expected .ecx to remain but it was cleaned up") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Cleanup test files for next iteration
|
||||
|
os.Remove(baseFileName + ".dat") |
||||
|
os.Remove(baseFileName + ".ecx") |
||||
|
os.Remove(baseFileName + ".ecj") |
||||
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
||||
|
os.Remove(baseFileName + erasure_coding.ToExt(i)) |
||||
|
} |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// TestValidateEcVolume tests the validateEcVolume function
|
||||
|
func TestValidateEcVolume(t *testing.T) { |
||||
|
tempDir, err := os.MkdirTemp("", "ec_validate_test") |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create temp dir: %v", err) |
||||
|
} |
||||
|
defer os.RemoveAll(tempDir) |
||||
|
|
||||
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} |
||||
|
diskLocation := &DiskLocation{ |
||||
|
Directory: tempDir, |
||||
|
DirectoryUuid: "test-uuid", |
||||
|
IdxDirectory: tempDir, |
||||
|
DiskType: types.HddType, |
||||
|
MinFreeSpace: minFreeSpace, |
||||
|
} |
||||
|
|
||||
|
tests := []struct { |
||||
|
name string |
||||
|
volumeId needle.VolumeId |
||||
|
collection string |
||||
|
createDatFile bool |
||||
|
numShards int |
||||
|
expectValid bool |
||||
|
}{ |
||||
|
{ |
||||
|
name: "Valid: .dat exists with 10+ shards", |
||||
|
volumeId: 200, |
||||
|
collection: "", |
||||
|
createDatFile: true, |
||||
|
numShards: 10, |
||||
|
expectValid: true, |
||||
|
}, |
||||
|
{ |
||||
|
name: "Invalid: .dat exists with < 10 shards", |
||||
|
volumeId: 201, |
||||
|
collection: "", |
||||
|
createDatFile: true, |
||||
|
numShards: 9, |
||||
|
expectValid: false, |
||||
|
}, |
||||
|
{ |
||||
|
name: "Valid: .dat deleted (distributed EC) with any shards", |
||||
|
volumeId: 202, |
||||
|
collection: "", |
||||
|
createDatFile: false, |
||||
|
numShards: 5, |
||||
|
expectValid: true, |
||||
|
}, |
||||
|
{ |
||||
|
name: "Valid: .dat deleted (distributed EC) with no shards", |
||||
|
volumeId: 203, |
||||
|
collection: "", |
||||
|
createDatFile: false, |
||||
|
numShards: 0, |
||||
|
expectValid: true, |
||||
|
}, |
||||
|
{ |
||||
|
name: "Invalid: zero-byte shard files should not count", |
||||
|
volumeId: 204, |
||||
|
collection: "", |
||||
|
createDatFile: true, |
||||
|
numShards: 0, // Will create 10 zero-byte files below
|
||||
|
expectValid: false, |
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
for _, tt := range tests { |
||||
|
t.Run(tt.name, func(t *testing.T) { |
||||
|
baseFileName := erasure_coding.EcShardFileName(tt.collection, tempDir, int(tt.volumeId)) |
||||
|
|
||||
|
// Create .dat file if needed
|
||||
|
if tt.createDatFile { |
||||
|
datFile, err := os.Create(baseFileName + ".dat") |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create .dat file: %v", err) |
||||
|
} |
||||
|
datFile.WriteString("dummy data") |
||||
|
datFile.Close() |
||||
|
} |
||||
|
|
||||
|
// Create EC shard files
|
||||
|
for i := 0; i < tt.numShards; i++ { |
||||
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i)) |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create shard file: %v", err) |
||||
|
} |
||||
|
shardFile.WriteString("dummy shard data") |
||||
|
shardFile.Close() |
||||
|
} |
||||
|
|
||||
|
// For zero-byte test case, create 10 empty files
|
||||
|
if tt.volumeId == 204 { |
||||
|
for i := 0; i < 10; i++ { |
||||
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i)) |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create empty shard file: %v", err) |
||||
|
} |
||||
|
// Don't write anything - leave as zero-byte
|
||||
|
shardFile.Close() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Test validation
|
||||
|
isValid := diskLocation.validateEcVolume(tt.collection, tt.volumeId) |
||||
|
if isValid != tt.expectValid { |
||||
|
t.Errorf("Expected validation result %v but got %v", tt.expectValid, isValid) |
||||
|
} |
||||
|
|
||||
|
// Cleanup
|
||||
|
os.Remove(baseFileName + ".dat") |
||||
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
||||
|
os.Remove(baseFileName + erasure_coding.ToExt(i)) |
||||
|
} |
||||
|
}) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// TestRemoveEcVolumeFiles tests the removeEcVolumeFiles function
|
||||
|
func TestRemoveEcVolumeFiles(t *testing.T) { |
||||
|
tempDir, err := os.MkdirTemp("", "ec_remove_test") |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create temp dir: %v", err) |
||||
|
} |
||||
|
defer os.RemoveAll(tempDir) |
||||
|
|
||||
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} |
||||
|
diskLocation := &DiskLocation{ |
||||
|
Directory: tempDir, |
||||
|
DirectoryUuid: "test-uuid", |
||||
|
IdxDirectory: tempDir, |
||||
|
DiskType: types.HddType, |
||||
|
MinFreeSpace: minFreeSpace, |
||||
|
} |
||||
|
|
||||
|
volumeId := needle.VolumeId(300) |
||||
|
collection := "" |
||||
|
baseFileName := erasure_coding.EcShardFileName(collection, tempDir, int(volumeId)) |
||||
|
|
||||
|
// Create all EC files
|
||||
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
||||
|
shardFile, err := os.Create(baseFileName + erasure_coding.ToExt(i)) |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create shard file: %v", err) |
||||
|
} |
||||
|
shardFile.WriteString("dummy shard data") |
||||
|
shardFile.Close() |
||||
|
} |
||||
|
|
||||
|
ecxFile, _ := os.Create(baseFileName + ".ecx") |
||||
|
ecxFile.WriteString("dummy ecx data") |
||||
|
ecxFile.Close() |
||||
|
|
||||
|
ecjFile, _ := os.Create(baseFileName + ".ecj") |
||||
|
ecjFile.WriteString("dummy ecj data") |
||||
|
ecjFile.Close() |
||||
|
|
||||
|
// Create .dat file that should NOT be removed
|
||||
|
datFile, _ := os.Create(baseFileName + ".dat") |
||||
|
datFile.WriteString("dummy dat data") |
||||
|
datFile.Close() |
||||
|
|
||||
|
// Call removeEcVolumeFiles
|
||||
|
diskLocation.removeEcVolumeFiles(collection, volumeId) |
||||
|
|
||||
|
// Verify all EC files are removed
|
||||
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
||||
|
shardFile := baseFileName + erasure_coding.ToExt(i) |
||||
|
if util.FileExists(shardFile) { |
||||
|
t.Errorf("Shard file %d should be removed but still exists", i) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if util.FileExists(baseFileName + ".ecx") { |
||||
|
t.Errorf(".ecx file should be removed but still exists") |
||||
|
} |
||||
|
|
||||
|
if util.FileExists(baseFileName + ".ecj") { |
||||
|
t.Errorf(".ecj file should be removed but still exists") |
||||
|
} |
||||
|
|
||||
|
// Verify .dat file is NOT removed
|
||||
|
if !util.FileExists(baseFileName + ".dat") { |
||||
|
t.Errorf(".dat file should NOT be removed but was deleted") |
||||
|
} |
||||
|
|
||||
|
// Cleanup
|
||||
|
os.Remove(baseFileName + ".dat") |
||||
|
} |
||||
|
|
||||
|
// TestEcCleanupWithSeparateIdxDirectory tests EC cleanup when idx directory is different
|
||||
|
func TestEcCleanupWithSeparateIdxDirectory(t *testing.T) { |
||||
|
tempDir, err := os.MkdirTemp("", "ec_cleanup_idx_test") |
||||
|
if err != nil { |
||||
|
t.Fatalf("Failed to create temp dir: %v", err) |
||||
|
} |
||||
|
defer os.RemoveAll(tempDir) |
||||
|
|
||||
|
idxDir := filepath.Join(tempDir, "idx") |
||||
|
dataDir := filepath.Join(tempDir, "data") |
||||
|
os.MkdirAll(idxDir, 0755) |
||||
|
os.MkdirAll(dataDir, 0755) |
||||
|
|
||||
|
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} |
||||
|
diskLocation := &DiskLocation{ |
||||
|
Directory: dataDir, |
||||
|
DirectoryUuid: "test-uuid", |
||||
|
IdxDirectory: idxDir, |
||||
|
DiskType: types.HddType, |
||||
|
MinFreeSpace: minFreeSpace, |
||||
|
} |
||||
|
diskLocation.volumes = make(map[needle.VolumeId]*Volume) |
||||
|
diskLocation.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) |
||||
|
|
||||
|
volumeId := needle.VolumeId(400) |
||||
|
collection := "" |
||||
|
|
||||
|
// Create shards in data directory
|
||||
|
dataBaseFileName := erasure_coding.EcShardFileName(collection, dataDir, int(volumeId)) |
||||
|
for i := 0; i < 14; i++ { |
||||
|
shardFile, _ := os.Create(dataBaseFileName + erasure_coding.ToExt(i)) |
||||
|
shardFile.WriteString("dummy shard data") |
||||
|
shardFile.Close() |
||||
|
} |
||||
|
|
||||
|
// Create .dat in data directory
|
||||
|
datFile, _ := os.Create(dataBaseFileName + ".dat") |
||||
|
datFile.WriteString("dummy data") |
||||
|
datFile.Close() |
||||
|
|
||||
|
// Create .ecx and .ecj in idx directory (but no .ecx to trigger cleanup)
|
||||
|
// Don't create .ecx to test orphaned shards cleanup
|
||||
|
|
||||
|
// Run loadAllEcShards
|
||||
|
err = diskLocation.loadAllEcShards() |
||||
|
if err != nil { |
||||
|
t.Logf("loadAllEcShards error: %v", err) |
||||
|
} |
||||
|
|
||||
|
// Verify cleanup occurred
|
||||
|
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
||||
|
shardFile := dataBaseFileName + erasure_coding.ToExt(i) |
||||
|
if util.FileExists(shardFile) { |
||||
|
t.Errorf("Shard file %d should be cleaned up but still exists", i) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// .dat should still exist
|
||||
|
if !util.FileExists(dataBaseFileName + ".dat") { |
||||
|
t.Errorf(".dat file should remain but was deleted") |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue