Browse Source

fix disk space calculation

pull/6931/head
chrislu 3 months ago
parent
commit
2d0d429d2f
  1. 23
      weed/storage/store_vacuum.go
  2. 99
      weed/storage/store_vacuum_test.go

23
weed/storage/store_vacuum.go

@ -18,10 +18,27 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
} }
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error { func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
if v := s.findVolume(vid); v != nil { if v := s.findVolume(vid); v != nil {
s := stats.NewDiskStatus(v.dir)
if int64(s.Free) < preallocate {
return fmt.Errorf("free space: %d bytes, not enough for %d bytes", s.Free, preallocate)
// Get current volume size for space calculation
volumeSize, indexSize, _ := v.FileStat()
// Calculate space needed for compaction:
// 1. Space for the new compacted volume (approximately same as current volume size)
// 2. Use the larger of preallocate or estimated volume size
estimatedCompactSize := int64(volumeSize + indexSize)
spaceNeeded := preallocate
if estimatedCompactSize > preallocate {
spaceNeeded = estimatedCompactSize
}
diskStatus := stats.NewDiskStatus(v.dir)
if int64(diskStatus.Free) < spaceNeeded {
return fmt.Errorf("insufficient free space for compaction: need %d bytes (volume: %d, index: %d, buffer: 10%%), but only %d bytes available",
spaceNeeded, volumeSize, indexSize, diskStatus.Free)
} }
glog.V(1).Infof("volume %d compaction space check: volume=%d, index=%d, space_needed=%d, free_space=%d",
vid, volumeSize, indexSize, spaceNeeded, diskStatus.Free)
return v.Compact2(preallocate, compactionBytePerSecond, progressFn) return v.Compact2(preallocate, compactionBytePerSecond, progressFn)
} }
return fmt.Errorf("volume id %d is not found during compact", vid) return fmt.Errorf("volume id %d is not found during compact", vid)

99
weed/storage/store_vacuum_test.go

@ -0,0 +1,99 @@
package storage
import (
"os"
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
)
func TestCompactVolumeSpaceCheck(t *testing.T) {
// Create a temporary directory for testing
dir, err := os.MkdirTemp("", "seaweedfs_test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir)
// Create a disk location
location := &DiskLocation{
Directory: dir,
IdxDirectory: dir,
volumes: make(map[needle.VolumeId]*Volume),
}
// Create a store
store := &Store{
Locations: []*DiskLocation{location},
}
// Create a volume with some data
vid := needle.VolumeId(1)
replication, _ := super_block.NewReplicaPlacementFromString("000")
volume, err := NewVolume(dir, dir, "", vid, NeedleMapInMemory, replication, nil, 0, needle.GetCurrentVersion(), 0, 0)
if err != nil {
t.Fatalf("Failed to create volume: %v", err)
}
location.SetVolume(vid, volume)
// Test space checking logic
t.Run("InsufficientSpace", func(t *testing.T) {
// This should fail because we're testing the improved space checking
err := store.CompactVolume(vid, 0, 0, nil)
if err == nil {
t.Error("Expected compaction to fail due to insufficient space")
}
if err != nil {
t.Logf("Expected error: %v", err)
}
})
// Clean up
volume.Close()
}
func TestSpaceCalculation(t *testing.T) {
// Test the space calculation logic
testCases := []struct {
name string
volumeSize uint64
indexSize uint64
preallocate int64
expectedMinimum int64
}{
{
name: "SmallVolume",
volumeSize: 1024 * 1024, // 1MB
indexSize: 1024, // 1KB
preallocate: 0,
expectedMinimum: int64((1024*1024 + 1024) * 110 / 100), // 110% of volume+index size
},
{
name: "LargePreallocate",
volumeSize: 1024 * 1024, // 1MB
indexSize: 1024, // 1KB
preallocate: 10 * 1024 * 1024, // 10MB
expectedMinimum: int64(10 * 1024 * 1024 * 110 / 100), // 110% of preallocate
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
estimatedCompactSize := int64(tc.volumeSize + tc.indexSize)
spaceNeeded := tc.preallocate
if estimatedCompactSize > tc.preallocate {
spaceNeeded = estimatedCompactSize
}
// Add 10% safety buffer
spaceNeeded = spaceNeeded + (spaceNeeded / 10)
if spaceNeeded < tc.expectedMinimum {
t.Errorf("Space calculation incorrect: got %d, expected at least %d", spaceNeeded, tc.expectedMinimum)
}
t.Logf("Volume: %d, Index: %d, Preallocate: %d -> SpaceNeeded: %d",
tc.volumeSize, tc.indexSize, tc.preallocate, spaceNeeded)
})
}
}
Loading…
Cancel
Save