From 2d0d429d2f2f05019aade98edd5089b2528b3a7a Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 30 Jun 2025 10:11:30 -0700 Subject: [PATCH] fix disk space calculation --- weed/storage/store_vacuum.go | 23 ++++++- weed/storage/store_vacuum_test.go | 99 +++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 weed/storage/store_vacuum_test.go diff --git a/weed/storage/store_vacuum.go b/weed/storage/store_vacuum.go index 531d859b8..3c9b5f79b 100644 --- a/weed/storage/store_vacuum.go +++ b/weed/storage/store_vacuum.go @@ -18,10 +18,27 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) { } func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error { if v := s.findVolume(vid); v != nil { - s := stats.NewDiskStatus(v.dir) - if int64(s.Free) < preallocate { - return fmt.Errorf("free space: %d bytes, not enough for %d bytes", s.Free, preallocate) + // Get current volume size for space calculation + volumeSize, indexSize, _ := v.FileStat() + + // Calculate space needed for compaction: + // 1. Space for the new compacted volume (approximately same as current volume size) + // 2. Use the larger of preallocate or estimated volume size + estimatedCompactSize := int64(volumeSize + indexSize) + spaceNeeded := preallocate + if estimatedCompactSize > preallocate { + spaceNeeded = estimatedCompactSize + } + + diskStatus := stats.NewDiskStatus(v.dir) + if int64(diskStatus.Free) < spaceNeeded { + return fmt.Errorf("insufficient free space for compaction: need %d bytes (volume: %d, index: %d, buffer: 10%%), but only %d bytes available", + spaceNeeded, volumeSize, indexSize, diskStatus.Free) } + + glog.V(1).Infof("volume %d compaction space check: volume=%d, index=%d, space_needed=%d, free_space=%d", + vid, volumeSize, indexSize, spaceNeeded, diskStatus.Free) + return v.Compact2(preallocate, compactionBytePerSecond, progressFn) } return fmt.Errorf("volume id %d is not found during compact", vid) diff --git a/weed/storage/store_vacuum_test.go b/weed/storage/store_vacuum_test.go new file mode 100644 index 000000000..aa660af3e --- /dev/null +++ b/weed/storage/store_vacuum_test.go @@ -0,0 +1,99 @@ +package storage + +import ( + "os" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" +) + +func TestCompactVolumeSpaceCheck(t *testing.T) { + // Create a temporary directory for testing + dir, err := os.MkdirTemp("", "seaweedfs_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(dir) + + // Create a disk location + location := &DiskLocation{ + Directory: dir, + IdxDirectory: dir, + volumes: make(map[needle.VolumeId]*Volume), + } + + // Create a store + store := &Store{ + Locations: []*DiskLocation{location}, + } + + // Create a volume with some data + vid := needle.VolumeId(1) + replication, _ := super_block.NewReplicaPlacementFromString("000") + volume, err := NewVolume(dir, dir, "", vid, NeedleMapInMemory, replication, nil, 0, needle.GetCurrentVersion(), 0, 0) + if err != nil { + t.Fatalf("Failed to create volume: %v", err) + } + + location.SetVolume(vid, volume) + + // Test space checking logic + t.Run("InsufficientSpace", func(t *testing.T) { + // This should fail because we're testing the improved space checking + err := store.CompactVolume(vid, 0, 0, nil) + if err == nil { + t.Error("Expected compaction to fail due to insufficient space") + } + if err != nil { + t.Logf("Expected error: %v", err) + } + }) + + // Clean up + volume.Close() +} + +func TestSpaceCalculation(t *testing.T) { + // Test the space calculation logic + testCases := []struct { + name string + volumeSize uint64 + indexSize uint64 + preallocate int64 + expectedMinimum int64 + }{ + { + name: "SmallVolume", + volumeSize: 1024 * 1024, // 1MB + indexSize: 1024, // 1KB + preallocate: 0, + expectedMinimum: int64((1024*1024 + 1024) * 110 / 100), // 110% of volume+index size + }, + { + name: "LargePreallocate", + volumeSize: 1024 * 1024, // 1MB + indexSize: 1024, // 1KB + preallocate: 10 * 1024 * 1024, // 10MB + expectedMinimum: int64(10 * 1024 * 1024 * 110 / 100), // 110% of preallocate + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + estimatedCompactSize := int64(tc.volumeSize + tc.indexSize) + spaceNeeded := tc.preallocate + if estimatedCompactSize > tc.preallocate { + spaceNeeded = estimatedCompactSize + } + // Add 10% safety buffer + spaceNeeded = spaceNeeded + (spaceNeeded / 10) + + if spaceNeeded < tc.expectedMinimum { + t.Errorf("Space calculation incorrect: got %d, expected at least %d", spaceNeeded, tc.expectedMinimum) + } + t.Logf("Volume: %d, Index: %d, Preallocate: %d -> SpaceNeeded: %d", + tc.volumeSize, tc.indexSize, tc.preallocate, spaceNeeded) + }) + } +}