From 965ec4f51bb12606e5896fff00d3dd6dc38ca1d9 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 26 Jul 2025 13:28:52 -0700 Subject: [PATCH] garbage percentage threshold --- .../create_vacuum_test_data.go | 5 +- weed/admin/task/master_sync.go | 24 ++++++-- weed/worker/tasks/vacuum/ui.go | 60 +++++++------------ 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/docker/admin_integration/create_vacuum_test_data.go b/docker/admin_integration/create_vacuum_test_data.go index d1ed7ff2b..46acdd4cd 100644 --- a/docker/admin_integration/create_vacuum_test_data.go +++ b/docker/admin_integration/create_vacuum_test_data.go @@ -245,7 +245,7 @@ func printTestingInstructions() { fmt.Println("1. Configure Vacuum for Testing:") fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum") fmt.Println(" Set:") - fmt.Printf(" - Garbage Threshold: 0.20 (20%% - lower than default)\n") + fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n") fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n") fmt.Printf(" - Min Volume Age: [0] [Minutes] (no age requirement)\n") fmt.Printf(" - Max Concurrent: 2\n") @@ -258,7 +258,8 @@ func printTestingInstructions() { fmt.Println() fmt.Println("3. Manual Vacuum (Optional):") - fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.01'") + fmt.Println(" curl -X POST 'http://localhost:9333/vol/vacuum?garbageThreshold=0.20'") + fmt.Println(" (Note: Master API still uses 0.0-1.0 decimal format)") fmt.Println() fmt.Println("4. Check Logs:") diff --git a/weed/admin/task/master_sync.go b/weed/admin/task/master_sync.go index 1ea8b6f5a..9abe33131 100644 --- a/weed/admin/task/master_sync.go +++ b/weed/admin/task/master_sync.go @@ -3,11 +3,13 @@ package task import ( "context" "fmt" + "strconv" "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/wdclient" + "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum" "github.com/seaweedfs/seaweedfs/weed/worker/types" ) @@ -311,11 +313,17 @@ func (ms *MasterSynchronizer) checkVacuumCandidate(volumeID uint32, state *Volum return nil } + // Get the current garbage threshold from the vacuum detector + vacuumDetector, _ := vacuum.GetSharedInstances() + var vacuumThresholdPercent float64 = 0.3 // Default fallback + if vacuumDetector != nil { + vacuumThresholdPercent = vacuumDetector.GetGarbageThreshold() + } + // Vacuum criteria: - // 1. Significant deleted bytes (> 30% of volume size or > 1GB) + // 1. Significant deleted bytes (> configured threshold or > 1GB) // 2. Not currently being written to heavily - const vacuumThresholdPercent = 0.3 const vacuumMinBytes = 1024 * 1024 * 1024 // 1GB deletedRatio := float64(volume.DeletedByteCount) / float64(volume.Size) @@ -331,8 +339,8 @@ func (ms *MasterSynchronizer) checkVacuumCandidate(volumeID uint32, state *Volum Server: volume.Server, TaskType: "vacuum", Priority: types.TaskPriorityNormal, - Reason: fmt.Sprintf("Deleted bytes %d (%.1f%%) exceed vacuum threshold", - volume.DeletedByteCount, deletedRatio*100), + Reason: fmt.Sprintf("Deleted bytes %d (%.1f%%) exceed vacuum threshold (%.1f%%)", + volume.DeletedByteCount, deletedRatio*100, vacuumThresholdPercent*100), VolumeInfo: volume, } } @@ -408,7 +416,13 @@ func (ms *MasterSynchronizer) createTaskFromCandidate(candidate *VolumeMaintenan task.Parameters["replication"] = "001" // Default replication for EC task.Parameters["collection"] = candidate.VolumeInfo.Collection case "vacuum": - task.Parameters["garbage_threshold"] = "0.3" // 30% threshold + // Get the current garbage threshold from the vacuum detector + vacuumDetector, _ := vacuum.GetSharedInstances() + var garbageThreshold float64 = 0.3 // Default fallback + if vacuumDetector != nil { + garbageThreshold = vacuumDetector.GetGarbageThreshold() + } + task.Parameters["garbage_threshold"] = strconv.FormatFloat(garbageThreshold, 'f', -1, 64) case "ec_rebuild": // Add info about which shards need rebuilding } diff --git a/weed/worker/tasks/vacuum/ui.go b/weed/worker/tasks/vacuum/ui.go index 7d1eebb6d..6e58f0914 100644 --- a/weed/worker/tasks/vacuum/ui.go +++ b/weed/worker/tasks/vacuum/ui.go @@ -95,25 +95,25 @@ func (ui *UIProvider) RenderConfigForm(currentConfig interface{}) (template.HTML form.AddNumberField( "garbage_threshold", - "Garbage Threshold (%)", - "Trigger vacuum when garbage ratio exceeds this percentage (0.0-1.0)", - config.GarbageThreshold, + "Garbage Percentage Threshold", + "Trigger vacuum when garbage ratio exceeds this percentage (0-100)", + config.GarbageThreshold*100, // Convert 0.0-1.0 to 0-100 for display true, ) - form.AddIntervalField( + form.AddDurationField( "scan_interval", "Scan Interval", "How often to scan for volumes needing vacuum", - config.ScanIntervalSeconds, + secondsToDuration(config.ScanIntervalSeconds), true, ) - form.AddIntervalField( + form.AddDurationField( "min_volume_age", "Minimum Volume Age", "Only vacuum volumes older than this duration", - config.MinVolumeAgeSeconds, + secondsToDuration(config.MinVolumeAgeSeconds), true, ) @@ -157,7 +157,7 @@ function resetForm() { if (confirm('Reset all vacuum settings to defaults?')) { // Reset to default values document.querySelector('input[name="enabled"]').checked = true; - document.querySelector('input[name="garbage_threshold"]').value = '0.3'; + document.querySelector('input[name="garbage_threshold"]').value = '30'; document.querySelector('input[name="scan_interval"]').value = '30m'; document.querySelector('input[name="min_volume_age"]').value = '1h'; document.querySelector('input[name="max_concurrent"]').value = '2'; @@ -177,47 +177,33 @@ func (ui *UIProvider) ParseConfigForm(formData map[string][]string) (interface{} // Parse enabled checkbox config.Enabled = len(formData["enabled"]) > 0 && formData["enabled"][0] == "on" - // Parse garbage threshold + // Parse garbage threshold (convert from 0-100 to 0.0-1.0) if thresholdStr := formData["garbage_threshold"]; len(thresholdStr) > 0 { if threshold, err := strconv.ParseFloat(thresholdStr[0], 64); err != nil { - return nil, fmt.Errorf("invalid garbage threshold: %w", err) - } else if threshold < 0 || threshold > 1 { - return nil, fmt.Errorf("garbage threshold must be between 0.0 and 1.0") + return nil, fmt.Errorf("invalid garbage percentage threshold: %w", err) + } else if threshold < 0 || threshold > 100 { + return nil, fmt.Errorf("garbage percentage threshold must be between 0 and 100") } else { - config.GarbageThreshold = threshold + config.GarbageThreshold = threshold / 100.0 // Convert percentage to decimal } } // Parse scan interval - if values, ok := formData["scan_interval_value"]; ok && len(values) > 0 { - value, err := strconv.Atoi(values[0]) - if err != nil { - return nil, fmt.Errorf("invalid scan interval value: %w", err) - } - - unit := "minute" // default - if units, ok := formData["scan_interval_unit"]; ok && len(units) > 0 { - unit = units[0] + if intervalStr := formData["scan_interval"]; len(intervalStr) > 0 { + if interval, err := time.ParseDuration(intervalStr[0]); err != nil { + return nil, fmt.Errorf("invalid scan interval: %w", err) + } else { + config.ScanIntervalSeconds = durationToSeconds(interval) } - - // Convert to seconds - config.ScanIntervalSeconds = types.IntervalValueUnitToSeconds(value, unit) } // Parse min volume age - if values, ok := formData["min_volume_age_value"]; ok && len(values) > 0 { - value, err := strconv.Atoi(values[0]) - if err != nil { - return nil, fmt.Errorf("invalid min volume age value: %w", err) - } - - unit := "minute" // default - if units, ok := formData["min_volume_age_unit"]; ok && len(units) > 0 { - unit = units[0] + if ageStr := formData["min_volume_age"]; len(ageStr) > 0 { + if age, err := time.ParseDuration(ageStr[0]); err != nil { + return nil, fmt.Errorf("invalid min volume age: %w", err) + } else { + config.MinVolumeAgeSeconds = durationToSeconds(age) } - - // Convert to seconds - config.MinVolumeAgeSeconds = types.IntervalValueUnitToSeconds(value, unit) } // Parse max concurrent