Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fix min volume age

add-ec-vacuum
chrislu 5 months ago
parent
424bb7fe11
commit
9df006b49d
5 changed files with 102 additions and 55 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      weed/admin/maintenance/maintenance_queue.go
  2. 74
      weed/admin/view/app/task_config_schema.templ
  3. 32
      weed/admin/view/app/task_config_schema_templ.go
  4. 26
      weed/worker/tasks/ec_vacuum/config.go
  5. 23
      weed/worker/tasks/ec_vacuum/detection.go

2
weed/admin/maintenance/maintenance_queue.go
View File

@ -272,7 +272,7 @@ func (mq *MaintenanceQueue) GetNextTask(workerID string, capabilities []Maintena
// If no task found, return nil
if selectedTask == nil {
glog.V(2).Infof("No suitable tasks available for worker %s (checked %d pending tasks)", workerID, len(mq.pendingTasks))
glog.V(3).Infof("No suitable tasks available for worker %s (checked %d pending tasks)", workerID, len(mq.pendingTasks))
return nil
}

74
weed/admin/view/app/task_config_schema.templ
View File

@ -6,9 +6,9 @@ import (
"fmt"
"reflect"
"strings"
"github.com/seaweedfs/seaweedfs/weed/admin/config"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/admin/config"
"github.com/seaweedfs/seaweedfs/weed/admin/view/components"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
@ -207,7 +207,7 @@ templ TaskConfigField(field *config.Field, config interface{}) {
class="form-control"
id={ field.JSONName + "_value" }
name={ field.JSONName + "_value" }
value={ fmt.Sprintf("%.0f", components.ConvertInt32SecondsToDisplayValue(getTaskConfigInt32Field(config, field.JSONName))) }
value={ fmt.Sprintf("%.0f", components.ConvertInt32SecondsToDisplayValue(getTaskConfigInt32FieldWithDefault(config, field))) }
step="1"
min="1"
if field.Required {
@ -223,30 +223,30 @@ templ TaskConfigField(field *config.Field, config interface{}) {
required
}
>
<option
value="minutes"
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "minutes" {
selected
}
>
Minutes
</option>
<option
value="hours"
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "hours" {
selected
}
>
Hours
</option>
<option
value="days"
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "days" {
selected
}
>
Days
</option>
<option
value="minutes"
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "minutes" {
selected
}
>
Minutes
</option>
<option
value="hours"
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "hours" {
selected
}
>
Hours
</option>
<option
value="days"
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "days" {
selected
}
>
Days
</option>
</select>
</div>
if field.Description != "" {
@ -388,6 +388,26 @@ func getTaskConfigInt32Field(config interface{}, fieldName string) int32 {
}
}
func getTaskConfigInt32FieldWithDefault(config interface{}, field *config.Field) int32 {
value := getTaskConfigInt32Field(config, field.JSONName)
// If no value is stored (value is 0), use the schema default
if value == 0 && field.DefaultValue != nil {
switch defaultVal := field.DefaultValue.(type) {
case int:
return int32(defaultVal)
case int32:
return defaultVal
case int64:
return int32(defaultVal)
case float64:
return int32(defaultVal)
}
}
return value
}
func getTaskConfigFloatField(config interface{}, fieldName string) float64 {
if value := getTaskFieldValue(config, fieldName); value != nil {
switch v := value.(type) {
@ -429,7 +449,7 @@ func getTaskConfigStringField(config interface{}, fieldName string) string {
}
func getTaskNumberStep(field *config.Field) string {
if field.Type == config.FieldTypeFloat {
if field.Type == "float" {
return "0.01"
}
return "1"

32
weed/admin/view/app/task_config_schema_templ.go
View File

@ -281,9 +281,9 @@ func TaskConfigField(field *config.Field, config interface{}) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var14 string
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f", components.ConvertInt32SecondsToDisplayValue(getTaskConfigInt32Field(config, field.JSONName))))
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f", components.ConvertInt32SecondsToDisplayValue(getTaskConfigInt32FieldWithDefault(config, field))))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/task_config_schema.templ`, Line: 210, Col: 142}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/task_config_schema.templ`, Line: 210, Col: 144}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
if templ_7745c5c3_Err != nil {
@ -339,7 +339,7 @@ func TaskConfigField(field *config.Field, config interface{}) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "minutes" {
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "minutes" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, " selected")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@ -349,7 +349,7 @@ func TaskConfigField(field *config.Field, config interface{}) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "hours" {
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "hours" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, " selected")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@ -359,7 +359,7 @@ func TaskConfigField(field *config.Field, config interface{}) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if components.GetInt32DisplayUnit(getTaskConfigInt32Field(config, field.JSONName)) == "days" {
if components.GetInt32DisplayUnit(getTaskConfigInt32FieldWithDefault(config, field)) == "days" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, " selected")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@ -849,6 +849,26 @@ func getTaskConfigInt32Field(config interface{}, fieldName string) int32 {
}
}
func getTaskConfigInt32FieldWithDefault(config interface{}, field *config.Field) int32 {
value := getTaskConfigInt32Field(config, field.JSONName)
// If no value is stored (value is 0), use the schema default
if value == 0 && field.DefaultValue != nil {
switch defaultVal := field.DefaultValue.(type) {
case int:
return int32(defaultVal)
case int32:
return defaultVal
case int64:
return int32(defaultVal)
case float64:
return int32(defaultVal)
}
}
return value
}
func getTaskConfigFloatField(config interface{}, fieldName string) float64 {
if value := getTaskFieldValue(config, fieldName); value != nil {
switch v := value.(type) {
@ -890,7 +910,7 @@ func getTaskConfigStringField(config interface{}, fieldName string) string {
}
func getTaskNumberStep(field *config.Field) string {
if field.Type == config.FieldTypeFloat {
if field.Type == "float" {
return "0.01"
}
return "1"

26
weed/worker/tasks/ec_vacuum/config.go
View File

@ -12,10 +12,10 @@ import (
// Config extends BaseConfig with EC vacuum specific settings
type Config struct {
base.BaseConfig
DeletionThreshold float64 `json:"deletion_threshold"` // Minimum deletion ratio to trigger vacuum
MinVolumeAgeHours int `json:"min_volume_age_hours"` // Minimum age before considering vacuum
CollectionFilter string `json:"collection_filter"` // Filter by collection
MinSizeMB int `json:"min_size_mb"` // Minimum original volume size
DeletionThreshold float64 `json:"deletion_threshold"` // Minimum deletion ratio to trigger vacuum
MinVolumeAgeSeconds int `json:"min_volume_age_seconds"` // Minimum age before considering vacuum (in seconds)
CollectionFilter string `json:"collection_filter"` // Filter by collection
MinSizeMB int `json:"min_size_mb"` // Minimum original volume size
}
// NewDefaultConfig creates a new default EC vacuum configuration
@ -26,10 +26,10 @@ func NewDefaultConfig() *Config {
ScanIntervalSeconds: 24 * 60 * 60, // 24 hours
MaxConcurrent: 1,
},
DeletionThreshold: 0.3, // 30% deletions trigger vacuum
MinVolumeAgeHours: 72, // 3 days minimum age
CollectionFilter: "", // No filter by default
MinSizeMB: 100, // 100MB minimum size
DeletionThreshold: 0.3, // 30% deletions trigger vacuum
MinVolumeAgeSeconds: 72 * 60 * 60, // 3 days minimum age (72 hours in seconds)
CollectionFilter: "", // No filter by default
MinSizeMB: 100, // 100MB minimum size
}
}
@ -98,12 +98,12 @@ func GetConfigSpec() base.ConfigSpec {
CSSClasses: "form-control",
},
{
Name: "min_volume_age_hours",
JSONName: "min_volume_age_hours",
Name: "min_volume_age_seconds",
JSONName: "min_volume_age_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 72,
MinValue: 24,
MaxValue: 30 * 24, // 30 days
DefaultValue: 72 * 60 * 60, // 72 hours in seconds
MinValue: 24 * 60 * 60, // 24 hours in seconds
MaxValue: 30 * 24 * 60 * 60, // 30 days in seconds
Required: true,
DisplayName: "Minimum Volume Age",
Description: "Minimum age before considering EC volume for vacuum",

23
weed/worker/tasks/ec_vacuum/detection.go
View File

@ -63,9 +63,9 @@ func Detection(metrics []*wtypes.VolumeHealthMetrics, info *wtypes.ClusterInfo,
Server: ecInfo.PrimaryNode,
Collection: ecInfo.Collection,
Priority: wtypes.TaskPriorityLow, // EC vacuum is not urgent
Reason: fmt.Sprintf("EC volume needs vacuum: deletion_ratio=%.1f%% (>%.1f%%), age=%.1fh (>%dh), size=%.1fMB (>%dMB)",
Reason: fmt.Sprintf("EC volume needs vacuum: deletion_ratio=%.1f%% (>%.1f%%), age=%.1fh (>%.1fh), size=%.1fMB (>%dMB)",
deletionRatio*100, ecVacuumConfig.DeletionThreshold*100,
ecInfo.Age.Hours(), ecVacuumConfig.MinVolumeAgeHours,
ecInfo.Age.Hours(), (time.Duration(ecVacuumConfig.MinVolumeAgeSeconds) * time.Second).Hours(),
float64(ecInfo.Size)/(1024*1024), ecVacuumConfig.MinSizeMB),
ScheduleAt: now,
}
@ -98,12 +98,18 @@ func Detection(metrics []*wtypes.VolumeHealthMetrics, info *wtypes.ClusterInfo,
deletionRatio := calculateDeletionRatio(ecInfo)
sizeMB := float64(ecInfo.Size) / (1024 * 1024)
deletedMB := deletionRatio * sizeMB
ageRequired := time.Duration(ecVacuumConfig.MinVolumeAgeHours) * time.Hour
ageRequired := time.Duration(ecVacuumConfig.MinVolumeAgeSeconds) * time.Second
glog.Infof("EC VACUUM: Volume %d: deleted=%.1fMB, ratio=%.1f%% (need ≥%.1f%%), age=%s (need ≥%s), size=%.1fMB (need ≥%dMB)",
// Check shard availability
totalShards := 0
for _, shardBits := range ecInfo.ShardNodes {
totalShards += shardBits.ShardIdCount()
}
glog.Infof("EC VACUUM: Volume %d: deleted=%.1fMB, ratio=%.1f%% (need ≥%.1f%%), age=%s (need ≥%s), size=%.1fMB (need ≥%dMB), shards=%d (need ≥%d)",
volumeID, deletedMB, deletionRatio*100, ecVacuumConfig.DeletionThreshold*100,
ecInfo.Age.Truncate(time.Minute), ageRequired.Truncate(time.Minute),
sizeMB, ecVacuumConfig.MinSizeMB)
sizeMB, ecVacuumConfig.MinSizeMB, totalShards, erasure_coding.DataShardsCount)
count++
}
}
@ -173,9 +179,10 @@ func collectEcVolumeInfo(metrics []*wtypes.VolumeHealthMetrics) map[uint32]*EcVo
// shouldVacuumEcVolume determines if an EC volume should be considered for vacuum
func shouldVacuumEcVolume(ecInfo *EcVolumeInfo, config *Config, now time.Time) bool {
// Check minimum age
if ecInfo.Age < time.Duration(config.MinVolumeAgeHours)*time.Hour {
glog.V(3).Infof("EC volume %d too young: age=%.1fh < %dh",
ecInfo.VolumeID, ecInfo.Age.Hours(), config.MinVolumeAgeHours)
minAge := time.Duration(config.MinVolumeAgeSeconds) * time.Second
if ecInfo.Age < minAge {
glog.V(3).Infof("EC volume %d too young: age=%.1fh < %.1fh",
ecInfo.VolumeID, ecInfo.Age.Hours(), minAge.Hours())
return false
}

Write Preview
Loading…
Cancel
Save