Browse Source

remove v2 reference

worker-execute-ec-tasks
chrislu 4 months ago
parent
commit
74146870c5
  1. 6
      weed/admin/handlers/maintenance_handlers.go
  2. 126
      weed/worker/tasks/balance/balance.go
  3. 2
      weed/worker/tasks/balance/balance_register.go
  4. 140
      weed/worker/tasks/erasure_coding/ec.go
  5. 2
      weed/worker/tasks/erasure_coding/ec_register.go
  6. 80
      weed/worker/tasks/vacuum/vacuum.go
  7. 2
      weed/worker/tasks/vacuum/vacuum_register.go

6
weed/admin/handlers/maintenance_handlers.go

@ -192,11 +192,11 @@ func (h *MaintenanceHandlers) UpdateTaskConfig(c *gin.Context) {
var config interface{}
switch taskType {
case types.TaskTypeVacuum:
config = &vacuum.VacuumConfigV2{}
config = &vacuum.VacuumConfig{}
case types.TaskTypeBalance:
config = &balance.BalanceConfigV2{}
config = &balance.BalanceConfig{}
case types.TaskTypeErasureCoding:
config = &erasure_coding.ErasureCodingConfigV2{}
config = &erasure_coding.ErasureCodingConfig{}
default:
c.JSON(http.StatusBadRequest, gin.H{"error": "Unsupported task type: " + taskTypeName})
return

126
weed/worker/tasks/balance/balance.go

@ -83,8 +83,8 @@ func (t *Task) EstimateTime(params types.TaskParams) time.Duration {
return baseTime
}
// BalanceConfigV2 extends BaseConfig with balance-specific settings
type BalanceConfigV2 struct {
// BalanceConfig extends BaseConfig with balance-specific settings
type BalanceConfig struct {
base.BaseConfig
ImbalanceThreshold float64 `json:"imbalance_threshold"`
MinServerCount int `json:"min_server_count"`
@ -96,7 +96,7 @@ func balanceDetection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.C
return nil, nil
}
balanceConfig := config.(*BalanceConfigV2)
balanceConfig := config.(*BalanceConfig)
// Skip if cluster is too small
minVolumeCount := 10
@ -166,7 +166,7 @@ func balanceDetection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.C
// balanceScheduling implements the scheduling logic for balance tasks
func balanceScheduling(task *types.Task, runningTasks []*types.Task, availableWorkers []*types.Worker, config base.TaskConfig) bool {
balanceConfig := config.(*BalanceConfigV2)
balanceConfig := config.(*BalanceConfig)
// Count running balance tasks
runningBalanceCount := 0
@ -195,19 +195,33 @@ func balanceScheduling(task *types.Task, runningTasks []*types.Task, availableWo
return availableWorkerCount > 0
}
// createBalanceTask creates a balance task instance
// createBalanceTask creates a new balance task instance
func createBalanceTask(params types.TaskParams) (types.TaskInterface, error) {
// Validate parameters
if params.VolumeID == 0 {
return nil, fmt.Errorf("volume_id is required")
// Extract configuration from params
var config *BalanceConfig
if configData, ok := params.Parameters["config"]; ok {
if configMap, ok := configData.(map[string]interface{}); ok {
config = &BalanceConfig{}
if err := config.FromMap(configMap); err != nil {
return nil, fmt.Errorf("failed to parse balance config: %v", err)
}
}
}
if params.Server == "" {
return nil, fmt.Errorf("server is required")
if config == nil {
config = &BalanceConfig{
BaseConfig: base.BaseConfig{
Enabled: true,
ScanIntervalSeconds: 30 * 60, // 30 minutes
MaxConcurrent: 1,
},
ImbalanceThreshold: 0.2, // 20%
MinServerCount: 2,
}
}
task := NewTask(params.Server, params.VolumeID, params.Collection)
task.SetEstimatedDuration(task.EstimateTime(params))
return task, nil
// Create and return the balance task using existing Task type
return NewTask(params.Server, params.VolumeID, params.Collection), nil
}
// getBalanceConfigSpec returns the configuration schema for balance tasks
@ -222,35 +236,23 @@ func getBalanceConfigSpec() base.ConfigSpec {
Required: false,
DisplayName: "Enable Balance Tasks",
Description: "Whether balance tasks should be automatically created",
HelpText: "Toggle this to enable or disable automatic balance task generation",
InputType: "checkbox",
CSSClasses: "form-check-input",
},
{
Name: "imbalance_threshold",
JSONName: "imbalance_threshold",
Type: config.FieldTypeFloat,
DefaultValue: 0.1, // 10%
MinValue: 0.01,
MaxValue: 0.5,
Required: true,
DisplayName: "Imbalance Threshold",
Description: "Trigger balance when storage imbalance exceeds this ratio",
Placeholder: "0.10 (10%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 6 * 60 * 60, // 6 hours
MinValue: 1 * 60 * 60, // 1 hour
MaxValue: 24 * 60 * 60, // 24 hours
DefaultValue: 30 * 60,
MinValue: 5 * 60,
MaxValue: 2 * 60 * 60,
Required: true,
DisplayName: "Scan Interval",
Description: "How often to scan for imbalanced volumes",
Unit: config.UnitHours,
Description: "How often to scan for volume distribution imbalances",
HelpText: "The system will check for volume distribution imbalances at this interval",
Placeholder: "30",
Unit: config.UnitMinutes,
InputType: "interval",
CSSClasses: "form-control",
},
@ -258,26 +260,46 @@ func getBalanceConfigSpec() base.ConfigSpec {
Name: "max_concurrent",
JSONName: "max_concurrent",
Type: config.FieldTypeInt,
DefaultValue: 2,
DefaultValue: 1,
MinValue: 1,
MaxValue: 5,
MaxValue: 3,
Required: true,
DisplayName: "Max Concurrent Tasks",
Description: "Maximum number of balance tasks that can run simultaneously",
HelpText: "Limits the number of balance operations running at the same time",
Placeholder: "1 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "imbalance_threshold",
JSONName: "imbalance_threshold",
Type: config.FieldTypeFloat,
DefaultValue: 0.2,
MinValue: 0.05,
MaxValue: 0.5,
Required: true,
DisplayName: "Imbalance Threshold",
Description: "Minimum imbalance ratio to trigger balancing",
HelpText: "Volume distribution imbalances above this threshold will trigger balancing",
Placeholder: "0.20 (20%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "min_server_count",
JSONName: "min_server_count",
Type: config.FieldTypeInt,
DefaultValue: 3,
DefaultValue: 2,
MinValue: 2,
MaxValue: 20,
MaxValue: 10,
Required: true,
DisplayName: "Minimum Server Count",
Description: "Only balance when at least this many servers are available",
Description: "Minimum number of servers required for balancing",
HelpText: "Balancing will only occur if there are at least this many servers",
Placeholder: "2 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
@ -286,17 +308,17 @@ func getBalanceConfigSpec() base.ConfigSpec {
}
}
// initBalanceV2 registers the refactored balance task
func initBalanceV2() {
// initBalance registers the refactored balance task
func initBalance() {
// Create configuration instance
config := &BalanceConfigV2{
config := &BalanceConfig{
BaseConfig: base.BaseConfig{
Enabled: false, // Conservative default
ScanIntervalSeconds: 6 * 60 * 60, // 6 hours
MaxConcurrent: 2,
Enabled: true,
ScanIntervalSeconds: 30 * 60, // 30 minutes
MaxConcurrent: 1,
},
ImbalanceThreshold: 0.1, // 10%
MinServerCount: 3,
ImbalanceThreshold: 0.2, // 20%
MinServerCount: 2,
}
// Create complete task definition
@ -304,18 +326,18 @@ func initBalanceV2() {
Type: types.TaskTypeBalance,
Name: "balance",
DisplayName: "Volume Balance",
Description: "Redistributes volumes across volume servers to optimize storage utilization",
Icon: "fas fa-balance-scale text-secondary",
Capabilities: []string{"balance", "storage", "optimization"},
Description: "Balances volume distribution across servers",
Icon: "fas fa-balance-scale text-warning",
Capabilities: []string{"balance", "distribution"},
Config: config,
ConfigSpec: getBalanceConfigSpec(),
CreateTask: createBalanceTask,
DetectionFunc: balanceDetection,
ScanInterval: 6 * time.Hour,
ScanInterval: 30 * time.Minute,
SchedulingFunc: balanceScheduling,
MaxConcurrent: 2,
RepeatInterval: 12 * time.Hour,
MaxConcurrent: 1,
RepeatInterval: 2 * time.Hour,
}
// Register everything with a single function call!

2
weed/worker/tasks/balance/balance_register.go

@ -3,5 +3,5 @@ package balance
// Auto-register this task when the package is imported
func init() {
// Use new architecture instead of old registration
initBalanceV2()
initBalance()
}

140
weed/worker/tasks/erasure_coding/ec.go

@ -1020,8 +1020,8 @@ func (t *Task) mountShardOnServer(targetServer pb.ServerAddress, shardId uint32)
return nil
}
// ErasureCodingConfigV2 extends BaseConfig with erasure coding specific settings
type ErasureCodingConfigV2 struct {
// ErasureCodingConfig extends BaseConfig with erasure coding specific settings
type ErasureCodingConfig struct {
base.BaseConfig
QuietForSeconds int `json:"quiet_for_seconds"`
FullnessRatio float64 `json:"fullness_ratio"`
@ -1034,7 +1034,7 @@ func ecDetection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.Cluste
return nil, nil
}
ecConfig := config.(*ErasureCodingConfigV2)
ecConfig := config.(*ErasureCodingConfig)
var results []*types.TaskDetectionResult
now := time.Now()
quietThreshold := time.Duration(ecConfig.QuietForSeconds) * time.Second
@ -1091,7 +1091,7 @@ func ecDetection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.Cluste
// ecScheduling implements the scheduling logic for erasure coding tasks
func ecScheduling(task *types.Task, runningTasks []*types.Task, availableWorkers []*types.Worker, config base.TaskConfig) bool {
ecConfig := config.(*ErasureCodingConfigV2)
ecConfig := config.(*ErasureCodingConfig)
// Check if we have available workers
if len(availableWorkers) == 0 {
@ -1123,37 +1123,34 @@ func ecScheduling(task *types.Task, runningTasks []*types.Task, availableWorkers
return false
}
// createErasureCodingTask creates an erasure coding task instance
// createErasureCodingTask creates a new erasure coding task instance
func createErasureCodingTask(params types.TaskParams) (types.TaskInterface, error) {
// Validate parameters
if params.VolumeID == 0 {
return nil, fmt.Errorf("volume_id is required")
}
if params.Server == "" {
return nil, fmt.Errorf("server is required")
}
// Extract additional parameters for comprehensive EC
masterClient := "localhost:9333" // Default master client
workDir := "/tmp/seaweedfs_ec_work" // Default work directory
if mc, ok := params.Parameters["master_client"].(string); ok && mc != "" {
masterClient = mc
}
if wd, ok := params.Parameters["work_dir"].(string); ok && wd != "" {
workDir = wd
// Extract configuration from params
var config *ErasureCodingConfig
if configData, ok := params.Parameters["config"]; ok {
if configMap, ok := configData.(map[string]interface{}); ok {
config = &ErasureCodingConfig{}
if err := config.FromMap(configMap); err != nil {
return nil, fmt.Errorf("failed to parse erasure coding config: %v", err)
}
}
}
// Create EC task with comprehensive capabilities
task := NewTaskWithParams(params.Server, params.VolumeID, masterClient, workDir)
// Set gRPC dial option if provided
if params.GrpcDialOption != nil {
task.SetDialOption(params.GrpcDialOption)
if config == nil {
config = &ErasureCodingConfig{
BaseConfig: base.BaseConfig{
Enabled: true,
ScanIntervalSeconds: 60 * 60, // 1 hour
MaxConcurrent: 1,
},
QuietForSeconds: 300, // 5 minutes
FullnessRatio: 0.8, // 80%
CollectionFilter: "",
}
}
task.SetEstimatedDuration(task.EstimateTime(params))
return task, nil
// Create and return the erasure coding task using existing Task type
return NewTask(params.Server, params.VolumeID), nil
}
// getErasureCodingConfigSpec returns the configuration schema for erasure coding tasks
@ -1168,33 +1165,22 @@ func getErasureCodingConfigSpec() base.ConfigSpec {
Required: false,
DisplayName: "Enable Erasure Coding Tasks",
Description: "Whether erasure coding tasks should be automatically created",
HelpText: "Toggle this to enable or disable automatic erasure coding task generation",
InputType: "checkbox",
CSSClasses: "form-check-input",
},
{
Name: "quiet_for_seconds",
JSONName: "quiet_for_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 7 * 24 * 60 * 60, // 7 days
MinValue: 1 * 24 * 60 * 60, // 1 day
MaxValue: 30 * 24 * 60 * 60, // 30 days
Required: true,
DisplayName: "Quiet For Duration",
Description: "Only apply erasure coding to volumes that have not been modified for this duration",
Unit: config.UnitDays,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 12 * 60 * 60, // 12 hours
MinValue: 2 * 60 * 60, // 2 hours
MaxValue: 24 * 60 * 60, // 24 hours
DefaultValue: 60 * 60,
MinValue: 10 * 60,
MaxValue: 24 * 60 * 60,
Required: true,
DisplayName: "Scan Interval",
Description: "How often to scan for volumes needing erasure coding",
HelpText: "The system will check for volumes that need erasure coding at this interval",
Placeholder: "1",
Unit: config.UnitHours,
InputType: "interval",
CSSClasses: "form-control",
@ -1205,25 +1191,44 @@ func getErasureCodingConfigSpec() base.ConfigSpec {
Type: config.FieldTypeInt,
DefaultValue: 1,
MinValue: 1,
MaxValue: 3,
MaxValue: 5,
Required: true,
DisplayName: "Max Concurrent Tasks",
Description: "Maximum number of erasure coding tasks that can run simultaneously",
HelpText: "Limits the number of erasure coding operations running at the same time",
Placeholder: "1 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "quiet_for_seconds",
JSONName: "quiet_for_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 300,
MinValue: 60,
MaxValue: 3600,
Required: true,
DisplayName: "Quiet Period",
Description: "Minimum time volume must be quiet before erasure coding",
HelpText: "Volume must not be modified for this duration before erasure coding",
Placeholder: "5",
Unit: config.UnitMinutes,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "fullness_ratio",
JSONName: "fullness_ratio",
Type: config.FieldTypeFloat,
DefaultValue: 0.9, // 90%
MinValue: 0.5,
DefaultValue: 0.8,
MinValue: 0.1,
MaxValue: 1.0,
Required: true,
DisplayName: "Fullness Ratio",
Description: "Only apply erasure coding to volumes with fullness ratio above this threshold",
Placeholder: "0.90 (90%)",
Description: "Minimum fullness ratio to trigger erasure coding",
HelpText: "Only volumes with this fullness ratio or higher will be erasure coded",
Placeholder: "0.80 (80%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
@ -1235,8 +1240,9 @@ func getErasureCodingConfigSpec() base.ConfigSpec {
DefaultValue: "",
Required: false,
DisplayName: "Collection Filter",
Description: "Only apply erasure coding to volumes in these collections (comma-separated, leave empty for all)",
Placeholder: "collection1,collection2",
Description: "Only process volumes from specific collections",
HelpText: "Leave empty to process all collections, or specify collection name",
Placeholder: "my_collection",
InputType: "text",
CSSClasses: "form-control",
},
@ -1244,18 +1250,18 @@ func getErasureCodingConfigSpec() base.ConfigSpec {
}
}
// initErasureCodingV2 registers the refactored erasure coding task
func initErasureCodingV2() {
// initErasureCoding registers the refactored erasure coding task
func initErasureCoding() {
// Create configuration instance
config := &ErasureCodingConfigV2{
config := &ErasureCodingConfig{
BaseConfig: base.BaseConfig{
Enabled: false, // Conservative default - enable via configuration
ScanIntervalSeconds: 12 * 60 * 60, // 12 hours
MaxConcurrent: 1, // Conservative default
Enabled: true,
ScanIntervalSeconds: 60 * 60, // 1 hour
MaxConcurrent: 1,
},
QuietForSeconds: 7 * 24 * 60 * 60, // 7 days quiet period
FullnessRatio: 0.90, // 90% full threshold
CollectionFilter: "", // No collection filter by default
QuietForSeconds: 300, // 5 minutes
FullnessRatio: 0.8, // 80%
CollectionFilter: "",
}
// Create complete task definition
@ -1263,15 +1269,15 @@ func initErasureCodingV2() {
Type: types.TaskTypeErasureCoding,
Name: "erasure_coding",
DisplayName: "Erasure Coding",
Description: "Converts volumes to erasure coded format for improved data durability",
Icon: "fas fa-shield-alt text-info",
Capabilities: []string{"erasure_coding", "storage", "durability"},
Description: "Applies erasure coding to volumes for data protection",
Icon: "fas fa-shield-alt text-success",
Capabilities: []string{"erasure_coding", "data_protection"},
Config: config,
ConfigSpec: getErasureCodingConfigSpec(),
CreateTask: createErasureCodingTask,
DetectionFunc: ecDetection,
ScanInterval: 12 * time.Hour,
ScanInterval: 1 * time.Hour,
SchedulingFunc: ecScheduling,
MaxConcurrent: 1,
RepeatInterval: 24 * time.Hour,

2
weed/worker/tasks/erasure_coding/ec_register.go

@ -3,5 +3,5 @@ package erasure_coding
// Auto-register this task when the package is imported
func init() {
// Use new architecture instead of old registration
initErasureCodingV2()
initErasureCoding()
}

80
weed/worker/tasks/vacuum/vacuum.go

@ -200,8 +200,8 @@ func (t *Task) Cancel() error {
return t.BaseTask.Cancel()
}
// VacuumConfigV2 extends BaseConfig with vacuum-specific settings
type VacuumConfigV2 struct {
// VacuumConfig extends BaseConfig with vacuum-specific settings
type VacuumConfig struct {
base.BaseConfig
GarbageThreshold float64 `json:"garbage_threshold"`
MinVolumeAgeSeconds int `json:"min_volume_age_seconds"`
@ -214,7 +214,7 @@ func vacuumDetection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.Cl
return nil, nil
}
vacuumConfig := config.(*VacuumConfigV2)
vacuumConfig := config.(*VacuumConfig)
var results []*types.TaskDetectionResult
minVolumeAge := time.Duration(vacuumConfig.MinVolumeAgeSeconds) * time.Second
@ -248,7 +248,7 @@ func vacuumDetection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.Cl
// vacuumScheduling implements the scheduling logic for vacuum tasks
func vacuumScheduling(task *types.Task, runningTasks []*types.Task, availableWorkers []*types.Worker, config base.TaskConfig) bool {
vacuumConfig := config.(*VacuumConfigV2)
vacuumConfig := config.(*VacuumConfig)
// Count running vacuum tasks
runningVacuumCount := 0
@ -277,20 +277,34 @@ func vacuumScheduling(task *types.Task, runningTasks []*types.Task, availableWor
return false
}
// createVacuumTask creates a vacuum task instance
// createVacuumTask creates a new vacuum task instance
func createVacuumTask(params types.TaskParams) (types.TaskInterface, error) {
// Validate parameters
if params.VolumeID == 0 {
return nil, fmt.Errorf("volume_id is required")
// Extract configuration from params
var config *VacuumConfig
if configData, ok := params.Parameters["config"]; ok {
if configMap, ok := configData.(map[string]interface{}); ok {
config = &VacuumConfig{}
if err := config.FromMap(configMap); err != nil {
return nil, fmt.Errorf("failed to parse vacuum config: %v", err)
}
}
}
if params.Server == "" {
return nil, fmt.Errorf("server is required")
if config == nil {
config = &VacuumConfig{
BaseConfig: base.BaseConfig{
Enabled: true,
ScanIntervalSeconds: 2 * 60 * 60, // 2 hours
MaxConcurrent: 2,
},
GarbageThreshold: 0.3, // 30%
MinVolumeAgeSeconds: 24 * 60 * 60, // 24 hours
MinIntervalSeconds: 7 * 24 * 60 * 60, // 7 days
}
}
// Use existing vacuum task implementation
task := NewTask(params.Server, params.VolumeID)
task.SetEstimatedDuration(task.EstimateTime(params))
return task, nil
// Create and return the vacuum task using existing Task type
return NewTask(params.Server, params.VolumeID), nil
}
// getVacuumConfigSpec returns the configuration schema for vacuum tasks
@ -309,22 +323,6 @@ func getVacuumConfigSpec() base.ConfigSpec {
InputType: "checkbox",
CSSClasses: "form-check-input",
},
{
Name: "garbage_threshold",
JSONName: "garbage_threshold",
Type: config.FieldTypeFloat,
DefaultValue: 0.3,
MinValue: 0.0,
MaxValue: 1.0,
Required: true,
DisplayName: "Garbage Percentage Threshold",
Description: "Trigger vacuum when garbage ratio exceeds this percentage",
HelpText: "Volumes with more deleted content than this threshold will be vacuumed",
Placeholder: "0.30 (30%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
@ -357,6 +355,22 @@ func getVacuumConfigSpec() base.ConfigSpec {
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "garbage_threshold",
JSONName: "garbage_threshold",
Type: config.FieldTypeFloat,
DefaultValue: 0.3,
MinValue: 0.0,
MaxValue: 1.0,
Required: true,
DisplayName: "Garbage Percentage Threshold",
Description: "Trigger vacuum when garbage ratio exceeds this percentage",
HelpText: "Volumes with more deleted content than this threshold will be vacuumed",
Placeholder: "0.30 (30%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "min_volume_age_seconds",
JSONName: "min_volume_age_seconds",
@ -393,10 +407,10 @@ func getVacuumConfigSpec() base.ConfigSpec {
}
}
// initVacuumV2 registers the refactored vacuum task (replaces the old registration)
func initVacuumV2() {
// initVacuum registers the refactored vacuum task (replaces the old registration)
func initVacuum() {
// Create configuration instance
config := &VacuumConfigV2{
config := &VacuumConfig{
BaseConfig: base.BaseConfig{
Enabled: true,
ScanIntervalSeconds: 2 * 60 * 60, // 2 hours

2
weed/worker/tasks/vacuum/vacuum_register.go

@ -3,5 +3,5 @@ package vacuum
// Auto-register this task when the package is imported
func init() {
// Use new architecture instead of old registration
initVacuumV2()
initVacuum()
}
Loading…
Cancel
Save