Browse Source
admin: Refactor task destination planning (#7063)
admin: Refactor task destination planning (#7063)
* refactor planning into task detection * refactoring worker tasks * refactor * compiles, but only balance task is registered * compiles, but has nil exception * avoid nil logger * add back ec task * setting ec log directory * implement balance and vacuum tasks * EC tasks will no longer fail with "file not found" errors * Use ReceiveFile API to send locally generated shards * distributing shard files and ecx,ecj,vif files * generate .ecx files correctly * do not mount all possible EC shards (0-13) on every destination * use constants * delete all replicas * rename files * pass in volume size to taskspull/7067/head
Chris Lu
2 months ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
43 changed files with 2901 additions and 2376 deletions
-
2docker/admin_integration/docker-compose-ec-test.yml
-
467weed/admin/maintenance/maintenance_integration.go
-
18weed/admin/maintenance/maintenance_scanner.go
-
17weed/admin/maintenance/maintenance_worker.go
-
301weed/admin/topology/active_topology.go
-
181weed/admin/topology/active_topology_test.go
-
1weed/pb/worker.proto
-
14weed/pb/worker_pb/worker.pb.go
-
22weed/worker/client.go
-
28weed/worker/registry.go
-
141weed/worker/tasks/balance/balance.go
-
248weed/worker/tasks/balance/balance_task.go
-
154weed/worker/tasks/balance/detection.go
-
0weed/worker/tasks/balance/execution.go
-
138weed/worker/tasks/balance/monitoring.go
-
17weed/worker/tasks/balance/register.go
-
37weed/worker/tasks/balance/scheduling.go
-
6weed/worker/tasks/base/generic_components.go
-
22weed/worker/tasks/base/registration.go
-
4weed/worker/tasks/base/task_definition.go
-
314weed/worker/tasks/erasure_coding/detection.go
-
785weed/worker/tasks/erasure_coding/ec.go
-
660weed/worker/tasks/erasure_coding/ec_task.go
-
229weed/worker/tasks/erasure_coding/monitoring.go
-
17weed/worker/tasks/erasure_coding/register.go
-
40weed/worker/tasks/erasure_coding/scheduling.go
-
64weed/worker/tasks/registry.go
-
53weed/worker/tasks/task.go
-
65weed/worker/tasks/vacuum/detection.go
-
151weed/worker/tasks/vacuum/monitoring.go
-
17weed/worker/tasks/vacuum/register.go
-
37weed/worker/tasks/vacuum/scheduling.go
-
214weed/worker/tasks/vacuum/vacuum.go
-
236weed/worker/tasks/vacuum/vacuum_task.go
-
99weed/worker/types/base/task.go
-
11weed/worker/types/data_types.go
-
52weed/worker/types/factory.go
-
189weed/worker/types/task.go
-
4weed/worker/types/task_scheduler.go
-
4weed/worker/types/task_types.go
-
76weed/worker/types/worker.go
-
53weed/worker/types/worker_types.go
-
89weed/worker/worker.go
@ -1,141 +0,0 @@ |
|||
package balance |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
) |
|||
|
|||
// Task implements balance operation to redistribute volumes across volume servers
|
|||
type Task struct { |
|||
*tasks.BaseTask |
|||
server string |
|||
volumeID uint32 |
|||
collection string |
|||
|
|||
// Task parameters for accessing planned destinations
|
|||
taskParams types.TaskParams |
|||
} |
|||
|
|||
// NewTask creates a new balance task instance
|
|||
func NewTask(server string, volumeID uint32, collection string) *Task { |
|||
task := &Task{ |
|||
BaseTask: tasks.NewBaseTask(types.TaskTypeBalance), |
|||
server: server, |
|||
volumeID: volumeID, |
|||
collection: collection, |
|||
} |
|||
return task |
|||
} |
|||
|
|||
// Execute executes the balance task
|
|||
func (t *Task) Execute(params types.TaskParams) error { |
|||
// Use BaseTask.ExecuteTask to handle logging initialization
|
|||
return t.ExecuteTask(context.Background(), params, t.executeImpl) |
|||
} |
|||
|
|||
// executeImpl is the actual balance implementation
|
|||
func (t *Task) executeImpl(ctx context.Context, params types.TaskParams) error { |
|||
// Store task parameters for accessing planned destinations
|
|||
t.taskParams = params |
|||
|
|||
// Get planned destination
|
|||
destNode := t.getPlannedDestination() |
|||
if destNode != "" { |
|||
t.LogWithFields("INFO", "Starting balance task with planned destination", map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"source": t.server, |
|||
"destination": destNode, |
|||
"collection": t.collection, |
|||
}) |
|||
} else { |
|||
t.LogWithFields("INFO", "Starting balance task without specific destination", map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"server": t.server, |
|||
"collection": t.collection, |
|||
}) |
|||
} |
|||
|
|||
// Simulate balance operation with progress updates
|
|||
steps := []struct { |
|||
name string |
|||
duration time.Duration |
|||
progress float64 |
|||
}{ |
|||
{"Analyzing cluster state", 2 * time.Second, 15}, |
|||
{"Identifying optimal placement", 3 * time.Second, 35}, |
|||
{"Moving volume data", 6 * time.Second, 75}, |
|||
{"Updating cluster metadata", 2 * time.Second, 95}, |
|||
{"Verifying balance", 1 * time.Second, 100}, |
|||
} |
|||
|
|||
for _, step := range steps { |
|||
select { |
|||
case <-ctx.Done(): |
|||
t.LogWarning("Balance task cancelled during step: %s", step.name) |
|||
return ctx.Err() |
|||
default: |
|||
} |
|||
|
|||
if t.IsCancelled() { |
|||
t.LogWarning("Balance task cancelled by request during step: %s", step.name) |
|||
return fmt.Errorf("balance task cancelled") |
|||
} |
|||
|
|||
t.LogWithFields("INFO", "Executing balance step", map[string]interface{}{ |
|||
"step": step.name, |
|||
"progress": step.progress, |
|||
"duration": step.duration.String(), |
|||
"volume_id": t.volumeID, |
|||
}) |
|||
t.SetProgress(step.progress) |
|||
|
|||
// Simulate work
|
|||
time.Sleep(step.duration) |
|||
} |
|||
|
|||
t.LogWithFields("INFO", "Balance task completed successfully", map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"server": t.server, |
|||
"collection": t.collection, |
|||
"final_progress": 100.0, |
|||
}) |
|||
return nil |
|||
} |
|||
|
|||
// Validate validates the task parameters
|
|||
func (t *Task) Validate(params types.TaskParams) error { |
|||
if params.VolumeID == 0 { |
|||
return fmt.Errorf("volume_id is required") |
|||
} |
|||
if params.Server == "" { |
|||
return fmt.Errorf("server is required") |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
// getPlannedDestination extracts the planned destination node from task parameters
|
|||
func (t *Task) getPlannedDestination() string { |
|||
if t.taskParams.TypedParams != nil { |
|||
if balanceParams := t.taskParams.TypedParams.GetBalanceParams(); balanceParams != nil { |
|||
if balanceParams.DestNode != "" { |
|||
glog.V(2).Infof("Found planned destination for volume %d: %s", t.volumeID, balanceParams.DestNode) |
|||
return balanceParams.DestNode |
|||
} |
|||
} |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
// EstimateTime estimates the time needed for the task
|
|||
func (t *Task) EstimateTime(params types.TaskParams) time.Duration { |
|||
// Base time for balance operation
|
|||
baseTime := 35 * time.Second |
|||
|
|||
// Could adjust based on volume size or cluster state
|
|||
return baseTime |
|||
} |
|||
@ -0,0 +1,248 @@ |
|||
package balance |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"io" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/operation" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/needle" |
|||
"github.com/seaweedfs/seaweedfs/weed/util" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types/base" |
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
// BalanceTask implements the Task interface
|
|||
type BalanceTask struct { |
|||
*base.BaseTask |
|||
server string |
|||
volumeID uint32 |
|||
collection string |
|||
progress float64 |
|||
} |
|||
|
|||
// NewBalanceTask creates a new balance task instance
|
|||
func NewBalanceTask(id string, server string, volumeID uint32, collection string) *BalanceTask { |
|||
return &BalanceTask{ |
|||
BaseTask: base.NewBaseTask(id, types.TaskTypeBalance), |
|||
server: server, |
|||
volumeID: volumeID, |
|||
collection: collection, |
|||
} |
|||
} |
|||
|
|||
// Execute implements the Task interface
|
|||
func (t *BalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error { |
|||
if params == nil { |
|||
return fmt.Errorf("task parameters are required") |
|||
} |
|||
|
|||
balanceParams := params.GetBalanceParams() |
|||
if balanceParams == nil { |
|||
return fmt.Errorf("balance parameters are required") |
|||
} |
|||
|
|||
// Get planned destination
|
|||
destNode := balanceParams.DestNode |
|||
|
|||
if destNode == "" { |
|||
return fmt.Errorf("destination node is required for balance task") |
|||
} |
|||
|
|||
t.GetLogger().WithFields(map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"source": t.server, |
|||
"destination": destNode, |
|||
"collection": t.collection, |
|||
}).Info("Starting balance task - moving volume") |
|||
|
|||
sourceServer := pb.ServerAddress(t.server) |
|||
targetServer := pb.ServerAddress(destNode) |
|||
volumeId := needle.VolumeId(t.volumeID) |
|||
|
|||
// Step 1: Mark volume readonly
|
|||
t.ReportProgress(10.0) |
|||
t.GetLogger().Info("Marking volume readonly for move") |
|||
if err := t.markVolumeReadonly(sourceServer, volumeId); err != nil { |
|||
return fmt.Errorf("failed to mark volume readonly: %v", err) |
|||
} |
|||
|
|||
// Step 2: Copy volume to destination
|
|||
t.ReportProgress(20.0) |
|||
t.GetLogger().Info("Copying volume to destination") |
|||
lastAppendAtNs, err := t.copyVolume(sourceServer, targetServer, volumeId) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to copy volume: %v", err) |
|||
} |
|||
|
|||
// Step 3: Mount volume on target and mark it readonly
|
|||
t.ReportProgress(60.0) |
|||
t.GetLogger().Info("Mounting volume on target server") |
|||
if err := t.mountVolume(targetServer, volumeId); err != nil { |
|||
return fmt.Errorf("failed to mount volume on target: %v", err) |
|||
} |
|||
|
|||
// Step 4: Tail for updates
|
|||
t.ReportProgress(70.0) |
|||
t.GetLogger().Info("Syncing final updates") |
|||
if err := t.tailVolume(sourceServer, targetServer, volumeId, lastAppendAtNs); err != nil { |
|||
glog.Warningf("Tail operation failed (may be normal): %v", err) |
|||
} |
|||
|
|||
// Step 5: Unmount from source
|
|||
t.ReportProgress(85.0) |
|||
t.GetLogger().Info("Unmounting volume from source server") |
|||
if err := t.unmountVolume(sourceServer, volumeId); err != nil { |
|||
return fmt.Errorf("failed to unmount volume from source: %v", err) |
|||
} |
|||
|
|||
// Step 6: Delete from source
|
|||
t.ReportProgress(95.0) |
|||
t.GetLogger().Info("Deleting volume from source server") |
|||
if err := t.deleteVolume(sourceServer, volumeId); err != nil { |
|||
return fmt.Errorf("failed to delete volume from source: %v", err) |
|||
} |
|||
|
|||
t.ReportProgress(100.0) |
|||
glog.Infof("Balance task completed successfully: volume %d moved from %s to %s", |
|||
t.volumeID, t.server, destNode) |
|||
return nil |
|||
} |
|||
|
|||
// Validate implements the UnifiedTask interface
|
|||
func (t *BalanceTask) Validate(params *worker_pb.TaskParams) error { |
|||
if params == nil { |
|||
return fmt.Errorf("task parameters are required") |
|||
} |
|||
|
|||
balanceParams := params.GetBalanceParams() |
|||
if balanceParams == nil { |
|||
return fmt.Errorf("balance parameters are required") |
|||
} |
|||
|
|||
if params.VolumeId != t.volumeID { |
|||
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId) |
|||
} |
|||
|
|||
if params.Server != t.server { |
|||
return fmt.Errorf("source server mismatch: expected %s, got %s", t.server, params.Server) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// EstimateTime implements the UnifiedTask interface
|
|||
func (t *BalanceTask) EstimateTime(params *worker_pb.TaskParams) time.Duration { |
|||
// Basic estimate based on simulated steps
|
|||
return 14 * time.Second // Sum of all step durations
|
|||
} |
|||
|
|||
// GetProgress returns current progress
|
|||
func (t *BalanceTask) GetProgress() float64 { |
|||
return t.progress |
|||
} |
|||
|
|||
// Helper methods for real balance operations
|
|||
|
|||
// markVolumeReadonly marks the volume readonly
|
|||
func (t *BalanceTask) markVolumeReadonly(server pb.ServerAddress, volumeId needle.VolumeId) error { |
|||
return operation.WithVolumeServerClient(false, server, grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, err := client.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{ |
|||
VolumeId: uint32(volumeId), |
|||
}) |
|||
return err |
|||
}) |
|||
} |
|||
|
|||
// copyVolume copies volume from source to target server
|
|||
func (t *BalanceTask) copyVolume(sourceServer, targetServer pb.ServerAddress, volumeId needle.VolumeId) (uint64, error) { |
|||
var lastAppendAtNs uint64 |
|||
|
|||
err := operation.WithVolumeServerClient(true, targetServer, grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
stream, err := client.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{ |
|||
VolumeId: uint32(volumeId), |
|||
SourceDataNode: string(sourceServer), |
|||
}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
for { |
|||
resp, recvErr := stream.Recv() |
|||
if recvErr != nil { |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
return recvErr |
|||
} |
|||
|
|||
if resp.LastAppendAtNs != 0 { |
|||
lastAppendAtNs = resp.LastAppendAtNs |
|||
} else { |
|||
// Report copy progress
|
|||
glog.V(1).Infof("Volume %d copy progress: %s", volumeId, |
|||
util.BytesToHumanReadable(uint64(resp.ProcessedBytes))) |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
}) |
|||
|
|||
return lastAppendAtNs, err |
|||
} |
|||
|
|||
// mountVolume mounts the volume on the target server
|
|||
func (t *BalanceTask) mountVolume(server pb.ServerAddress, volumeId needle.VolumeId) error { |
|||
return operation.WithVolumeServerClient(false, server, grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, err := client.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{ |
|||
VolumeId: uint32(volumeId), |
|||
}) |
|||
return err |
|||
}) |
|||
} |
|||
|
|||
// tailVolume syncs remaining updates from source to target
|
|||
func (t *BalanceTask) tailVolume(sourceServer, targetServer pb.ServerAddress, volumeId needle.VolumeId, sinceNs uint64) error { |
|||
return operation.WithVolumeServerClient(true, targetServer, grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, err := client.VolumeTailReceiver(context.Background(), &volume_server_pb.VolumeTailReceiverRequest{ |
|||
VolumeId: uint32(volumeId), |
|||
SinceNs: sinceNs, |
|||
IdleTimeoutSeconds: 60, // 1 minute timeout
|
|||
SourceVolumeServer: string(sourceServer), |
|||
}) |
|||
return err |
|||
}) |
|||
} |
|||
|
|||
// unmountVolume unmounts the volume from the server
|
|||
func (t *BalanceTask) unmountVolume(server pb.ServerAddress, volumeId needle.VolumeId) error { |
|||
return operation.WithVolumeServerClient(false, server, grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, err := client.VolumeUnmount(context.Background(), &volume_server_pb.VolumeUnmountRequest{ |
|||
VolumeId: uint32(volumeId), |
|||
}) |
|||
return err |
|||
}) |
|||
} |
|||
|
|||
// deleteVolume deletes the volume from the server
|
|||
func (t *BalanceTask) deleteVolume(server pb.ServerAddress, volumeId needle.VolumeId) error { |
|||
return operation.WithVolumeServerClient(false, server, grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, err := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{ |
|||
VolumeId: uint32(volumeId), |
|||
OnlyEmpty: false, |
|||
}) |
|||
return err |
|||
}) |
|||
} |
|||
@ -0,0 +1,138 @@ |
|||
package balance |
|||
|
|||
import ( |
|||
"sync" |
|||
"time" |
|||
) |
|||
|
|||
// BalanceMetrics contains balance-specific monitoring data
|
|||
type BalanceMetrics struct { |
|||
// Execution metrics
|
|||
VolumesBalanced int64 `json:"volumes_balanced"` |
|||
TotalDataTransferred int64 `json:"total_data_transferred"` |
|||
AverageImbalance float64 `json:"average_imbalance"` |
|||
LastBalanceTime time.Time `json:"last_balance_time"` |
|||
|
|||
// Performance metrics
|
|||
AverageTransferSpeed float64 `json:"average_transfer_speed_mbps"` |
|||
TotalExecutionTime int64 `json:"total_execution_time_seconds"` |
|||
SuccessfulOperations int64 `json:"successful_operations"` |
|||
FailedOperations int64 `json:"failed_operations"` |
|||
|
|||
// Current task metrics
|
|||
CurrentImbalanceScore float64 `json:"current_imbalance_score"` |
|||
PlannedDestinations int `json:"planned_destinations"` |
|||
|
|||
mutex sync.RWMutex |
|||
} |
|||
|
|||
// NewBalanceMetrics creates a new balance metrics instance
|
|||
func NewBalanceMetrics() *BalanceMetrics { |
|||
return &BalanceMetrics{ |
|||
LastBalanceTime: time.Now(), |
|||
} |
|||
} |
|||
|
|||
// RecordVolumeBalanced records a successful volume balance operation
|
|||
func (m *BalanceMetrics) RecordVolumeBalanced(volumeSize int64, transferTime time.Duration) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.VolumesBalanced++ |
|||
m.TotalDataTransferred += volumeSize |
|||
m.SuccessfulOperations++ |
|||
m.LastBalanceTime = time.Now() |
|||
m.TotalExecutionTime += int64(transferTime.Seconds()) |
|||
|
|||
// Calculate average transfer speed (MB/s)
|
|||
if transferTime > 0 { |
|||
speedMBps := float64(volumeSize) / (1024 * 1024) / transferTime.Seconds() |
|||
if m.AverageTransferSpeed == 0 { |
|||
m.AverageTransferSpeed = speedMBps |
|||
} else { |
|||
// Exponential moving average
|
|||
m.AverageTransferSpeed = 0.8*m.AverageTransferSpeed + 0.2*speedMBps |
|||
} |
|||
} |
|||
} |
|||
|
|||
// RecordFailure records a failed balance operation
|
|||
func (m *BalanceMetrics) RecordFailure() { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.FailedOperations++ |
|||
} |
|||
|
|||
// UpdateImbalanceScore updates the current cluster imbalance score
|
|||
func (m *BalanceMetrics) UpdateImbalanceScore(score float64) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.CurrentImbalanceScore = score |
|||
|
|||
// Update average imbalance with exponential moving average
|
|||
if m.AverageImbalance == 0 { |
|||
m.AverageImbalance = score |
|||
} else { |
|||
m.AverageImbalance = 0.9*m.AverageImbalance + 0.1*score |
|||
} |
|||
} |
|||
|
|||
// SetPlannedDestinations sets the number of planned destinations
|
|||
func (m *BalanceMetrics) SetPlannedDestinations(count int) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.PlannedDestinations = count |
|||
} |
|||
|
|||
// GetMetrics returns a copy of the current metrics (without the mutex)
|
|||
func (m *BalanceMetrics) GetMetrics() BalanceMetrics { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
// Create a copy without the mutex to avoid copying lock value
|
|||
return BalanceMetrics{ |
|||
VolumesBalanced: m.VolumesBalanced, |
|||
TotalDataTransferred: m.TotalDataTransferred, |
|||
AverageImbalance: m.AverageImbalance, |
|||
LastBalanceTime: m.LastBalanceTime, |
|||
AverageTransferSpeed: m.AverageTransferSpeed, |
|||
TotalExecutionTime: m.TotalExecutionTime, |
|||
SuccessfulOperations: m.SuccessfulOperations, |
|||
FailedOperations: m.FailedOperations, |
|||
CurrentImbalanceScore: m.CurrentImbalanceScore, |
|||
PlannedDestinations: m.PlannedDestinations, |
|||
} |
|||
} |
|||
|
|||
// GetSuccessRate returns the success rate as a percentage
|
|||
func (m *BalanceMetrics) GetSuccessRate() float64 { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
total := m.SuccessfulOperations + m.FailedOperations |
|||
if total == 0 { |
|||
return 100.0 |
|||
} |
|||
return float64(m.SuccessfulOperations) / float64(total) * 100.0 |
|||
} |
|||
|
|||
// Reset resets all metrics to zero
|
|||
func (m *BalanceMetrics) Reset() { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
*m = BalanceMetrics{ |
|||
LastBalanceTime: time.Now(), |
|||
} |
|||
} |
|||
|
|||
// Global metrics instance for balance tasks
|
|||
var globalBalanceMetrics = NewBalanceMetrics() |
|||
|
|||
// GetGlobalBalanceMetrics returns the global balance metrics instance
|
|||
func GetGlobalBalanceMetrics() *BalanceMetrics { |
|||
return globalBalanceMetrics |
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
package balance |
|||
|
|||
import ( |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
) |
|||
|
|||
// Scheduling implements the scheduling logic for balance tasks
|
|||
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool { |
|||
balanceConfig := config.(*Config) |
|||
|
|||
// Count running balance tasks
|
|||
runningBalanceCount := 0 |
|||
for _, runningTask := range runningTasks { |
|||
if runningTask.Type == types.TaskTypeBalance { |
|||
runningBalanceCount++ |
|||
} |
|||
} |
|||
|
|||
// Check concurrency limit
|
|||
if runningBalanceCount >= balanceConfig.MaxConcurrent { |
|||
return false |
|||
} |
|||
|
|||
// Check if we have available workers
|
|||
availableWorkerCount := 0 |
|||
for _, worker := range availableWorkers { |
|||
for _, capability := range worker.Capabilities { |
|||
if capability == types.TaskTypeBalance { |
|||
availableWorkerCount++ |
|||
break |
|||
} |
|||
} |
|||
} |
|||
|
|||
return availableWorkerCount > 0 |
|||
} |
|||
@ -1,785 +0,0 @@ |
|||
package erasure_coding |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"io" |
|||
"math" |
|||
"os" |
|||
"path/filepath" |
|||
"strings" |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/operation" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/needle" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
"google.golang.org/grpc" |
|||
"google.golang.org/grpc/credentials/insecure" |
|||
) |
|||
|
|||
// Task implements comprehensive erasure coding with protobuf parameters
|
|||
type Task struct { |
|||
*base.BaseTypedTask |
|||
|
|||
// Current task state
|
|||
sourceServer string |
|||
volumeID uint32 |
|||
collection string |
|||
workDir string |
|||
masterClient string |
|||
grpcDialOpt grpc.DialOption |
|||
|
|||
// EC parameters from protobuf
|
|||
destinations []*worker_pb.ECDestination // Disk-aware destinations
|
|||
existingShardLocations []*worker_pb.ExistingECShardLocation // Existing shards to cleanup
|
|||
estimatedShardSize uint64 |
|||
dataShards int |
|||
parityShards int |
|||
cleanupSource bool |
|||
|
|||
// Progress tracking
|
|||
currentStep string |
|||
stepProgress map[string]float64 |
|||
} |
|||
|
|||
// NewTask creates a new erasure coding task
|
|||
func NewTask() types.TypedTaskInterface { |
|||
task := &Task{ |
|||
BaseTypedTask: base.NewBaseTypedTask(types.TaskTypeErasureCoding), |
|||
masterClient: "localhost:9333", // Default master client
|
|||
workDir: "/tmp/seaweedfs_ec_work", // Default work directory
|
|||
grpcDialOpt: grpc.WithTransportCredentials(insecure.NewCredentials()), // Default to insecure
|
|||
dataShards: erasure_coding.DataShardsCount, // Use package constant
|
|||
parityShards: erasure_coding.ParityShardsCount, // Use package constant
|
|||
stepProgress: make(map[string]float64), |
|||
} |
|||
return task |
|||
} |
|||
|
|||
// ValidateTyped validates the typed parameters for EC task
|
|||
func (t *Task) ValidateTyped(params *worker_pb.TaskParams) error { |
|||
// Basic validation from base class
|
|||
if err := t.BaseTypedTask.ValidateTyped(params); err != nil { |
|||
return err |
|||
} |
|||
|
|||
// Check that we have EC-specific parameters
|
|||
ecParams := params.GetErasureCodingParams() |
|||
if ecParams == nil { |
|||
return fmt.Errorf("erasure_coding_params is required for EC task") |
|||
} |
|||
|
|||
// Require destinations
|
|||
if len(ecParams.Destinations) == 0 { |
|||
return fmt.Errorf("destinations must be specified for EC task") |
|||
} |
|||
|
|||
// DataShards and ParityShards are constants from erasure_coding package
|
|||
expectedDataShards := int32(erasure_coding.DataShardsCount) |
|||
expectedParityShards := int32(erasure_coding.ParityShardsCount) |
|||
|
|||
if ecParams.DataShards > 0 && ecParams.DataShards != expectedDataShards { |
|||
return fmt.Errorf("data_shards must be %d (fixed constant), got %d", expectedDataShards, ecParams.DataShards) |
|||
} |
|||
if ecParams.ParityShards > 0 && ecParams.ParityShards != expectedParityShards { |
|||
return fmt.Errorf("parity_shards must be %d (fixed constant), got %d", expectedParityShards, ecParams.ParityShards) |
|||
} |
|||
|
|||
// Validate destination count
|
|||
destinationCount := len(ecParams.Destinations) |
|||
totalShards := expectedDataShards + expectedParityShards |
|||
if totalShards > int32(destinationCount) { |
|||
return fmt.Errorf("insufficient destinations: need %d, have %d", totalShards, destinationCount) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// EstimateTimeTyped estimates the time needed for EC processing based on protobuf parameters
|
|||
func (t *Task) EstimateTimeTyped(params *worker_pb.TaskParams) time.Duration { |
|||
baseTime := 20 * time.Minute // Processing takes time due to comprehensive operations
|
|||
|
|||
ecParams := params.GetErasureCodingParams() |
|||
if ecParams != nil && ecParams.EstimatedShardSize > 0 { |
|||
// More accurate estimate based on shard size
|
|||
// Account for copying, encoding, and distribution
|
|||
gbSize := ecParams.EstimatedShardSize / (1024 * 1024 * 1024) |
|||
estimatedTime := time.Duration(gbSize*2) * time.Minute // 2 minutes per GB
|
|||
if estimatedTime > baseTime { |
|||
return estimatedTime |
|||
} |
|||
} |
|||
|
|||
return baseTime |
|||
} |
|||
|
|||
// ExecuteTyped implements the actual erasure coding workflow with typed parameters
|
|||
func (t *Task) ExecuteTyped(params *worker_pb.TaskParams) error { |
|||
// Extract basic parameters
|
|||
t.volumeID = params.VolumeId |
|||
t.sourceServer = params.Server |
|||
t.collection = params.Collection |
|||
|
|||
// Extract EC-specific parameters
|
|||
ecParams := params.GetErasureCodingParams() |
|||
if ecParams != nil { |
|||
t.destinations = ecParams.Destinations // Store disk-aware destinations
|
|||
t.existingShardLocations = ecParams.ExistingShardLocations // Store existing shards for cleanup
|
|||
t.estimatedShardSize = ecParams.EstimatedShardSize |
|||
t.cleanupSource = ecParams.CleanupSource |
|||
|
|||
// DataShards and ParityShards are constants, don't override from parameters
|
|||
// t.dataShards and t.parityShards are already set to constants in NewTask
|
|||
|
|||
if ecParams.WorkingDir != "" { |
|||
t.workDir = ecParams.WorkingDir |
|||
} |
|||
if ecParams.MasterClient != "" { |
|||
t.masterClient = ecParams.MasterClient |
|||
} |
|||
} |
|||
|
|||
// Determine available destinations for logging
|
|||
var availableDestinations []string |
|||
for _, dest := range t.destinations { |
|||
availableDestinations = append(availableDestinations, fmt.Sprintf("%s(disk:%d)", dest.Node, dest.DiskId)) |
|||
} |
|||
|
|||
glog.V(1).Infof("Starting EC task for volume %d: %s -> %v (data:%d, parity:%d)", |
|||
t.volumeID, t.sourceServer, availableDestinations, t.dataShards, t.parityShards) |
|||
|
|||
// Create unique working directory for this task
|
|||
taskWorkDir := filepath.Join(t.workDir, fmt.Sprintf("vol_%d_%d", t.volumeID, time.Now().Unix())) |
|||
if err := os.MkdirAll(taskWorkDir, 0755); err != nil { |
|||
return fmt.Errorf("failed to create task working directory %s: %v", taskWorkDir, err) |
|||
} |
|||
glog.V(1).Infof("WORKFLOW: Created working directory: %s", taskWorkDir) |
|||
|
|||
// Ensure cleanup of working directory
|
|||
defer func() { |
|||
if err := os.RemoveAll(taskWorkDir); err != nil { |
|||
glog.Warningf("Failed to cleanup working directory %s: %v", taskWorkDir, err) |
|||
} else { |
|||
glog.V(1).Infof("WORKFLOW: Cleaned up working directory: %s", taskWorkDir) |
|||
} |
|||
}() |
|||
|
|||
// Step 1: Collect volume locations from master
|
|||
glog.V(1).Infof("WORKFLOW STEP 1: Collecting volume locations from master") |
|||
t.SetProgress(5.0) |
|||
volumeId := needle.VolumeId(t.volumeID) |
|||
volumeLocations, err := t.collectVolumeLocations(volumeId) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to collect volume locations before EC encoding: %v", err) |
|||
} |
|||
glog.V(1).Infof("WORKFLOW: Found volume %d on %d servers: %v", t.volumeID, len(volumeLocations), volumeLocations) |
|||
|
|||
// Convert ServerAddress slice to string slice
|
|||
var locationStrings []string |
|||
for _, addr := range volumeLocations { |
|||
locationStrings = append(locationStrings, string(addr)) |
|||
} |
|||
|
|||
// Step 2: Check if volume has sufficient size for EC encoding
|
|||
if !t.shouldPerformECEncoding(locationStrings) { |
|||
glog.Infof("Volume %d does not meet EC encoding criteria, skipping", t.volumeID) |
|||
t.SetProgress(100.0) |
|||
return nil |
|||
} |
|||
|
|||
// Step 2A: Cleanup existing EC shards if any
|
|||
glog.V(1).Infof("WORKFLOW STEP 2A: Cleaning up existing EC shards for volume %d", t.volumeID) |
|||
t.SetProgress(10.0) |
|||
err = t.cleanupExistingEcShards() |
|||
if err != nil { |
|||
glog.Warningf("Failed to cleanup existing EC shards (continuing anyway): %v", err) |
|||
// Don't fail the task - this is just cleanup
|
|||
} |
|||
glog.V(1).Infof("WORKFLOW: Existing EC shards cleanup completed for volume %d", t.volumeID) |
|||
|
|||
// Step 3: Mark volume readonly on all servers
|
|||
glog.V(1).Infof("WORKFLOW STEP 2B: Marking volume %d readonly on all replica servers", t.volumeID) |
|||
t.SetProgress(15.0) |
|||
err = t.markVolumeReadonlyOnAllReplicas(needle.VolumeId(t.volumeID), locationStrings) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to mark volume readonly: %v", err) |
|||
} |
|||
glog.V(1).Infof("WORKFLOW: Volume %d marked readonly on all replicas", t.volumeID) |
|||
|
|||
// Step 5: Copy volume files (.dat, .idx) to EC worker
|
|||
glog.V(1).Infof("WORKFLOW STEP 3: Copying volume files from source server %s to EC worker", t.sourceServer) |
|||
t.SetProgress(25.0) |
|||
localVolumeFiles, err := t.copyVolumeFilesToWorker(taskWorkDir) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to copy volume files to EC worker: %v", err) |
|||
} |
|||
glog.V(1).Infof("WORKFLOW: Volume files copied to EC worker: %v", localVolumeFiles) |
|||
|
|||
// Step 6: Generate EC shards locally on EC worker
|
|||
glog.V(1).Infof("WORKFLOW STEP 4: Generating EC shards locally on EC worker") |
|||
t.SetProgress(40.0) |
|||
localShardFiles, err := t.generateEcShardsLocally(localVolumeFiles, taskWorkDir) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to generate EC shards locally: %v", err) |
|||
} |
|||
glog.V(1).Infof("WORKFLOW: EC shards generated locally: %d shard files", len(localShardFiles)) |
|||
|
|||
// Step 7: Distribute shards from EC worker to destination servers
|
|||
glog.V(1).Infof("WORKFLOW STEP 5: Distributing EC shards from worker to destination servers") |
|||
t.SetProgress(60.0) |
|||
err = t.distributeEcShardsFromWorker(localShardFiles) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to distribute EC shards from worker: %v", err) |
|||
} |
|||
glog.V(1).Infof("WORKFLOW: EC shards distributed to all destination servers") |
|||
|
|||
// Step 8: Mount EC shards on destination servers
|
|||
glog.V(1).Infof("WORKFLOW STEP 6: Mounting EC shards on destination servers") |
|||
t.SetProgress(80.0) |
|||
err = t.mountEcShardsOnDestinations() |
|||
if err != nil { |
|||
return fmt.Errorf("failed to mount EC shards: %v", err) |
|||
} |
|||
glog.V(1).Infof("WORKFLOW: EC shards mounted successfully") |
|||
|
|||
// Step 9: Delete original volume from all locations
|
|||
glog.V(1).Infof("WORKFLOW STEP 7: Deleting original volume %d from all replica servers", t.volumeID) |
|||
t.SetProgress(90.0) |
|||
err = t.deleteVolumeFromAllLocations(needle.VolumeId(t.volumeID), locationStrings) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to delete original volume: %v", err) |
|||
} |
|||
glog.V(1).Infof("WORKFLOW: Original volume %d deleted from all locations", t.volumeID) |
|||
|
|||
t.SetProgress(100.0) |
|||
glog.Infof("EC task completed successfully for volume %d", t.volumeID) |
|||
return nil |
|||
} |
|||
|
|||
// collectVolumeLocations gets volume location from master (placeholder implementation)
|
|||
func (t *Task) collectVolumeLocations(volumeId needle.VolumeId) ([]pb.ServerAddress, error) { |
|||
// For now, return a placeholder implementation
|
|||
// Full implementation would call master to get volume locations
|
|||
return []pb.ServerAddress{pb.ServerAddress(t.sourceServer)}, nil |
|||
} |
|||
|
|||
// cleanupExistingEcShards deletes existing EC shards using planned locations
|
|||
func (t *Task) cleanupExistingEcShards() error { |
|||
if len(t.existingShardLocations) == 0 { |
|||
glog.V(1).Infof("No existing EC shards to cleanup for volume %d", t.volumeID) |
|||
return nil |
|||
} |
|||
|
|||
glog.V(1).Infof("Cleaning up existing EC shards for volume %d on %d servers", t.volumeID, len(t.existingShardLocations)) |
|||
|
|||
// Delete existing shards from each location using planned shard locations
|
|||
for _, location := range t.existingShardLocations { |
|||
if len(location.ShardIds) == 0 { |
|||
continue |
|||
} |
|||
|
|||
glog.V(1).Infof("Deleting existing EC shards %v from %s for volume %d", location.ShardIds, location.Node, t.volumeID) |
|||
|
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(location.Node), t.grpcDialOpt, |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, deleteErr := client.VolumeEcShardsDelete(context.Background(), &volume_server_pb.VolumeEcShardsDeleteRequest{ |
|||
VolumeId: t.volumeID, |
|||
Collection: t.collection, |
|||
ShardIds: location.ShardIds, |
|||
}) |
|||
return deleteErr |
|||
}) |
|||
|
|||
if err != nil { |
|||
glog.Errorf("Failed to delete existing EC shards %v from %s for volume %d: %v", location.ShardIds, location.Node, t.volumeID, err) |
|||
// Continue with other servers - don't fail the entire cleanup
|
|||
} else { |
|||
glog.V(1).Infof("Successfully deleted existing EC shards %v from %s for volume %d", location.ShardIds, location.Node, t.volumeID) |
|||
} |
|||
} |
|||
|
|||
glog.V(1).Infof("Completed cleanup of existing EC shards for volume %d", t.volumeID) |
|||
return nil |
|||
} |
|||
|
|||
// shouldPerformECEncoding checks if the volume meets criteria for EC encoding
|
|||
func (t *Task) shouldPerformECEncoding(volumeLocations []string) bool { |
|||
// For now, always proceed with EC encoding if volume exists
|
|||
// This can be extended with volume size checks, etc.
|
|||
return len(volumeLocations) > 0 |
|||
} |
|||
|
|||
// markVolumeReadonlyOnAllReplicas marks the volume as readonly on all replica servers
|
|||
func (t *Task) markVolumeReadonlyOnAllReplicas(volumeId needle.VolumeId, volumeLocations []string) error { |
|||
glog.V(1).Infof("Marking volume %d readonly on %d servers", volumeId, len(volumeLocations)) |
|||
|
|||
// Mark volume readonly on all replica servers
|
|||
for _, location := range volumeLocations { |
|||
glog.V(1).Infof("Marking volume %d readonly on %s", volumeId, location) |
|||
|
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(location), t.grpcDialOpt, |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, markErr := client.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{ |
|||
VolumeId: uint32(volumeId), |
|||
}) |
|||
return markErr |
|||
}) |
|||
|
|||
if err != nil { |
|||
glog.Errorf("Failed to mark volume %d readonly on %s: %v", volumeId, location, err) |
|||
return fmt.Errorf("failed to mark volume %d readonly on %s: %v", volumeId, location, err) |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully marked volume %d readonly on %s", volumeId, location) |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully marked volume %d readonly on all %d locations", volumeId, len(volumeLocations)) |
|||
return nil |
|||
} |
|||
|
|||
// copyVolumeFilesToWorker copies .dat and .idx files from source server to local worker
|
|||
func (t *Task) copyVolumeFilesToWorker(workDir string) (map[string]string, error) { |
|||
localFiles := make(map[string]string) |
|||
|
|||
// Copy .dat file
|
|||
datFile := fmt.Sprintf("%s.dat", filepath.Join(workDir, fmt.Sprintf("%d", t.volumeID))) |
|||
err := t.copyFileFromSource(".dat", datFile) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("failed to copy .dat file: %v", err) |
|||
} |
|||
localFiles["dat"] = datFile |
|||
glog.V(1).Infof("Copied .dat file to: %s", datFile) |
|||
|
|||
// Copy .idx file
|
|||
idxFile := fmt.Sprintf("%s.idx", filepath.Join(workDir, fmt.Sprintf("%d", t.volumeID))) |
|||
err = t.copyFileFromSource(".idx", idxFile) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("failed to copy .idx file: %v", err) |
|||
} |
|||
localFiles["idx"] = idxFile |
|||
glog.V(1).Infof("Copied .idx file to: %s", idxFile) |
|||
|
|||
return localFiles, nil |
|||
} |
|||
|
|||
// copyFileFromSource copies a file from source server to local path using gRPC streaming
|
|||
func (t *Task) copyFileFromSource(ext, localPath string) error { |
|||
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.sourceServer), t.grpcDialOpt, |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
stream, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: t.volumeID, |
|||
Collection: t.collection, |
|||
Ext: ext, |
|||
StopOffset: uint64(math.MaxInt64), |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to initiate file copy: %v", err) |
|||
} |
|||
|
|||
// Create local file
|
|||
localFile, err := os.Create(localPath) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to create local file %s: %v", localPath, err) |
|||
} |
|||
defer localFile.Close() |
|||
|
|||
// Stream data and write to local file
|
|||
totalBytes := int64(0) |
|||
for { |
|||
resp, err := stream.Recv() |
|||
if err == io.EOF { |
|||
break |
|||
} |
|||
if err != nil { |
|||
return fmt.Errorf("failed to receive file data: %v", err) |
|||
} |
|||
|
|||
if len(resp.FileContent) > 0 { |
|||
written, writeErr := localFile.Write(resp.FileContent) |
|||
if writeErr != nil { |
|||
return fmt.Errorf("failed to write to local file: %v", writeErr) |
|||
} |
|||
totalBytes += int64(written) |
|||
} |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully copied %s (%d bytes) from %s to %s", ext, totalBytes, t.sourceServer, localPath) |
|||
return nil |
|||
}) |
|||
} |
|||
|
|||
// generateEcShardsLocally generates EC shards from local volume files
|
|||
func (t *Task) generateEcShardsLocally(localFiles map[string]string, workDir string) (map[string]string, error) { |
|||
datFile := localFiles["dat"] |
|||
idxFile := localFiles["idx"] |
|||
|
|||
if datFile == "" || idxFile == "" { |
|||
return nil, fmt.Errorf("missing required volume files: dat=%s, idx=%s", datFile, idxFile) |
|||
} |
|||
|
|||
// Get base name without extension for EC operations
|
|||
baseName := strings.TrimSuffix(datFile, ".dat") |
|||
|
|||
shardFiles := make(map[string]string) |
|||
|
|||
glog.V(1).Infof("Generating EC shards from local files: dat=%s, idx=%s", datFile, idxFile) |
|||
|
|||
// Generate EC shard files (.ec00 ~ .ec13)
|
|||
if err := erasure_coding.WriteEcFiles(baseName); err != nil { |
|||
return nil, fmt.Errorf("failed to generate EC shard files: %v", err) |
|||
} |
|||
|
|||
// Generate .ecx file from .idx
|
|||
if err := erasure_coding.WriteSortedFileFromIdx(idxFile, ".ecx"); err != nil { |
|||
return nil, fmt.Errorf("failed to generate .ecx file: %v", err) |
|||
} |
|||
|
|||
// Collect generated shard file paths
|
|||
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
|||
shardFile := fmt.Sprintf("%s.ec%02d", baseName, i) |
|||
if _, err := os.Stat(shardFile); err == nil { |
|||
shardFiles[fmt.Sprintf("ec%02d", i)] = shardFile |
|||
} |
|||
} |
|||
|
|||
// Add metadata files
|
|||
ecxFile := idxFile + ".ecx" |
|||
if _, err := os.Stat(ecxFile); err == nil { |
|||
shardFiles["ecx"] = ecxFile |
|||
} |
|||
|
|||
// Generate .vif file (volume info)
|
|||
vifFile := baseName + ".vif" |
|||
// Create basic volume info - in a real implementation, this would come from the original volume
|
|||
volumeInfo := &volume_server_pb.VolumeInfo{ |
|||
Version: uint32(needle.GetCurrentVersion()), |
|||
} |
|||
if err := volume_info.SaveVolumeInfo(vifFile, volumeInfo); err != nil { |
|||
glog.Warningf("Failed to create .vif file: %v", err) |
|||
} else { |
|||
shardFiles["vif"] = vifFile |
|||
} |
|||
|
|||
glog.V(1).Infof("Generated %d EC files locally", len(shardFiles)) |
|||
return shardFiles, nil |
|||
} |
|||
|
|||
func (t *Task) copyEcShardsToDestinations() error { |
|||
if len(t.destinations) == 0 { |
|||
return fmt.Errorf("no destinations specified for EC shard distribution") |
|||
} |
|||
|
|||
destinations := t.destinations |
|||
|
|||
glog.V(1).Infof("Copying EC shards for volume %d to %d destinations", t.volumeID, len(destinations)) |
|||
|
|||
// Prepare shard IDs (0-13 for EC shards)
|
|||
var shardIds []uint32 |
|||
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
|||
shardIds = append(shardIds, uint32(i)) |
|||
} |
|||
|
|||
// Distribute shards across destinations
|
|||
var wg sync.WaitGroup |
|||
errorChan := make(chan error, len(destinations)) |
|||
|
|||
// Track which disks have already received metadata files (server+disk)
|
|||
metadataFilesCopied := make(map[string]bool) |
|||
var metadataMutex sync.Mutex |
|||
|
|||
// For each destination, copy a subset of shards
|
|||
shardsPerDest := len(shardIds) / len(destinations) |
|||
remainder := len(shardIds) % len(destinations) |
|||
|
|||
shardOffset := 0 |
|||
for i, dest := range destinations { |
|||
wg.Add(1) |
|||
|
|||
shardsForThisDest := shardsPerDest |
|||
if i < remainder { |
|||
shardsForThisDest++ // Distribute remainder shards
|
|||
} |
|||
|
|||
destShardIds := shardIds[shardOffset : shardOffset+shardsForThisDest] |
|||
shardOffset += shardsForThisDest |
|||
|
|||
go func(destination *worker_pb.ECDestination, targetShardIds []uint32) { |
|||
defer wg.Done() |
|||
|
|||
if t.IsCancelled() { |
|||
errorChan <- fmt.Errorf("task cancelled during shard copy") |
|||
return |
|||
} |
|||
|
|||
// Create disk-specific metadata key (server+disk)
|
|||
diskKey := fmt.Sprintf("%s:%d", destination.Node, destination.DiskId) |
|||
|
|||
glog.V(1).Infof("Copying shards %v from %s to %s (disk %d)", |
|||
targetShardIds, t.sourceServer, destination.Node, destination.DiskId) |
|||
|
|||
// Check if this disk needs metadata files (only once per disk)
|
|||
metadataMutex.Lock() |
|||
needsMetadataFiles := !metadataFilesCopied[diskKey] |
|||
if needsMetadataFiles { |
|||
metadataFilesCopied[diskKey] = true |
|||
} |
|||
metadataMutex.Unlock() |
|||
|
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(destination.Node), t.grpcDialOpt, |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, copyErr := client.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ |
|||
VolumeId: uint32(t.volumeID), |
|||
Collection: t.collection, |
|||
ShardIds: targetShardIds, |
|||
CopyEcxFile: needsMetadataFiles, // Copy .ecx only once per disk
|
|||
CopyEcjFile: needsMetadataFiles, // Copy .ecj only once per disk
|
|||
CopyVifFile: needsMetadataFiles, // Copy .vif only once per disk
|
|||
SourceDataNode: t.sourceServer, |
|||
DiskId: destination.DiskId, // Pass target disk ID
|
|||
}) |
|||
return copyErr |
|||
}) |
|||
|
|||
if err != nil { |
|||
errorChan <- fmt.Errorf("failed to copy shards to %s disk %d: %v", destination.Node, destination.DiskId, err) |
|||
return |
|||
} |
|||
|
|||
if needsMetadataFiles { |
|||
glog.V(1).Infof("Successfully copied shards %v and metadata files (.ecx, .ecj, .vif) to %s disk %d", |
|||
targetShardIds, destination.Node, destination.DiskId) |
|||
} else { |
|||
glog.V(1).Infof("Successfully copied shards %v to %s disk %d (metadata files already present)", |
|||
targetShardIds, destination.Node, destination.DiskId) |
|||
} |
|||
}(dest, destShardIds) |
|||
} |
|||
|
|||
wg.Wait() |
|||
close(errorChan) |
|||
|
|||
// Check for any copy errors
|
|||
if err := <-errorChan; err != nil { |
|||
return err |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully copied all EC shards for volume %d", t.volumeID) |
|||
return nil |
|||
} |
|||
|
|||
// distributeEcShardsFromWorker distributes locally generated EC shards to destination servers
|
|||
func (t *Task) distributeEcShardsFromWorker(localShardFiles map[string]string) error { |
|||
if len(t.destinations) == 0 { |
|||
return fmt.Errorf("no destinations specified for EC shard distribution") |
|||
} |
|||
|
|||
destinations := t.destinations |
|||
|
|||
glog.V(1).Infof("Distributing EC shards for volume %d from worker to %d destinations", t.volumeID, len(destinations)) |
|||
|
|||
// Prepare shard IDs (0-13 for EC shards)
|
|||
var shardIds []uint32 |
|||
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
|||
shardIds = append(shardIds, uint32(i)) |
|||
} |
|||
|
|||
// Distribute shards across destinations
|
|||
var wg sync.WaitGroup |
|||
errorChan := make(chan error, len(destinations)) |
|||
|
|||
// Track which disks have already received metadata files (server+disk)
|
|||
metadataFilesCopied := make(map[string]bool) |
|||
var metadataMutex sync.Mutex |
|||
|
|||
// For each destination, send a subset of shards
|
|||
shardsPerDest := len(shardIds) / len(destinations) |
|||
remainder := len(shardIds) % len(destinations) |
|||
|
|||
shardOffset := 0 |
|||
for i, dest := range destinations { |
|||
wg.Add(1) |
|||
|
|||
shardsForThisDest := shardsPerDest |
|||
if i < remainder { |
|||
shardsForThisDest++ // Distribute remainder shards
|
|||
} |
|||
|
|||
destShardIds := shardIds[shardOffset : shardOffset+shardsForThisDest] |
|||
shardOffset += shardsForThisDest |
|||
|
|||
go func(destination *worker_pb.ECDestination, targetShardIds []uint32) { |
|||
defer wg.Done() |
|||
|
|||
if t.IsCancelled() { |
|||
errorChan <- fmt.Errorf("task cancelled during shard distribution") |
|||
return |
|||
} |
|||
|
|||
// Create disk-specific metadata key (server+disk)
|
|||
diskKey := fmt.Sprintf("%s:%d", destination.Node, destination.DiskId) |
|||
|
|||
glog.V(1).Infof("Distributing shards %v from worker to %s (disk %d)", |
|||
targetShardIds, destination.Node, destination.DiskId) |
|||
|
|||
// Check if this disk needs metadata files (only once per disk)
|
|||
metadataMutex.Lock() |
|||
needsMetadataFiles := !metadataFilesCopied[diskKey] |
|||
if needsMetadataFiles { |
|||
metadataFilesCopied[diskKey] = true |
|||
} |
|||
metadataMutex.Unlock() |
|||
|
|||
// Send shard files to destination using HTTP upload (simplified for now)
|
|||
err := t.sendShardsToDestination(destination, targetShardIds, localShardFiles, needsMetadataFiles) |
|||
if err != nil { |
|||
errorChan <- fmt.Errorf("failed to send shards to %s disk %d: %v", destination.Node, destination.DiskId, err) |
|||
return |
|||
} |
|||
|
|||
if needsMetadataFiles { |
|||
glog.V(1).Infof("Successfully distributed shards %v and metadata files (.ecx, .vif) to %s disk %d", |
|||
targetShardIds, destination.Node, destination.DiskId) |
|||
} else { |
|||
glog.V(1).Infof("Successfully distributed shards %v to %s disk %d (metadata files already present)", |
|||
targetShardIds, destination.Node, destination.DiskId) |
|||
} |
|||
}(dest, destShardIds) |
|||
} |
|||
|
|||
wg.Wait() |
|||
close(errorChan) |
|||
|
|||
// Check for any distribution errors
|
|||
if err := <-errorChan; err != nil { |
|||
return err |
|||
} |
|||
|
|||
glog.V(1).Infof("Completed distributing EC shards for volume %d", t.volumeID) |
|||
return nil |
|||
} |
|||
|
|||
// sendShardsToDestination sends specific shard files from worker to a destination server (simplified)
|
|||
func (t *Task) sendShardsToDestination(destination *worker_pb.ECDestination, shardIds []uint32, localFiles map[string]string, includeMetadata bool) error { |
|||
// For now, use a simplified approach - just upload the files
|
|||
// In a full implementation, this would use proper file upload mechanisms
|
|||
glog.V(2).Infof("Would send shards %v and metadata=%v to %s disk %d", shardIds, includeMetadata, destination.Node, destination.DiskId) |
|||
|
|||
// TODO: Implement actual file upload to volume server
|
|||
// This is a placeholder - actual implementation would:
|
|||
// 1. Open each shard file locally
|
|||
// 2. Upload via HTTP POST or gRPC stream to destination volume server
|
|||
// 3. Volume server would save to the specified disk_id
|
|||
|
|||
return nil |
|||
} |
|||
|
|||
// mountEcShardsOnDestinations mounts EC shards on all destination servers
|
|||
func (t *Task) mountEcShardsOnDestinations() error { |
|||
if len(t.destinations) == 0 { |
|||
return fmt.Errorf("no destinations specified for mounting EC shards") |
|||
} |
|||
|
|||
destinations := t.destinations |
|||
|
|||
glog.V(1).Infof("Mounting EC shards for volume %d on %d destinations", t.volumeID, len(destinations)) |
|||
|
|||
// Prepare all shard IDs (0-13)
|
|||
var allShardIds []uint32 |
|||
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
|||
allShardIds = append(allShardIds, uint32(i)) |
|||
} |
|||
|
|||
var wg sync.WaitGroup |
|||
errorChan := make(chan error, len(destinations)) |
|||
|
|||
// Mount shards on each destination server
|
|||
for _, dest := range destinations { |
|||
wg.Add(1) |
|||
|
|||
go func(destination *worker_pb.ECDestination) { |
|||
defer wg.Done() |
|||
|
|||
if t.IsCancelled() { |
|||
errorChan <- fmt.Errorf("task cancelled during shard mounting") |
|||
return |
|||
} |
|||
|
|||
glog.V(1).Infof("Mounting EC shards on %s disk %d", destination.Node, destination.DiskId) |
|||
|
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(destination.Node), t.grpcDialOpt, |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: uint32(t.volumeID), |
|||
Collection: t.collection, |
|||
ShardIds: allShardIds, // Mount all available shards on each server
|
|||
}) |
|||
return mountErr |
|||
}) |
|||
|
|||
if err != nil { |
|||
// It's normal for some servers to not have all shards, so log as warning rather than error
|
|||
glog.Warningf("Failed to mount some shards on %s disk %d (this may be normal): %v", destination.Node, destination.DiskId, err) |
|||
} else { |
|||
glog.V(1).Infof("Successfully mounted EC shards on %s disk %d", destination.Node, destination.DiskId) |
|||
} |
|||
}(dest) |
|||
} |
|||
|
|||
wg.Wait() |
|||
close(errorChan) |
|||
|
|||
// Check for any critical mounting errors
|
|||
select { |
|||
case err := <-errorChan: |
|||
if err != nil { |
|||
glog.Warningf("Some shard mounting issues occurred: %v", err) |
|||
} |
|||
default: |
|||
// No errors
|
|||
} |
|||
|
|||
glog.V(1).Infof("Completed mounting EC shards for volume %d", t.volumeID) |
|||
return nil |
|||
} |
|||
|
|||
// deleteVolumeFromAllLocations deletes the original volume from all replica servers
|
|||
func (t *Task) deleteVolumeFromAllLocations(volumeId needle.VolumeId, volumeLocations []string) error { |
|||
glog.V(1).Infof("Deleting original volume %d from %d locations", volumeId, len(volumeLocations)) |
|||
|
|||
for _, location := range volumeLocations { |
|||
glog.V(1).Infof("Deleting volume %d from %s", volumeId, location) |
|||
|
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(location), t.grpcDialOpt, |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, deleteErr := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{ |
|||
VolumeId: uint32(volumeId), |
|||
OnlyEmpty: false, // Force delete even if not empty since we've already created EC shards
|
|||
}) |
|||
return deleteErr |
|||
}) |
|||
|
|||
if err != nil { |
|||
glog.Errorf("Failed to delete volume %d from %s: %v", volumeId, location, err) |
|||
return fmt.Errorf("failed to delete volume %d from %s: %v", volumeId, location, err) |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully deleted volume %d from %s", volumeId, location) |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully deleted volume %d from all %d locations", volumeId, len(volumeLocations)) |
|||
return nil |
|||
} |
|||
|
|||
// Register the task in the global registry
|
|||
func init() { |
|||
types.RegisterGlobalTypedTask(types.TaskTypeErasureCoding, NewTask) |
|||
glog.V(1).Infof("Registered EC task") |
|||
} |
|||
@ -0,0 +1,660 @@ |
|||
package erasure_coding |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"io" |
|||
"math" |
|||
"os" |
|||
"path/filepath" |
|||
"sort" |
|||
"strings" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/operation" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/needle" |
|||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types/base" |
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
// ErasureCodingTask implements the Task interface
|
|||
type ErasureCodingTask struct { |
|||
*base.BaseTask |
|||
server string |
|||
volumeID uint32 |
|||
collection string |
|||
workDir string |
|||
progress float64 |
|||
|
|||
// EC parameters
|
|||
dataShards int32 |
|||
parityShards int32 |
|||
destinations []*worker_pb.ECDestination |
|||
shardAssignment map[string][]string // destination -> assigned shard types
|
|||
replicas []string // volume replica servers for deletion
|
|||
} |
|||
|
|||
// NewErasureCodingTask creates a new unified EC task instance
|
|||
func NewErasureCodingTask(id string, server string, volumeID uint32, collection string) *ErasureCodingTask { |
|||
return &ErasureCodingTask{ |
|||
BaseTask: base.NewBaseTask(id, types.TaskTypeErasureCoding), |
|||
server: server, |
|||
volumeID: volumeID, |
|||
collection: collection, |
|||
dataShards: erasure_coding.DataShardsCount, // Default values
|
|||
parityShards: erasure_coding.ParityShardsCount, // Default values
|
|||
} |
|||
} |
|||
|
|||
// Execute implements the UnifiedTask interface
|
|||
func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error { |
|||
if params == nil { |
|||
return fmt.Errorf("task parameters are required") |
|||
} |
|||
|
|||
ecParams := params.GetErasureCodingParams() |
|||
if ecParams == nil { |
|||
return fmt.Errorf("erasure coding parameters are required") |
|||
} |
|||
|
|||
t.dataShards = ecParams.DataShards |
|||
t.parityShards = ecParams.ParityShards |
|||
t.workDir = ecParams.WorkingDir |
|||
t.destinations = ecParams.Destinations |
|||
t.replicas = params.Replicas // Get replicas from task parameters
|
|||
|
|||
t.GetLogger().WithFields(map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"server": t.server, |
|||
"collection": t.collection, |
|||
"data_shards": t.dataShards, |
|||
"parity_shards": t.parityShards, |
|||
"destinations": len(t.destinations), |
|||
}).Info("Starting erasure coding task") |
|||
|
|||
// Use the working directory from task parameters, or fall back to a default
|
|||
baseWorkDir := t.workDir |
|||
if baseWorkDir == "" { |
|||
baseWorkDir = "/tmp/seaweedfs_ec_work" |
|||
} |
|||
|
|||
// Create unique working directory for this task
|
|||
taskWorkDir := filepath.Join(baseWorkDir, fmt.Sprintf("vol_%d_%d", t.volumeID, time.Now().Unix())) |
|||
if err := os.MkdirAll(taskWorkDir, 0755); err != nil { |
|||
return fmt.Errorf("failed to create task working directory %s: %v", taskWorkDir, err) |
|||
} |
|||
glog.V(1).Infof("Created working directory: %s", taskWorkDir) |
|||
|
|||
// Update the task's working directory to the specific instance directory
|
|||
t.workDir = taskWorkDir |
|||
glog.V(1).Infof("Task working directory configured: %s (logs will be written here)", taskWorkDir) |
|||
|
|||
// Ensure cleanup of working directory (but preserve logs)
|
|||
defer func() { |
|||
// Clean up volume files and EC shards, but preserve the directory structure and any logs
|
|||
patterns := []string{"*.dat", "*.idx", "*.ec*", "*.vif"} |
|||
for _, pattern := range patterns { |
|||
matches, err := filepath.Glob(filepath.Join(taskWorkDir, pattern)) |
|||
if err != nil { |
|||
continue |
|||
} |
|||
for _, match := range matches { |
|||
if err := os.Remove(match); err != nil { |
|||
glog.V(2).Infof("Could not remove %s: %v", match, err) |
|||
} |
|||
} |
|||
} |
|||
glog.V(1).Infof("Cleaned up volume files from working directory: %s (logs preserved)", taskWorkDir) |
|||
}() |
|||
|
|||
// Step 1: Mark volume readonly
|
|||
t.ReportProgress(10.0) |
|||
t.GetLogger().Info("Marking volume readonly") |
|||
if err := t.markVolumeReadonly(); err != nil { |
|||
return fmt.Errorf("failed to mark volume readonly: %v", err) |
|||
} |
|||
|
|||
// Step 2: Copy volume files to worker
|
|||
t.ReportProgress(25.0) |
|||
t.GetLogger().Info("Copying volume files to worker") |
|||
localFiles, err := t.copyVolumeFilesToWorker(taskWorkDir) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to copy volume files: %v", err) |
|||
} |
|||
|
|||
// Step 3: Generate EC shards locally
|
|||
t.ReportProgress(40.0) |
|||
t.GetLogger().Info("Generating EC shards locally") |
|||
shardFiles, err := t.generateEcShardsLocally(localFiles, taskWorkDir) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to generate EC shards: %v", err) |
|||
} |
|||
|
|||
// Step 4: Distribute shards to destinations
|
|||
t.ReportProgress(60.0) |
|||
t.GetLogger().Info("Distributing EC shards to destinations") |
|||
if err := t.distributeEcShards(shardFiles); err != nil { |
|||
return fmt.Errorf("failed to distribute EC shards: %v", err) |
|||
} |
|||
|
|||
// Step 5: Mount EC shards
|
|||
t.ReportProgress(80.0) |
|||
t.GetLogger().Info("Mounting EC shards") |
|||
if err := t.mountEcShards(); err != nil { |
|||
return fmt.Errorf("failed to mount EC shards: %v", err) |
|||
} |
|||
|
|||
// Step 6: Delete original volume
|
|||
t.ReportProgress(90.0) |
|||
t.GetLogger().Info("Deleting original volume") |
|||
if err := t.deleteOriginalVolume(); err != nil { |
|||
return fmt.Errorf("failed to delete original volume: %v", err) |
|||
} |
|||
|
|||
t.ReportProgress(100.0) |
|||
glog.Infof("EC task completed successfully: volume %d from %s with %d shards distributed", |
|||
t.volumeID, t.server, len(shardFiles)) |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// Validate implements the UnifiedTask interface
|
|||
func (t *ErasureCodingTask) Validate(params *worker_pb.TaskParams) error { |
|||
if params == nil { |
|||
return fmt.Errorf("task parameters are required") |
|||
} |
|||
|
|||
ecParams := params.GetErasureCodingParams() |
|||
if ecParams == nil { |
|||
return fmt.Errorf("erasure coding parameters are required") |
|||
} |
|||
|
|||
if params.VolumeId != t.volumeID { |
|||
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId) |
|||
} |
|||
|
|||
if params.Server != t.server { |
|||
return fmt.Errorf("source server mismatch: expected %s, got %s", t.server, params.Server) |
|||
} |
|||
|
|||
if ecParams.DataShards < 1 { |
|||
return fmt.Errorf("invalid data shards: %d (must be >= 1)", ecParams.DataShards) |
|||
} |
|||
|
|||
if ecParams.ParityShards < 1 { |
|||
return fmt.Errorf("invalid parity shards: %d (must be >= 1)", ecParams.ParityShards) |
|||
} |
|||
|
|||
if len(ecParams.Destinations) < int(ecParams.DataShards+ecParams.ParityShards) { |
|||
return fmt.Errorf("insufficient destinations: got %d, need %d", len(ecParams.Destinations), ecParams.DataShards+ecParams.ParityShards) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// EstimateTime implements the UnifiedTask interface
|
|||
func (t *ErasureCodingTask) EstimateTime(params *worker_pb.TaskParams) time.Duration { |
|||
// Basic estimate based on simulated steps
|
|||
return 20 * time.Second // Sum of all step durations
|
|||
} |
|||
|
|||
// GetProgress returns current progress
|
|||
func (t *ErasureCodingTask) GetProgress() float64 { |
|||
return t.progress |
|||
} |
|||
|
|||
// Helper methods for actual EC operations
|
|||
|
|||
// markVolumeReadonly marks the volume as readonly on the source server
|
|||
func (t *ErasureCodingTask) markVolumeReadonly() error { |
|||
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, err := client.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
return err |
|||
}) |
|||
} |
|||
|
|||
// copyVolumeFilesToWorker copies .dat and .idx files from source server to local worker
|
|||
func (t *ErasureCodingTask) copyVolumeFilesToWorker(workDir string) (map[string]string, error) { |
|||
localFiles := make(map[string]string) |
|||
|
|||
// Copy .dat file
|
|||
datFile := filepath.Join(workDir, fmt.Sprintf("%d.dat", t.volumeID)) |
|||
if err := t.copyFileFromSource(".dat", datFile); err != nil { |
|||
return nil, fmt.Errorf("failed to copy .dat file: %v", err) |
|||
} |
|||
localFiles["dat"] = datFile |
|||
|
|||
// Copy .idx file
|
|||
idxFile := filepath.Join(workDir, fmt.Sprintf("%d.idx", t.volumeID)) |
|||
if err := t.copyFileFromSource(".idx", idxFile); err != nil { |
|||
return nil, fmt.Errorf("failed to copy .idx file: %v", err) |
|||
} |
|||
localFiles["idx"] = idxFile |
|||
|
|||
return localFiles, nil |
|||
} |
|||
|
|||
// copyFileFromSource copies a file from source server to local path using gRPC streaming
|
|||
func (t *ErasureCodingTask) copyFileFromSource(ext, localPath string) error { |
|||
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
stream, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{ |
|||
VolumeId: t.volumeID, |
|||
Collection: t.collection, |
|||
Ext: ext, |
|||
StopOffset: uint64(math.MaxInt64), |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to initiate file copy: %v", err) |
|||
} |
|||
|
|||
// Create local file
|
|||
localFile, err := os.Create(localPath) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to create local file %s: %v", localPath, err) |
|||
} |
|||
defer localFile.Close() |
|||
|
|||
// Stream data and write to local file
|
|||
totalBytes := int64(0) |
|||
for { |
|||
resp, err := stream.Recv() |
|||
if err == io.EOF { |
|||
break |
|||
} |
|||
if err != nil { |
|||
return fmt.Errorf("failed to receive file data: %v", err) |
|||
} |
|||
|
|||
if len(resp.FileContent) > 0 { |
|||
written, writeErr := localFile.Write(resp.FileContent) |
|||
if writeErr != nil { |
|||
return fmt.Errorf("failed to write to local file: %v", writeErr) |
|||
} |
|||
totalBytes += int64(written) |
|||
} |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully copied %s (%d bytes) from %s to %s", ext, totalBytes, t.server, localPath) |
|||
return nil |
|||
}) |
|||
} |
|||
|
|||
// generateEcShardsLocally generates EC shards from local volume files
|
|||
func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string, workDir string) (map[string]string, error) { |
|||
datFile := localFiles["dat"] |
|||
idxFile := localFiles["idx"] |
|||
|
|||
if datFile == "" || idxFile == "" { |
|||
return nil, fmt.Errorf("missing required volume files: dat=%s, idx=%s", datFile, idxFile) |
|||
} |
|||
|
|||
// Get base name without extension for EC operations
|
|||
baseName := strings.TrimSuffix(datFile, ".dat") |
|||
shardFiles := make(map[string]string) |
|||
|
|||
glog.V(1).Infof("Generating EC shards from local files: dat=%s, idx=%s", datFile, idxFile) |
|||
|
|||
// Generate EC shard files (.ec00 ~ .ec13)
|
|||
if err := erasure_coding.WriteEcFiles(baseName); err != nil { |
|||
return nil, fmt.Errorf("failed to generate EC shard files: %v", err) |
|||
} |
|||
|
|||
// Generate .ecx file from .idx (use baseName, not full idx path)
|
|||
if err := erasure_coding.WriteSortedFileFromIdx(baseName, ".ecx"); err != nil { |
|||
return nil, fmt.Errorf("failed to generate .ecx file: %v", err) |
|||
} |
|||
|
|||
// Collect generated shard file paths
|
|||
for i := 0; i < erasure_coding.TotalShardsCount; i++ { |
|||
shardFile := fmt.Sprintf("%s.ec%02d", baseName, i) |
|||
if _, err := os.Stat(shardFile); err == nil { |
|||
shardFiles[fmt.Sprintf("ec%02d", i)] = shardFile |
|||
} |
|||
} |
|||
|
|||
// Add metadata files
|
|||
ecxFile := baseName + ".ecx" |
|||
if _, err := os.Stat(ecxFile); err == nil { |
|||
shardFiles["ecx"] = ecxFile |
|||
} |
|||
|
|||
// Generate .vif file (volume info)
|
|||
vifFile := baseName + ".vif" |
|||
volumeInfo := &volume_server_pb.VolumeInfo{ |
|||
Version: uint32(needle.GetCurrentVersion()), |
|||
} |
|||
if err := volume_info.SaveVolumeInfo(vifFile, volumeInfo); err != nil { |
|||
glog.Warningf("Failed to create .vif file: %v", err) |
|||
} else { |
|||
shardFiles["vif"] = vifFile |
|||
} |
|||
|
|||
glog.V(1).Infof("Generated %d EC files locally", len(shardFiles)) |
|||
return shardFiles, nil |
|||
} |
|||
|
|||
// distributeEcShards distributes locally generated EC shards to destination servers
|
|||
func (t *ErasureCodingTask) distributeEcShards(shardFiles map[string]string) error { |
|||
if len(t.destinations) == 0 { |
|||
return fmt.Errorf("no destinations specified for EC shard distribution") |
|||
} |
|||
|
|||
if len(shardFiles) == 0 { |
|||
return fmt.Errorf("no shard files available for distribution") |
|||
} |
|||
|
|||
// Create shard assignment: assign specific shards to specific destinations
|
|||
shardAssignment := t.createShardAssignment(shardFiles) |
|||
if len(shardAssignment) == 0 { |
|||
return fmt.Errorf("failed to create shard assignment") |
|||
} |
|||
|
|||
// Store assignment for use during mounting
|
|||
t.shardAssignment = shardAssignment |
|||
|
|||
// Send assigned shards to each destination
|
|||
for destNode, assignedShards := range shardAssignment { |
|||
t.GetLogger().WithFields(map[string]interface{}{ |
|||
"destination": destNode, |
|||
"assigned_shards": len(assignedShards), |
|||
"shard_ids": assignedShards, |
|||
}).Info("Distributing assigned EC shards to destination") |
|||
|
|||
// Send only the assigned shards to this destination
|
|||
for _, shardType := range assignedShards { |
|||
filePath, exists := shardFiles[shardType] |
|||
if !exists { |
|||
return fmt.Errorf("shard file %s not found for destination %s", shardType, destNode) |
|||
} |
|||
|
|||
if err := t.sendShardFileToDestination(destNode, filePath, shardType); err != nil { |
|||
return fmt.Errorf("failed to send %s to %s: %v", shardType, destNode, err) |
|||
} |
|||
} |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully distributed EC shards to %d destinations", len(shardAssignment)) |
|||
return nil |
|||
} |
|||
|
|||
// createShardAssignment assigns specific EC shards to specific destination servers
|
|||
// Each destination gets a subset of shards based on availability and placement rules
|
|||
func (t *ErasureCodingTask) createShardAssignment(shardFiles map[string]string) map[string][]string { |
|||
assignment := make(map[string][]string) |
|||
|
|||
// Collect all available EC shards (ec00-ec13)
|
|||
var availableShards []string |
|||
for shardType := range shardFiles { |
|||
if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 { |
|||
availableShards = append(availableShards, shardType) |
|||
} |
|||
} |
|||
|
|||
// Sort shards for consistent assignment
|
|||
sort.Strings(availableShards) |
|||
|
|||
if len(availableShards) == 0 { |
|||
glog.Warningf("No EC shards found for assignment") |
|||
return assignment |
|||
} |
|||
|
|||
// Calculate shards per destination
|
|||
numDestinations := len(t.destinations) |
|||
if numDestinations == 0 { |
|||
return assignment |
|||
} |
|||
|
|||
// Strategy: Distribute shards as evenly as possible across destinations
|
|||
// With 14 shards and N destinations, some destinations get ⌈14/N⌉ shards, others get ⌊14/N⌋
|
|||
shardsPerDest := len(availableShards) / numDestinations |
|||
extraShards := len(availableShards) % numDestinations |
|||
|
|||
shardIndex := 0 |
|||
for i, dest := range t.destinations { |
|||
var destShards []string |
|||
|
|||
// Assign base number of shards
|
|||
shardsToAssign := shardsPerDest |
|||
|
|||
// Assign one extra shard to first 'extraShards' destinations
|
|||
if i < extraShards { |
|||
shardsToAssign++ |
|||
} |
|||
|
|||
// Assign the shards
|
|||
for j := 0; j < shardsToAssign && shardIndex < len(availableShards); j++ { |
|||
destShards = append(destShards, availableShards[shardIndex]) |
|||
shardIndex++ |
|||
} |
|||
|
|||
assignment[dest.Node] = destShards |
|||
|
|||
glog.V(2).Infof("Assigned shards %v to destination %s", destShards, dest.Node) |
|||
} |
|||
|
|||
// Assign metadata files (.ecx, .vif) to each destination that has shards
|
|||
// Note: .ecj files are created during mount, not during initial generation
|
|||
for destNode, destShards := range assignment { |
|||
if len(destShards) > 0 { |
|||
// Add .ecx file if available
|
|||
if _, hasEcx := shardFiles["ecx"]; hasEcx { |
|||
assignment[destNode] = append(assignment[destNode], "ecx") |
|||
} |
|||
|
|||
// Add .vif file if available
|
|||
if _, hasVif := shardFiles["vif"]; hasVif { |
|||
assignment[destNode] = append(assignment[destNode], "vif") |
|||
} |
|||
|
|||
glog.V(2).Infof("Assigned metadata files (.ecx, .vif) to destination %s", destNode) |
|||
} |
|||
} |
|||
|
|||
return assignment |
|||
} |
|||
|
|||
// sendShardFileToDestination sends a single shard file to a destination server using ReceiveFile API
|
|||
func (t *ErasureCodingTask) sendShardFileToDestination(destServer, filePath, shardType string) error { |
|||
return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
// Open the local shard file
|
|||
file, err := os.Open(filePath) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to open shard file %s: %v", filePath, err) |
|||
} |
|||
defer file.Close() |
|||
|
|||
// Get file size
|
|||
fileInfo, err := file.Stat() |
|||
if err != nil { |
|||
return fmt.Errorf("failed to get file info for %s: %v", filePath, err) |
|||
} |
|||
|
|||
// Determine file extension and shard ID
|
|||
var ext string |
|||
var shardId uint32 |
|||
if shardType == "ecx" { |
|||
ext = ".ecx" |
|||
shardId = 0 // ecx file doesn't have a specific shard ID
|
|||
} else if shardType == "vif" { |
|||
ext = ".vif" |
|||
shardId = 0 // vif file doesn't have a specific shard ID
|
|||
} else if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 { |
|||
// EC shard file like "ec00", "ec01", etc.
|
|||
ext = "." + shardType |
|||
fmt.Sscanf(shardType[2:], "%d", &shardId) |
|||
} else { |
|||
return fmt.Errorf("unknown shard type: %s", shardType) |
|||
} |
|||
|
|||
// Create streaming client
|
|||
stream, err := client.ReceiveFile(context.Background()) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to create receive stream: %v", err) |
|||
} |
|||
|
|||
// Send file info first
|
|||
err = stream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_Info{ |
|||
Info: &volume_server_pb.ReceiveFileInfo{ |
|||
VolumeId: t.volumeID, |
|||
Ext: ext, |
|||
Collection: t.collection, |
|||
IsEcVolume: true, |
|||
ShardId: shardId, |
|||
FileSize: uint64(fileInfo.Size()), |
|||
}, |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to send file info: %v", err) |
|||
} |
|||
|
|||
// Send file content in chunks
|
|||
buffer := make([]byte, 64*1024) // 64KB chunks
|
|||
for { |
|||
n, readErr := file.Read(buffer) |
|||
if n > 0 { |
|||
err = stream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_FileContent{ |
|||
FileContent: buffer[:n], |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to send file content: %v", err) |
|||
} |
|||
} |
|||
if readErr == io.EOF { |
|||
break |
|||
} |
|||
if readErr != nil { |
|||
return fmt.Errorf("failed to read file: %v", readErr) |
|||
} |
|||
} |
|||
|
|||
// Close stream and get response
|
|||
resp, err := stream.CloseAndRecv() |
|||
if err != nil { |
|||
return fmt.Errorf("failed to close stream: %v", err) |
|||
} |
|||
|
|||
if resp.Error != "" { |
|||
return fmt.Errorf("server error: %s", resp.Error) |
|||
} |
|||
|
|||
glog.V(2).Infof("Successfully sent %s (%d bytes) to %s", shardType, resp.BytesWritten, destServer) |
|||
return nil |
|||
}) |
|||
} |
|||
|
|||
// mountEcShards mounts EC shards on destination servers
|
|||
func (t *ErasureCodingTask) mountEcShards() error { |
|||
if t.shardAssignment == nil { |
|||
return fmt.Errorf("shard assignment not available for mounting") |
|||
} |
|||
|
|||
// Mount only assigned shards on each destination
|
|||
for destNode, assignedShards := range t.shardAssignment { |
|||
// Convert shard names to shard IDs for mounting
|
|||
var shardIds []uint32 |
|||
for _, shardType := range assignedShards { |
|||
// Skip metadata files (.ecx, .vif) - only mount EC shards
|
|||
if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 { |
|||
// Parse shard ID from "ec00", "ec01", etc.
|
|||
var shardId uint32 |
|||
if _, err := fmt.Sscanf(shardType[2:], "%d", &shardId); err == nil { |
|||
shardIds = append(shardIds, shardId) |
|||
} |
|||
} |
|||
} |
|||
|
|||
if len(shardIds) == 0 { |
|||
glog.V(1).Infof("No EC shards to mount on %s (only metadata files)", destNode) |
|||
continue |
|||
} |
|||
|
|||
glog.V(1).Infof("Mounting shards %v on %s", shardIds, destNode) |
|||
|
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: t.volumeID, |
|||
Collection: t.collection, |
|||
ShardIds: shardIds, |
|||
}) |
|||
return mountErr |
|||
}) |
|||
|
|||
if err != nil { |
|||
glog.Warningf("Failed to mount shards %v on %s: %v", shardIds, destNode, err) |
|||
} else { |
|||
glog.V(1).Infof("Successfully mounted EC shards %v on %s", shardIds, destNode) |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// deleteOriginalVolume deletes the original volume and all its replicas from all servers
|
|||
func (t *ErasureCodingTask) deleteOriginalVolume() error { |
|||
// Get replicas from task parameters (set during detection)
|
|||
replicas := t.getReplicas() |
|||
|
|||
if len(replicas) == 0 { |
|||
glog.Warningf("No replicas found for volume %d, falling back to source server only", t.volumeID) |
|||
replicas = []string{t.server} |
|||
} |
|||
|
|||
glog.V(1).Infof("Deleting volume %d from %d replica servers: %v", t.volumeID, len(replicas), replicas) |
|||
|
|||
// Delete volume from all replica locations
|
|||
var deleteErrors []string |
|||
successCount := 0 |
|||
|
|||
for _, replicaServer := range replicas { |
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(replicaServer), grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, err := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{ |
|||
VolumeId: t.volumeID, |
|||
OnlyEmpty: false, // Force delete since we've created EC shards
|
|||
}) |
|||
return err |
|||
}) |
|||
|
|||
if err != nil { |
|||
deleteErrors = append(deleteErrors, fmt.Sprintf("failed to delete volume %d from %s: %v", t.volumeID, replicaServer, err)) |
|||
glog.Warningf("Failed to delete volume %d from replica server %s: %v", t.volumeID, replicaServer, err) |
|||
} else { |
|||
successCount++ |
|||
glog.V(1).Infof("Successfully deleted volume %d from replica server %s", t.volumeID, replicaServer) |
|||
} |
|||
} |
|||
|
|||
// Report results
|
|||
if len(deleteErrors) > 0 { |
|||
glog.Warningf("Some volume deletions failed (%d/%d successful): %v", successCount, len(replicas), deleteErrors) |
|||
// Don't return error - EC task should still be considered successful if shards are mounted
|
|||
} else { |
|||
glog.V(1).Infof("Successfully deleted volume %d from all %d replica servers", t.volumeID, len(replicas)) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// getReplicas extracts replica servers from task parameters
|
|||
func (t *ErasureCodingTask) getReplicas() []string { |
|||
// Access replicas from the parameters passed during Execute
|
|||
// We'll need to store these during Execute - let me add a field to the task
|
|||
return t.replicas |
|||
} |
|||
@ -0,0 +1,229 @@ |
|||
package erasure_coding |
|||
|
|||
import ( |
|||
"sync" |
|||
"time" |
|||
) |
|||
|
|||
// ErasureCodingMetrics contains erasure coding-specific monitoring data
|
|||
type ErasureCodingMetrics struct { |
|||
// Execution metrics
|
|||
VolumesEncoded int64 `json:"volumes_encoded"` |
|||
TotalShardsCreated int64 `json:"total_shards_created"` |
|||
TotalDataProcessed int64 `json:"total_data_processed"` |
|||
TotalSourcesRemoved int64 `json:"total_sources_removed"` |
|||
LastEncodingTime time.Time `json:"last_encoding_time"` |
|||
|
|||
// Performance metrics
|
|||
AverageEncodingTime int64 `json:"average_encoding_time_seconds"` |
|||
AverageShardSize int64 `json:"average_shard_size"` |
|||
AverageDataShards int `json:"average_data_shards"` |
|||
AverageParityShards int `json:"average_parity_shards"` |
|||
SuccessfulOperations int64 `json:"successful_operations"` |
|||
FailedOperations int64 `json:"failed_operations"` |
|||
|
|||
// Distribution metrics
|
|||
ShardsPerDataCenter map[string]int64 `json:"shards_per_datacenter"` |
|||
ShardsPerRack map[string]int64 `json:"shards_per_rack"` |
|||
PlacementSuccessRate float64 `json:"placement_success_rate"` |
|||
|
|||
// Current task metrics
|
|||
CurrentVolumeSize int64 `json:"current_volume_size"` |
|||
CurrentShardCount int `json:"current_shard_count"` |
|||
VolumesPendingEncoding int `json:"volumes_pending_encoding"` |
|||
|
|||
mutex sync.RWMutex |
|||
} |
|||
|
|||
// NewErasureCodingMetrics creates a new erasure coding metrics instance
|
|||
func NewErasureCodingMetrics() *ErasureCodingMetrics { |
|||
return &ErasureCodingMetrics{ |
|||
LastEncodingTime: time.Now(), |
|||
ShardsPerDataCenter: make(map[string]int64), |
|||
ShardsPerRack: make(map[string]int64), |
|||
} |
|||
} |
|||
|
|||
// RecordVolumeEncoded records a successful volume encoding operation
|
|||
func (m *ErasureCodingMetrics) RecordVolumeEncoded(volumeSize int64, shardsCreated int, dataShards int, parityShards int, encodingTime time.Duration, sourceRemoved bool) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.VolumesEncoded++ |
|||
m.TotalShardsCreated += int64(shardsCreated) |
|||
m.TotalDataProcessed += volumeSize |
|||
m.SuccessfulOperations++ |
|||
m.LastEncodingTime = time.Now() |
|||
|
|||
if sourceRemoved { |
|||
m.TotalSourcesRemoved++ |
|||
} |
|||
|
|||
// Update average encoding time
|
|||
if m.AverageEncodingTime == 0 { |
|||
m.AverageEncodingTime = int64(encodingTime.Seconds()) |
|||
} else { |
|||
// Exponential moving average
|
|||
newTime := int64(encodingTime.Seconds()) |
|||
m.AverageEncodingTime = (m.AverageEncodingTime*4 + newTime) / 5 |
|||
} |
|||
|
|||
// Update average shard size
|
|||
if shardsCreated > 0 { |
|||
avgShardSize := volumeSize / int64(shardsCreated) |
|||
if m.AverageShardSize == 0 { |
|||
m.AverageShardSize = avgShardSize |
|||
} else { |
|||
m.AverageShardSize = (m.AverageShardSize*4 + avgShardSize) / 5 |
|||
} |
|||
} |
|||
|
|||
// Update average data/parity shards
|
|||
if m.AverageDataShards == 0 { |
|||
m.AverageDataShards = dataShards |
|||
m.AverageParityShards = parityShards |
|||
} else { |
|||
m.AverageDataShards = (m.AverageDataShards*4 + dataShards) / 5 |
|||
m.AverageParityShards = (m.AverageParityShards*4 + parityShards) / 5 |
|||
} |
|||
} |
|||
|
|||
// RecordFailure records a failed erasure coding operation
|
|||
func (m *ErasureCodingMetrics) RecordFailure() { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.FailedOperations++ |
|||
} |
|||
|
|||
// RecordShardPlacement records shard placement for distribution tracking
|
|||
func (m *ErasureCodingMetrics) RecordShardPlacement(dataCenter string, rack string) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.ShardsPerDataCenter[dataCenter]++ |
|||
rackKey := dataCenter + ":" + rack |
|||
m.ShardsPerRack[rackKey]++ |
|||
} |
|||
|
|||
// UpdateCurrentVolumeInfo updates current volume processing information
|
|||
func (m *ErasureCodingMetrics) UpdateCurrentVolumeInfo(volumeSize int64, shardCount int) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.CurrentVolumeSize = volumeSize |
|||
m.CurrentShardCount = shardCount |
|||
} |
|||
|
|||
// SetVolumesPendingEncoding sets the number of volumes pending encoding
|
|||
func (m *ErasureCodingMetrics) SetVolumesPendingEncoding(count int) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.VolumesPendingEncoding = count |
|||
} |
|||
|
|||
// UpdatePlacementSuccessRate updates the placement success rate
|
|||
func (m *ErasureCodingMetrics) UpdatePlacementSuccessRate(rate float64) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
if m.PlacementSuccessRate == 0 { |
|||
m.PlacementSuccessRate = rate |
|||
} else { |
|||
// Exponential moving average
|
|||
m.PlacementSuccessRate = 0.8*m.PlacementSuccessRate + 0.2*rate |
|||
} |
|||
} |
|||
|
|||
// GetMetrics returns a copy of the current metrics (without the mutex)
|
|||
func (m *ErasureCodingMetrics) GetMetrics() ErasureCodingMetrics { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
// Create deep copy of maps
|
|||
shardsPerDC := make(map[string]int64) |
|||
for k, v := range m.ShardsPerDataCenter { |
|||
shardsPerDC[k] = v |
|||
} |
|||
|
|||
shardsPerRack := make(map[string]int64) |
|||
for k, v := range m.ShardsPerRack { |
|||
shardsPerRack[k] = v |
|||
} |
|||
|
|||
// Create a copy without the mutex to avoid copying lock value
|
|||
return ErasureCodingMetrics{ |
|||
VolumesEncoded: m.VolumesEncoded, |
|||
TotalShardsCreated: m.TotalShardsCreated, |
|||
TotalDataProcessed: m.TotalDataProcessed, |
|||
TotalSourcesRemoved: m.TotalSourcesRemoved, |
|||
LastEncodingTime: m.LastEncodingTime, |
|||
AverageEncodingTime: m.AverageEncodingTime, |
|||
AverageShardSize: m.AverageShardSize, |
|||
AverageDataShards: m.AverageDataShards, |
|||
AverageParityShards: m.AverageParityShards, |
|||
SuccessfulOperations: m.SuccessfulOperations, |
|||
FailedOperations: m.FailedOperations, |
|||
ShardsPerDataCenter: shardsPerDC, |
|||
ShardsPerRack: shardsPerRack, |
|||
PlacementSuccessRate: m.PlacementSuccessRate, |
|||
CurrentVolumeSize: m.CurrentVolumeSize, |
|||
CurrentShardCount: m.CurrentShardCount, |
|||
VolumesPendingEncoding: m.VolumesPendingEncoding, |
|||
} |
|||
} |
|||
|
|||
// GetSuccessRate returns the success rate as a percentage
|
|||
func (m *ErasureCodingMetrics) GetSuccessRate() float64 { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
total := m.SuccessfulOperations + m.FailedOperations |
|||
if total == 0 { |
|||
return 100.0 |
|||
} |
|||
return float64(m.SuccessfulOperations) / float64(total) * 100.0 |
|||
} |
|||
|
|||
// GetAverageDataProcessed returns the average data processed per volume
|
|||
func (m *ErasureCodingMetrics) GetAverageDataProcessed() float64 { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
if m.VolumesEncoded == 0 { |
|||
return 0 |
|||
} |
|||
return float64(m.TotalDataProcessed) / float64(m.VolumesEncoded) |
|||
} |
|||
|
|||
// GetSourceRemovalRate returns the percentage of sources removed after encoding
|
|||
func (m *ErasureCodingMetrics) GetSourceRemovalRate() float64 { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
if m.VolumesEncoded == 0 { |
|||
return 0 |
|||
} |
|||
return float64(m.TotalSourcesRemoved) / float64(m.VolumesEncoded) * 100.0 |
|||
} |
|||
|
|||
// Reset resets all metrics to zero
|
|||
func (m *ErasureCodingMetrics) Reset() { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
*m = ErasureCodingMetrics{ |
|||
LastEncodingTime: time.Now(), |
|||
ShardsPerDataCenter: make(map[string]int64), |
|||
ShardsPerRack: make(map[string]int64), |
|||
} |
|||
} |
|||
|
|||
// Global metrics instance for erasure coding tasks
|
|||
var globalErasureCodingMetrics = NewErasureCodingMetrics() |
|||
|
|||
// GetGlobalErasureCodingMetrics returns the global erasure coding metrics instance
|
|||
func GetGlobalErasureCodingMetrics() *ErasureCodingMetrics { |
|||
return globalErasureCodingMetrics |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
package erasure_coding |
|||
|
|||
import ( |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
) |
|||
|
|||
// Scheduling implements the scheduling logic for erasure coding tasks
|
|||
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool { |
|||
ecConfig := config.(*Config) |
|||
|
|||
// Check if we have available workers
|
|||
if len(availableWorkers) == 0 { |
|||
return false |
|||
} |
|||
|
|||
// Count running EC tasks
|
|||
runningCount := 0 |
|||
for _, runningTask := range runningTasks { |
|||
if runningTask.Type == types.TaskTypeErasureCoding { |
|||
runningCount++ |
|||
} |
|||
} |
|||
|
|||
// Check concurrency limit
|
|||
if runningCount >= ecConfig.MaxConcurrent { |
|||
return false |
|||
} |
|||
|
|||
// Check if any worker can handle EC tasks
|
|||
for _, worker := range availableWorkers { |
|||
for _, capability := range worker.Capabilities { |
|||
if capability == types.TaskTypeErasureCoding { |
|||
return true |
|||
} |
|||
} |
|||
} |
|||
|
|||
return false |
|||
} |
|||
@ -0,0 +1,151 @@ |
|||
package vacuum |
|||
|
|||
import ( |
|||
"sync" |
|||
"time" |
|||
) |
|||
|
|||
// VacuumMetrics contains vacuum-specific monitoring data
|
|||
type VacuumMetrics struct { |
|||
// Execution metrics
|
|||
VolumesVacuumed int64 `json:"volumes_vacuumed"` |
|||
TotalSpaceReclaimed int64 `json:"total_space_reclaimed"` |
|||
TotalFilesProcessed int64 `json:"total_files_processed"` |
|||
TotalGarbageCollected int64 `json:"total_garbage_collected"` |
|||
LastVacuumTime time.Time `json:"last_vacuum_time"` |
|||
|
|||
// Performance metrics
|
|||
AverageVacuumTime int64 `json:"average_vacuum_time_seconds"` |
|||
AverageGarbageRatio float64 `json:"average_garbage_ratio"` |
|||
SuccessfulOperations int64 `json:"successful_operations"` |
|||
FailedOperations int64 `json:"failed_operations"` |
|||
|
|||
// Current task metrics
|
|||
CurrentGarbageRatio float64 `json:"current_garbage_ratio"` |
|||
VolumesPendingVacuum int `json:"volumes_pending_vacuum"` |
|||
|
|||
mutex sync.RWMutex |
|||
} |
|||
|
|||
// NewVacuumMetrics creates a new vacuum metrics instance
|
|||
func NewVacuumMetrics() *VacuumMetrics { |
|||
return &VacuumMetrics{ |
|||
LastVacuumTime: time.Now(), |
|||
} |
|||
} |
|||
|
|||
// RecordVolumeVacuumed records a successful volume vacuum operation
|
|||
func (m *VacuumMetrics) RecordVolumeVacuumed(spaceReclaimed int64, filesProcessed int64, garbageCollected int64, vacuumTime time.Duration, garbageRatio float64) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.VolumesVacuumed++ |
|||
m.TotalSpaceReclaimed += spaceReclaimed |
|||
m.TotalFilesProcessed += filesProcessed |
|||
m.TotalGarbageCollected += garbageCollected |
|||
m.SuccessfulOperations++ |
|||
m.LastVacuumTime = time.Now() |
|||
|
|||
// Update average vacuum time
|
|||
if m.AverageVacuumTime == 0 { |
|||
m.AverageVacuumTime = int64(vacuumTime.Seconds()) |
|||
} else { |
|||
// Exponential moving average
|
|||
newTime := int64(vacuumTime.Seconds()) |
|||
m.AverageVacuumTime = (m.AverageVacuumTime*4 + newTime) / 5 |
|||
} |
|||
|
|||
// Update average garbage ratio
|
|||
if m.AverageGarbageRatio == 0 { |
|||
m.AverageGarbageRatio = garbageRatio |
|||
} else { |
|||
// Exponential moving average
|
|||
m.AverageGarbageRatio = 0.8*m.AverageGarbageRatio + 0.2*garbageRatio |
|||
} |
|||
} |
|||
|
|||
// RecordFailure records a failed vacuum operation
|
|||
func (m *VacuumMetrics) RecordFailure() { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.FailedOperations++ |
|||
} |
|||
|
|||
// UpdateCurrentGarbageRatio updates the current volume's garbage ratio
|
|||
func (m *VacuumMetrics) UpdateCurrentGarbageRatio(ratio float64) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.CurrentGarbageRatio = ratio |
|||
} |
|||
|
|||
// SetVolumesPendingVacuum sets the number of volumes pending vacuum
|
|||
func (m *VacuumMetrics) SetVolumesPendingVacuum(count int) { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
m.VolumesPendingVacuum = count |
|||
} |
|||
|
|||
// GetMetrics returns a copy of the current metrics (without the mutex)
|
|||
func (m *VacuumMetrics) GetMetrics() VacuumMetrics { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
// Create a copy without the mutex to avoid copying lock value
|
|||
return VacuumMetrics{ |
|||
VolumesVacuumed: m.VolumesVacuumed, |
|||
TotalSpaceReclaimed: m.TotalSpaceReclaimed, |
|||
TotalFilesProcessed: m.TotalFilesProcessed, |
|||
TotalGarbageCollected: m.TotalGarbageCollected, |
|||
LastVacuumTime: m.LastVacuumTime, |
|||
AverageVacuumTime: m.AverageVacuumTime, |
|||
AverageGarbageRatio: m.AverageGarbageRatio, |
|||
SuccessfulOperations: m.SuccessfulOperations, |
|||
FailedOperations: m.FailedOperations, |
|||
CurrentGarbageRatio: m.CurrentGarbageRatio, |
|||
VolumesPendingVacuum: m.VolumesPendingVacuum, |
|||
} |
|||
} |
|||
|
|||
// GetSuccessRate returns the success rate as a percentage
|
|||
func (m *VacuumMetrics) GetSuccessRate() float64 { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
total := m.SuccessfulOperations + m.FailedOperations |
|||
if total == 0 { |
|||
return 100.0 |
|||
} |
|||
return float64(m.SuccessfulOperations) / float64(total) * 100.0 |
|||
} |
|||
|
|||
// GetAverageSpaceReclaimed returns the average space reclaimed per volume
|
|||
func (m *VacuumMetrics) GetAverageSpaceReclaimed() float64 { |
|||
m.mutex.RLock() |
|||
defer m.mutex.RUnlock() |
|||
|
|||
if m.VolumesVacuumed == 0 { |
|||
return 0 |
|||
} |
|||
return float64(m.TotalSpaceReclaimed) / float64(m.VolumesVacuumed) |
|||
} |
|||
|
|||
// Reset resets all metrics to zero
|
|||
func (m *VacuumMetrics) Reset() { |
|||
m.mutex.Lock() |
|||
defer m.mutex.Unlock() |
|||
|
|||
*m = VacuumMetrics{ |
|||
LastVacuumTime: time.Now(), |
|||
} |
|||
} |
|||
|
|||
// Global metrics instance for vacuum tasks
|
|||
var globalVacuumMetrics = NewVacuumMetrics() |
|||
|
|||
// GetGlobalVacuumMetrics returns the global vacuum metrics instance
|
|||
func GetGlobalVacuumMetrics() *VacuumMetrics { |
|||
return globalVacuumMetrics |
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
package vacuum |
|||
|
|||
import ( |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
) |
|||
|
|||
// Scheduling implements the scheduling logic for vacuum tasks
|
|||
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool { |
|||
vacuumConfig := config.(*Config) |
|||
|
|||
// Count running vacuum tasks
|
|||
runningVacuumCount := 0 |
|||
for _, runningTask := range runningTasks { |
|||
if runningTask.Type == types.TaskTypeVacuum { |
|||
runningVacuumCount++ |
|||
} |
|||
} |
|||
|
|||
// Check concurrency limit
|
|||
if runningVacuumCount >= vacuumConfig.MaxConcurrent { |
|||
return false |
|||
} |
|||
|
|||
// Check for available workers with vacuum capability
|
|||
for _, worker := range availableWorkers { |
|||
if worker.CurrentLoad < worker.MaxConcurrent { |
|||
for _, capability := range worker.Capabilities { |
|||
if capability == types.TaskTypeVacuum { |
|||
return true |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
return false |
|||
} |
|||
@ -1,214 +0,0 @@ |
|||
package vacuum |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"io" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
"google.golang.org/grpc" |
|||
"google.golang.org/grpc/credentials/insecure" |
|||
) |
|||
|
|||
// Task implements vacuum operation to reclaim disk space
|
|||
type Task struct { |
|||
*tasks.BaseTask |
|||
server string |
|||
volumeID uint32 |
|||
garbageThreshold float64 |
|||
} |
|||
|
|||
// NewTask creates a new vacuum task instance
|
|||
func NewTask(server string, volumeID uint32) *Task { |
|||
task := &Task{ |
|||
BaseTask: tasks.NewBaseTask(types.TaskTypeVacuum), |
|||
server: server, |
|||
volumeID: volumeID, |
|||
garbageThreshold: 0.3, // Default 30% threshold
|
|||
} |
|||
return task |
|||
} |
|||
|
|||
// Execute performs the vacuum operation
|
|||
func (t *Task) Execute(params types.TaskParams) error { |
|||
// Use BaseTask.ExecuteTask to handle logging initialization
|
|||
return t.ExecuteTask(context.Background(), params, t.executeImpl) |
|||
} |
|||
|
|||
// executeImpl is the actual vacuum implementation
|
|||
func (t *Task) executeImpl(ctx context.Context, params types.TaskParams) error { |
|||
t.LogInfo("Starting vacuum for volume %d on server %s", t.volumeID, t.server) |
|||
|
|||
// Parse garbage threshold from typed parameters
|
|||
if params.TypedParams != nil { |
|||
if vacuumParams := params.TypedParams.GetVacuumParams(); vacuumParams != nil { |
|||
t.garbageThreshold = vacuumParams.GarbageThreshold |
|||
t.LogWithFields("INFO", "Using garbage threshold from parameters", map[string]interface{}{ |
|||
"threshold": t.garbageThreshold, |
|||
}) |
|||
} |
|||
} |
|||
|
|||
// Convert server address to gRPC address and use proper dial option
|
|||
grpcAddress := pb.ServerToGrpcAddress(t.server) |
|||
var dialOpt grpc.DialOption = grpc.WithTransportCredentials(insecure.NewCredentials()) |
|||
if params.GrpcDialOption != nil { |
|||
dialOpt = params.GrpcDialOption |
|||
} |
|||
|
|||
conn, err := grpc.NewClient(grpcAddress, dialOpt) |
|||
if err != nil { |
|||
t.LogError("Failed to connect to volume server %s: %v", t.server, err) |
|||
return fmt.Errorf("failed to connect to volume server %s: %v", t.server, err) |
|||
} |
|||
defer conn.Close() |
|||
|
|||
client := volume_server_pb.NewVolumeServerClient(conn) |
|||
|
|||
// Step 1: Check vacuum eligibility
|
|||
t.SetProgress(10.0) |
|||
t.LogDebug("Checking vacuum eligibility for volume %d", t.volumeID) |
|||
|
|||
checkResp, err := client.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
t.LogError("Vacuum check failed for volume %d: %v", t.volumeID, err) |
|||
return fmt.Errorf("vacuum check failed for volume %d: %v", t.volumeID, err) |
|||
} |
|||
|
|||
// Check if garbage ratio meets threshold
|
|||
if checkResp.GarbageRatio < t.garbageThreshold { |
|||
t.LogWarning("Volume %d garbage ratio %.2f%% is below threshold %.2f%%, skipping vacuum", |
|||
t.volumeID, checkResp.GarbageRatio*100, t.garbageThreshold*100) |
|||
return fmt.Errorf("volume %d garbage ratio %.2f%% is below threshold %.2f%%, skipping vacuum", |
|||
t.volumeID, checkResp.GarbageRatio*100, t.garbageThreshold*100) |
|||
} |
|||
|
|||
t.LogWithFields("INFO", "Volume eligible for vacuum", map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"garbage_ratio": checkResp.GarbageRatio, |
|||
"threshold": t.garbageThreshold, |
|||
"garbage_percent": checkResp.GarbageRatio * 100, |
|||
}) |
|||
|
|||
// Step 2: Compact volume
|
|||
t.SetProgress(30.0) |
|||
t.LogInfo("Starting compact for volume %d", t.volumeID) |
|||
|
|||
compactStream, err := client.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
t.LogError("Vacuum compact failed for volume %d: %v", t.volumeID, err) |
|||
return fmt.Errorf("vacuum compact failed for volume %d: %v", t.volumeID, err) |
|||
} |
|||
|
|||
// Process compact stream and track progress
|
|||
var processedBytes int64 |
|||
var totalBytes int64 |
|||
|
|||
for { |
|||
resp, err := compactStream.Recv() |
|||
if err != nil { |
|||
if err == io.EOF { |
|||
break |
|||
} |
|||
t.LogError("Vacuum compact stream error for volume %d: %v", t.volumeID, err) |
|||
return fmt.Errorf("vacuum compact stream error for volume %d: %v", t.volumeID, err) |
|||
} |
|||
|
|||
processedBytes = resp.ProcessedBytes |
|||
if resp.LoadAvg_1M > 0 { |
|||
totalBytes = int64(resp.LoadAvg_1M) // This is a rough approximation
|
|||
} |
|||
|
|||
// Update progress based on processed bytes (30% to 70% of total progress)
|
|||
if totalBytes > 0 { |
|||
compactProgress := float64(processedBytes) / float64(totalBytes) |
|||
if compactProgress > 1.0 { |
|||
compactProgress = 1.0 |
|||
} |
|||
progress := 30.0 + (compactProgress * 40.0) // 30% to 70%
|
|||
t.SetProgress(progress) |
|||
} |
|||
|
|||
t.LogWithFields("DEBUG", "Volume compact progress", map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"processed_bytes": processedBytes, |
|||
"total_bytes": totalBytes, |
|||
"compact_progress": fmt.Sprintf("%.1f%%", (float64(processedBytes)/float64(totalBytes))*100), |
|||
}) |
|||
} |
|||
|
|||
// Step 3: Commit vacuum changes
|
|||
t.SetProgress(80.0) |
|||
t.LogInfo("Committing vacuum for volume %d", t.volumeID) |
|||
|
|||
commitResp, err := client.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
t.LogError("Vacuum commit failed for volume %d: %v", t.volumeID, err) |
|||
return fmt.Errorf("vacuum commit failed for volume %d: %v", t.volumeID, err) |
|||
} |
|||
|
|||
// Step 4: Cleanup temporary files
|
|||
t.SetProgress(90.0) |
|||
t.LogInfo("Cleaning up vacuum files for volume %d", t.volumeID) |
|||
|
|||
_, err = client.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
// Log warning but don't fail the task
|
|||
t.LogWarning("Vacuum cleanup warning for volume %d: %v", t.volumeID, err) |
|||
} |
|||
|
|||
t.SetProgress(100.0) |
|||
|
|||
newVolumeSize := commitResp.VolumeSize |
|||
t.LogWithFields("INFO", "Successfully completed vacuum", map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"server": t.server, |
|||
"new_volume_size": newVolumeSize, |
|||
"garbage_reclaimed": true, |
|||
}) |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// Validate validates the task parameters
|
|||
func (t *Task) Validate(params types.TaskParams) error { |
|||
if params.VolumeID == 0 { |
|||
return fmt.Errorf("volume_id is required") |
|||
} |
|||
if params.Server == "" { |
|||
return fmt.Errorf("server is required") |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
// EstimateTime estimates the time needed for the task
|
|||
func (t *Task) EstimateTime(params types.TaskParams) time.Duration { |
|||
// Base time for vacuum operations - varies by volume size and garbage ratio
|
|||
// Typically vacuum is faster than EC encoding
|
|||
baseTime := 5 * time.Minute |
|||
|
|||
// Use default estimation since volume size is not available in typed params
|
|||
return baseTime |
|||
} |
|||
|
|||
// GetProgress returns the current progress
|
|||
func (t *Task) GetProgress() float64 { |
|||
return t.BaseTask.GetProgress() |
|||
} |
|||
|
|||
// Cancel cancels the task
|
|||
func (t *Task) Cancel() error { |
|||
return t.BaseTask.Cancel() |
|||
} |
|||
@ -0,0 +1,236 @@ |
|||
package vacuum |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"io" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/operation" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types/base" |
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
// VacuumTask implements the Task interface
|
|||
type VacuumTask struct { |
|||
*base.BaseTask |
|||
server string |
|||
volumeID uint32 |
|||
collection string |
|||
garbageThreshold float64 |
|||
progress float64 |
|||
} |
|||
|
|||
// NewVacuumTask creates a new unified vacuum task instance
|
|||
func NewVacuumTask(id string, server string, volumeID uint32, collection string) *VacuumTask { |
|||
return &VacuumTask{ |
|||
BaseTask: base.NewBaseTask(id, types.TaskTypeVacuum), |
|||
server: server, |
|||
volumeID: volumeID, |
|||
collection: collection, |
|||
garbageThreshold: 0.3, // Default 30% threshold
|
|||
} |
|||
} |
|||
|
|||
// Execute implements the UnifiedTask interface
|
|||
func (t *VacuumTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error { |
|||
if params == nil { |
|||
return fmt.Errorf("task parameters are required") |
|||
} |
|||
|
|||
vacuumParams := params.GetVacuumParams() |
|||
if vacuumParams == nil { |
|||
return fmt.Errorf("vacuum parameters are required") |
|||
} |
|||
|
|||
t.garbageThreshold = vacuumParams.GarbageThreshold |
|||
|
|||
t.GetLogger().WithFields(map[string]interface{}{ |
|||
"volume_id": t.volumeID, |
|||
"server": t.server, |
|||
"collection": t.collection, |
|||
"garbage_threshold": t.garbageThreshold, |
|||
}).Info("Starting vacuum task") |
|||
|
|||
// Step 1: Check volume status and garbage ratio
|
|||
t.ReportProgress(10.0) |
|||
t.GetLogger().Info("Checking volume status") |
|||
eligible, currentGarbageRatio, err := t.checkVacuumEligibility() |
|||
if err != nil { |
|||
return fmt.Errorf("failed to check vacuum eligibility: %v", err) |
|||
} |
|||
|
|||
if !eligible { |
|||
t.GetLogger().WithFields(map[string]interface{}{ |
|||
"current_garbage_ratio": currentGarbageRatio, |
|||
"required_threshold": t.garbageThreshold, |
|||
}).Info("Volume does not meet vacuum criteria, skipping") |
|||
t.ReportProgress(100.0) |
|||
return nil |
|||
} |
|||
|
|||
// Step 2: Perform vacuum operation
|
|||
t.ReportProgress(50.0) |
|||
t.GetLogger().WithFields(map[string]interface{}{ |
|||
"garbage_ratio": currentGarbageRatio, |
|||
"threshold": t.garbageThreshold, |
|||
}).Info("Performing vacuum operation") |
|||
|
|||
if err := t.performVacuum(); err != nil { |
|||
return fmt.Errorf("failed to perform vacuum: %v", err) |
|||
} |
|||
|
|||
// Step 3: Verify vacuum results
|
|||
t.ReportProgress(90.0) |
|||
t.GetLogger().Info("Verifying vacuum results") |
|||
if err := t.verifyVacuumResults(); err != nil { |
|||
glog.Warningf("Vacuum verification failed: %v", err) |
|||
// Don't fail the task - vacuum operation itself succeeded
|
|||
} |
|||
|
|||
t.ReportProgress(100.0) |
|||
glog.Infof("Vacuum task completed successfully: volume %d from %s (garbage ratio was %.2f%%)", |
|||
t.volumeID, t.server, currentGarbageRatio*100) |
|||
return nil |
|||
} |
|||
|
|||
// Validate implements the UnifiedTask interface
|
|||
func (t *VacuumTask) Validate(params *worker_pb.TaskParams) error { |
|||
if params == nil { |
|||
return fmt.Errorf("task parameters are required") |
|||
} |
|||
|
|||
vacuumParams := params.GetVacuumParams() |
|||
if vacuumParams == nil { |
|||
return fmt.Errorf("vacuum parameters are required") |
|||
} |
|||
|
|||
if params.VolumeId != t.volumeID { |
|||
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId) |
|||
} |
|||
|
|||
if params.Server != t.server { |
|||
return fmt.Errorf("source server mismatch: expected %s, got %s", t.server, params.Server) |
|||
} |
|||
|
|||
if vacuumParams.GarbageThreshold < 0 || vacuumParams.GarbageThreshold > 1.0 { |
|||
return fmt.Errorf("invalid garbage threshold: %f (must be between 0.0 and 1.0)", vacuumParams.GarbageThreshold) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// EstimateTime implements the UnifiedTask interface
|
|||
func (t *VacuumTask) EstimateTime(params *worker_pb.TaskParams) time.Duration { |
|||
// Basic estimate based on simulated steps
|
|||
return 14 * time.Second // Sum of all step durations
|
|||
} |
|||
|
|||
// GetProgress returns current progress
|
|||
func (t *VacuumTask) GetProgress() float64 { |
|||
return t.progress |
|||
} |
|||
|
|||
// Helper methods for real vacuum operations
|
|||
|
|||
// checkVacuumEligibility checks if the volume meets vacuum criteria
|
|||
func (t *VacuumTask) checkVacuumEligibility() (bool, float64, error) { |
|||
var garbageRatio float64 |
|||
|
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
resp, err := client.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to check volume vacuum status: %v", err) |
|||
} |
|||
|
|||
garbageRatio = resp.GarbageRatio |
|||
|
|||
return nil |
|||
}) |
|||
|
|||
if err != nil { |
|||
return false, 0, err |
|||
} |
|||
|
|||
eligible := garbageRatio >= t.garbageThreshold |
|||
glog.V(1).Infof("Volume %d garbage ratio: %.2f%%, threshold: %.2f%%, eligible: %v", |
|||
t.volumeID, garbageRatio*100, t.garbageThreshold*100, eligible) |
|||
|
|||
return eligible, garbageRatio, nil |
|||
} |
|||
|
|||
// performVacuum executes the actual vacuum operation
|
|||
func (t *VacuumTask) performVacuum() error { |
|||
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
// Step 1: Compact the volume
|
|||
t.GetLogger().Info("Compacting volume") |
|||
stream, err := client.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("vacuum compact failed: %v", err) |
|||
} |
|||
|
|||
// Read compact progress
|
|||
for { |
|||
resp, recvErr := stream.Recv() |
|||
if recvErr != nil { |
|||
if recvErr == io.EOF { |
|||
break |
|||
} |
|||
return fmt.Errorf("vacuum compact stream error: %v", recvErr) |
|||
} |
|||
glog.V(2).Infof("Volume %d compact progress: %d bytes processed", t.volumeID, resp.ProcessedBytes) |
|||
} |
|||
|
|||
// Step 2: Commit the vacuum
|
|||
t.GetLogger().Info("Committing vacuum operation") |
|||
_, err = client.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("vacuum commit failed: %v", err) |
|||
} |
|||
|
|||
// Step 3: Cleanup old files
|
|||
t.GetLogger().Info("Cleaning up vacuum files") |
|||
_, err = client.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("vacuum cleanup failed: %v", err) |
|||
} |
|||
|
|||
glog.V(1).Infof("Volume %d vacuum operation completed successfully", t.volumeID) |
|||
return nil |
|||
}) |
|||
} |
|||
|
|||
// verifyVacuumResults checks the volume status after vacuum
|
|||
func (t *VacuumTask) verifyVacuumResults() error { |
|||
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(), |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
resp, err := client.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{ |
|||
VolumeId: t.volumeID, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to verify vacuum results: %v", err) |
|||
} |
|||
|
|||
postVacuumGarbageRatio := resp.GarbageRatio |
|||
|
|||
glog.V(1).Infof("Volume %d post-vacuum garbage ratio: %.2f%%", |
|||
t.volumeID, postVacuumGarbageRatio*100) |
|||
|
|||
return nil |
|||
}) |
|||
} |
|||
@ -0,0 +1,99 @@ |
|||
package base |
|||
|
|||
import ( |
|||
"context" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/worker/types" |
|||
) |
|||
|
|||
// BaseTask provides common task functionality
|
|||
type BaseTask struct { |
|||
id string |
|||
taskType types.TaskType |
|||
progressCallback func(float64) |
|||
logger types.Logger |
|||
cancelled bool |
|||
} |
|||
|
|||
// NewBaseTask creates a new base task
|
|||
func NewBaseTask(id string, taskType types.TaskType) *BaseTask { |
|||
return &BaseTask{ |
|||
id: id, |
|||
taskType: taskType, |
|||
logger: &types.GlogFallbackLogger{}, // Default fallback logger
|
|||
} |
|||
} |
|||
|
|||
// ID returns the task ID
|
|||
func (t *BaseTask) ID() string { |
|||
return t.id |
|||
} |
|||
|
|||
// Type returns the task type
|
|||
func (t *BaseTask) Type() types.TaskType { |
|||
return t.taskType |
|||
} |
|||
|
|||
// SetProgressCallback sets the progress callback
|
|||
func (t *BaseTask) SetProgressCallback(callback func(float64)) { |
|||
t.progressCallback = callback |
|||
} |
|||
|
|||
// ReportProgress reports current progress through the callback
|
|||
func (t *BaseTask) ReportProgress(progress float64) { |
|||
if t.progressCallback != nil { |
|||
t.progressCallback(progress) |
|||
} |
|||
} |
|||
|
|||
// GetProgress returns current progress
|
|||
func (t *BaseTask) GetProgress() float64 { |
|||
// Subclasses should override this
|
|||
return 0 |
|||
} |
|||
|
|||
// Cancel marks the task as cancelled
|
|||
func (t *BaseTask) Cancel() error { |
|||
t.cancelled = true |
|||
return nil |
|||
} |
|||
|
|||
// IsCancellable returns true if the task can be cancelled
|
|||
func (t *BaseTask) IsCancellable() bool { |
|||
return true |
|||
} |
|||
|
|||
// IsCancelled returns true if the task has been cancelled
|
|||
func (t *BaseTask) IsCancelled() bool { |
|||
return t.cancelled |
|||
} |
|||
|
|||
// SetLogger sets the task logger
|
|||
func (t *BaseTask) SetLogger(logger types.Logger) { |
|||
t.logger = logger |
|||
} |
|||
|
|||
// GetLogger returns the task logger
|
|||
func (t *BaseTask) GetLogger() types.Logger { |
|||
return t.logger |
|||
} |
|||
|
|||
// Execute implements the Task interface
|
|||
func (t *BaseTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error { |
|||
// Subclasses must implement this
|
|||
return nil |
|||
} |
|||
|
|||
// Validate implements the UnifiedTask interface
|
|||
func (t *BaseTask) Validate(params *worker_pb.TaskParams) error { |
|||
// Subclasses must implement this
|
|||
return nil |
|||
} |
|||
|
|||
// EstimateTime implements the UnifiedTask interface
|
|||
func (t *BaseTask) EstimateTime(params *worker_pb.TaskParams) time.Duration { |
|||
// Subclasses must implement this
|
|||
return 0 |
|||
} |
|||
@ -0,0 +1,52 @@ |
|||
package types |
|||
|
|||
// This file contains the unified factory interfaces.
|
|||
|
|||
import ( |
|||
"context" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
) |
|||
|
|||
// Factory defines a generic factory interface
|
|||
type Factory[T any, C any] interface { |
|||
// Create new instance with context and config
|
|||
Create(ctx context.Context, config C) (T, error) |
|||
|
|||
// Metadata
|
|||
Type() string |
|||
Description() string |
|||
Capabilities() []string |
|||
} |
|||
|
|||
// TaskFactory creates new task instances
|
|||
type TaskFactory interface { |
|||
Create(params *worker_pb.TaskParams) (Task, error) |
|||
Type() string |
|||
Description() string |
|||
Capabilities() []string |
|||
} |
|||
|
|||
// TaskCreationConfig defines task creation configuration
|
|||
type TaskCreationConfig struct { |
|||
ID string |
|||
Type TaskType |
|||
Server string |
|||
Collection string |
|||
VolumeID uint32 |
|||
Logger Logger |
|||
} |
|||
|
|||
// WorkerCreationConfig encapsulates all worker configuration
|
|||
type WorkerCreationConfig struct { |
|||
ID string |
|||
Capabilities []TaskType |
|||
MaxConcurrent int |
|||
HeartbeatInterval time.Duration |
|||
TaskRequestInterval time.Duration |
|||
LoggerFactory LoggerFactory |
|||
} |
|||
|
|||
// WorkerFactory creates new worker instances
|
|||
type WorkerFactory = Factory[Worker, WorkerConfig] |
|||
@ -0,0 +1,189 @@ |
|||
package types |
|||
|
|||
// This file contains the new unified task interfaces that will replace
|
|||
// the existing TaskInterface and TypedTaskInterface.
|
|||
|
|||
import ( |
|||
"context" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
) |
|||
|
|||
// Task defines the core task interface that all tasks must implement
|
|||
type Task interface { |
|||
// Core identity
|
|||
ID() string |
|||
Type() TaskType |
|||
|
|||
// Execution
|
|||
Execute(ctx context.Context, params *worker_pb.TaskParams) error |
|||
Validate(params *worker_pb.TaskParams) error |
|||
EstimateTime(params *worker_pb.TaskParams) time.Duration |
|||
|
|||
// Control
|
|||
Cancel() error |
|||
IsCancellable() bool |
|||
|
|||
// Progress
|
|||
GetProgress() float64 |
|||
SetProgressCallback(func(float64)) |
|||
} |
|||
|
|||
// TaskWithLogging extends Task with logging capabilities
|
|||
type TaskWithLogging interface { |
|||
Task |
|||
Logger |
|||
} |
|||
|
|||
// Logger defines standard logging interface
|
|||
type Logger interface { |
|||
Info(msg string, args ...interface{}) |
|||
Warning(msg string, args ...interface{}) |
|||
Error(msg string, args ...interface{}) |
|||
Debug(msg string, args ...interface{}) |
|||
WithFields(fields map[string]interface{}) Logger |
|||
} |
|||
|
|||
// NoOpLogger is a logger that does nothing (silent)
|
|||
type NoOpLogger struct{} |
|||
|
|||
func (l *NoOpLogger) Info(msg string, args ...interface{}) {} |
|||
func (l *NoOpLogger) Warning(msg string, args ...interface{}) {} |
|||
func (l *NoOpLogger) Error(msg string, args ...interface{}) {} |
|||
func (l *NoOpLogger) Debug(msg string, args ...interface{}) {} |
|||
func (l *NoOpLogger) WithFields(fields map[string]interface{}) Logger { |
|||
return l // Return self since we're doing nothing anyway
|
|||
} |
|||
|
|||
// GlogFallbackLogger is a logger that falls back to glog
|
|||
type GlogFallbackLogger struct{} |
|||
|
|||
func (l *GlogFallbackLogger) Info(msg string, args ...interface{}) { |
|||
if len(args) > 0 { |
|||
glog.Infof(msg, args...) |
|||
} else { |
|||
glog.Info(msg) |
|||
} |
|||
} |
|||
|
|||
func (l *GlogFallbackLogger) Warning(msg string, args ...interface{}) { |
|||
if len(args) > 0 { |
|||
glog.Warningf(msg, args...) |
|||
} else { |
|||
glog.Warning(msg) |
|||
} |
|||
} |
|||
|
|||
func (l *GlogFallbackLogger) Error(msg string, args ...interface{}) { |
|||
if len(args) > 0 { |
|||
glog.Errorf(msg, args...) |
|||
} else { |
|||
glog.Error(msg) |
|||
} |
|||
} |
|||
|
|||
func (l *GlogFallbackLogger) Debug(msg string, args ...interface{}) { |
|||
if len(args) > 0 { |
|||
glog.V(1).Infof(msg, args...) |
|||
} else { |
|||
glog.V(1).Info(msg) |
|||
} |
|||
} |
|||
|
|||
func (l *GlogFallbackLogger) WithFields(fields map[string]interface{}) Logger { |
|||
// For glog fallback, we'll just return self and ignore fields for simplicity
|
|||
// A more sophisticated implementation could format the fields into the message
|
|||
return l |
|||
} |
|||
|
|||
// LogLevel represents logging severity levels
|
|||
type LogLevel int |
|||
|
|||
const ( |
|||
LogLevelDebug LogLevel = iota |
|||
LogLevelInfo |
|||
LogLevelWarning |
|||
LogLevelError |
|||
) |
|||
|
|||
// LoggerConfig defines logger configuration
|
|||
type LoggerConfig struct { |
|||
MinLevel LogLevel |
|||
MaxSize int64 |
|||
MaxFiles int |
|||
Directory string |
|||
ServiceName string |
|||
EnableJSON bool |
|||
} |
|||
|
|||
// LoggerFactory creates configured loggers
|
|||
type LoggerFactory interface { |
|||
CreateLogger(ctx context.Context, config LoggerConfig) (Logger, error) |
|||
} |
|||
|
|||
// BaseTask provides common task functionality
|
|||
type UnifiedBaseTask struct { |
|||
id string |
|||
taskType TaskType |
|||
progressCallback func(float64) |
|||
logger Logger |
|||
cancelled bool |
|||
} |
|||
|
|||
// NewBaseTask creates a new base task
|
|||
func NewUnifiedBaseTask(id string, taskType TaskType) *UnifiedBaseTask { |
|||
return &UnifiedBaseTask{ |
|||
id: id, |
|||
taskType: taskType, |
|||
} |
|||
} |
|||
|
|||
// ID returns the task ID
|
|||
func (t *UnifiedBaseTask) ID() string { |
|||
return t.id |
|||
} |
|||
|
|||
// Type returns the task type
|
|||
func (t *UnifiedBaseTask) Type() TaskType { |
|||
return t.taskType |
|||
} |
|||
|
|||
// SetProgressCallback sets the progress callback
|
|||
func (t *UnifiedBaseTask) SetProgressCallback(callback func(float64)) { |
|||
t.progressCallback = callback |
|||
} |
|||
|
|||
// ReportProgress reports current progress through the callback
|
|||
func (t *UnifiedBaseTask) ReportProgress(progress float64) { |
|||
if t.progressCallback != nil { |
|||
t.progressCallback(progress) |
|||
} |
|||
} |
|||
|
|||
// Cancel marks the task as cancelled
|
|||
func (t *UnifiedBaseTask) Cancel() error { |
|||
t.cancelled = true |
|||
return nil |
|||
} |
|||
|
|||
// IsCancellable returns true if the task can be cancelled
|
|||
func (t *UnifiedBaseTask) IsCancellable() bool { |
|||
return true |
|||
} |
|||
|
|||
// IsCancelled returns true if the task has been cancelled
|
|||
func (t *UnifiedBaseTask) IsCancelled() bool { |
|||
return t.cancelled |
|||
} |
|||
|
|||
// SetLogger sets the task logger
|
|||
func (t *UnifiedBaseTask) SetLogger(logger Logger) { |
|||
t.logger = logger |
|||
} |
|||
|
|||
// GetLogger returns the task logger
|
|||
func (t *UnifiedBaseTask) GetLogger() Logger { |
|||
return t.logger |
|||
} |
|||
@ -0,0 +1,76 @@ |
|||
package types |
|||
|
|||
// This file contains the new unified worker interfaces that will replace
|
|||
// the existing WorkerInterface.
|
|||
|
|||
import ( |
|||
"context" |
|||
) |
|||
|
|||
// Worker defines core worker functionality
|
|||
type Worker interface { |
|||
// Core operations
|
|||
Start(ctx context.Context) error |
|||
Stop(ctx context.Context) error |
|||
HandleTask(ctx context.Context, task Task) error |
|||
|
|||
// Status
|
|||
GetStatus() WorkerStatus |
|||
GetCapabilities() []TaskType |
|||
|
|||
// Configuration
|
|||
Configure(config WorkerCreationConfig) error |
|||
} |
|||
|
|||
// BaseWorker provides common worker functionality
|
|||
type BaseWorker struct { |
|||
id string |
|||
capabilities []TaskType |
|||
maxConcurrent int |
|||
currentTasks map[string]Task |
|||
logger Logger |
|||
} |
|||
|
|||
// NewBaseWorker creates a new base worker
|
|||
func NewBaseWorker(id string) *BaseWorker { |
|||
return &BaseWorker{ |
|||
id: id, |
|||
currentTasks: make(map[string]Task), |
|||
} |
|||
} |
|||
|
|||
// Configure applies worker configuration
|
|||
func (w *BaseWorker) Configure(config WorkerCreationConfig) error { |
|||
w.id = config.ID |
|||
w.capabilities = config.Capabilities |
|||
w.maxConcurrent = config.MaxConcurrent |
|||
|
|||
if config.LoggerFactory != nil { |
|||
logger, err := config.LoggerFactory.CreateLogger(context.Background(), LoggerConfig{ |
|||
ServiceName: "worker-" + w.id, |
|||
MinLevel: LogLevelInfo, |
|||
}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
w.logger = logger |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// GetCapabilities returns worker capabilities
|
|||
func (w *BaseWorker) GetCapabilities() []TaskType { |
|||
return w.capabilities |
|||
} |
|||
|
|||
// GetStatus returns current worker status
|
|||
func (w *BaseWorker) GetStatus() WorkerStatus { |
|||
return WorkerStatus{ |
|||
WorkerID: w.id, |
|||
Status: "active", |
|||
Capabilities: w.capabilities, |
|||
MaxConcurrent: w.maxConcurrent, |
|||
CurrentLoad: len(w.currentTasks), |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue