Browse Source
refactor ec shard distribution (#8465)
refactor ec shard distribution (#8465)
* refactor ec shard distribution * fix shard assignment merge and mount errors * fix mount error aggregation scope * make WithFields compatible and wrap errorspull/8473/head
Chris Lu
3 days ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 382 additions and 250 deletions
-
367weed/storage/erasure_coding/shard_distribution.go
-
265weed/worker/tasks/erasure_coding/ec_task.go
@ -0,0 +1,367 @@ |
|||
package erasure_coding |
|||
|
|||
import ( |
|||
"context" |
|||
"errors" |
|||
"fmt" |
|||
"io" |
|||
"os" |
|||
"reflect" |
|||
"strings" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/glog" |
|||
"github.com/seaweedfs/seaweedfs/weed/operation" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" |
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
func ensureLogger(logger logger) logger { |
|||
if logger == nil { |
|||
return &glogFallbackLogger{} |
|||
} |
|||
return logger |
|||
} |
|||
|
|||
type logger interface { |
|||
Info(string, ...interface{}) |
|||
Warning(string, ...interface{}) |
|||
Error(string, ...interface{}) |
|||
} |
|||
|
|||
type withFieldLogger interface { |
|||
WithFields(map[string]interface{}) logger |
|||
} |
|||
|
|||
func withFields(log logger, fields map[string]interface{}) logger { |
|||
if fields == nil { |
|||
return log |
|||
} |
|||
if wf, ok := log.(withFieldLogger); ok { |
|||
return wf.WithFields(fields) |
|||
} |
|||
|
|||
val := reflect.ValueOf(log) |
|||
method := val.MethodByName("WithFields") |
|||
if !method.IsValid() { |
|||
return log |
|||
} |
|||
methodType := method.Type() |
|||
if methodType.NumIn() != 1 || methodType.NumOut() != 1 { |
|||
return log |
|||
} |
|||
argType := methodType.In(0) |
|||
mapType := reflect.TypeOf(map[string]interface{}{}) |
|||
var arg reflect.Value |
|||
if mapType.AssignableTo(argType) { |
|||
arg = reflect.ValueOf(fields) |
|||
} else if mapType.ConvertibleTo(argType) { |
|||
arg = reflect.ValueOf(fields).Convert(argType) |
|||
} else { |
|||
return log |
|||
} |
|||
result := method.Call([]reflect.Value{arg})[0] |
|||
if result.IsValid() && result.CanInterface() { |
|||
if enhanced, ok := result.Interface().(logger); ok { |
|||
return enhanced |
|||
} |
|||
} |
|||
return log |
|||
} |
|||
|
|||
type glogFallbackLogger struct{} |
|||
|
|||
func (g *glogFallbackLogger) Info(msg string, args ...interface{}) { |
|||
if len(args) > 0 { |
|||
glog.Infof(msg, args...) |
|||
return |
|||
} |
|||
glog.Info(msg) |
|||
} |
|||
|
|||
func (g *glogFallbackLogger) Warning(msg string, args ...interface{}) { |
|||
if len(args) > 0 { |
|||
glog.Warningf(msg, args...) |
|||
return |
|||
} |
|||
glog.Warning(msg) |
|||
} |
|||
|
|||
func (g *glogFallbackLogger) Error(msg string, args ...interface{}) { |
|||
if len(args) > 0 { |
|||
glog.Errorf(msg, args...) |
|||
return |
|||
} |
|||
glog.Error(msg) |
|||
} |
|||
|
|||
// DistributeEcShards distributes locally generated EC shards to destination servers.
|
|||
// Returns the shard assignment map used for mounting.
|
|||
func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb.TaskTarget, shardFiles map[string]string, dialOption grpc.DialOption, logger logger) (map[string][]string, error) { |
|||
if len(targets) == 0 { |
|||
return nil, fmt.Errorf("no targets specified for EC shard distribution") |
|||
} |
|||
|
|||
if len(shardFiles) == 0 { |
|||
return nil, fmt.Errorf("no shard files available for distribution") |
|||
} |
|||
|
|||
log := ensureLogger(logger) |
|||
|
|||
shardAssignment := make(map[string][]string) |
|||
|
|||
for _, target := range targets { |
|||
if len(target.ShardIds) == 0 { |
|||
continue |
|||
} |
|||
|
|||
var assignedShards []string |
|||
for _, shardId := range target.ShardIds { |
|||
shardType := fmt.Sprintf("ec%02d", shardId) |
|||
assignedShards = append(assignedShards, shardType) |
|||
} |
|||
|
|||
if len(assignedShards) > 0 { |
|||
if _, hasEcx := shardFiles["ecx"]; hasEcx { |
|||
assignedShards = append(assignedShards, "ecx") |
|||
} |
|||
if _, hasEcj := shardFiles["ecj"]; hasEcj { |
|||
assignedShards = append(assignedShards, "ecj") |
|||
} |
|||
if _, hasVif := shardFiles["vif"]; hasVif { |
|||
assignedShards = append(assignedShards, "vif") |
|||
} |
|||
} |
|||
|
|||
existing := shardAssignment[target.Node] |
|||
if len(existing) == 0 { |
|||
shardAssignment[target.Node] = assignedShards |
|||
continue |
|||
} |
|||
|
|||
seen := make(map[string]struct{}, len(existing)) |
|||
for _, shard := range existing { |
|||
seen[shard] = struct{}{} |
|||
} |
|||
for _, shard := range assignedShards { |
|||
if _, ok := seen[shard]; ok { |
|||
continue |
|||
} |
|||
seen[shard] = struct{}{} |
|||
existing = append(existing, shard) |
|||
} |
|||
shardAssignment[target.Node] = existing |
|||
} |
|||
|
|||
if len(shardAssignment) == 0 { |
|||
return nil, fmt.Errorf("no shard assignments found from planning phase") |
|||
} |
|||
|
|||
for destNode, assignedShards := range shardAssignment { |
|||
withFields(log, map[string]interface{}{ |
|||
"destination": destNode, |
|||
"assigned_shards": len(assignedShards), |
|||
"shard_types": assignedShards, |
|||
}).Info("Starting shard distribution to destination server") |
|||
|
|||
var transferredBytes int64 |
|||
for _, shardType := range assignedShards { |
|||
filePath, exists := shardFiles[shardType] |
|||
if !exists { |
|||
return nil, fmt.Errorf("shard file %s not found for destination %s", shardType, destNode) |
|||
} |
|||
|
|||
if info, err := os.Stat(filePath); err == nil { |
|||
transferredBytes += info.Size() |
|||
withFields(log, map[string]interface{}{ |
|||
"destination": destNode, |
|||
"shard_type": shardType, |
|||
"file_path": filePath, |
|||
"size_bytes": info.Size(), |
|||
"size_kb": float64(info.Size()) / 1024, |
|||
}).Info("Starting shard file transfer") |
|||
} |
|||
|
|||
if err := sendShardFileToDestination(volumeID, collection, dialOption, destNode, filePath, shardType); err != nil { |
|||
return nil, fmt.Errorf("failed to send %s to %s: %w", shardType, destNode, err) |
|||
} |
|||
|
|||
withFields(log, map[string]interface{}{ |
|||
"destination": destNode, |
|||
"shard_type": shardType, |
|||
}).Info("Shard file transfer completed") |
|||
} |
|||
|
|||
withFields(log, map[string]interface{}{ |
|||
"destination": destNode, |
|||
"shards_transferred": len(assignedShards), |
|||
"total_bytes": transferredBytes, |
|||
"total_mb": float64(transferredBytes) / (1024 * 1024), |
|||
}).Info("All shards distributed to destination server") |
|||
} |
|||
|
|||
glog.V(1).Infof("Successfully distributed EC shards to %d destinations", len(shardAssignment)) |
|||
return shardAssignment, nil |
|||
} |
|||
|
|||
// MountEcShards mounts EC shards on destination servers using an assignment map.
|
|||
func MountEcShards(volumeID uint32, collection string, shardAssignment map[string][]string, dialOption grpc.DialOption, logger logger) error { |
|||
if shardAssignment == nil { |
|||
return fmt.Errorf("shard assignment not available for mounting") |
|||
} |
|||
|
|||
log := ensureLogger(logger) |
|||
|
|||
var mountErrors []error |
|||
for destNode, assignedShards := range shardAssignment { |
|||
var shardIds []uint32 |
|||
var metadataFiles []string |
|||
|
|||
for _, shardType := range assignedShards { |
|||
if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 { |
|||
var shardId uint32 |
|||
if _, err := fmt.Sscanf(shardType[2:], "%d", &shardId); err == nil { |
|||
shardIds = append(shardIds, shardId) |
|||
} |
|||
} else { |
|||
metadataFiles = append(metadataFiles, shardType) |
|||
} |
|||
} |
|||
|
|||
withFields(log, map[string]interface{}{ |
|||
"destination": destNode, |
|||
"shard_ids": shardIds, |
|||
"shard_count": len(shardIds), |
|||
"metadata_files": metadataFiles, |
|||
}).Info("Starting EC shard mount operation") |
|||
|
|||
if len(shardIds) == 0 { |
|||
withFields(log, map[string]interface{}{ |
|||
"destination": destNode, |
|||
"metadata_files": metadataFiles, |
|||
}).Info("No EC shards to mount (only metadata files)") |
|||
continue |
|||
} |
|||
|
|||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), dialOption, |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ |
|||
VolumeId: volumeID, |
|||
Collection: collection, |
|||
ShardIds: shardIds, |
|||
}) |
|||
return mountErr |
|||
}) |
|||
|
|||
if err != nil { |
|||
mountErrors = append(mountErrors, fmt.Errorf("mount %s shards %v: %w", destNode, shardIds, err)) |
|||
withFields(log, map[string]interface{}{ |
|||
"destination": destNode, |
|||
"shard_ids": shardIds, |
|||
"error": err.Error(), |
|||
}).Error("Failed to mount EC shards") |
|||
} else { |
|||
withFields(log, map[string]interface{}{ |
|||
"destination": destNode, |
|||
"shard_ids": shardIds, |
|||
"volume_id": volumeID, |
|||
"collection": collection, |
|||
}).Info("Successfully mounted EC shards") |
|||
} |
|||
} |
|||
|
|||
if len(mountErrors) > 0 { |
|||
return errors.Join(mountErrors...) |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
// sendShardFileToDestination sends a single shard file to a destination server using ReceiveFile API.
|
|||
func sendShardFileToDestination(volumeID uint32, collection string, dialOption grpc.DialOption, destServer, filePath, shardType string) error { |
|||
return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), dialOption, |
|||
func(client volume_server_pb.VolumeServerClient) error { |
|||
file, err := os.Open(filePath) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to open shard file %s: %v", filePath, err) |
|||
} |
|||
defer file.Close() |
|||
|
|||
fileInfo, err := file.Stat() |
|||
if err != nil { |
|||
return fmt.Errorf("failed to get file info for %s: %v", filePath, err) |
|||
} |
|||
|
|||
var ext string |
|||
var shardId uint32 |
|||
if shardType == "ecx" { |
|||
ext = ".ecx" |
|||
shardId = 0 |
|||
} else if shardType == "ecj" { |
|||
ext = ".ecj" |
|||
shardId = 0 |
|||
} else if shardType == "vif" { |
|||
ext = ".vif" |
|||
shardId = 0 |
|||
} else if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 { |
|||
ext = "." + shardType |
|||
fmt.Sscanf(shardType[2:], "%d", &shardId) |
|||
} else { |
|||
return fmt.Errorf("unknown shard type: %s", shardType) |
|||
} |
|||
|
|||
stream, err := client.ReceiveFile(context.Background()) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to create receive stream: %v", err) |
|||
} |
|||
|
|||
err = stream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_Info{ |
|||
Info: &volume_server_pb.ReceiveFileInfo{ |
|||
VolumeId: volumeID, |
|||
Ext: ext, |
|||
Collection: collection, |
|||
IsEcVolume: true, |
|||
ShardId: shardId, |
|||
FileSize: uint64(fileInfo.Size()), |
|||
}, |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to send file info: %v", err) |
|||
} |
|||
|
|||
buffer := make([]byte, 64*1024) |
|||
for { |
|||
n, readErr := file.Read(buffer) |
|||
if n > 0 { |
|||
err = stream.Send(&volume_server_pb.ReceiveFileRequest{ |
|||
Data: &volume_server_pb.ReceiveFileRequest_FileContent{ |
|||
FileContent: buffer[:n], |
|||
}, |
|||
}) |
|||
if err != nil { |
|||
return fmt.Errorf("failed to send file content: %v", err) |
|||
} |
|||
} |
|||
if readErr == io.EOF { |
|||
break |
|||
} |
|||
if readErr != nil { |
|||
return fmt.Errorf("failed to read file: %v", readErr) |
|||
} |
|||
} |
|||
|
|||
resp, err := stream.CloseAndRecv() |
|||
if err != nil { |
|||
return fmt.Errorf("failed to close stream: %v", err) |
|||
} |
|||
|
|||
if resp.Error != "" { |
|||
return fmt.Errorf("server error: %s", resp.Error) |
|||
} |
|||
|
|||
glog.V(2).Infof("Successfully sent %s (%d bytes) to %s", shardType, resp.BytesWritten, destServer) |
|||
return nil |
|||
}) |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue