Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Add support for TLS in gRPC communication between worker and volume server (#8370) 

* Add support for TLS in gRPC communication between worker and volume server

* address comments

* worker: capture shared grpc.DialOption in BalanceTask registration closure

* worker: capture shared grpc.DialOption in ErasureCodingTask registration closure

* worker: capture shared grpc.DialOption in VacuumTask registration closure

* worker: use grpc.worker security configuration section for tasks

* plugin/worker: fix compilation errors by passing grpc.DialOption to task constructors

* plugin/worker: prevent double-counting in EC skip counters

---------

Co-authored-by: Chris Lu <chris.lu@gmail.com>
pull/8373/head
Аlexey Medvedev 22 hours ago
committed by GitHub
parent
8ec9ff4a12
commit
6a3a97333f
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
9 changed files with 65 additions and 47 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      weed/plugin/worker/erasure_coding_handler.go
  2. 1
      weed/plugin/worker/vacuum_handler.go
  3. 1
      weed/plugin/worker/volume_balance_handler.go
  4. 43
      weed/worker/tasks/balance/balance_task.go
  5. 6
      weed/worker/tasks/balance/register.go
  6. 36
      weed/worker/tasks/erasure_coding/ec_task.go
  7. 6
      weed/worker/tasks/erasure_coding/register.go
  8. 6
      weed/worker/tasks/vacuum/register.go
  9. 10
      weed/worker/tasks/vacuum/vacuum_task.go

3
weed/plugin/worker/erasure_coding_handler.go
View File

@ -332,9 +332,11 @@ func emitErasureCodingDetectionDecisionTrace(
}
if metric.Age < quietThreshold {
skippedQuietTime++
continue
}
if metric.FullnessRatio < taskConfig.FullnessRatio {
skippedFullness++
continue
}
}
@ -482,6 +484,7 @@ func (h *ErasureCodingHandler) Execute(
params.Sources[0].Node,
params.VolumeId,
params.Collection,
h.grpcDialOption,
)
task.SetProgressCallback(func(progress float64, stage string) {
message := fmt.Sprintf("erasure coding progress %.0f%%", progress)

1
weed/plugin/worker/vacuum_handler.go
View File

@ -403,6 +403,7 @@ func (h *VacuumHandler) Execute(ctx context.Context, request *plugin_pb.ExecuteJ
params.Sources[0].Node,
params.VolumeId,
params.Collection,
h.grpcDialOption,
)
task.SetProgressCallback(func(progress float64, stage string) {
message := fmt.Sprintf("vacuum progress %.0f%%", progress)

1
weed/plugin/worker/volume_balance_handler.go
View File

@ -517,6 +517,7 @@ func (h *VolumeBalanceHandler) Execute(
params.Sources[0].Node,
params.VolumeId,
params.Collection,
h.grpcDialOption,
)
task.SetProgressCallback(func(progress float64, stage string) {
message := fmt.Sprintf("balance progress %.0f%%", progress)

43
weed/worker/tasks/balance/balance_task.go
View File

@ -21,19 +21,21 @@ import (
// BalanceTask implements the Task interface
type BalanceTask struct {
*base.BaseTask
server string
volumeID uint32
collection string
progress float64
server string
volumeID uint32
collection string
progress float64
grpcDialOption grpc.DialOption
}
// NewBalanceTask creates a new balance task instance
func NewBalanceTask(id string, server string, volumeID uint32, collection string) *BalanceTask {
func NewBalanceTask(id string, server string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *BalanceTask {
return &BalanceTask{
BaseTask: base.NewBaseTask(id, types.TaskTypeBalance),
server: server,
volumeID: volumeID,
collection: collection,
BaseTask: base.NewBaseTask(id, types.TaskTypeBalance),
server: server,
volumeID: volumeID,
collection: collection,
grpcDialOption: grpcDialOption,
}
}
@ -115,7 +117,7 @@ func (t *BalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParams)
t.ReportProgress(100.0)
glog.Infof("Balance task completed successfully: volume %d moved from %s to %s",
t.volumeID, t.server, destNode)
t.volumeID, sourceNode, destNode)
return nil
}
@ -164,7 +166,7 @@ func (t *BalanceTask) GetProgress() float64 {
// markVolumeReadonly marks the volume readonly
func (t *BalanceTask) markVolumeReadonly(server pb.ServerAddress, volumeId needle.VolumeId) error {
return operation.WithVolumeServerClient(false, server, grpc.WithInsecure(),
return operation.WithVolumeServerClient(false, server, t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: uint32(volumeId),
@ -177,7 +179,7 @@ func (t *BalanceTask) markVolumeReadonly(server pb.ServerAddress, volumeId needl
func (t *BalanceTask) copyVolume(sourceServer, targetServer pb.ServerAddress, volumeId needle.VolumeId) (uint64, error) {
var lastAppendAtNs uint64
err := operation.WithVolumeServerClient(true, targetServer, grpc.WithInsecure(),
err := operation.WithVolumeServerClient(true, targetServer, t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
stream, err := client.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{
VolumeId: uint32(volumeId),
@ -213,7 +215,7 @@ func (t *BalanceTask) copyVolume(sourceServer, targetServer pb.ServerAddress, vo
// mountVolume mounts the volume on the target server
func (t *BalanceTask) mountVolume(server pb.ServerAddress, volumeId needle.VolumeId) error {
return operation.WithVolumeServerClient(false, server, grpc.WithInsecure(),
return operation.WithVolumeServerClient(false, server, t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{
VolumeId: uint32(volumeId),
@ -224,7 +226,7 @@ func (t *BalanceTask) mountVolume(server pb.ServerAddress, volumeId needle.Volum
// tailVolume syncs remaining updates from source to target
func (t *BalanceTask) tailVolume(sourceServer, targetServer pb.ServerAddress, volumeId needle.VolumeId, sinceNs uint64) error {
return operation.WithVolumeServerClient(true, targetServer, grpc.WithInsecure(),
return operation.WithVolumeServerClient(true, targetServer, t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeTailReceiver(context.Background(), &volume_server_pb.VolumeTailReceiverRequest{
VolumeId: uint32(volumeId),
@ -236,20 +238,9 @@ func (t *BalanceTask) tailVolume(sourceServer, targetServer pb.ServerAddress, vo
})
}
// unmountVolume unmounts the volume from the server
func (t *BalanceTask) unmountVolume(server pb.ServerAddress, volumeId needle.VolumeId) error {
return operation.WithVolumeServerClient(false, server, grpc.WithInsecure(),
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeUnmount(context.Background(), &volume_server_pb.VolumeUnmountRequest{
VolumeId: uint32(volumeId),
})
return err
})
}
// deleteVolume deletes the volume from the server
func (t *BalanceTask) deleteVolume(server pb.ServerAddress, volumeId needle.VolumeId) error {
return operation.WithVolumeServerClient(false, server, grpc.WithInsecure(),
return operation.WithVolumeServerClient(false, server, t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{
VolumeId: uint32(volumeId),

6
weed/worker/tasks/balance/register.go
View File

@ -6,6 +6,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
@ -27,6 +29,9 @@ func RegisterBalanceTask() {
// Create configuration instance
config := NewDefaultConfig()
// Create shared gRPC dial option using TLS configuration
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
// Create complete task definition
taskDef := &base.TaskDefinition{
Type: types.TaskTypeBalance,
@ -50,6 +55,7 @@ func RegisterBalanceTask() {
params.Sources[0].Node, // Use first source node
params.VolumeId,
params.Collection,
dialOpt,
), nil
},
DetectionFunc: Detection,

36
weed/worker/tasks/erasure_coding/ec_task.go
View File

@ -26,11 +26,12 @@ import (
// ErasureCodingTask implements the Task interface
type ErasureCodingTask struct {
*base.BaseTask
server string
volumeID uint32
collection string
workDir string
progress float64
server string
volumeID uint32
collection string
workDir string
progress float64
grpcDialOption grpc.DialOption
// EC parameters
dataShards int32
@ -41,14 +42,15 @@ type ErasureCodingTask struct {
}
// NewErasureCodingTask creates a new unified EC task instance
func NewErasureCodingTask(id string, server string, volumeID uint32, collection string) *ErasureCodingTask {
func NewErasureCodingTask(id string, server string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *ErasureCodingTask {
return &ErasureCodingTask{
BaseTask: base.NewBaseTask(id, types.TaskTypeErasureCoding),
server: server,
volumeID: volumeID,
collection: collection,
dataShards: erasure_coding.DataShardsCount, // Default values
parityShards: erasure_coding.ParityShardsCount, // Default values
BaseTask: base.NewBaseTask(id, types.TaskTypeErasureCoding),
server: server,
volumeID: volumeID,
collection: collection,
dataShards: erasure_coding.DataShardsCount, // Default values
parityShards: erasure_coding.ParityShardsCount, // Default values
grpcDialOption: grpcDialOption,
}
}
@ -243,7 +245,7 @@ func (t *ErasureCodingTask) GetProgress() float64 {
// markVolumeReadonly marks the volume as readonly on the source server
func (t *ErasureCodingTask) markVolumeReadonly() error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: t.volumeID,
@ -301,7 +303,7 @@ func (t *ErasureCodingTask) copyVolumeFilesToWorker(workDir string) (map[string]
// copyFileFromSource copies a file from source server to local path using gRPC streaming
func (t *ErasureCodingTask) copyFileFromSource(ext, localPath string) error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
stream, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
VolumeId: t.volumeID,
@ -533,7 +535,7 @@ func (t *ErasureCodingTask) distributeEcShards(shardFiles map[string]string) err
// sendShardFileToDestination sends a single shard file to a destination server using ReceiveFile API
func (t *ErasureCodingTask) sendShardFileToDestination(destServer, filePath, shardType string) error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), grpc.WithInsecure(),
return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
// Open the local shard file
file, err := os.Open(filePath)
@ -665,7 +667,7 @@ func (t *ErasureCodingTask) mountEcShards() error {
continue
}
err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), grpc.WithInsecure(),
err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: t.volumeID,
@ -722,7 +724,7 @@ func (t *ErasureCodingTask) deleteOriginalVolume() error {
"volume_id": t.volumeID,
}).Info("Deleting volume from replica server")
err := operation.WithVolumeServerClient(false, pb.ServerAddress(replicaServer), grpc.WithInsecure(),
err := operation.WithVolumeServerClient(false, pb.ServerAddress(replicaServer), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
_, err := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{
VolumeId: t.volumeID,

6
weed/worker/tasks/erasure_coding/register.go
View File

@ -6,6 +6,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
@ -27,6 +29,9 @@ func RegisterErasureCodingTask() {
// Create configuration instance
config := NewDefaultConfig()
// Create shared gRPC dial option using TLS configuration
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
// Create complete task definition
taskDef := &base.TaskDefinition{
Type: types.TaskTypeErasureCoding,
@ -50,6 +55,7 @@ func RegisterErasureCodingTask() {
params.Sources[0].Node, // Use first source node
params.VolumeId,
params.Collection,
dialOpt,
), nil
},
DetectionFunc: Detection,

6
weed/worker/tasks/vacuum/register.go
View File

@ -6,6 +6,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
@ -27,6 +29,9 @@ func RegisterVacuumTask() {
// Create configuration instance
config := NewDefaultConfig()
// Create shared gRPC dial option using TLS configuration
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
// Create complete task definition
taskDef := &base.TaskDefinition{
Type: types.TaskTypeVacuum,
@ -50,6 +55,7 @@ func RegisterVacuumTask() {
params.Sources[0].Node, // Use first source node
params.VolumeId,
params.Collection,
dialOpt,
), nil
},
DetectionFunc: Detection,

10
weed/worker/tasks/vacuum/vacuum_task.go
View File

@ -24,16 +24,18 @@ type VacuumTask struct {
collection string
garbageThreshold float64
progress float64
grpcDialOption grpc.DialOption
}
// NewVacuumTask creates a new unified vacuum task instance
func NewVacuumTask(id string, server string, volumeID uint32, collection string) *VacuumTask {
func NewVacuumTask(id string, server string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *VacuumTask {
return &VacuumTask{
BaseTask: base.NewBaseTask(id, types.TaskTypeVacuum),
server: server,
volumeID: volumeID,
collection: collection,
garbageThreshold: 0.3, // Default 30% threshold
grpcDialOption: grpcDialOption,
}
}
@ -150,7 +152,7 @@ func (t *VacuumTask) GetProgress() float64 {
func (t *VacuumTask) checkVacuumEligibility() (bool, float64, error) {
var garbageRatio float64
err := operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
err := operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
resp, err := client.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
VolumeId: t.volumeID,
@ -177,7 +179,7 @@ func (t *VacuumTask) checkVacuumEligibility() (bool, float64, error) {
// performVacuum executes the actual vacuum operation
func (t *VacuumTask) performVacuum() error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
// Step 1: Compact the volume
t.GetLogger().Info("Compacting volume")
@ -225,7 +227,7 @@ func (t *VacuumTask) performVacuum() error {
// verifyVacuumResults checks the volume status after vacuum
func (t *VacuumTask) verifyVacuumResults() error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
resp, err := client.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
VolumeId: t.volumeID,

Write Preview
Loading…
Cancel
Save