Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

generate, balance, delete copied shards, delete old volume

pull/991/head
Chris Lu 6 years ago
parent
f0e6574d5e
commit
41e8ae61f8
4 changed files with 389 additions and 204 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 23
      weed/pb/volume_server.proto
  2. 407
      weed/pb/volume_server_pb/volume_server.pb.go
  3. 54
      weed/server/volume_grpc_erasure_coding.go
  4. 109
      weed/shell/command_ec_encode.go

23
weed/pb/volume_server.proto
View File

@ -48,9 +48,11 @@ service VolumeServer {
} }
// erasure coding // erasure coding
rpc VolumeEcGenerateSlices (VolumeEcGenerateSlicesRequest) returns (VolumeEcGenerateSlicesResponse) {
rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
} }
rpc VolumeEcCopy (VolumeEcCopyRequest) returns (VolumeEcCopyResponse) {
rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
}
rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
} }
@ -197,19 +199,28 @@ message VolumeTailReceiverRequest {
message VolumeTailReceiverResponse { message VolumeTailReceiverResponse {
} }
message VolumeEcGenerateSlicesRequest {
message VolumeEcShardsGenerateRequest {
uint32 volume_id = 1; uint32 volume_id = 1;
string collection = 2;
} }
message VolumeEcGenerateSlicesResponse {
message VolumeEcShardsGenerateResponse {
} }
message VolumeEcCopyRequest {
message VolumeEcShardsCopyRequest {
uint32 volume_id = 1; uint32 volume_id = 1;
string collection = 2; string collection = 2;
repeated uint32 ec_indexes = 3; repeated uint32 ec_indexes = 3;
string source_data_node = 5; string source_data_node = 5;
} }
message VolumeEcCopyResponse {
message VolumeEcShardsCopyResponse {
}
message VolumeEcShardsDeleteRequest {
uint32 volume_id = 1;
bool should_delete_ecx = 2;
repeated uint32 ec_indexes = 3;
}
message VolumeEcShardsDeleteResponse {
} }
message ReadVolumeFileStatusRequest { message ReadVolumeFileStatusRequest {

407
weed/pb/volume_server_pb/volume_server.pb.go
View File

@ -43,10 +43,12 @@ It has these top-level messages:
VolumeTailSenderResponse VolumeTailSenderResponse
VolumeTailReceiverRequest VolumeTailReceiverRequest
VolumeTailReceiverResponse VolumeTailReceiverResponse
VolumeEcGenerateSlicesRequest
VolumeEcGenerateSlicesResponse
VolumeEcCopyRequest
VolumeEcCopyResponse
VolumeEcShardsGenerateRequest
VolumeEcShardsGenerateResponse
VolumeEcShardsCopyRequest
VolumeEcShardsCopyResponse
VolumeEcShardsDeleteRequest
VolumeEcShardsDeleteResponse
ReadVolumeFileStatusRequest ReadVolumeFileStatusRequest
ReadVolumeFileStatusResponse ReadVolumeFileStatusResponse
DiskStatus DiskStatus
@ -770,77 +772,125 @@ func (m *VolumeTailReceiverResponse) String() string { return proto.C
func (*VolumeTailReceiverResponse) ProtoMessage() {} func (*VolumeTailReceiverResponse) ProtoMessage() {}
func (*VolumeTailReceiverResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} } func (*VolumeTailReceiverResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} }
type VolumeEcGenerateSlicesRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
type VolumeEcShardsGenerateRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
} }
func (m *VolumeEcGenerateSlicesRequest) Reset() { *m = VolumeEcGenerateSlicesRequest{} }
func (m *VolumeEcGenerateSlicesRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcGenerateSlicesRequest) ProtoMessage() {}
func (*VolumeEcGenerateSlicesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} }
func (m *VolumeEcShardsGenerateRequest) Reset() { *m = VolumeEcShardsGenerateRequest{} }
func (m *VolumeEcShardsGenerateRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsGenerateRequest) ProtoMessage() {}
func (*VolumeEcShardsGenerateRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} }
func (m *VolumeEcGenerateSlicesRequest) GetVolumeId() uint32 {
func (m *VolumeEcShardsGenerateRequest) GetVolumeId() uint32 {
if m != nil { if m != nil {
return m.VolumeId return m.VolumeId
} }
return 0 return 0
} }
type VolumeEcGenerateSlicesResponse struct {
func (m *VolumeEcShardsGenerateRequest) GetCollection() string {
if m != nil {
return m.Collection
}
return ""
}
type VolumeEcShardsGenerateResponse struct {
} }
func (m *VolumeEcGenerateSlicesResponse) Reset() { *m = VolumeEcGenerateSlicesResponse{} }
func (m *VolumeEcGenerateSlicesResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcGenerateSlicesResponse) ProtoMessage() {}
func (*VolumeEcGenerateSlicesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} }
func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} }
func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsGenerateResponse) ProtoMessage() {}
func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} }
type VolumeEcCopyRequest struct {
type VolumeEcShardsCopyRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"` EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"`
SourceDataNode string `protobuf:"bytes,5,opt,name=source_data_node,json=sourceDataNode" json:"source_data_node,omitempty"` SourceDataNode string `protobuf:"bytes,5,opt,name=source_data_node,json=sourceDataNode" json:"source_data_node,omitempty"`
} }
func (m *VolumeEcCopyRequest) Reset() { *m = VolumeEcCopyRequest{} }
func (m *VolumeEcCopyRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcCopyRequest) ProtoMessage() {}
func (*VolumeEcCopyRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} }
func (m *VolumeEcShardsCopyRequest) Reset() { *m = VolumeEcShardsCopyRequest{} }
func (m *VolumeEcShardsCopyRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsCopyRequest) ProtoMessage() {}
func (*VolumeEcShardsCopyRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} }
func (m *VolumeEcCopyRequest) GetVolumeId() uint32 {
func (m *VolumeEcShardsCopyRequest) GetVolumeId() uint32 {
if m != nil { if m != nil {
return m.VolumeId return m.VolumeId
} }
return 0 return 0
} }
func (m *VolumeEcCopyRequest) GetCollection() string {
func (m *VolumeEcShardsCopyRequest) GetCollection() string {
if m != nil { if m != nil {
return m.Collection return m.Collection
} }
return "" return ""
} }
func (m *VolumeEcCopyRequest) GetEcIndexes() []uint32 {
func (m *VolumeEcShardsCopyRequest) GetEcIndexes() []uint32 {
if m != nil { if m != nil {
return m.EcIndexes return m.EcIndexes
} }
return nil return nil
} }
func (m *VolumeEcCopyRequest) GetSourceDataNode() string {
func (m *VolumeEcShardsCopyRequest) GetSourceDataNode() string {
if m != nil { if m != nil {
return m.SourceDataNode return m.SourceDataNode
} }
return "" return ""
} }
type VolumeEcCopyResponse struct {
type VolumeEcShardsCopyResponse struct {
}
func (m *VolumeEcShardsCopyResponse) Reset() { *m = VolumeEcShardsCopyResponse{} }
func (m *VolumeEcShardsCopyResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsCopyResponse) ProtoMessage() {}
func (*VolumeEcShardsCopyResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{37} }
type VolumeEcShardsDeleteRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
ShouldDeleteEcx bool `protobuf:"varint,2,opt,name=should_delete_ecx,json=shouldDeleteEcx" json:"should_delete_ecx,omitempty"`
EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"`
}
func (m *VolumeEcShardsDeleteRequest) Reset() { *m = VolumeEcShardsDeleteRequest{} }
func (m *VolumeEcShardsDeleteRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsDeleteRequest) ProtoMessage() {}
func (*VolumeEcShardsDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{38} }
func (m *VolumeEcShardsDeleteRequest) GetVolumeId() uint32 {
if m != nil {
return m.VolumeId
}
return 0
} }
func (m *VolumeEcCopyResponse) Reset() { *m = VolumeEcCopyResponse{} }
func (m *VolumeEcCopyResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcCopyResponse) ProtoMessage() {}
func (*VolumeEcCopyResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{37} }
func (m *VolumeEcShardsDeleteRequest) GetShouldDeleteEcx() bool {
if m != nil {
return m.ShouldDeleteEcx
}
return false
}
func (m *VolumeEcShardsDeleteRequest) GetEcIndexes() []uint32 {
if m != nil {
return m.EcIndexes
}
return nil
}
type VolumeEcShardsDeleteResponse struct {
}
func (m *VolumeEcShardsDeleteResponse) Reset() { *m = VolumeEcShardsDeleteResponse{} }
func (m *VolumeEcShardsDeleteResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsDeleteResponse) ProtoMessage() {}
func (*VolumeEcShardsDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{39} }
type ReadVolumeFileStatusRequest struct { type ReadVolumeFileStatusRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
@ -849,7 +899,7 @@ type ReadVolumeFileStatusRequest struct {
func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} } func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} }
func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) } func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) }
func (*ReadVolumeFileStatusRequest) ProtoMessage() {} func (*ReadVolumeFileStatusRequest) ProtoMessage() {}
func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{38} }
func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{40} }
func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 { func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 {
if m != nil { if m != nil {
@ -872,7 +922,7 @@ type ReadVolumeFileStatusResponse struct {
func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} } func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} }
func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) } func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) }
func (*ReadVolumeFileStatusResponse) ProtoMessage() {} func (*ReadVolumeFileStatusResponse) ProtoMessage() {}
func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{39} }
func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} }
func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 { func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 {
if m != nil { if m != nil {
@ -940,7 +990,7 @@ type DiskStatus struct {
func (m *DiskStatus) Reset() { *m = DiskStatus{} } func (m *DiskStatus) Reset() { *m = DiskStatus{} }
func (m *DiskStatus) String() string { return proto.CompactTextString(m) } func (m *DiskStatus) String() string { return proto.CompactTextString(m) }
func (*DiskStatus) ProtoMessage() {} func (*DiskStatus) ProtoMessage() {}
func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{40} }
func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{42} }
func (m *DiskStatus) GetDir() string { func (m *DiskStatus) GetDir() string {
if m != nil { if m != nil {
@ -983,7 +1033,7 @@ type MemStatus struct {
func (m *MemStatus) Reset() { *m = MemStatus{} } func (m *MemStatus) Reset() { *m = MemStatus{} }
func (m *MemStatus) String() string { return proto.CompactTextString(m) } func (m *MemStatus) String() string { return proto.CompactTextString(m) }
func (*MemStatus) ProtoMessage() {} func (*MemStatus) ProtoMessage() {}
func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} }
func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{43} }
func (m *MemStatus) GetGoroutines() int32 { func (m *MemStatus) GetGoroutines() int32 {
if m != nil { if m != nil {
@ -1069,10 +1119,12 @@ func init() {
proto.RegisterType((*VolumeTailSenderResponse)(nil), "volume_server_pb.VolumeTailSenderResponse") proto.RegisterType((*VolumeTailSenderResponse)(nil), "volume_server_pb.VolumeTailSenderResponse")
proto.RegisterType((*VolumeTailReceiverRequest)(nil), "volume_server_pb.VolumeTailReceiverRequest") proto.RegisterType((*VolumeTailReceiverRequest)(nil), "volume_server_pb.VolumeTailReceiverRequest")
proto.RegisterType((*VolumeTailReceiverResponse)(nil), "volume_server_pb.VolumeTailReceiverResponse") proto.RegisterType((*VolumeTailReceiverResponse)(nil), "volume_server_pb.VolumeTailReceiverResponse")
proto.RegisterType((*VolumeEcGenerateSlicesRequest)(nil), "volume_server_pb.VolumeEcGenerateSlicesRequest")
proto.RegisterType((*VolumeEcGenerateSlicesResponse)(nil), "volume_server_pb.VolumeEcGenerateSlicesResponse")
proto.RegisterType((*VolumeEcCopyRequest)(nil), "volume_server_pb.VolumeEcCopyRequest")
proto.RegisterType((*VolumeEcCopyResponse)(nil), "volume_server_pb.VolumeEcCopyResponse")
proto.RegisterType((*VolumeEcShardsGenerateRequest)(nil), "volume_server_pb.VolumeEcShardsGenerateRequest")
proto.RegisterType((*VolumeEcShardsGenerateResponse)(nil), "volume_server_pb.VolumeEcShardsGenerateResponse")
proto.RegisterType((*VolumeEcShardsCopyRequest)(nil), "volume_server_pb.VolumeEcShardsCopyRequest")
proto.RegisterType((*VolumeEcShardsCopyResponse)(nil), "volume_server_pb.VolumeEcShardsCopyResponse")
proto.RegisterType((*VolumeEcShardsDeleteRequest)(nil), "volume_server_pb.VolumeEcShardsDeleteRequest")
proto.RegisterType((*VolumeEcShardsDeleteResponse)(nil), "volume_server_pb.VolumeEcShardsDeleteResponse")
proto.RegisterType((*ReadVolumeFileStatusRequest)(nil), "volume_server_pb.ReadVolumeFileStatusRequest") proto.RegisterType((*ReadVolumeFileStatusRequest)(nil), "volume_server_pb.ReadVolumeFileStatusRequest")
proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse") proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse")
proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus") proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus")
@ -1110,8 +1162,9 @@ type VolumeServerClient interface {
VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error) VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error)
VolumeTailReceiver(ctx context.Context, in *VolumeTailReceiverRequest, opts ...grpc.CallOption) (*VolumeTailReceiverResponse, error) VolumeTailReceiver(ctx context.Context, in *VolumeTailReceiverRequest, opts ...grpc.CallOption) (*VolumeTailReceiverResponse, error)
// erasure coding // erasure coding
VolumeEcGenerateSlices(ctx context.Context, in *VolumeEcGenerateSlicesRequest, opts ...grpc.CallOption) (*VolumeEcGenerateSlicesResponse, error)
VolumeEcCopy(ctx context.Context, in *VolumeEcCopyRequest, opts ...grpc.CallOption) (*VolumeEcCopyResponse, error)
VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error)
VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error)
VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error)
} }
type volumeServerClient struct { type volumeServerClient struct {
@ -1344,18 +1397,27 @@ func (c *volumeServerClient) VolumeTailReceiver(ctx context.Context, in *VolumeT
return out, nil return out, nil
} }
func (c *volumeServerClient) VolumeEcGenerateSlices(ctx context.Context, in *VolumeEcGenerateSlicesRequest, opts ...grpc.CallOption) (*VolumeEcGenerateSlicesResponse, error) {
out := new(VolumeEcGenerateSlicesResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcGenerateSlices", in, out, c.cc, opts...)
func (c *volumeServerClient) VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error) {
out := new(VolumeEcShardsGenerateResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate", in, out, c.cc, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return out, nil return out, nil
} }
func (c *volumeServerClient) VolumeEcCopy(ctx context.Context, in *VolumeEcCopyRequest, opts ...grpc.CallOption) (*VolumeEcCopyResponse, error) {
out := new(VolumeEcCopyResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcCopy", in, out, c.cc, opts...)
func (c *volumeServerClient) VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error) {
out := new(VolumeEcShardsCopyResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsCopy", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error) {
out := new(VolumeEcShardsDeleteResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsDelete", in, out, c.cc, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1385,8 +1447,9 @@ type VolumeServerServer interface {
VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error
VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error) VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error)
// erasure coding // erasure coding
VolumeEcGenerateSlices(context.Context, *VolumeEcGenerateSlicesRequest) (*VolumeEcGenerateSlicesResponse, error)
VolumeEcCopy(context.Context, *VolumeEcCopyRequest) (*VolumeEcCopyResponse, error)
VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error)
VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error)
VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error)
} }
func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) { func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) {
@ -1708,38 +1771,56 @@ func _VolumeServer_VolumeTailReceiver_Handler(srv interface{}, ctx context.Conte
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _VolumeServer_VolumeEcGenerateSlices_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcGenerateSlicesRequest)
func _VolumeServer_VolumeEcShardsGenerate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcShardsGenerateRequest)
if err := dec(in); err != nil { if err := dec(in); err != nil {
return nil, err return nil, err
} }
if interceptor == nil { if interceptor == nil {
return srv.(VolumeServerServer).VolumeEcGenerateSlices(ctx, in)
return srv.(VolumeServerServer).VolumeEcShardsGenerate(ctx, in)
} }
info := &grpc.UnaryServerInfo{ info := &grpc.UnaryServerInfo{
Server: srv, Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcGenerateSlices",
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate",
} }
handler := func(ctx context.Context, req interface{}) (interface{}, error) { handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).VolumeEcGenerateSlices(ctx, req.(*VolumeEcGenerateSlicesRequest))
return srv.(VolumeServerServer).VolumeEcShardsGenerate(ctx, req.(*VolumeEcShardsGenerateRequest))
} }
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _VolumeServer_VolumeEcCopy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcCopyRequest)
func _VolumeServer_VolumeEcShardsCopy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcShardsCopyRequest)
if err := dec(in); err != nil { if err := dec(in); err != nil {
return nil, err return nil, err
} }
if interceptor == nil { if interceptor == nil {
return srv.(VolumeServerServer).VolumeEcCopy(ctx, in)
return srv.(VolumeServerServer).VolumeEcShardsCopy(ctx, in)
} }
info := &grpc.UnaryServerInfo{ info := &grpc.UnaryServerInfo{
Server: srv, Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcCopy",
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsCopy",
} }
handler := func(ctx context.Context, req interface{}) (interface{}, error) { handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).VolumeEcCopy(ctx, req.(*VolumeEcCopyRequest))
return srv.(VolumeServerServer).VolumeEcShardsCopy(ctx, req.(*VolumeEcShardsCopyRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_VolumeEcShardsDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcShardsDeleteRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).VolumeEcShardsDelete(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsDelete",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).VolumeEcShardsDelete(ctx, req.(*VolumeEcShardsDeleteRequest))
} }
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
@ -1805,12 +1886,16 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
Handler: _VolumeServer_VolumeTailReceiver_Handler, Handler: _VolumeServer_VolumeTailReceiver_Handler,
}, },
{ {
MethodName: "VolumeEcGenerateSlices",
Handler: _VolumeServer_VolumeEcGenerateSlices_Handler,
MethodName: "VolumeEcShardsGenerate",
Handler: _VolumeServer_VolumeEcShardsGenerate_Handler,
},
{
MethodName: "VolumeEcShardsCopy",
Handler: _VolumeServer_VolumeEcShardsCopy_Handler,
}, },
{ {
MethodName: "VolumeEcCopy",
Handler: _VolumeServer_VolumeEcCopy_Handler,
MethodName: "VolumeEcShardsDelete",
Handler: _VolumeServer_VolumeEcShardsDelete_Handler,
}, },
}, },
Streams: []grpc.StreamDesc{ Streams: []grpc.StreamDesc{
@ -1836,101 +1921,105 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) } func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 1534 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x58, 0xcb, 0x6f, 0xd4, 0xd6,
0x1a, 0x8f, 0x99, 0xc9, 0xeb, 0x9b, 0x04, 0x86, 0x93, 0x90, 0x4c, 0x0c, 0x09, 0x83, 0xe1, 0xc2,
0xf0, 0xb8, 0x81, 0x0b, 0xba, 0xf7, 0xd2, 0x87, 0xd4, 0x42, 0xa0, 0x6d, 0xa4, 0x02, 0x92, 0x03,
0xa8, 0x55, 0x2b, 0x59, 0x27, 0xf6, 0x17, 0x62, 0xc5, 0x63, 0x1b, 0xfb, 0x38, 0x4d, 0xaa, 0x76,
0xd5, 0xae, 0xba, 0xec, 0xb6, 0xcb, 0xee, 0xba, 0xe8, 0xb6, 0x7f, 0x55, 0xff, 0x82, 0x6e, 0xaa,
0xf3, 0xb0, 0x67, 0xfc, 0xca, 0x98, 0x52, 0xa9, 0xbb, 0x33, 0xdf, 0xf9, 0x5e, 0xe7, 0xf3, 0xf7,
0xfa, 0x0d, 0x2c, 0x1d, 0x06, 0x5e, 0x32, 0x44, 0x2b, 0xc6, 0xe8, 0x10, 0xa3, 0xcd, 0x30, 0x0a,
0x58, 0x40, 0xba, 0x39, 0xa2, 0x15, 0xee, 0x1a, 0xb7, 0x81, 0x3c, 0xa4, 0xcc, 0xde, 0x7f, 0x84,
0x1e, 0x32, 0x34, 0xf1, 0x75, 0x82, 0x31, 0x23, 0x6b, 0x30, 0xb7, 0xe7, 0x7a, 0x68, 0xb9, 0x4e,
0xdc, 0xd3, 0xfa, 0xad, 0xc1, 0xbc, 0x39, 0xcb, 0x7f, 0x6f, 0x3b, 0xb1, 0xf1, 0x0c, 0x96, 0x72,
0x02, 0x71, 0x18, 0xf8, 0x31, 0x92, 0xfb, 0x30, 0x1b, 0x61, 0x9c, 0x78, 0x4c, 0x0a, 0x74, 0xee,
0x6e, 0x6c, 0x16, 0x6d, 0x6d, 0x66, 0x22, 0x89, 0xc7, 0xcc, 0x94, 0xdd, 0x70, 0x61, 0x61, 0xfc,
0x82, 0xac, 0xc2, 0xac, 0xb2, 0xdd, 0xd3, 0xfa, 0xda, 0x60, 0xde, 0x9c, 0x91, 0xa6, 0xc9, 0x0a,
0xcc, 0xc4, 0x8c, 0xb2, 0x24, 0xee, 0x9d, 0xea, 0x6b, 0x83, 0x69, 0x53, 0xfd, 0x22, 0xcb, 0x30,
0x8d, 0x51, 0x14, 0x44, 0xbd, 0x96, 0x60, 0x97, 0x3f, 0x08, 0x81, 0x76, 0xec, 0x7e, 0x8d, 0xbd,
0x76, 0x5f, 0x1b, 0x2c, 0x9a, 0xe2, 0x6c, 0xcc, 0xc2, 0xf4, 0xe3, 0x61, 0xc8, 0x8e, 0x8d, 0xff,
0x43, 0xef, 0x25, 0xb5, 0x93, 0x64, 0xf8, 0x52, 0xf8, 0xb8, 0xb5, 0x8f, 0xf6, 0x41, 0xfa, 0xf6,
0xf3, 0x30, 0xaf, 0x3c, 0x57, 0x1e, 0x2c, 0x9a, 0x73, 0x92, 0xb0, 0xed, 0x18, 0x1f, 0xc2, 0x5a,
0x85, 0xa0, 0x8a, 0xc1, 0x65, 0x58, 0x7c, 0x45, 0xa3, 0x5d, 0xfa, 0x0a, 0xad, 0x88, 0x32, 0x37,
0x10, 0xd2, 0x9a, 0xb9, 0xa0, 0x88, 0x26, 0xa7, 0x19, 0x5f, 0x80, 0x9e, 0xd3, 0x10, 0x0c, 0x43,
0x6a, 0xb3, 0x26, 0xc6, 0x49, 0x1f, 0x3a, 0x61, 0x84, 0xd4, 0xf3, 0x02, 0x9b, 0x32, 0x14, 0x51,
0x68, 0x99, 0xe3, 0x24, 0x63, 0x1d, 0xce, 0x57, 0x2a, 0x97, 0x0e, 0x1a, 0xf7, 0x0b, 0xde, 0x07,
0xc3, 0xa1, 0xdb, 0xc8, 0xb4, 0x71, 0xa1, 0xe4, 0xb5, 0x90, 0x54, 0x7a, 0xdf, 0x29, 0xdc, 0x7a,
0x48, 0xfd, 0x24, 0x6c, 0xa4, 0xb8, 0xe8, 0x71, 0x2a, 0x9a, 0x69, 0x5e, 0x95, 0xc9, 0xb1, 0x15,
0x78, 0x1e, 0xda, 0xcc, 0x0d, 0xfc, 0x54, 0xed, 0x06, 0x80, 0x9d, 0x11, 0x55, 0xaa, 0x8c, 0x51,
0x0c, 0x1d, 0x7a, 0x65, 0x51, 0xa5, 0xf6, 0x17, 0x0d, 0xce, 0x3d, 0x50, 0x41, 0x93, 0x86, 0x1b,
0x7d, 0x80, 0xbc, 0xc9, 0x53, 0x45, 0x93, 0xc5, 0x0f, 0xd4, 0x2a, 0x7d, 0x20, 0xce, 0x11, 0x61,
0xe8, 0xb9, 0x36, 0x15, 0x2a, 0xda, 0x42, 0xc5, 0x38, 0x89, 0x74, 0xa1, 0xc5, 0x98, 0xd7, 0x9b,
0x16, 0x37, 0xfc, 0x68, 0xf4, 0x60, 0xa5, 0xe8, 0xab, 0x7a, 0xc6, 0xff, 0x60, 0x55, 0x52, 0x76,
0x8e, 0x7d, 0x7b, 0x47, 0x54, 0x43, 0xa3, 0xa0, 0xff, 0xa1, 0x41, 0xaf, 0x2c, 0xa8, 0xb2, 0xf8,
0x6d, 0x23, 0xf0, 0xa6, 0xef, 0x23, 0x17, 0xa1, 0xc3, 0xa8, 0xeb, 0x59, 0xc1, 0xde, 0x5e, 0x8c,
0xac, 0x37, 0xd3, 0xd7, 0x06, 0x6d, 0x13, 0x38, 0xe9, 0x99, 0xa0, 0x90, 0xeb, 0xd0, 0xb5, 0x65,
0x26, 0x5b, 0x11, 0x1e, 0xba, 0x31, 0xd7, 0x3c, 0x2b, 0x1c, 0x3b, 0x63, 0xa7, 0x19, 0x2e, 0xc9,
0xc4, 0x80, 0x45, 0xd7, 0x39, 0xb2, 0x44, 0x03, 0x11, 0xe5, 0x3f, 0x27, 0xb4, 0x75, 0x5c, 0xe7,
0xe8, 0x23, 0xd7, 0xc3, 0x1d, 0xde, 0x05, 0x5e, 0xc2, 0x05, 0xf9, 0xf8, 0x6d, 0xdf, 0x8e, 0x70,
0x88, 0x3e, 0xa3, 0xde, 0x56, 0x10, 0x1e, 0x37, 0x4a, 0x81, 0x35, 0x98, 0x8b, 0x5d, 0xdf, 0x46,
0xcb, 0x97, 0x6d, 0xa8, 0x6d, 0xce, 0x8a, 0xdf, 0x4f, 0x63, 0xe3, 0x21, 0xac, 0xd7, 0xe8, 0x55,
0x91, 0xbd, 0x04, 0x0b, 0xc2, 0x31, 0x3b, 0xf0, 0x19, 0xfa, 0x4c, 0xe8, 0x5e, 0x30, 0x3b, 0x9c,
0xb6, 0x25, 0x49, 0xc6, 0x7f, 0x80, 0x48, 0x1d, 0x4f, 0x82, 0xc4, 0x6f, 0x56, 0x9a, 0xe7, 0x60,
0x29, 0x27, 0xa2, 0x72, 0xe3, 0x1e, 0x2c, 0x4b, 0xf2, 0x0b, 0x7f, 0xd8, 0x58, 0xd7, 0x2a, 0x9c,
0x2b, 0x08, 0x29, 0x6d, 0x77, 0x53, 0x23, 0xf9, 0x39, 0x71, 0xa2, 0xb2, 0x95, 0xd4, 0x83, 0xfc,
0xa8, 0x30, 0x7e, 0xd5, 0xe0, 0x6c, 0xda, 0x46, 0x1a, 0x46, 0xfd, 0x0d, 0xd3, 0xae, 0x55, 0x9b,
0x76, 0xed, 0x51, 0xda, 0x0d, 0xa0, 0x1b, 0x07, 0x49, 0x64, 0xa3, 0xe5, 0x50, 0x46, 0x2d, 0x3f,
0x70, 0x50, 0x65, 0xe5, 0x69, 0x49, 0x7f, 0x44, 0x19, 0x7d, 0x1a, 0x38, 0x68, 0x7c, 0x90, 0x7e,
0x94, 0xdc, 0xd7, 0xbc, 0x0e, 0x67, 0x3d, 0x1a, 0x33, 0x8b, 0x86, 0x21, 0xfa, 0x8e, 0x45, 0x19,
0x4f, 0x09, 0x4d, 0xa4, 0xc4, 0x69, 0x7e, 0xf1, 0x40, 0xd0, 0x1f, 0xb0, 0xa7, 0xb1, 0xf1, 0xa3,
0x06, 0x67, 0xb8, 0x2c, 0x4f, 0xc1, 0x46, 0xef, 0xed, 0x42, 0x0b, 0x8f, 0x98, 0x7a, 0x28, 0x3f,
0x92, 0xdb, 0xb0, 0xa4, 0x72, 0xdd, 0x0d, 0xfc, 0x51, 0x19, 0xb4, 0x84, 0x20, 0x19, 0x5d, 0x65,
0x95, 0x70, 0x11, 0x3a, 0x31, 0x0b, 0xc2, 0xb4, 0xaa, 0xda, 0xb2, 0xaa, 0x38, 0x49, 0x56, 0x95,
0xf1, 0x5f, 0xe8, 0x8e, 0x7c, 0x6a, 0x9e, 0xa1, 0xdf, 0x69, 0x69, 0xd3, 0x79, 0x4e, 0x5d, 0x6f,
0x07, 0x7d, 0x07, 0xa3, 0xb7, 0xac, 0x1c, 0x72, 0x07, 0x96, 0x5d, 0xc7, 0x43, 0x8b, 0xb9, 0x43,
0x0c, 0x12, 0x66, 0xc5, 0x68, 0x07, 0xbe, 0x13, 0xa7, 0xaf, 0xe3, 0x77, 0xcf, 0xe5, 0xd5, 0x8e,
0xbc, 0x31, 0xbe, 0xcf, 0x3a, 0xd8, 0xb8, 0x17, 0xa3, 0x39, 0xec, 0x23, 0x72, 0x85, 0xfb, 0x48,
0x1d, 0x8c, 0xd4, 0x33, 0x16, 0x24, 0xf1, 0x13, 0x41, 0xe3, 0xf1, 0x51, 0x4c, 0xbb, 0x81, 0x73,
0x2c, 0x3c, 0x5a, 0x30, 0x41, 0x92, 0x1e, 0x06, 0xce, 0xb1, 0x68, 0x25, 0xb1, 0x25, 0x3e, 0xb1,
0xbd, 0x9f, 0xf8, 0x07, 0xc2, 0x9b, 0x39, 0xb3, 0xe3, 0xc6, 0x9f, 0xd2, 0x98, 0x6d, 0x71, 0x92,
0xf1, 0x9b, 0x06, 0x6b, 0x23, 0x37, 0x4c, 0xb4, 0xd1, 0x3d, 0xfc, 0x07, 0xc2, 0xc1, 0x25, 0x54,
0x2e, 0xe7, 0x96, 0x2e, 0x95, 0xee, 0x44, 0xde, 0xa9, 0x8e, 0x2f, 0x6e, 0xc4, 0x40, 0xaf, 0x70,
0x5c, 0x95, 0xe8, 0xfb, 0x69, 0x2b, 0x7b, 0x6c, 0x7f, 0x8c, 0x3e, 0x46, 0x94, 0xe1, 0x8e, 0xe7,
0xda, 0xd8, 0x6c, 0xbc, 0xf4, 0x61, 0xa3, 0x4e, 0x5a, 0xe9, 0xff, 0x49, 0x4b, 0xfb, 0xc9, 0x63,
0xfb, 0x6f, 0x6b, 0x02, 0xeb, 0x00, 0x68, 0x5b, 0xae, 0xef, 0xe0, 0x11, 0xf2, 0x60, 0xb5, 0x06,
0x8b, 0xe6, 0x3c, 0xda, 0xdb, 0x92, 0xf0, 0x06, 0xf5, 0x9e, 0x35, 0xae, 0xd4, 0x39, 0xe5, 0xf5,
0xbb, 0x70, 0xde, 0x44, 0xea, 0xc8, 0x3b, 0x31, 0x4e, 0x9a, 0x8f, 0xdc, 0xdf, 0x4f, 0xc1, 0x85,
0x6a, 0xe1, 0x26, 0x63, 0xf7, 0x3d, 0xd0, 0xb3, 0xb1, 0xc6, 0xb3, 0x22, 0x66, 0x74, 0x18, 0x66,
0x79, 0x21, 0xd3, 0x67, 0x55, 0xcd, 0xb8, 0xe7, 0xe9, 0x7d, 0x9a, 0x1c, 0xa5, 0x99, 0xd8, 0x2a,
0xcd, 0x44, 0x6e, 0xc0, 0xa1, 0xac, 0xce, 0x80, 0x6c, 0x1e, 0xab, 0x0e, 0x65, 0x75, 0x06, 0x32,
0x61, 0x61, 0x60, 0x5a, 0x1a, 0x50, 0xfc, 0xc2, 0xc0, 0x3a, 0x80, 0xea, 0x2c, 0x89, 0x9f, 0xce,
0xf8, 0x79, 0xd9, 0x57, 0x12, 0xbf, 0xb6, 0xbd, 0xcd, 0xd6, 0xb6, 0xb7, 0x7c, 0x32, 0xcc, 0x95,
0xb6, 0xbf, 0xcf, 0x00, 0x1e, 0xb9, 0xf1, 0x81, 0x0c, 0x32, 0xef, 0xa7, 0x8e, 0x1b, 0xa9, 0x25,
0x91, 0x1f, 0x39, 0x85, 0x7a, 0x9e, 0x0a, 0x1d, 0x3f, 0x72, 0xc0, 0x90, 0xc4, 0xe8, 0xa8, 0xe8,
0x88, 0x33, 0xa7, 0xed, 0x45, 0x88, 0x2a, 0x00, 0xe2, 0x6c, 0xfc, 0xac, 0xc1, 0xfc, 0x13, 0x1c,
0x2a, 0xcd, 0x1b, 0x00, 0xaf, 0x82, 0x28, 0x48, 0x98, 0xeb, 0xa3, 0x6c, 0xff, 0xd3, 0xe6, 0x18,
0xe5, 0xaf, 0xdb, 0x11, 0x00, 0x06, 0xbd, 0x3d, 0x15, 0x4c, 0x71, 0xe6, 0xb4, 0x7d, 0xa4, 0xa1,
0x8a, 0x9f, 0x38, 0x73, 0xf8, 0x13, 0x33, 0x6a, 0x1f, 0x88, 0x60, 0xb5, 0x4d, 0xf9, 0xe3, 0xee,
0x0f, 0x5d, 0x58, 0x18, 0x2f, 0x78, 0xf2, 0x25, 0x74, 0xc6, 0x70, 0x1b, 0xb9, 0x52, 0x86, 0x67,
0x65, 0x1c, 0xa8, 0xff, 0x6b, 0x02, 0x97, 0x2a, 0x8c, 0x29, 0xe2, 0xc3, 0xd9, 0x12, 0x2e, 0x22,
0x37, 0xca, 0xd2, 0x75, 0xa8, 0x4b, 0xbf, 0xd9, 0x88, 0x37, 0xb3, 0xc7, 0x60, 0xa9, 0x02, 0xe8,
0x90, 0x5b, 0x13, 0xb4, 0xe4, 0xc0, 0x96, 0xfe, 0xef, 0x86, 0xdc, 0x99, 0xd5, 0xd7, 0x40, 0xca,
0x28, 0x88, 0xdc, 0x9c, 0xa8, 0x66, 0x84, 0xb2, 0xf4, 0x5b, 0xcd, 0x98, 0x6b, 0x1f, 0x2a, 0xf1,
0xd1, 0xc4, 0x87, 0xe6, 0x10, 0xd8, 0xc4, 0x87, 0x16, 0x40, 0xd7, 0x14, 0x39, 0x80, 0x6e, 0x11,
0x3b, 0x91, 0xeb, 0x75, 0x80, 0xbe, 0x04, 0xcd, 0xf4, 0x1b, 0x4d, 0x58, 0x33, 0x63, 0x08, 0xa7,
0xf3, 0xf8, 0x86, 0x5c, 0x2b, 0xcb, 0x57, 0xa2, 0x35, 0x7d, 0x30, 0x99, 0x71, 0xfc, 0x4d, 0x45,
0xcc, 0x53, 0xf5, 0xa6, 0x1a, 0x40, 0x55, 0xf5, 0xa6, 0x3a, 0x08, 0x65, 0x4c, 0x91, 0x6f, 0xd2,
0x45, 0xba, 0x80, 0x05, 0xc8, 0x66, 0x9d, 0x9a, 0x6a, 0x30, 0xa2, 0xdf, 0x6e, 0xcc, 0x9f, 0xda,
0xbe, 0xa3, 0xf1, 0x5a, 0x1f, 0x83, 0x04, 0x55, 0xb5, 0x5e, 0x06, 0x19, 0x55, 0xb5, 0x5e, 0x85,
0x2b, 0xa6, 0xc8, 0x2e, 0x2c, 0xe6, 0x40, 0x02, 0xb9, 0x5a, 0x27, 0x99, 0x87, 0x1e, 0xfa, 0xb5,
0x89, 0x7c, 0x99, 0x0d, 0x2b, 0xed, 0x5e, 0xaa, 0x5d, 0xd5, 0x3a, 0x97, 0xef, 0x57, 0x57, 0x27,
0xb1, 0x65, 0x06, 0x3e, 0x07, 0x18, 0xed, 0xf4, 0xe4, 0x72, 0x9d, 0xdc, 0xf8, 0xa7, 0xb8, 0x72,
0x32, 0x53, 0xa6, 0xfa, 0x2b, 0x58, 0xae, 0x9a, 0xf4, 0xa4, 0xa2, 0x0a, 0x4f, 0x58, 0x27, 0xf4,
0xcd, 0xa6, 0xec, 0x99, 0xe1, 0x17, 0x30, 0x97, 0x6e, 0xf4, 0xe4, 0x52, 0x59, 0xba, 0x80, 0x40,
0x74, 0xe3, 0x24, 0x96, 0xb1, 0x6c, 0x1a, 0xa6, 0x85, 0x33, 0x5a, 0xb5, 0xeb, 0x0b, 0xa7, 0x04,
0x0a, 0xea, 0x0b, 0xa7, 0xbc, 0xb9, 0x0b, 0x73, 0xaf, 0x53, 0xb4, 0x35, 0xbe, 0x99, 0x56, 0x36,
0xd9, 0xba, 0xc5, 0xbb, 0xb2, 0xc9, 0xd6, 0x2f, 0xbb, 0x53, 0xe4, 0x5b, 0x58, 0xa9, 0x5e, 0x58,
0x49, 0x6d, 0xf9, 0xd5, 0x2c, 0xc6, 0xfa, 0x9d, 0xe6, 0x02, 0xe5, 0x64, 0x97, 0xfb, 0x66, 0x7d,
0xb2, 0xe7, 0x96, 0xe5, 0xfa, 0x64, 0x2f, 0xac, 0xad, 0x53, 0xbb, 0x33, 0xe2, 0xdf, 0xdf, 0x7b,
0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0x2f, 0xf9, 0x4c, 0xc9, 0x14, 0x16, 0x00, 0x00,
// 1598 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x58, 0x5f, 0x6f, 0xd4, 0x56,
0x16, 0x8f, 0x99, 0x49, 0x32, 0x39, 0x93, 0x40, 0x72, 0x13, 0x92, 0x89, 0x43, 0x42, 0x30, 0x2c,
0x0c, 0x7f, 0x36, 0xb0, 0xa0, 0xdd, 0x65, 0x77, 0x1f, 0xb6, 0x10, 0x68, 0x1b, 0xa9, 0x80, 0xe4,
0x00, 0x6a, 0x55, 0x24, 0xeb, 0xc6, 0x3e, 0x21, 0x56, 0x3c, 0xf6, 0xe0, 0x7b, 0x9d, 0x26, 0x55,
0xfb, 0xd4, 0xf6, 0x0b, 0x54, 0xea, 0x07, 0xa8, 0xfa, 0xd6, 0x87, 0xbe, 0xf6, 0x53, 0xf5, 0x13,
0xf4, 0xa5, 0xba, 0x7f, 0xec, 0x19, 0x8f, 0xed, 0x8c, 0x29, 0x48, 0x7d, 0xbb, 0x73, 0xee, 0xf9,
0xef, 0x73, 0xce, 0x3d, 0xbf, 0x81, 0xc5, 0xa3, 0x28, 0x48, 0x7a, 0xe8, 0x30, 0x8c, 0x8f, 0x30,
0xde, 0xea, 0xc7, 0x11, 0x8f, 0xc8, 0x7c, 0x8e, 0xe8, 0xf4, 0xf7, 0xac, 0xdb, 0x40, 0x1e, 0x52,
0xee, 0x1e, 0x3c, 0xc2, 0x00, 0x39, 0xda, 0xf8, 0x26, 0x41, 0xc6, 0xc9, 0x2a, 0xb4, 0xf6, 0xfd,
0x00, 0x1d, 0xdf, 0x63, 0x1d, 0x63, 0xb3, 0xd1, 0x9d, 0xb1, 0xa7, 0xc5, 0xef, 0x1d, 0x8f, 0x59,
0xcf, 0x60, 0x31, 0x27, 0xc0, 0xfa, 0x51, 0xc8, 0x90, 0xdc, 0x87, 0xe9, 0x18, 0x59, 0x12, 0x70,
0x25, 0xd0, 0xbe, 0xbb, 0xb1, 0x35, 0x6a, 0x6b, 0x2b, 0x13, 0x49, 0x02, 0x6e, 0xa7, 0xec, 0x96,
0x0f, 0xb3, 0xc3, 0x17, 0x64, 0x05, 0xa6, 0xb5, 0xed, 0x8e, 0xb1, 0x69, 0x74, 0x67, 0xec, 0x29,
0x65, 0x9a, 0x2c, 0xc3, 0x14, 0xe3, 0x94, 0x27, 0xac, 0x73, 0x66, 0xd3, 0xe8, 0x4e, 0xda, 0xfa,
0x17, 0x59, 0x82, 0x49, 0x8c, 0xe3, 0x28, 0xee, 0x34, 0x24, 0xbb, 0xfa, 0x41, 0x08, 0x34, 0x99,
0xff, 0x25, 0x76, 0x9a, 0x9b, 0x46, 0x77, 0xce, 0x96, 0x67, 0x6b, 0x1a, 0x26, 0x1f, 0xf7, 0xfa,
0xfc, 0xc4, 0xfa, 0x37, 0x74, 0x5e, 0x52, 0x37, 0x49, 0x7a, 0x2f, 0xa5, 0x8f, 0xdb, 0x07, 0xe8,
0x1e, 0xa6, 0xb1, 0xaf, 0xc1, 0x8c, 0xf6, 0x5c, 0x7b, 0x30, 0x67, 0xb7, 0x14, 0x61, 0xc7, 0xb3,
0x3e, 0x80, 0xd5, 0x12, 0x41, 0x9d, 0x83, 0xcb, 0x30, 0xf7, 0x9a, 0xc6, 0x7b, 0xf4, 0x35, 0x3a,
0x31, 0xe5, 0x7e, 0x24, 0xa5, 0x0d, 0x7b, 0x56, 0x13, 0x6d, 0x41, 0xb3, 0x3e, 0x07, 0x33, 0xa7,
0x21, 0xea, 0xf5, 0xa9, 0xcb, 0xeb, 0x18, 0x27, 0x9b, 0xd0, 0xee, 0xc7, 0x48, 0x83, 0x20, 0x72,
0x29, 0x47, 0x99, 0x85, 0x86, 0x3d, 0x4c, 0xb2, 0xd6, 0x61, 0xad, 0x54, 0xb9, 0x72, 0xd0, 0xba,
0x3f, 0xe2, 0x7d, 0xd4, 0xeb, 0xf9, 0xb5, 0x4c, 0x5b, 0x17, 0x0a, 0x5e, 0x4b, 0x49, 0xad, 0xf7,
0x3f, 0x23, 0xb7, 0x01, 0xd2, 0x30, 0xe9, 0xd7, 0x52, 0x3c, 0xea, 0x71, 0x2a, 0x9a, 0x69, 0x5e,
0x51, 0xc5, 0xb1, 0x1d, 0x05, 0x01, 0xba, 0xdc, 0x8f, 0xc2, 0x54, 0xed, 0x06, 0x80, 0x9b, 0x11,
0x75, 0xa9, 0x0c, 0x51, 0x2c, 0x13, 0x3a, 0x45, 0x51, 0xad, 0xf6, 0x67, 0x03, 0xce, 0x3f, 0xd0,
0x49, 0x53, 0x86, 0x6b, 0x7d, 0x80, 0xbc, 0xc9, 0x33, 0xa3, 0x26, 0x47, 0x3f, 0x50, 0xa3, 0xf0,
0x81, 0x04, 0x47, 0x8c, 0xfd, 0xc0, 0x77, 0xa9, 0x54, 0xd1, 0x94, 0x2a, 0x86, 0x49, 0x64, 0x1e,
0x1a, 0x9c, 0x07, 0x9d, 0x49, 0x79, 0x23, 0x8e, 0x56, 0x07, 0x96, 0x47, 0x7d, 0xd5, 0x61, 0xfc,
0x0b, 0x56, 0x14, 0x65, 0xf7, 0x24, 0x74, 0x77, 0x65, 0x37, 0xd4, 0x4a, 0xfa, 0xef, 0x06, 0x74,
0x8a, 0x82, 0xba, 0x8a, 0xdf, 0x35, 0x03, 0x6f, 0x1b, 0x1f, 0xb9, 0x08, 0x6d, 0x4e, 0xfd, 0xc0,
0x89, 0xf6, 0xf7, 0x19, 0xf2, 0xce, 0xd4, 0xa6, 0xd1, 0x6d, 0xda, 0x20, 0x48, 0xcf, 0x24, 0x85,
0x5c, 0x87, 0x79, 0x57, 0x55, 0xb2, 0x13, 0xe3, 0x91, 0xcf, 0x84, 0xe6, 0x69, 0xe9, 0xd8, 0x39,
0x37, 0xad, 0x70, 0x45, 0x26, 0x16, 0xcc, 0xf9, 0xde, 0xb1, 0x23, 0x07, 0x88, 0x6c, 0xff, 0x96,
0xd4, 0xd6, 0xf6, 0xbd, 0xe3, 0x0f, 0xfd, 0x00, 0x77, 0xc5, 0x14, 0x78, 0x09, 0x17, 0x54, 0xf0,
0x3b, 0xa1, 0x1b, 0x63, 0x0f, 0x43, 0x4e, 0x83, 0xed, 0xa8, 0x7f, 0x52, 0xab, 0x04, 0x56, 0xa1,
0xc5, 0xfc, 0xd0, 0x45, 0x27, 0x54, 0x63, 0xa8, 0x69, 0x4f, 0xcb, 0xdf, 0x4f, 0x99, 0xf5, 0x10,
0xd6, 0x2b, 0xf4, 0xea, 0xcc, 0x5e, 0x82, 0x59, 0xe9, 0x98, 0x1b, 0x85, 0x1c, 0x43, 0x2e, 0x75,
0xcf, 0xda, 0x6d, 0x41, 0xdb, 0x56, 0x24, 0xeb, 0x1f, 0x40, 0x94, 0x8e, 0x27, 0x51, 0x12, 0xd6,
0x6b, 0xcd, 0xf3, 0xb0, 0x98, 0x13, 0xd1, 0xb5, 0x71, 0x0f, 0x96, 0x14, 0xf9, 0x45, 0xd8, 0xab,
0xad, 0x6b, 0x05, 0xce, 0x8f, 0x08, 0x69, 0x6d, 0x77, 0x53, 0x23, 0xf9, 0x77, 0xe2, 0x54, 0x65,
0xcb, 0xa9, 0x07, 0xf9, 0xa7, 0xc2, 0xfa, 0xc5, 0x80, 0x85, 0x74, 0x8c, 0xd4, 0xcc, 0xfa, 0x5b,
0x96, 0x5d, 0xa3, 0xb2, 0xec, 0x9a, 0x83, 0xb2, 0xeb, 0xc2, 0x3c, 0x8b, 0x92, 0xd8, 0x45, 0xc7,
0xa3, 0x9c, 0x3a, 0x61, 0xe4, 0xa1, 0xae, 0xca, 0xb3, 0x8a, 0xfe, 0x88, 0x72, 0xfa, 0x34, 0xf2,
0xd0, 0xfa, 0x7f, 0xfa, 0x51, 0x72, 0x5f, 0xf3, 0x3a, 0x2c, 0x04, 0x94, 0x71, 0x87, 0xf6, 0xfb,
0x18, 0x7a, 0x0e, 0xe5, 0xa2, 0x24, 0x0c, 0x59, 0x12, 0x67, 0xc5, 0xc5, 0x03, 0x49, 0x7f, 0xc0,
0x9f, 0x32, 0xeb, 0x7b, 0x03, 0xce, 0x09, 0x59, 0x51, 0x82, 0xb5, 0xe2, 0x9d, 0x87, 0x06, 0x1e,
0x73, 0x1d, 0xa8, 0x38, 0x92, 0xdb, 0xb0, 0xa8, 0x6b, 0xdd, 0x8f, 0xc2, 0x41, 0x1b, 0x34, 0xa4,
0x20, 0x19, 0x5c, 0x65, 0x9d, 0x70, 0x11, 0xda, 0x8c, 0x47, 0xfd, 0xb4, 0xab, 0x9a, 0xaa, 0xab,
0x04, 0x49, 0x75, 0x95, 0xf5, 0x4f, 0x98, 0x1f, 0xf8, 0x54, 0xbf, 0x42, 0xbf, 0x31, 0xd2, 0xa1,
0xf3, 0x9c, 0xfa, 0xc1, 0x2e, 0x86, 0x1e, 0xc6, 0xef, 0xd8, 0x39, 0xe4, 0x0e, 0x2c, 0xf9, 0x5e,
0x80, 0x0e, 0xf7, 0x7b, 0x18, 0x25, 0xdc, 0x61, 0xe8, 0x46, 0xa1, 0xc7, 0xd2, 0xe8, 0xc4, 0xdd,
0x73, 0x75, 0xb5, 0xab, 0x6e, 0xac, 0x6f, 0xb3, 0x09, 0x36, 0xec, 0xc5, 0xe0, 0x1d, 0x0e, 0x11,
0x85, 0xc2, 0x03, 0xa4, 0x1e, 0xc6, 0x3a, 0x8c, 0x59, 0x45, 0xfc, 0x58, 0xd2, 0x44, 0x7e, 0x34,
0xd3, 0x5e, 0xe4, 0x9d, 0x48, 0x8f, 0x66, 0x6d, 0x50, 0xa4, 0x87, 0x91, 0x77, 0x22, 0x47, 0x09,
0x73, 0xe4, 0x27, 0x76, 0x0f, 0x92, 0xf0, 0x50, 0x7a, 0xd3, 0xb2, 0xdb, 0x3e, 0xfb, 0x84, 0x32,
0xbe, 0x2d, 0x48, 0xd6, 0xaf, 0x06, 0xac, 0x0e, 0xdc, 0xb0, 0xd1, 0x45, 0xff, 0xe8, 0x2f, 0x48,
0x87, 0x90, 0xd0, 0xb5, 0x9c, 0x5b, 0xba, 0x74, 0xb9, 0x13, 0x75, 0xa7, 0x27, 0xbe, 0xbc, 0x91,
0x0f, 0x7a, 0x89, 0xe3, 0xba, 0x45, 0x5f, 0xa5, 0xa3, 0xec, 0xb1, 0xbb, 0x7b, 0x40, 0x63, 0x8f,
0x7d, 0x84, 0x21, 0xc6, 0x94, 0xbf, 0x97, 0x67, 0xd2, 0xda, 0x84, 0x8d, 0x2a, 0xed, 0xda, 0xfe,
0x8f, 0x59, 0x5e, 0x53, 0x96, 0xf7, 0x36, 0x2a, 0xd6, 0x01, 0xd0, 0x75, 0xfc, 0xd0, 0xc3, 0x63,
0x14, 0x29, 0x6d, 0x74, 0xe7, 0xec, 0x19, 0x74, 0x77, 0x14, 0xe1, 0x2d, 0xa6, 0x42, 0x96, 0xc1,
0xbc, 0x8b, 0x3a, 0x82, 0xef, 0x0c, 0x58, 0xcb, 0x5f, 0xd7, 0x9f, 0x9c, 0xe4, 0x06, 0x2c, 0xb0,
0x83, 0x28, 0x09, 0x3c, 0xc7, 0x93, 0x42, 0x0e, 0xba, 0xc7, 0x32, 0x94, 0x96, 0x7d, 0x4e, 0x5d,
0x28, 0x65, 0x8f, 0xdd, 0xe3, 0x31, 0xf1, 0x58, 0x1b, 0xe9, 0x63, 0x37, 0xea, 0x86, 0xf6, 0xf3,
0xbf, 0xb0, 0x66, 0x23, 0xf5, 0x14, 0x8f, 0x7c, 0x22, 0xeb, 0xaf, 0x11, 0xbf, 0x9d, 0x81, 0x0b,
0xe5, 0xc2, 0x75, 0x56, 0x89, 0xff, 0x81, 0x99, 0x3d, 0xd5, 0xa2, 0xd2, 0x19, 0xa7, 0xbd, 0x7e,
0x56, 0xeb, 0xaa, 0x25, 0x56, 0xf4, 0xbb, 0xfd, 0x3c, 0xbd, 0x4f, 0x0b, 0xbe, 0xf0, 0xce, 0x37,
0x0a, 0xef, 0xbc, 0x30, 0xe0, 0x51, 0x5e, 0x65, 0x40, 0x0d, 0xc4, 0x15, 0x8f, 0xf2, 0x2a, 0x03,
0x99, 0xb0, 0x34, 0x30, 0xa9, 0x0c, 0x68, 0x7e, 0x69, 0x60, 0x1d, 0x40, 0x4f, 0xcb, 0x24, 0x4c,
0xf7, 0x96, 0x19, 0x35, 0x2b, 0x93, 0xb0, 0x72, 0x64, 0x4f, 0x57, 0x8e, 0xec, 0x7c, 0xe9, 0xb6,
0x0a, 0x7d, 0xf3, 0x29, 0xc0, 0x23, 0x9f, 0x1d, 0xaa, 0x24, 0x8b, 0x37, 0xc2, 0xf3, 0x63, 0xbd,
0xf8, 0x8a, 0xa3, 0xa0, 0xd0, 0x20, 0xd0, 0xa9, 0x13, 0x47, 0x01, 0x82, 0x12, 0x86, 0x9e, 0xce,
0x8e, 0x3c, 0x0b, 0xda, 0x7e, 0x8c, 0xa8, 0x13, 0x20, 0xcf, 0xd6, 0x4f, 0x06, 0xcc, 0x3c, 0xc1,
0x9e, 0xd6, 0xbc, 0x01, 0xf0, 0x3a, 0x8a, 0xa3, 0x84, 0xfb, 0x21, 0xaa, 0x27, 0x6d, 0xd2, 0x1e,
0xa2, 0xfc, 0x79, 0x3b, 0x12, 0x94, 0x61, 0xb0, 0xaf, 0x93, 0x29, 0xcf, 0x82, 0x76, 0x80, 0xb4,
0xaf, 0xf3, 0x27, 0xcf, 0x02, 0xd2, 0x31, 0x4e, 0xdd, 0x43, 0x99, 0xac, 0xa6, 0xad, 0x7e, 0xdc,
0xfd, 0x61, 0x01, 0x66, 0x87, 0x87, 0x18, 0x79, 0x05, 0xed, 0x21, 0x2c, 0x4a, 0xae, 0x14, 0x21,
0x67, 0x11, 0xdb, 0x9a, 0x7f, 0x1b, 0xc3, 0xa5, 0x1b, 0x63, 0x82, 0x84, 0xb0, 0x50, 0xc0, 0x7a,
0xe4, 0x46, 0x51, 0xba, 0x0a, 0x49, 0x9a, 0x37, 0x6b, 0xf1, 0x66, 0xf6, 0x38, 0x2c, 0x96, 0x80,
0x37, 0x72, 0x6b, 0x8c, 0x96, 0x1c, 0x80, 0x34, 0xff, 0x5e, 0x93, 0x3b, 0xb3, 0xfa, 0x06, 0x48,
0x11, 0xd9, 0x91, 0x9b, 0x63, 0xd5, 0x0c, 0x90, 0xa3, 0x79, 0xab, 0x1e, 0x73, 0x65, 0xa0, 0x0a,
0xf3, 0x8d, 0x0d, 0x34, 0x87, 0x2a, 0xc7, 0x06, 0x3a, 0x02, 0x24, 0x27, 0xc8, 0x21, 0xcc, 0x8f,
0xe2, 0x41, 0x72, 0xbd, 0xea, 0x4f, 0x8a, 0x02, 0xdc, 0x34, 0x6f, 0xd4, 0x61, 0xcd, 0x8c, 0x21,
0x9c, 0xcd, 0x63, 0x36, 0x72, 0xad, 0x28, 0x5f, 0x8a, 0x40, 0xcd, 0xee, 0x78, 0xc6, 0xe1, 0x98,
0x46, 0x71, 0x5c, 0x59, 0x4c, 0x15, 0x20, 0xb1, 0x2c, 0xa6, 0x2a, 0x58, 0x68, 0x4d, 0x90, 0xaf,
0x52, 0x70, 0x30, 0x82, 0x6f, 0xc8, 0x56, 0x95, 0x9a, 0x72, 0x80, 0x65, 0xde, 0xae, 0xcd, 0x9f,
0xda, 0xbe, 0x63, 0x88, 0x5e, 0x1f, 0x82, 0x39, 0x65, 0xbd, 0x5e, 0x04, 0x4e, 0x65, 0xbd, 0x5e,
0x86, 0x95, 0x26, 0xc8, 0x1e, 0xcc, 0xe5, 0x80, 0x0f, 0xb9, 0x5a, 0x25, 0x99, 0x87, 0x53, 0xe6,
0xb5, 0xb1, 0x7c, 0x99, 0x0d, 0x27, 0x9d, 0x5e, 0x7a, 0x5c, 0x55, 0x3a, 0x97, 0x9f, 0x57, 0x57,
0xc7, 0xb1, 0x65, 0x06, 0x3e, 0x03, 0x18, 0xe0, 0x14, 0x72, 0xb9, 0x4a, 0x6e, 0xf8, 0x53, 0x5c,
0x39, 0x9d, 0x29, 0x53, 0xfd, 0x05, 0x2c, 0x95, 0xbd, 0xf4, 0xa4, 0xa4, 0x0b, 0x4f, 0x59, 0x27,
0xcc, 0xad, 0xba, 0xec, 0x99, 0xe1, 0x17, 0xd0, 0x4a, 0x51, 0x0a, 0xb9, 0x54, 0x94, 0x1e, 0x41,
0x55, 0xa6, 0x75, 0x1a, 0xcb, 0x50, 0x35, 0xf5, 0xd2, 0xc6, 0x19, 0xc0, 0x87, 0xea, 0xc6, 0x29,
0x00, 0x9d, 0xea, 0xc6, 0x29, 0xa2, 0x11, 0x69, 0xee, 0x4d, 0x8a, 0x20, 0x87, 0xb7, 0xed, 0xd2,
0x21, 0x5b, 0x05, 0x26, 0x4a, 0x87, 0x6c, 0xf5, 0x02, 0x3f, 0x41, 0xbe, 0x86, 0xe5, 0xf2, 0x25,
0x9b, 0x54, 0xb6, 0x5f, 0xc5, 0xb2, 0x6f, 0xde, 0xa9, 0x2f, 0x90, 0x7b, 0x56, 0x0a, 0xdb, 0x71,
0x75, 0xc4, 0x25, 0x6b, 0x7e, 0x75, 0xc4, 0xa5, 0x0b, 0xb7, 0xac, 0xd1, 0xb2, 0x55, 0xb7, 0xac,
0x46, 0x4f, 0xd9, 0xcc, 0xcd, 0xad, 0xba, 0xec, 0xa9, 0xe1, 0xbd, 0x29, 0xf9, 0xe7, 0xfa, 0xbd,
0x3f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x1a, 0xed, 0xaf, 0x31, 0x73, 0x17, 0x00, 0x00,
} }

54
weed/server/volume_grpc_erasure_coding.go
View File

@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"math" "math"
"os"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
@ -16,17 +17,17 @@ import (
Steps to apply erasure coding to .dat .idx files Steps to apply erasure coding to .dat .idx files
0. ensure the volume is readonly 0. ensure the volume is readonly
1. client call VolumeEcGenerateSlices to generate the .ecx and .ec01~.ec14 files
1. client call VolumeEcShardsGenerate to generate the .ecx and .ec01~.ec14 files
2. client ask master for possible servers to hold the ec files, at least 4 servers 2. client ask master for possible servers to hold the ec files, at least 4 servers
3. client call VolumeEcCopy on above target servers to copy ec files from the source server
3. client call VolumeEcShardsCopy on above target servers to copy ec files from the source server
4. target servers report the new ec files to the master 4. target servers report the new ec files to the master
5. master stores vid -> [14]*DataNode 5. master stores vid -> [14]*DataNode
6. client checks master. If all 14 slices are ready, delete the original .idx, .idx files 6. client checks master. If all 14 slices are ready, delete the original .idx, .idx files
*/
*/
// VolumeEcGenerateSlices generates the .ecx and .ec01 ~ .ec14 files
func (vs *VolumeServer) VolumeEcGenerateSlices(ctx context.Context, req *volume_server_pb.VolumeEcGenerateSlicesRequest) (*volume_server_pb.VolumeEcGenerateSlicesResponse, error) {
// VolumeEcShardsGenerate generates the .ecx and .ec01 ~ .ec14 files
func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_server_pb.VolumeEcShardsGenerateRequest) (*volume_server_pb.VolumeEcShardsGenerateResponse, error) {
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil { if v == nil {
@ -34,6 +35,10 @@ func (vs *VolumeServer) VolumeEcGenerateSlices(ctx context.Context, req *volume_
} }
baseFileName := v.FileName() baseFileName := v.FileName()
if v.Collection != req.Collection {
return nil, fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection)
}
// write .ecx file // write .ecx file
if err := erasure_coding.WriteSortedEcxFile(baseFileName); err != nil { if err := erasure_coding.WriteSortedEcxFile(baseFileName); err != nil {
return nil, fmt.Errorf("WriteSortedEcxFile %s: %v", baseFileName, err) return nil, fmt.Errorf("WriteSortedEcxFile %s: %v", baseFileName, err)
@ -44,12 +49,11 @@ func (vs *VolumeServer) VolumeEcGenerateSlices(ctx context.Context, req *volume_
return nil, fmt.Errorf("WriteEcFiles %s: %v", baseFileName, err) return nil, fmt.Errorf("WriteEcFiles %s: %v", baseFileName, err)
} }
return &volume_server_pb.VolumeEcGenerateSlicesResponse{}, nil
return &volume_server_pb.VolumeEcShardsGenerateResponse{}, nil
} }
// VolumeEcCopy copy the .ecx and some ec data slices
func (vs *VolumeServer) VolumeEcCopy(ctx context.Context, req *volume_server_pb.VolumeEcCopyRequest) (*volume_server_pb.VolumeEcCopyResponse, error) {
// VolumeEcShardsCopy copy the .ecx and some ec data slices
func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_server_pb.VolumeEcShardsCopyRequest) (*volume_server_pb.VolumeEcShardsCopyResponse, error) {
location := vs.store.FindFreeLocation() location := vs.store.FindFreeLocation()
if location == nil { if location == nil {
@ -61,13 +65,13 @@ func (vs *VolumeServer) VolumeEcCopy(ctx context.Context, req *volume_server_pb.
err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
// copy ecx file // copy ecx file
if err:=vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxUint64, baseFileName, ".ecx"); err!=nil{
if err := vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, ".ecx"); err != nil {
return err return err
} }
// copy ec data slices // copy ec data slices
for _, ecIndex := range req.EcIndexes { for _, ecIndex := range req.EcIndexes {
if err:=vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxUint64, baseFileName, erasure_coding.ToExt(int(ecIndex))); err!=nil{
if err := vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, erasure_coding.ToExt(int(ecIndex))); err != nil {
return err return err
} }
} }
@ -75,8 +79,32 @@ func (vs *VolumeServer) VolumeEcCopy(ctx context.Context, req *volume_server_pb.
return nil return nil
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("VolumeEcCopy volume %d: %v", req.VolumeId, err)
return nil, fmt.Errorf("VolumeEcShardsCopy volume %d: %v", req.VolumeId, err)
}
return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil
}
// VolumeEcShardsDelete local delete the .ecx and some ec data slices if not needed, assuming current server has the source volume
func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
return nil, fmt.Errorf("volume %d not found", req.VolumeId)
}
baseFileName := v.FileName()
for _, shardIndex := range req.EcIndexes {
if err := os.Remove(baseFileName + erasure_coding.ToExt(int(shardIndex))); err != nil {
return nil, err
}
}
if req.ShouldDeleteEcx {
if err := os.Remove(baseFileName + ".ecx"); err != nil {
return nil, err
}
} }
return &volume_server_pb.VolumeEcCopyResponse{}, nil
return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
} }

109
weed/shell/command_ec_encode.go
View File

@ -51,7 +51,8 @@ func (c *commandEcEncode) Help() string {
func (c *commandEcEncode) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) { func (c *commandEcEncode) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
volumeId := encodeCommand.Int("vid", 0, "the volume id")
volumeId := encodeCommand.Int("volumeId", 0, "the volume id")
collection := encodeCommand.String("collection", "", "the collection name")
if err = encodeCommand.Parse(args); err != nil { if err = encodeCommand.Parse(args); err != nil {
return nil return nil
} }
@ -65,13 +66,13 @@ func (c *commandEcEncode) Do(args []string, commandEnv *commandEnv, writer io.Wr
} }
// generate ec shards // generate ec shards
err = generateEcShards(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(*volumeId), locations[0].Url)
err = generateEcShards(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(*volumeId), *collection, locations[0].Url)
if err != nil { if err != nil {
return fmt.Errorf("generate ec shards for volume %d on %s: %v", *volumeId, locations[0].Url, err) return fmt.Errorf("generate ec shards for volume %d on %s: %v", *volumeId, locations[0].Url, err)
} }
// balance the ec shards to current cluster // balance the ec shards to current cluster
err = balanceEcShards(ctx, commandEnv, needle.VolumeId(*volumeId), locations[0])
err = balanceEcShards(ctx, commandEnv, needle.VolumeId(*volumeId), *collection, locations)
if err != nil { if err != nil {
return fmt.Errorf("balance ec shards for volume %d on %s: %v", *volumeId, locations[0].Url, err) return fmt.Errorf("balance ec shards for volume %d on %s: %v", *volumeId, locations[0].Url, err)
} }
@ -79,11 +80,12 @@ func (c *commandEcEncode) Do(args []string, commandEnv *commandEnv, writer io.Wr
return err return err
} }
func generateEcShards(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer string) error {
func generateEcShards(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer string) error {
err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, genErr := volumeServerClient.VolumeEcGenerateSlices(ctx, &volume_server_pb.VolumeEcGenerateSlicesRequest{
VolumeId: uint32(volumeId),
_, genErr := volumeServerClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: uint32(volumeId),
Collection: collection,
}) })
return genErr return genErr
}) })
@ -92,7 +94,7 @@ func generateEcShards(ctx context.Context, grpcDialOption grpc.DialOption, volum
} }
func balanceEcShards(ctx context.Context, commandEnv *commandEnv, volumeId needle.VolumeId, existingLocation wdclient.Location) (err error) {
func balanceEcShards(ctx context.Context, commandEnv *commandEnv, volumeId needle.VolumeId, collection string, existingLocations []wdclient.Location) (err error) {
// list all possible locations // list all possible locations
var resp *master_pb.VolumeListResponse var resp *master_pb.VolumeListResponse
@ -106,7 +108,7 @@ func balanceEcShards(ctx context.Context, commandEnv *commandEnv, volumeId needl
// find out all volume servers with one volume slot left. // find out all volume servers with one volume slot left.
var allDataNodes []*master_pb.DataNodeInfo var allDataNodes []*master_pb.DataNodeInfo
var totalFreeEcSlots int
var totalFreeEcSlots uint32
eachDataNode(resp.TopologyInfo, func(dn *master_pb.DataNodeInfo) { eachDataNode(resp.TopologyInfo, func(dn *master_pb.DataNodeInfo) {
if freeEcSlots := countFreeShardSlots(dn); freeEcSlots > 0 { if freeEcSlots := countFreeShardSlots(dn); freeEcSlots > 0 {
allDataNodes = append(allDataNodes, dn) allDataNodes = append(allDataNodes, dn)
@ -127,63 +129,118 @@ func balanceEcShards(ctx context.Context, commandEnv *commandEnv, volumeId needl
allocated := balancedEcDistribution(allDataNodes) allocated := balancedEcDistribution(allDataNodes)
// ask the data nodes to copy from the source volume server // ask the data nodes to copy from the source volume server
err = parallelCopyEcShardsFromSource(ctx, commandEnv.option.GrpcDialOption, allDataNodes, allocated, volumeId, existingLocation)
copiedShardIds, err := parallelCopyEcShardsFromSource(ctx, commandEnv.option.GrpcDialOption, allDataNodes, allocated, volumeId, collection, existingLocations[0])
if err != nil { if err != nil {
return nil return nil
} }
// ask the source volume server to clean up copied ec shards // ask the source volume server to clean up copied ec shards
err = sourceServerDeleteEcShards(ctx, commandEnv.option.GrpcDialOption, volumeId, existingLocations[0], copiedShardIds)
if err != nil {
return fmt.Errorf("sourceServerDeleteEcShards %s %d.%v: %v", existingLocations[0], volumeId, copiedShardIds, err)
}
// ask the source volume server to delete the original volume // ask the source volume server to delete the original volume
for _, location := range existingLocations {
err = deleteVolume(ctx, commandEnv.option.GrpcDialOption, volumeId, location.Url)
if err != nil {
return fmt.Errorf("deleteVolume %s volume %d: %v", location.Url, volumeId, err)
}
}
return err return err
} }
func parallelCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.DialOption, func parallelCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.DialOption,
targetServers []*master_pb.DataNodeInfo, allocated []int,
volumeId needle.VolumeId, existingLocation wdclient.Location) (err error) {
targetServers []*master_pb.DataNodeInfo, allocated []uint32,
volumeId needle.VolumeId, collection string, existingLocation wdclient.Location) (actuallyCopied []uint32, err error) {
// parallelize // parallelize
shardIdChan := make(chan []uint32, len(targetServers))
var wg sync.WaitGroup var wg sync.WaitGroup
startFromShardId := 0
startFromShardId := uint32(0)
for i, server := range targetServers { for i, server := range targetServers {
if allocated[i] <= 0 { if allocated[i] <= 0 {
continue continue
} }
wg.Add(1) wg.Add(1)
go func(server *master_pb.DataNodeInfo, startFromShardId int, shardCount int) {
go func(server *master_pb.DataNodeInfo, startFromShardId uint32, shardCount uint32) {
defer wg.Done() defer wg.Done()
copyErr := oneServerCopyEcShardsFromSource(ctx, grpcDialOption, server, startFromShardId, shardCount, volumeId, existingLocation)
copiedShardIds, copyErr := oneServerCopyEcShardsFromSource(ctx, grpcDialOption, server,
startFromShardId, shardCount, volumeId, collection, existingLocation)
if copyErr != nil { if copyErr != nil {
err = copyErr err = copyErr
} else {
shardIdChan <- copiedShardIds
} }
}(server, startFromShardId, allocated[i]) }(server, startFromShardId, allocated[i])
startFromShardId += allocated[i] startFromShardId += allocated[i]
} }
wg.Wait() wg.Wait()
close(shardIdChan)
return err
if err != nil {
return nil, err
}
for shardIds := range shardIdChan {
actuallyCopied = append(actuallyCopied, shardIds...)
}
return
} }
func oneServerCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.DialOption, func oneServerCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.DialOption,
targetServer *master_pb.DataNodeInfo, startFromShardId int, shardCount int,
volumeId needle.VolumeId, existingLocation wdclient.Location) (err error) {
targetServer *master_pb.DataNodeInfo, startFromShardId uint32, shardCount uint32,
volumeId needle.VolumeId, collection string, existingLocation wdclient.Location) (copiedShardIds []uint32, err error) {
if targetServer.Id == existingLocation.Url { if targetServer.Id == existingLocation.Url {
return nil
return nil, nil
} }
for shardId := startFromShardId; shardId < startFromShardId+shardCount; shardId++ { for shardId := startFromShardId; shardId < startFromShardId+shardCount; shardId++ {
fmt.Printf("copy %d.%d %s => %s\n", volumeId, shardId, existingLocation.Url, targetServer.Id) fmt.Printf("copy %d.%d %s => %s\n", volumeId, shardId, existingLocation.Url, targetServer.Id)
copiedShardIds = append(copiedShardIds, shardId)
} }
return nil
err = operation.WithVolumeServerClient(targetServer.Id, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, copyErr := volumeServerClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
VolumeId: uint32(volumeId),
Collection: collection,
EcIndexes: copiedShardIds,
SourceDataNode: existingLocation.Url,
})
return copyErr
})
if err != nil {
return
}
return
} }
func balancedEcDistribution(servers []*master_pb.DataNodeInfo) (allocated []int) {
freeSlots := make([]int, len(servers))
allocated = make([]int, len(servers))
func sourceServerDeleteEcShards(ctx context.Context, grpcDialOption grpc.DialOption,
volumeId needle.VolumeId, sourceLocation wdclient.Location, toBeDeletedShardIds []uint32) error {
shouldDeleteEcx := len(toBeDeletedShardIds) == erasure_coding.TotalShardsCount
return operation.WithVolumeServerClient(sourceLocation.Url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, deleteErr := volumeServerClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: uint32(volumeId),
EcIndexes: toBeDeletedShardIds,
ShouldDeleteEcx: shouldDeleteEcx,
})
return deleteErr
})
}
func balancedEcDistribution(servers []*master_pb.DataNodeInfo) (allocated []uint32) {
freeSlots := make([]uint32, len(servers))
allocated = make([]uint32, len(servers))
for i, server := range servers { for i, server := range servers {
freeSlots[i] = countFreeShardSlots(server) freeSlots[i] = countFreeShardSlots(server)
} }
@ -213,14 +270,14 @@ func eachDataNode(topo *master_pb.TopologyInfo, fn func(*master_pb.DataNodeInfo)
} }
} }
func countShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count int) {
func countShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count uint32) {
for _, ecShardInfo := range ecShardInfos { for _, ecShardInfo := range ecShardInfos {
shardBits := erasure_coding.ShardBits(ecShardInfo.EcIndexBits) shardBits := erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
count += shardBits.ShardIdCount()
count += uint32(shardBits.ShardIdCount())
} }
return return
} }
func countFreeShardSlots(dn *master_pb.DataNodeInfo) (count int) {
return int(dn.FreeVolumeCount)*10 - countShards(dn.EcShardInfos)
func countFreeShardSlots(dn *master_pb.DataNodeInfo) (count uint32) {
return uint32(dn.FreeVolumeCount)*10 - countShards(dn.EcShardInfos)
} }
Write Preview
Loading…
Cancel
Save