Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

generate, balance, delete copied shards, delete old volume

pull/991/head
Chris Lu 6 years ago
parent
f0e6574d5e
commit
41e8ae61f8
4 changed files with 389 additions and 204 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 23
      weed/pb/volume_server.proto
  2. 407
      weed/pb/volume_server_pb/volume_server.pb.go
  3. 54
      weed/server/volume_grpc_erasure_coding.go
  4. 109
      weed/shell/command_ec_encode.go

23
weed/pb/volume_server.proto
View File

@ -48,9 +48,11 @@ service VolumeServer {
}
// erasure coding
rpc VolumeEcGenerateSlices (VolumeEcGenerateSlicesRequest) returns (VolumeEcGenerateSlicesResponse) {
rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
}
rpc VolumeEcCopy (VolumeEcCopyRequest) returns (VolumeEcCopyResponse) {
rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
}
rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
}
@ -197,19 +199,28 @@ message VolumeTailReceiverRequest {
message VolumeTailReceiverResponse {
}
message VolumeEcGenerateSlicesRequest {
message VolumeEcShardsGenerateRequest {
uint32 volume_id = 1;
string collection = 2;
}
message VolumeEcGenerateSlicesResponse {
message VolumeEcShardsGenerateResponse {
}
message VolumeEcCopyRequest {
message VolumeEcShardsCopyRequest {
uint32 volume_id = 1;
string collection = 2;
repeated uint32 ec_indexes = 3;
string source_data_node = 5;
}
message VolumeEcCopyResponse {
message VolumeEcShardsCopyResponse {
}
message VolumeEcShardsDeleteRequest {
uint32 volume_id = 1;
bool should_delete_ecx = 2;
repeated uint32 ec_indexes = 3;
}
message VolumeEcShardsDeleteResponse {
}
message ReadVolumeFileStatusRequest {

407
weed/pb/volume_server_pb/volume_server.pb.go
View File

@ -43,10 +43,12 @@ It has these top-level messages:
VolumeTailSenderResponse
VolumeTailReceiverRequest
VolumeTailReceiverResponse
VolumeEcGenerateSlicesRequest
VolumeEcGenerateSlicesResponse
VolumeEcCopyRequest
VolumeEcCopyResponse
VolumeEcShardsGenerateRequest
VolumeEcShardsGenerateResponse
VolumeEcShardsCopyRequest
VolumeEcShardsCopyResponse
VolumeEcShardsDeleteRequest
VolumeEcShardsDeleteResponse
ReadVolumeFileStatusRequest
ReadVolumeFileStatusResponse
DiskStatus
@ -770,77 +772,125 @@ func (m *VolumeTailReceiverResponse) String() string { return proto.C
func (*VolumeTailReceiverResponse) ProtoMessage() {}
func (*VolumeTailReceiverResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} }
type VolumeEcGenerateSlicesRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
type VolumeEcShardsGenerateRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
}
func (m *VolumeEcGenerateSlicesRequest) Reset() { *m = VolumeEcGenerateSlicesRequest{} }
func (m *VolumeEcGenerateSlicesRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcGenerateSlicesRequest) ProtoMessage() {}
func (*VolumeEcGenerateSlicesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} }
func (m *VolumeEcShardsGenerateRequest) Reset() { *m = VolumeEcShardsGenerateRequest{} }
func (m *VolumeEcShardsGenerateRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsGenerateRequest) ProtoMessage() {}
func (*VolumeEcShardsGenerateRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} }
func (m *VolumeEcGenerateSlicesRequest) GetVolumeId() uint32 {
func (m *VolumeEcShardsGenerateRequest) GetVolumeId() uint32 {
if m != nil {
return m.VolumeId
}
return 0
}
type VolumeEcGenerateSlicesResponse struct {
func (m *VolumeEcShardsGenerateRequest) GetCollection() string {
if m != nil {
return m.Collection
}
return ""
}
type VolumeEcShardsGenerateResponse struct {
}
func (m *VolumeEcGenerateSlicesResponse) Reset() { *m = VolumeEcGenerateSlicesResponse{} }
func (m *VolumeEcGenerateSlicesResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcGenerateSlicesResponse) ProtoMessage() {}
func (*VolumeEcGenerateSlicesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} }
func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} }
func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsGenerateResponse) ProtoMessage() {}
func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} }
type VolumeEcCopyRequest struct {
type VolumeEcShardsCopyRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"`
SourceDataNode string `protobuf:"bytes,5,opt,name=source_data_node,json=sourceDataNode" json:"source_data_node,omitempty"`
}
func (m *VolumeEcCopyRequest) Reset() { *m = VolumeEcCopyRequest{} }
func (m *VolumeEcCopyRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcCopyRequest) ProtoMessage() {}
func (*VolumeEcCopyRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} }
func (m *VolumeEcShardsCopyRequest) Reset() { *m = VolumeEcShardsCopyRequest{} }
func (m *VolumeEcShardsCopyRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsCopyRequest) ProtoMessage() {}
func (*VolumeEcShardsCopyRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} }
func (m *VolumeEcCopyRequest) GetVolumeId() uint32 {
func (m *VolumeEcShardsCopyRequest) GetVolumeId() uint32 {
if m != nil {
return m.VolumeId
}
return 0
}
func (m *VolumeEcCopyRequest) GetCollection() string {
func (m *VolumeEcShardsCopyRequest) GetCollection() string {
if m != nil {
return m.Collection
}
return ""
}
func (m *VolumeEcCopyRequest) GetEcIndexes() []uint32 {
func (m *VolumeEcShardsCopyRequest) GetEcIndexes() []uint32 {
if m != nil {
return m.EcIndexes
}
return nil
}
func (m *VolumeEcCopyRequest) GetSourceDataNode() string {
func (m *VolumeEcShardsCopyRequest) GetSourceDataNode() string {
if m != nil {
return m.SourceDataNode
}
return ""
}
type VolumeEcCopyResponse struct {
type VolumeEcShardsCopyResponse struct {
}
func (m *VolumeEcShardsCopyResponse) Reset() { *m = VolumeEcShardsCopyResponse{} }
func (m *VolumeEcShardsCopyResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsCopyResponse) ProtoMessage() {}
func (*VolumeEcShardsCopyResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{37} }
type VolumeEcShardsDeleteRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
ShouldDeleteEcx bool `protobuf:"varint,2,opt,name=should_delete_ecx,json=shouldDeleteEcx" json:"should_delete_ecx,omitempty"`
EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"`
}
func (m *VolumeEcShardsDeleteRequest) Reset() { *m = VolumeEcShardsDeleteRequest{} }
func (m *VolumeEcShardsDeleteRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsDeleteRequest) ProtoMessage() {}
func (*VolumeEcShardsDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{38} }
func (m *VolumeEcShardsDeleteRequest) GetVolumeId() uint32 {
if m != nil {
return m.VolumeId
}
return 0
}
func (m *VolumeEcCopyResponse) Reset() { *m = VolumeEcCopyResponse{} }
func (m *VolumeEcCopyResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcCopyResponse) ProtoMessage() {}
func (*VolumeEcCopyResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{37} }
func (m *VolumeEcShardsDeleteRequest) GetShouldDeleteEcx() bool {
if m != nil {
return m.ShouldDeleteEcx
}
return false
}
func (m *VolumeEcShardsDeleteRequest) GetEcIndexes() []uint32 {
if m != nil {
return m.EcIndexes
}
return nil
}
type VolumeEcShardsDeleteResponse struct {
}
func (m *VolumeEcShardsDeleteResponse) Reset() { *m = VolumeEcShardsDeleteResponse{} }
func (m *VolumeEcShardsDeleteResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardsDeleteResponse) ProtoMessage() {}
func (*VolumeEcShardsDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{39} }
type ReadVolumeFileStatusRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
@ -849,7 +899,7 @@ type ReadVolumeFileStatusRequest struct {
func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} }
func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) }
func (*ReadVolumeFileStatusRequest) ProtoMessage() {}
func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{38} }
func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{40} }
func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 {
if m != nil {
@ -872,7 +922,7 @@ type ReadVolumeFileStatusResponse struct {
func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} }
func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) }
func (*ReadVolumeFileStatusResponse) ProtoMessage() {}
func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{39} }
func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} }
func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 {
if m != nil {
@ -940,7 +990,7 @@ type DiskStatus struct {
func (m *DiskStatus) Reset() { *m = DiskStatus{} }
func (m *DiskStatus) String() string { return proto.CompactTextString(m) }
func (*DiskStatus) ProtoMessage() {}
func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{40} }
func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{42} }
func (m *DiskStatus) GetDir() string {
if m != nil {
@ -983,7 +1033,7 @@ type MemStatus struct {
func (m *MemStatus) Reset() { *m = MemStatus{} }
func (m *MemStatus) String() string { return proto.CompactTextString(m) }
func (*MemStatus) ProtoMessage() {}
func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} }
func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{43} }
func (m *MemStatus) GetGoroutines() int32 {
if m != nil {
@ -1069,10 +1119,12 @@ func init() {
proto.RegisterType((*VolumeTailSenderResponse)(nil), "volume_server_pb.VolumeTailSenderResponse")
proto.RegisterType((*VolumeTailReceiverRequest)(nil), "volume_server_pb.VolumeTailReceiverRequest")
proto.RegisterType((*VolumeTailReceiverResponse)(nil), "volume_server_pb.VolumeTailReceiverResponse")
proto.RegisterType((*VolumeEcGenerateSlicesRequest)(nil), "volume_server_pb.VolumeEcGenerateSlicesRequest")
proto.RegisterType((*VolumeEcGenerateSlicesResponse)(nil), "volume_server_pb.VolumeEcGenerateSlicesResponse")
proto.RegisterType((*VolumeEcCopyRequest)(nil), "volume_server_pb.VolumeEcCopyRequest")
proto.RegisterType((*VolumeEcCopyResponse)(nil), "volume_server_pb.VolumeEcCopyResponse")
proto.RegisterType((*VolumeEcShardsGenerateRequest)(nil), "volume_server_pb.VolumeEcShardsGenerateRequest")
proto.RegisterType((*VolumeEcShardsGenerateResponse)(nil), "volume_server_pb.VolumeEcShardsGenerateResponse")
proto.RegisterType((*VolumeEcShardsCopyRequest)(nil), "volume_server_pb.VolumeEcShardsCopyRequest")
proto.RegisterType((*VolumeEcShardsCopyResponse)(nil), "volume_server_pb.VolumeEcShardsCopyResponse")
proto.RegisterType((*VolumeEcShardsDeleteRequest)(nil), "volume_server_pb.VolumeEcShardsDeleteRequest")
proto.RegisterType((*VolumeEcShardsDeleteResponse)(nil), "volume_server_pb.VolumeEcShardsDeleteResponse")
proto.RegisterType((*ReadVolumeFileStatusRequest)(nil), "volume_server_pb.ReadVolumeFileStatusRequest")
proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse")
proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus")
@ -1110,8 +1162,9 @@ type VolumeServerClient interface {
VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error)
VolumeTailReceiver(ctx context.Context, in *VolumeTailReceiverRequest, opts ...grpc.CallOption) (*VolumeTailReceiverResponse, error)
// erasure coding
VolumeEcGenerateSlices(ctx context.Context, in *VolumeEcGenerateSlicesRequest, opts ...grpc.CallOption) (*VolumeEcGenerateSlicesResponse, error)
VolumeEcCopy(ctx context.Context, in *VolumeEcCopyRequest, opts ...grpc.CallOption) (*VolumeEcCopyResponse, error)
VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error)
VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error)
VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error)
}
type volumeServerClient struct {
@ -1344,18 +1397,27 @@ func (c *volumeServerClient) VolumeTailReceiver(ctx context.Context, in *VolumeT
return out, nil
}
func (c *volumeServerClient) VolumeEcGenerateSlices(ctx context.Context, in *VolumeEcGenerateSlicesRequest, opts ...grpc.CallOption) (*VolumeEcGenerateSlicesResponse, error) {
out := new(VolumeEcGenerateSlicesResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcGenerateSlices", in, out, c.cc, opts...)
func (c *volumeServerClient) VolumeEcShardsGenerate(ctx context.Context, in *VolumeEcShardsGenerateRequest, opts ...grpc.CallOption) (*VolumeEcShardsGenerateResponse, error) {
out := new(VolumeEcShardsGenerateResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) VolumeEcCopy(ctx context.Context, in *VolumeEcCopyRequest, opts ...grpc.CallOption) (*VolumeEcCopyResponse, error) {
out := new(VolumeEcCopyResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcCopy", in, out, c.cc, opts...)
func (c *volumeServerClient) VolumeEcShardsCopy(ctx context.Context, in *VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*VolumeEcShardsCopyResponse, error) {
out := new(VolumeEcShardsCopyResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsCopy", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *volumeServerClient) VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error) {
out := new(VolumeEcShardsDeleteResponse)
err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeEcShardsDelete", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
@ -1385,8 +1447,9 @@ type VolumeServerServer interface {
VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error
VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error)
// erasure coding
VolumeEcGenerateSlices(context.Context, *VolumeEcGenerateSlicesRequest) (*VolumeEcGenerateSlicesResponse, error)
VolumeEcCopy(context.Context, *VolumeEcCopyRequest) (*VolumeEcCopyResponse, error)
VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error)
VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error)
VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error)
}
func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) {
@ -1708,38 +1771,56 @@ func _VolumeServer_VolumeTailReceiver_Handler(srv interface{}, ctx context.Conte
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_VolumeEcGenerateSlices_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcGenerateSlicesRequest)
func _VolumeServer_VolumeEcShardsGenerate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcShardsGenerateRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).VolumeEcGenerateSlices(ctx, in)
return srv.(VolumeServerServer).VolumeEcShardsGenerate(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcGenerateSlices",
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).VolumeEcGenerateSlices(ctx, req.(*VolumeEcGenerateSlicesRequest))
return srv.(VolumeServerServer).VolumeEcShardsGenerate(ctx, req.(*VolumeEcShardsGenerateRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_VolumeEcCopy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcCopyRequest)
func _VolumeServer_VolumeEcShardsCopy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcShardsCopyRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).VolumeEcCopy(ctx, in)
return srv.(VolumeServerServer).VolumeEcShardsCopy(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcCopy",
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsCopy",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).VolumeEcCopy(ctx, req.(*VolumeEcCopyRequest))
return srv.(VolumeServerServer).VolumeEcShardsCopy(ctx, req.(*VolumeEcShardsCopyRequest))
}
return interceptor(ctx, in, info, handler)
}
func _VolumeServer_VolumeEcShardsDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(VolumeEcShardsDeleteRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(VolumeServerServer).VolumeEcShardsDelete(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/volume_server_pb.VolumeServer/VolumeEcShardsDelete",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(VolumeServerServer).VolumeEcShardsDelete(ctx, req.(*VolumeEcShardsDeleteRequest))
}
return interceptor(ctx, in, info, handler)
}
@ -1805,12 +1886,16 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
Handler: _VolumeServer_VolumeTailReceiver_Handler,
},
{
MethodName: "VolumeEcGenerateSlices",
Handler: _VolumeServer_VolumeEcGenerateSlices_Handler,
MethodName: "VolumeEcShardsGenerate",
Handler: _VolumeServer_VolumeEcShardsGenerate_Handler,
},
{
MethodName: "VolumeEcShardsCopy",
Handler: _VolumeServer_VolumeEcShardsCopy_Handler,
},
{
MethodName: "VolumeEcCopy",
Handler: _VolumeServer_VolumeEcCopy_Handler,
MethodName: "VolumeEcShardsDelete",
Handler: _VolumeServer_VolumeEcShardsDelete_Handler,
},
},
Streams: []grpc.StreamDesc{
@ -1836,101 +1921,105 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 1534 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x58, 0xcb, 0x6f, 0xd4, 0xd6,
0x1a, 0x8f, 0x99, 0xc9, 0xeb, 0x9b, 0x04, 0x86, 0x93, 0x90, 0x4c, 0x0c, 0x09, 0x83, 0xe1, 0xc2,
0xf0, 0xb8, 0x81, 0x0b, 0xba, 0xf7, 0xd2, 0x87, 0xd4, 0x42, 0xa0, 0x6d, 0xa4, 0x02, 0x92, 0x03,
0xa8, 0x55, 0x2b, 0x59, 0x27, 0xf6, 0x17, 0x62, 0xc5, 0x63, 0x1b, 0xfb, 0x38, 0x4d, 0xaa, 0x76,
0xd5, 0xae, 0xba, 0xec, 0xb6, 0xcb, 0xee, 0xba, 0xe8, 0xb6, 0x7f, 0x55, 0xff, 0x82, 0x6e, 0xaa,
0xf3, 0xb0, 0x67, 0xfc, 0xca, 0x98, 0x52, 0xa9, 0xbb, 0x33, 0xdf, 0xf9, 0x5e, 0xe7, 0xf3, 0xf7,
0xfa, 0x0d, 0x2c, 0x1d, 0x06, 0x5e, 0x32, 0x44, 0x2b, 0xc6, 0xe8, 0x10, 0xa3, 0xcd, 0x30, 0x0a,
0x58, 0x40, 0xba, 0x39, 0xa2, 0x15, 0xee, 0x1a, 0xb7, 0x81, 0x3c, 0xa4, 0xcc, 0xde, 0x7f, 0x84,
0x1e, 0x32, 0x34, 0xf1, 0x75, 0x82, 0x31, 0x23, 0x6b, 0x30, 0xb7, 0xe7, 0x7a, 0x68, 0xb9, 0x4e,
0xdc, 0xd3, 0xfa, 0xad, 0xc1, 0xbc, 0x39, 0xcb, 0x7f, 0x6f, 0x3b, 0xb1, 0xf1, 0x0c, 0x96, 0x72,
0x02, 0x71, 0x18, 0xf8, 0x31, 0x92, 0xfb, 0x30, 0x1b, 0x61, 0x9c, 0x78, 0x4c, 0x0a, 0x74, 0xee,
0x6e, 0x6c, 0x16, 0x6d, 0x6d, 0x66, 0x22, 0x89, 0xc7, 0xcc, 0x94, 0xdd, 0x70, 0x61, 0x61, 0xfc,
0x82, 0xac, 0xc2, 0xac, 0xb2, 0xdd, 0xd3, 0xfa, 0xda, 0x60, 0xde, 0x9c, 0x91, 0xa6, 0xc9, 0x0a,
0xcc, 0xc4, 0x8c, 0xb2, 0x24, 0xee, 0x9d, 0xea, 0x6b, 0x83, 0x69, 0x53, 0xfd, 0x22, 0xcb, 0x30,
0x8d, 0x51, 0x14, 0x44, 0xbd, 0x96, 0x60, 0x97, 0x3f, 0x08, 0x81, 0x76, 0xec, 0x7e, 0x8d, 0xbd,
0x76, 0x5f, 0x1b, 0x2c, 0x9a, 0xe2, 0x6c, 0xcc, 0xc2, 0xf4, 0xe3, 0x61, 0xc8, 0x8e, 0x8d, 0xff,
0x43, 0xef, 0x25, 0xb5, 0x93, 0x64, 0xf8, 0x52, 0xf8, 0xb8, 0xb5, 0x8f, 0xf6, 0x41, 0xfa, 0xf6,
0xf3, 0x30, 0xaf, 0x3c, 0x57, 0x1e, 0x2c, 0x9a, 0x73, 0x92, 0xb0, 0xed, 0x18, 0x1f, 0xc2, 0x5a,
0x85, 0xa0, 0x8a, 0xc1, 0x65, 0x58, 0x7c, 0x45, 0xa3, 0x5d, 0xfa, 0x0a, 0xad, 0x88, 0x32, 0x37,
0x10, 0xd2, 0x9a, 0xb9, 0xa0, 0x88, 0x26, 0xa7, 0x19, 0x5f, 0x80, 0x9e, 0xd3, 0x10, 0x0c, 0x43,
0x6a, 0xb3, 0x26, 0xc6, 0x49, 0x1f, 0x3a, 0x61, 0x84, 0xd4, 0xf3, 0x02, 0x9b, 0x32, 0x14, 0x51,
0x68, 0x99, 0xe3, 0x24, 0x63, 0x1d, 0xce, 0x57, 0x2a, 0x97, 0x0e, 0x1a, 0xf7, 0x0b, 0xde, 0x07,
0xc3, 0xa1, 0xdb, 0xc8, 0xb4, 0x71, 0xa1, 0xe4, 0xb5, 0x90, 0x54, 0x7a, 0xdf, 0x29, 0xdc, 0x7a,
0x48, 0xfd, 0x24, 0x6c, 0xa4, 0xb8, 0xe8, 0x71, 0x2a, 0x9a, 0x69, 0x5e, 0x95, 0xc9, 0xb1, 0x15,
0x78, 0x1e, 0xda, 0xcc, 0x0d, 0xfc, 0x54, 0xed, 0x06, 0x80, 0x9d, 0x11, 0x55, 0xaa, 0x8c, 0x51,
0x0c, 0x1d, 0x7a, 0x65, 0x51, 0xa5, 0xf6, 0x17, 0x0d, 0xce, 0x3d, 0x50, 0x41, 0x93, 0x86, 0x1b,
0x7d, 0x80, 0xbc, 0xc9, 0x53, 0x45, 0x93, 0xc5, 0x0f, 0xd4, 0x2a, 0x7d, 0x20, 0xce, 0x11, 0x61,
0xe8, 0xb9, 0x36, 0x15, 0x2a, 0xda, 0x42, 0xc5, 0x38, 0x89, 0x74, 0xa1, 0xc5, 0x98, 0xd7, 0x9b,
0x16, 0x37, 0xfc, 0x68, 0xf4, 0x60, 0xa5, 0xe8, 0xab, 0x7a, 0xc6, 0xff, 0x60, 0x55, 0x52, 0x76,
0x8e, 0x7d, 0x7b, 0x47, 0x54, 0x43, 0xa3, 0xa0, 0xff, 0xa1, 0x41, 0xaf, 0x2c, 0xa8, 0xb2, 0xf8,
0x6d, 0x23, 0xf0, 0xa6, 0xef, 0x23, 0x17, 0xa1, 0xc3, 0xa8, 0xeb, 0x59, 0xc1, 0xde, 0x5e, 0x8c,
0xac, 0x37, 0xd3, 0xd7, 0x06, 0x6d, 0x13, 0x38, 0xe9, 0x99, 0xa0, 0x90, 0xeb, 0xd0, 0xb5, 0x65,
0x26, 0x5b, 0x11, 0x1e, 0xba, 0x31, 0xd7, 0x3c, 0x2b, 0x1c, 0x3b, 0x63, 0xa7, 0x19, 0x2e, 0xc9,
0xc4, 0x80, 0x45, 0xd7, 0x39, 0xb2, 0x44, 0x03, 0x11, 0xe5, 0x3f, 0x27, 0xb4, 0x75, 0x5c, 0xe7,
0xe8, 0x23, 0xd7, 0xc3, 0x1d, 0xde, 0x05, 0x5e, 0xc2, 0x05, 0xf9, 0xf8, 0x6d, 0xdf, 0x8e, 0x70,
0x88, 0x3e, 0xa3, 0xde, 0x56, 0x10, 0x1e, 0x37, 0x4a, 0x81, 0x35, 0x98, 0x8b, 0x5d, 0xdf, 0x46,
0xcb, 0x97, 0x6d, 0xa8, 0x6d, 0xce, 0x8a, 0xdf, 0x4f, 0x63, 0xe3, 0x21, 0xac, 0xd7, 0xe8, 0x55,
0x91, 0xbd, 0x04, 0x0b, 0xc2, 0x31, 0x3b, 0xf0, 0x19, 0xfa, 0x4c, 0xe8, 0x5e, 0x30, 0x3b, 0x9c,
0xb6, 0x25, 0x49, 0xc6, 0x7f, 0x80, 0x48, 0x1d, 0x4f, 0x82, 0xc4, 0x6f, 0x56, 0x9a, 0xe7, 0x60,
0x29, 0x27, 0xa2, 0x72, 0xe3, 0x1e, 0x2c, 0x4b, 0xf2, 0x0b, 0x7f, 0xd8, 0x58, 0xd7, 0x2a, 0x9c,
0x2b, 0x08, 0x29, 0x6d, 0x77, 0x53, 0x23, 0xf9, 0x39, 0x71, 0xa2, 0xb2, 0x95, 0xd4, 0x83, 0xfc,
0xa8, 0x30, 0x7e, 0xd5, 0xe0, 0x6c, 0xda, 0x46, 0x1a, 0x46, 0xfd, 0x0d, 0xd3, 0xae, 0x55, 0x9b,
0x76, 0xed, 0x51, 0xda, 0x0d, 0xa0, 0x1b, 0x07, 0x49, 0x64, 0xa3, 0xe5, 0x50, 0x46, 0x2d, 0x3f,
0x70, 0x50, 0x65, 0xe5, 0x69, 0x49, 0x7f, 0x44, 0x19, 0x7d, 0x1a, 0x38, 0x68, 0x7c, 0x90, 0x7e,
0x94, 0xdc, 0xd7, 0xbc, 0x0e, 0x67, 0x3d, 0x1a, 0x33, 0x8b, 0x86, 0x21, 0xfa, 0x8e, 0x45, 0x19,
0x4f, 0x09, 0x4d, 0xa4, 0xc4, 0x69, 0x7e, 0xf1, 0x40, 0xd0, 0x1f, 0xb0, 0xa7, 0xb1, 0xf1, 0xa3,
0x06, 0x67, 0xb8, 0x2c, 0x4f, 0xc1, 0x46, 0xef, 0xed, 0x42, 0x0b, 0x8f, 0x98, 0x7a, 0x28, 0x3f,
0x92, 0xdb, 0xb0, 0xa4, 0x72, 0xdd, 0x0d, 0xfc, 0x51, 0x19, 0xb4, 0x84, 0x20, 0x19, 0x5d, 0x65,
0x95, 0x70, 0x11, 0x3a, 0x31, 0x0b, 0xc2, 0xb4, 0xaa, 0xda, 0xb2, 0xaa, 0x38, 0x49, 0x56, 0x95,
0xf1, 0x5f, 0xe8, 0x8e, 0x7c, 0x6a, 0x9e, 0xa1, 0xdf, 0x69, 0x69, 0xd3, 0x79, 0x4e, 0x5d, 0x6f,
0x07, 0x7d, 0x07, 0xa3, 0xb7, 0xac, 0x1c, 0x72, 0x07, 0x96, 0x5d, 0xc7, 0x43, 0x8b, 0xb9, 0x43,
0x0c, 0x12, 0x66, 0xc5, 0x68, 0x07, 0xbe, 0x13, 0xa7, 0xaf, 0xe3, 0x77, 0xcf, 0xe5, 0xd5, 0x8e,
0xbc, 0x31, 0xbe, 0xcf, 0x3a, 0xd8, 0xb8, 0x17, 0xa3, 0x39, 0xec, 0x23, 0x72, 0x85, 0xfb, 0x48,
0x1d, 0x8c, 0xd4, 0x33, 0x16, 0x24, 0xf1, 0x13, 0x41, 0xe3, 0xf1, 0x51, 0x4c, 0xbb, 0x81, 0x73,
0x2c, 0x3c, 0x5a, 0x30, 0x41, 0x92, 0x1e, 0x06, 0xce, 0xb1, 0x68, 0x25, 0xb1, 0x25, 0x3e, 0xb1,
0xbd, 0x9f, 0xf8, 0x07, 0xc2, 0x9b, 0x39, 0xb3, 0xe3, 0xc6, 0x9f, 0xd2, 0x98, 0x6d, 0x71, 0x92,
0xf1, 0x9b, 0x06, 0x6b, 0x23, 0x37, 0x4c, 0xb4, 0xd1, 0x3d, 0xfc, 0x07, 0xc2, 0xc1, 0x25, 0x54,
0x2e, 0xe7, 0x96, 0x2e, 0x95, 0xee, 0x44, 0xde, 0xa9, 0x8e, 0x2f, 0x6e, 0xc4, 0x40, 0xaf, 0x70,
0x5c, 0x95, 0xe8, 0xfb, 0x69, 0x2b, 0x7b, 0x6c, 0x7f, 0x8c, 0x3e, 0x46, 0x94, 0xe1, 0x8e, 0xe7,
0xda, 0xd8, 0x6c, 0xbc, 0xf4, 0x61, 0xa3, 0x4e, 0x5a, 0xe9, 0xff, 0x49, 0x4b, 0xfb, 0xc9, 0x63,
0xfb, 0x6f, 0x6b, 0x02, 0xeb, 0x00, 0x68, 0x5b, 0xae, 0xef, 0xe0, 0x11, 0xf2, 0x60, 0xb5, 0x06,
0x8b, 0xe6, 0x3c, 0xda, 0xdb, 0x92, 0xf0, 0x06, 0xf5, 0x9e, 0x35, 0xae, 0xd4, 0x39, 0xe5, 0xf5,
0xbb, 0x70, 0xde, 0x44, 0xea, 0xc8, 0x3b, 0x31, 0x4e, 0x9a, 0x8f, 0xdc, 0xdf, 0x4f, 0xc1, 0x85,
0x6a, 0xe1, 0x26, 0x63, 0xf7, 0x3d, 0xd0, 0xb3, 0xb1, 0xc6, 0xb3, 0x22, 0x66, 0x74, 0x18, 0x66,
0x79, 0x21, 0xd3, 0x67, 0x55, 0xcd, 0xb8, 0xe7, 0xe9, 0x7d, 0x9a, 0x1c, 0xa5, 0x99, 0xd8, 0x2a,
0xcd, 0x44, 0x6e, 0xc0, 0xa1, 0xac, 0xce, 0x80, 0x6c, 0x1e, 0xab, 0x0e, 0x65, 0x75, 0x06, 0x32,
0x61, 0x61, 0x60, 0x5a, 0x1a, 0x50, 0xfc, 0xc2, 0xc0, 0x3a, 0x80, 0xea, 0x2c, 0x89, 0x9f, 0xce,
0xf8, 0x79, 0xd9, 0x57, 0x12, 0xbf, 0xb6, 0xbd, 0xcd, 0xd6, 0xb6, 0xb7, 0x7c, 0x32, 0xcc, 0x95,
0xb6, 0xbf, 0xcf, 0x00, 0x1e, 0xb9, 0xf1, 0x81, 0x0c, 0x32, 0xef, 0xa7, 0x8e, 0x1b, 0xa9, 0x25,
0x91, 0x1f, 0x39, 0x85, 0x7a, 0x9e, 0x0a, 0x1d, 0x3f, 0x72, 0xc0, 0x90, 0xc4, 0xe8, 0xa8, 0xe8,
0x88, 0x33, 0xa7, 0xed, 0x45, 0x88, 0x2a, 0x00, 0xe2, 0x6c, 0xfc, 0xac, 0xc1, 0xfc, 0x13, 0x1c,
0x2a, 0xcd, 0x1b, 0x00, 0xaf, 0x82, 0x28, 0x48, 0x98, 0xeb, 0xa3, 0x6c, 0xff, 0xd3, 0xe6, 0x18,
0xe5, 0xaf, 0xdb, 0x11, 0x00, 0x06, 0xbd, 0x3d, 0x15, 0x4c, 0x71, 0xe6, 0xb4, 0x7d, 0xa4, 0xa1,
0x8a, 0x9f, 0x38, 0x73, 0xf8, 0x13, 0x33, 0x6a, 0x1f, 0x88, 0x60, 0xb5, 0x4d, 0xf9, 0xe3, 0xee,
0x0f, 0x5d, 0x58, 0x18, 0x2f, 0x78, 0xf2, 0x25, 0x74, 0xc6, 0x70, 0x1b, 0xb9, 0x52, 0x86, 0x67,
0x65, 0x1c, 0xa8, 0xff, 0x6b, 0x02, 0x97, 0x2a, 0x8c, 0x29, 0xe2, 0xc3, 0xd9, 0x12, 0x2e, 0x22,
0x37, 0xca, 0xd2, 0x75, 0xa8, 0x4b, 0xbf, 0xd9, 0x88, 0x37, 0xb3, 0xc7, 0x60, 0xa9, 0x02, 0xe8,
0x90, 0x5b, 0x13, 0xb4, 0xe4, 0xc0, 0x96, 0xfe, 0xef, 0x86, 0xdc, 0x99, 0xd5, 0xd7, 0x40, 0xca,
0x28, 0x88, 0xdc, 0x9c, 0xa8, 0x66, 0x84, 0xb2, 0xf4, 0x5b, 0xcd, 0x98, 0x6b, 0x1f, 0x2a, 0xf1,
0xd1, 0xc4, 0x87, 0xe6, 0x10, 0xd8, 0xc4, 0x87, 0x16, 0x40, 0xd7, 0x14, 0x39, 0x80, 0x6e, 0x11,
0x3b, 0x91, 0xeb, 0x75, 0x80, 0xbe, 0x04, 0xcd, 0xf4, 0x1b, 0x4d, 0x58, 0x33, 0x63, 0x08, 0xa7,
0xf3, 0xf8, 0x86, 0x5c, 0x2b, 0xcb, 0x57, 0xa2, 0x35, 0x7d, 0x30, 0x99, 0x71, 0xfc, 0x4d, 0x45,
0xcc, 0x53, 0xf5, 0xa6, 0x1a, 0x40, 0x55, 0xf5, 0xa6, 0x3a, 0x08, 0x65, 0x4c, 0x91, 0x6f, 0xd2,
0x45, 0xba, 0x80, 0x05, 0xc8, 0x66, 0x9d, 0x9a, 0x6a, 0x30, 0xa2, 0xdf, 0x6e, 0xcc, 0x9f, 0xda,
0xbe, 0xa3, 0xf1, 0x5a, 0x1f, 0x83, 0x04, 0x55, 0xb5, 0x5e, 0x06, 0x19, 0x55, 0xb5, 0x5e, 0x85,
0x2b, 0xa6, 0xc8, 0x2e, 0x2c, 0xe6, 0x40, 0x02, 0xb9, 0x5a, 0x27, 0x99, 0x87, 0x1e, 0xfa, 0xb5,
0x89, 0x7c, 0x99, 0x0d, 0x2b, 0xed, 0x5e, 0xaa, 0x5d, 0xd5, 0x3a, 0x97, 0xef, 0x57, 0x57, 0x27,
0xb1, 0x65, 0x06, 0x3e, 0x07, 0x18, 0xed, 0xf4, 0xe4, 0x72, 0x9d, 0xdc, 0xf8, 0xa7, 0xb8, 0x72,
0x32, 0x53, 0xa6, 0xfa, 0x2b, 0x58, 0xae, 0x9a, 0xf4, 0xa4, 0xa2, 0x0a, 0x4f, 0x58, 0x27, 0xf4,
0xcd, 0xa6, 0xec, 0x99, 0xe1, 0x17, 0x30, 0x97, 0x6e, 0xf4, 0xe4, 0x52, 0x59, 0xba, 0x80, 0x40,
0x74, 0xe3, 0x24, 0x96, 0xb1, 0x6c, 0x1a, 0xa6, 0x85, 0x33, 0x5a, 0xb5, 0xeb, 0x0b, 0xa7, 0x04,
0x0a, 0xea, 0x0b, 0xa7, 0xbc, 0xb9, 0x0b, 0x73, 0xaf, 0x53, 0xb4, 0x35, 0xbe, 0x99, 0x56, 0x36,
0xd9, 0xba, 0xc5, 0xbb, 0xb2, 0xc9, 0xd6, 0x2f, 0xbb, 0x53, 0xe4, 0x5b, 0x58, 0xa9, 0x5e, 0x58,
0x49, 0x6d, 0xf9, 0xd5, 0x2c, 0xc6, 0xfa, 0x9d, 0xe6, 0x02, 0xe5, 0x64, 0x97, 0xfb, 0x66, 0x7d,
0xb2, 0xe7, 0x96, 0xe5, 0xfa, 0x64, 0x2f, 0xac, 0xad, 0x53, 0xbb, 0x33, 0xe2, 0xdf, 0xdf, 0x7b,
0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0x2f, 0xf9, 0x4c, 0xc9, 0x14, 0x16, 0x00, 0x00,
// 1598 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x58, 0x5f, 0x6f, 0xd4, 0x56,
0x16, 0x8f, 0x99, 0x49, 0x32, 0x39, 0x93, 0x40, 0x72, 0x13, 0x92, 0x89, 0x43, 0x42, 0x30, 0x2c,
0x0c, 0x7f, 0x36, 0xb0, 0xa0, 0xdd, 0x65, 0x77, 0x1f, 0xb6, 0x10, 0x68, 0x1b, 0xa9, 0x80, 0xe4,
0x00, 0x6a, 0x55, 0x24, 0xeb, 0xc6, 0x3e, 0x21, 0x56, 0x3c, 0xf6, 0xe0, 0x7b, 0x9d, 0x26, 0x55,
0xfb, 0xd4, 0xf6, 0x0b, 0x54, 0xea, 0x07, 0xa8, 0xfa, 0xd6, 0x87, 0xbe, 0xf6, 0x53, 0xf5, 0x13,
0xf4, 0xa5, 0xba, 0x7f, 0xec, 0x19, 0x8f, 0xed, 0x8c, 0x29, 0x48, 0x7d, 0xbb, 0x73, 0xee, 0xf9,
0xef, 0x73, 0xce, 0x3d, 0xbf, 0x81, 0xc5, 0xa3, 0x28, 0x48, 0x7a, 0xe8, 0x30, 0x8c, 0x8f, 0x30,
0xde, 0xea, 0xc7, 0x11, 0x8f, 0xc8, 0x7c, 0x8e, 0xe8, 0xf4, 0xf7, 0xac, 0xdb, 0x40, 0x1e, 0x52,
0xee, 0x1e, 0x3c, 0xc2, 0x00, 0x39, 0xda, 0xf8, 0x26, 0x41, 0xc6, 0xc9, 0x2a, 0xb4, 0xf6, 0xfd,
0x00, 0x1d, 0xdf, 0x63, 0x1d, 0x63, 0xb3, 0xd1, 0x9d, 0xb1, 0xa7, 0xc5, 0xef, 0x1d, 0x8f, 0x59,
0xcf, 0x60, 0x31, 0x27, 0xc0, 0xfa, 0x51, 0xc8, 0x90, 0xdc, 0x87, 0xe9, 0x18, 0x59, 0x12, 0x70,
0x25, 0xd0, 0xbe, 0xbb, 0xb1, 0x35, 0x6a, 0x6b, 0x2b, 0x13, 0x49, 0x02, 0x6e, 0xa7, 0xec, 0x96,
0x0f, 0xb3, 0xc3, 0x17, 0x64, 0x05, 0xa6, 0xb5, 0xed, 0x8e, 0xb1, 0x69, 0x74, 0x67, 0xec, 0x29,
0x65, 0x9a, 0x2c, 0xc3, 0x14, 0xe3, 0x94, 0x27, 0xac, 0x73, 0x66, 0xd3, 0xe8, 0x4e, 0xda, 0xfa,
0x17, 0x59, 0x82, 0x49, 0x8c, 0xe3, 0x28, 0xee, 0x34, 0x24, 0xbb, 0xfa, 0x41, 0x08, 0x34, 0x99,
0xff, 0x25, 0x76, 0x9a, 0x9b, 0x46, 0x77, 0xce, 0x96, 0x67, 0x6b, 0x1a, 0x26, 0x1f, 0xf7, 0xfa,
0xfc, 0xc4, 0xfa, 0x37, 0x74, 0x5e, 0x52, 0x37, 0x49, 0x7a, 0x2f, 0xa5, 0x8f, 0xdb, 0x07, 0xe8,
0x1e, 0xa6, 0xb1, 0xaf, 0xc1, 0x8c, 0xf6, 0x5c, 0x7b, 0x30, 0x67, 0xb7, 0x14, 0x61, 0xc7, 0xb3,
0x3e, 0x80, 0xd5, 0x12, 0x41, 0x9d, 0x83, 0xcb, 0x30, 0xf7, 0x9a, 0xc6, 0x7b, 0xf4, 0x35, 0x3a,
0x31, 0xe5, 0x7e, 0x24, 0xa5, 0x0d, 0x7b, 0x56, 0x13, 0x6d, 0x41, 0xb3, 0x3e, 0x07, 0x33, 0xa7,
0x21, 0xea, 0xf5, 0xa9, 0xcb, 0xeb, 0x18, 0x27, 0x9b, 0xd0, 0xee, 0xc7, 0x48, 0x83, 0x20, 0x72,
0x29, 0x47, 0x99, 0x85, 0x86, 0x3d, 0x4c, 0xb2, 0xd6, 0x61, 0xad, 0x54, 0xb9, 0x72, 0xd0, 0xba,
0x3f, 0xe2, 0x7d, 0xd4, 0xeb, 0xf9, 0xb5, 0x4c, 0x5b, 0x17, 0x0a, 0x5e, 0x4b, 0x49, 0xad, 0xf7,
0x3f, 0x23, 0xb7, 0x01, 0xd2, 0x30, 0xe9, 0xd7, 0x52, 0x3c, 0xea, 0x71, 0x2a, 0x9a, 0x69, 0x5e,
0x51, 0xc5, 0xb1, 0x1d, 0x05, 0x01, 0xba, 0xdc, 0x8f, 0xc2, 0x54, 0xed, 0x06, 0x80, 0x9b, 0x11,
0x75, 0xa9, 0x0c, 0x51, 0x2c, 0x13, 0x3a, 0x45, 0x51, 0xad, 0xf6, 0x67, 0x03, 0xce, 0x3f, 0xd0,
0x49, 0x53, 0x86, 0x6b, 0x7d, 0x80, 0xbc, 0xc9, 0x33, 0xa3, 0x26, 0x47, 0x3f, 0x50, 0xa3, 0xf0,
0x81, 0x04, 0x47, 0x8c, 0xfd, 0xc0, 0x77, 0xa9, 0x54, 0xd1, 0x94, 0x2a, 0x86, 0x49, 0x64, 0x1e,
0x1a, 0x9c, 0x07, 0x9d, 0x49, 0x79, 0x23, 0x8e, 0x56, 0x07, 0x96, 0x47, 0x7d, 0xd5, 0x61, 0xfc,
0x0b, 0x56, 0x14, 0x65, 0xf7, 0x24, 0x74, 0x77, 0x65, 0x37, 0xd4, 0x4a, 0xfa, 0xef, 0x06, 0x74,
0x8a, 0x82, 0xba, 0x8a, 0xdf, 0x35, 0x03, 0x6f, 0x1b, 0x1f, 0xb9, 0x08, 0x6d, 0x4e, 0xfd, 0xc0,
0x89, 0xf6, 0xf7, 0x19, 0xf2, 0xce, 0xd4, 0xa6, 0xd1, 0x6d, 0xda, 0x20, 0x48, 0xcf, 0x24, 0x85,
0x5c, 0x87, 0x79, 0x57, 0x55, 0xb2, 0x13, 0xe3, 0x91, 0xcf, 0x84, 0xe6, 0x69, 0xe9, 0xd8, 0x39,
0x37, 0xad, 0x70, 0x45, 0x26, 0x16, 0xcc, 0xf9, 0xde, 0xb1, 0x23, 0x07, 0x88, 0x6c, 0xff, 0x96,
0xd4, 0xd6, 0xf6, 0xbd, 0xe3, 0x0f, 0xfd, 0x00, 0x77, 0xc5, 0x14, 0x78, 0x09, 0x17, 0x54, 0xf0,
0x3b, 0xa1, 0x1b, 0x63, 0x0f, 0x43, 0x4e, 0x83, 0xed, 0xa8, 0x7f, 0x52, 0xab, 0x04, 0x56, 0xa1,
0xc5, 0xfc, 0xd0, 0x45, 0x27, 0x54, 0x63, 0xa8, 0x69, 0x4f, 0xcb, 0xdf, 0x4f, 0x99, 0xf5, 0x10,
0xd6, 0x2b, 0xf4, 0xea, 0xcc, 0x5e, 0x82, 0x59, 0xe9, 0x98, 0x1b, 0x85, 0x1c, 0x43, 0x2e, 0x75,
0xcf, 0xda, 0x6d, 0x41, 0xdb, 0x56, 0x24, 0xeb, 0x1f, 0x40, 0x94, 0x8e, 0x27, 0x51, 0x12, 0xd6,
0x6b, 0xcd, 0xf3, 0xb0, 0x98, 0x13, 0xd1, 0xb5, 0x71, 0x0f, 0x96, 0x14, 0xf9, 0x45, 0xd8, 0xab,
0xad, 0x6b, 0x05, 0xce, 0x8f, 0x08, 0x69, 0x6d, 0x77, 0x53, 0x23, 0xf9, 0x77, 0xe2, 0x54, 0x65,
0xcb, 0xa9, 0x07, 0xf9, 0xa7, 0xc2, 0xfa, 0xc5, 0x80, 0x85, 0x74, 0x8c, 0xd4, 0xcc, 0xfa, 0x5b,
0x96, 0x5d, 0xa3, 0xb2, 0xec, 0x9a, 0x83, 0xb2, 0xeb, 0xc2, 0x3c, 0x8b, 0x92, 0xd8, 0x45, 0xc7,
0xa3, 0x9c, 0x3a, 0x61, 0xe4, 0xa1, 0xae, 0xca, 0xb3, 0x8a, 0xfe, 0x88, 0x72, 0xfa, 0x34, 0xf2,
0xd0, 0xfa, 0x7f, 0xfa, 0x51, 0x72, 0x5f, 0xf3, 0x3a, 0x2c, 0x04, 0x94, 0x71, 0x87, 0xf6, 0xfb,
0x18, 0x7a, 0x0e, 0xe5, 0xa2, 0x24, 0x0c, 0x59, 0x12, 0x67, 0xc5, 0xc5, 0x03, 0x49, 0x7f, 0xc0,
0x9f, 0x32, 0xeb, 0x7b, 0x03, 0xce, 0x09, 0x59, 0x51, 0x82, 0xb5, 0xe2, 0x9d, 0x87, 0x06, 0x1e,
0x73, 0x1d, 0xa8, 0x38, 0x92, 0xdb, 0xb0, 0xa8, 0x6b, 0xdd, 0x8f, 0xc2, 0x41, 0x1b, 0x34, 0xa4,
0x20, 0x19, 0x5c, 0x65, 0x9d, 0x70, 0x11, 0xda, 0x8c, 0x47, 0xfd, 0xb4, 0xab, 0x9a, 0xaa, 0xab,
0x04, 0x49, 0x75, 0x95, 0xf5, 0x4f, 0x98, 0x1f, 0xf8, 0x54, 0xbf, 0x42, 0xbf, 0x31, 0xd2, 0xa1,
0xf3, 0x9c, 0xfa, 0xc1, 0x2e, 0x86, 0x1e, 0xc6, 0xef, 0xd8, 0x39, 0xe4, 0x0e, 0x2c, 0xf9, 0x5e,
0x80, 0x0e, 0xf7, 0x7b, 0x18, 0x25, 0xdc, 0x61, 0xe8, 0x46, 0xa1, 0xc7, 0xd2, 0xe8, 0xc4, 0xdd,
0x73, 0x75, 0xb5, 0xab, 0x6e, 0xac, 0x6f, 0xb3, 0x09, 0x36, 0xec, 0xc5, 0xe0, 0x1d, 0x0e, 0x11,
0x85, 0xc2, 0x03, 0xa4, 0x1e, 0xc6, 0x3a, 0x8c, 0x59, 0x45, 0xfc, 0x58, 0xd2, 0x44, 0x7e, 0x34,
0xd3, 0x5e, 0xe4, 0x9d, 0x48, 0x8f, 0x66, 0x6d, 0x50, 0xa4, 0x87, 0x91, 0x77, 0x22, 0x47, 0x09,
0x73, 0xe4, 0x27, 0x76, 0x0f, 0x92, 0xf0, 0x50, 0x7a, 0xd3, 0xb2, 0xdb, 0x3e, 0xfb, 0x84, 0x32,
0xbe, 0x2d, 0x48, 0xd6, 0xaf, 0x06, 0xac, 0x0e, 0xdc, 0xb0, 0xd1, 0x45, 0xff, 0xe8, 0x2f, 0x48,
0x87, 0x90, 0xd0, 0xb5, 0x9c, 0x5b, 0xba, 0x74, 0xb9, 0x13, 0x75, 0xa7, 0x27, 0xbe, 0xbc, 0x91,
0x0f, 0x7a, 0x89, 0xe3, 0xba, 0x45, 0x5f, 0xa5, 0xa3, 0xec, 0xb1, 0xbb, 0x7b, 0x40, 0x63, 0x8f,
0x7d, 0x84, 0x21, 0xc6, 0x94, 0xbf, 0x97, 0x67, 0xd2, 0xda, 0x84, 0x8d, 0x2a, 0xed, 0xda, 0xfe,
0x8f, 0x59, 0x5e, 0x53, 0x96, 0xf7, 0x36, 0x2a, 0xd6, 0x01, 0xd0, 0x75, 0xfc, 0xd0, 0xc3, 0x63,
0x14, 0x29, 0x6d, 0x74, 0xe7, 0xec, 0x19, 0x74, 0x77, 0x14, 0xe1, 0x2d, 0xa6, 0x42, 0x96, 0xc1,
0xbc, 0x8b, 0x3a, 0x82, 0xef, 0x0c, 0x58, 0xcb, 0x5f, 0xd7, 0x9f, 0x9c, 0xe4, 0x06, 0x2c, 0xb0,
0x83, 0x28, 0x09, 0x3c, 0xc7, 0x93, 0x42, 0x0e, 0xba, 0xc7, 0x32, 0x94, 0x96, 0x7d, 0x4e, 0x5d,
0x28, 0x65, 0x8f, 0xdd, 0xe3, 0x31, 0xf1, 0x58, 0x1b, 0xe9, 0x63, 0x37, 0xea, 0x86, 0xf6, 0xf3,
0xbf, 0xb0, 0x66, 0x23, 0xf5, 0x14, 0x8f, 0x7c, 0x22, 0xeb, 0xaf, 0x11, 0xbf, 0x9d, 0x81, 0x0b,
0xe5, 0xc2, 0x75, 0x56, 0x89, 0xff, 0x81, 0x99, 0x3d, 0xd5, 0xa2, 0xd2, 0x19, 0xa7, 0xbd, 0x7e,
0x56, 0xeb, 0xaa, 0x25, 0x56, 0xf4, 0xbb, 0xfd, 0x3c, 0xbd, 0x4f, 0x0b, 0xbe, 0xf0, 0xce, 0x37,
0x0a, 0xef, 0xbc, 0x30, 0xe0, 0x51, 0x5e, 0x65, 0x40, 0x0d, 0xc4, 0x15, 0x8f, 0xf2, 0x2a, 0x03,
0x99, 0xb0, 0x34, 0x30, 0xa9, 0x0c, 0x68, 0x7e, 0x69, 0x60, 0x1d, 0x40, 0x4f, 0xcb, 0x24, 0x4c,
0xf7, 0x96, 0x19, 0x35, 0x2b, 0x93, 0xb0, 0x72, 0x64, 0x4f, 0x57, 0x8e, 0xec, 0x7c, 0xe9, 0xb6,
0x0a, 0x7d, 0xf3, 0x29, 0xc0, 0x23, 0x9f, 0x1d, 0xaa, 0x24, 0x8b, 0x37, 0xc2, 0xf3, 0x63, 0xbd,
0xf8, 0x8a, 0xa3, 0xa0, 0xd0, 0x20, 0xd0, 0xa9, 0x13, 0x47, 0x01, 0x82, 0x12, 0x86, 0x9e, 0xce,
0x8e, 0x3c, 0x0b, 0xda, 0x7e, 0x8c, 0xa8, 0x13, 0x20, 0xcf, 0xd6, 0x4f, 0x06, 0xcc, 0x3c, 0xc1,
0x9e, 0xd6, 0xbc, 0x01, 0xf0, 0x3a, 0x8a, 0xa3, 0x84, 0xfb, 0x21, 0xaa, 0x27, 0x6d, 0xd2, 0x1e,
0xa2, 0xfc, 0x79, 0x3b, 0x12, 0x94, 0x61, 0xb0, 0xaf, 0x93, 0x29, 0xcf, 0x82, 0x76, 0x80, 0xb4,
0xaf, 0xf3, 0x27, 0xcf, 0x02, 0xd2, 0x31, 0x4e, 0xdd, 0x43, 0x99, 0xac, 0xa6, 0xad, 0x7e, 0xdc,
0xfd, 0x61, 0x01, 0x66, 0x87, 0x87, 0x18, 0x79, 0x05, 0xed, 0x21, 0x2c, 0x4a, 0xae, 0x14, 0x21,
0x67, 0x11, 0xdb, 0x9a, 0x7f, 0x1b, 0xc3, 0xa5, 0x1b, 0x63, 0x82, 0x84, 0xb0, 0x50, 0xc0, 0x7a,
0xe4, 0x46, 0x51, 0xba, 0x0a, 0x49, 0x9a, 0x37, 0x6b, 0xf1, 0x66, 0xf6, 0x38, 0x2c, 0x96, 0x80,
0x37, 0x72, 0x6b, 0x8c, 0x96, 0x1c, 0x80, 0x34, 0xff, 0x5e, 0x93, 0x3b, 0xb3, 0xfa, 0x06, 0x48,
0x11, 0xd9, 0x91, 0x9b, 0x63, 0xd5, 0x0c, 0x90, 0xa3, 0x79, 0xab, 0x1e, 0x73, 0x65, 0xa0, 0x0a,
0xf3, 0x8d, 0x0d, 0x34, 0x87, 0x2a, 0xc7, 0x06, 0x3a, 0x02, 0x24, 0x27, 0xc8, 0x21, 0xcc, 0x8f,
0xe2, 0x41, 0x72, 0xbd, 0xea, 0x4f, 0x8a, 0x02, 0xdc, 0x34, 0x6f, 0xd4, 0x61, 0xcd, 0x8c, 0x21,
0x9c, 0xcd, 0x63, 0x36, 0x72, 0xad, 0x28, 0x5f, 0x8a, 0x40, 0xcd, 0xee, 0x78, 0xc6, 0xe1, 0x98,
0x46, 0x71, 0x5c, 0x59, 0x4c, 0x15, 0x20, 0xb1, 0x2c, 0xa6, 0x2a, 0x58, 0x68, 0x4d, 0x90, 0xaf,
0x52, 0x70, 0x30, 0x82, 0x6f, 0xc8, 0x56, 0x95, 0x9a, 0x72, 0x80, 0x65, 0xde, 0xae, 0xcd, 0x9f,
0xda, 0xbe, 0x63, 0x88, 0x5e, 0x1f, 0x82, 0x39, 0x65, 0xbd, 0x5e, 0x04, 0x4e, 0x65, 0xbd, 0x5e,
0x86, 0x95, 0x26, 0xc8, 0x1e, 0xcc, 0xe5, 0x80, 0x0f, 0xb9, 0x5a, 0x25, 0x99, 0x87, 0x53, 0xe6,
0xb5, 0xb1, 0x7c, 0x99, 0x0d, 0x27, 0x9d, 0x5e, 0x7a, 0x5c, 0x55, 0x3a, 0x97, 0x9f, 0x57, 0x57,
0xc7, 0xb1, 0x65, 0x06, 0x3e, 0x03, 0x18, 0xe0, 0x14, 0x72, 0xb9, 0x4a, 0x6e, 0xf8, 0x53, 0x5c,
0x39, 0x9d, 0x29, 0x53, 0xfd, 0x05, 0x2c, 0x95, 0xbd, 0xf4, 0xa4, 0xa4, 0x0b, 0x4f, 0x59, 0x27,
0xcc, 0xad, 0xba, 0xec, 0x99, 0xe1, 0x17, 0xd0, 0x4a, 0x51, 0x0a, 0xb9, 0x54, 0x94, 0x1e, 0x41,
0x55, 0xa6, 0x75, 0x1a, 0xcb, 0x50, 0x35, 0xf5, 0xd2, 0xc6, 0x19, 0xc0, 0x87, 0xea, 0xc6, 0x29,
0x00, 0x9d, 0xea, 0xc6, 0x29, 0xa2, 0x11, 0x69, 0xee, 0x4d, 0x8a, 0x20, 0x87, 0xb7, 0xed, 0xd2,
0x21, 0x5b, 0x05, 0x26, 0x4a, 0x87, 0x6c, 0xf5, 0x02, 0x3f, 0x41, 0xbe, 0x86, 0xe5, 0xf2, 0x25,
0x9b, 0x54, 0xb6, 0x5f, 0xc5, 0xb2, 0x6f, 0xde, 0xa9, 0x2f, 0x90, 0x7b, 0x56, 0x0a, 0xdb, 0x71,
0x75, 0xc4, 0x25, 0x6b, 0x7e, 0x75, 0xc4, 0xa5, 0x0b, 0xb7, 0xac, 0xd1, 0xb2, 0x55, 0xb7, 0xac,
0x46, 0x4f, 0xd9, 0xcc, 0xcd, 0xad, 0xba, 0xec, 0xa9, 0xe1, 0xbd, 0x29, 0xf9, 0xe7, 0xfa, 0xbd,
0x3f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x1a, 0xed, 0xaf, 0x31, 0x73, 0x17, 0x00, 0x00,
}

54
weed/server/volume_grpc_erasure_coding.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"math"
"os"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
@ -16,17 +17,17 @@ import (
Steps to apply erasure coding to .dat .idx files
0. ensure the volume is readonly
1. client call VolumeEcGenerateSlices to generate the .ecx and .ec01~.ec14 files
1. client call VolumeEcShardsGenerate to generate the .ecx and .ec01~.ec14 files
2. client ask master for possible servers to hold the ec files, at least 4 servers
3. client call VolumeEcCopy on above target servers to copy ec files from the source server
3. client call VolumeEcShardsCopy on above target servers to copy ec files from the source server
4. target servers report the new ec files to the master
5. master stores vid -> [14]*DataNode
6. client checks master. If all 14 slices are ready, delete the original .idx, .idx files
*/
*/
// VolumeEcGenerateSlices generates the .ecx and .ec01 ~ .ec14 files
func (vs *VolumeServer) VolumeEcGenerateSlices(ctx context.Context, req *volume_server_pb.VolumeEcGenerateSlicesRequest) (*volume_server_pb.VolumeEcGenerateSlicesResponse, error) {
// VolumeEcShardsGenerate generates the .ecx and .ec01 ~ .ec14 files
func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_server_pb.VolumeEcShardsGenerateRequest) (*volume_server_pb.VolumeEcShardsGenerateResponse, error) {
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
@ -34,6 +35,10 @@ func (vs *VolumeServer) VolumeEcGenerateSlices(ctx context.Context, req *volume_
}
baseFileName := v.FileName()
if v.Collection != req.Collection {
return nil, fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection)
}
// write .ecx file
if err := erasure_coding.WriteSortedEcxFile(baseFileName); err != nil {
return nil, fmt.Errorf("WriteSortedEcxFile %s: %v", baseFileName, err)
@ -44,12 +49,11 @@ func (vs *VolumeServer) VolumeEcGenerateSlices(ctx context.Context, req *volume_
return nil, fmt.Errorf("WriteEcFiles %s: %v", baseFileName, err)
}
return &volume_server_pb.VolumeEcGenerateSlicesResponse{}, nil
return &volume_server_pb.VolumeEcShardsGenerateResponse{}, nil
}
// VolumeEcCopy copy the .ecx and some ec data slices
func (vs *VolumeServer) VolumeEcCopy(ctx context.Context, req *volume_server_pb.VolumeEcCopyRequest) (*volume_server_pb.VolumeEcCopyResponse, error) {
// VolumeEcShardsCopy copy the .ecx and some ec data slices
func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_server_pb.VolumeEcShardsCopyRequest) (*volume_server_pb.VolumeEcShardsCopyResponse, error) {
location := vs.store.FindFreeLocation()
if location == nil {
@ -61,13 +65,13 @@ func (vs *VolumeServer) VolumeEcCopy(ctx context.Context, req *volume_server_pb.
err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
// copy ecx file
if err:=vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxUint64, baseFileName, ".ecx"); err!=nil{
if err := vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, ".ecx"); err != nil {
return err
}
// copy ec data slices
for _, ecIndex := range req.EcIndexes {
if err:=vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxUint64, baseFileName, erasure_coding.ToExt(int(ecIndex))); err!=nil{
if err := vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, erasure_coding.ToExt(int(ecIndex))); err != nil {
return err
}
}
@ -75,8 +79,32 @@ func (vs *VolumeServer) VolumeEcCopy(ctx context.Context, req *volume_server_pb.
return nil
})
if err != nil {
return nil, fmt.Errorf("VolumeEcCopy volume %d: %v", req.VolumeId, err)
return nil, fmt.Errorf("VolumeEcShardsCopy volume %d: %v", req.VolumeId, err)
}
return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil
}
// VolumeEcShardsDelete local delete the .ecx and some ec data slices if not needed, assuming current server has the source volume
func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
return nil, fmt.Errorf("volume %d not found", req.VolumeId)
}
baseFileName := v.FileName()
for _, shardIndex := range req.EcIndexes {
if err := os.Remove(baseFileName + erasure_coding.ToExt(int(shardIndex))); err != nil {
return nil, err
}
}
if req.ShouldDeleteEcx {
if err := os.Remove(baseFileName + ".ecx"); err != nil {
return nil, err
}
}
return &volume_server_pb.VolumeEcCopyResponse{}, nil
return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
}

109
weed/shell/command_ec_encode.go
View File

@ -51,7 +51,8 @@ func (c *commandEcEncode) Help() string {
func (c *commandEcEncode) Do(args []string, commandEnv *commandEnv, writer io.Writer) (err error) {
encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
volumeId := encodeCommand.Int("vid", 0, "the volume id")
volumeId := encodeCommand.Int("volumeId", 0, "the volume id")
collection := encodeCommand.String("collection", "", "the collection name")
if err = encodeCommand.Parse(args); err != nil {
return nil
}
@ -65,13 +66,13 @@ func (c *commandEcEncode) Do(args []string, commandEnv *commandEnv, writer io.Wr
}
// generate ec shards
err = generateEcShards(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(*volumeId), locations[0].Url)
err = generateEcShards(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(*volumeId), *collection, locations[0].Url)
if err != nil {
return fmt.Errorf("generate ec shards for volume %d on %s: %v", *volumeId, locations[0].Url, err)
}
// balance the ec shards to current cluster
err = balanceEcShards(ctx, commandEnv, needle.VolumeId(*volumeId), locations[0])
err = balanceEcShards(ctx, commandEnv, needle.VolumeId(*volumeId), *collection, locations)
if err != nil {
return fmt.Errorf("balance ec shards for volume %d on %s: %v", *volumeId, locations[0].Url, err)
}
@ -79,11 +80,12 @@ func (c *commandEcEncode) Do(args []string, commandEnv *commandEnv, writer io.Wr
return err
}
func generateEcShards(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer string) error {
func generateEcShards(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer string) error {
err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, genErr := volumeServerClient.VolumeEcGenerateSlices(ctx, &volume_server_pb.VolumeEcGenerateSlicesRequest{
VolumeId: uint32(volumeId),
_, genErr := volumeServerClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: uint32(volumeId),
Collection: collection,
})
return genErr
})
@ -92,7 +94,7 @@ func generateEcShards(ctx context.Context, grpcDialOption grpc.DialOption, volum
}
func balanceEcShards(ctx context.Context, commandEnv *commandEnv, volumeId needle.VolumeId, existingLocation wdclient.Location) (err error) {
func balanceEcShards(ctx context.Context, commandEnv *commandEnv, volumeId needle.VolumeId, collection string, existingLocations []wdclient.Location) (err error) {
// list all possible locations
var resp *master_pb.VolumeListResponse
@ -106,7 +108,7 @@ func balanceEcShards(ctx context.Context, commandEnv *commandEnv, volumeId needl
// find out all volume servers with one volume slot left.
var allDataNodes []*master_pb.DataNodeInfo
var totalFreeEcSlots int
var totalFreeEcSlots uint32
eachDataNode(resp.TopologyInfo, func(dn *master_pb.DataNodeInfo) {
if freeEcSlots := countFreeShardSlots(dn); freeEcSlots > 0 {
allDataNodes = append(allDataNodes, dn)
@ -127,63 +129,118 @@ func balanceEcShards(ctx context.Context, commandEnv *commandEnv, volumeId needl
allocated := balancedEcDistribution(allDataNodes)
// ask the data nodes to copy from the source volume server
err = parallelCopyEcShardsFromSource(ctx, commandEnv.option.GrpcDialOption, allDataNodes, allocated, volumeId, existingLocation)
copiedShardIds, err := parallelCopyEcShardsFromSource(ctx, commandEnv.option.GrpcDialOption, allDataNodes, allocated, volumeId, collection, existingLocations[0])
if err != nil {
return nil
}
// ask the source volume server to clean up copied ec shards
err = sourceServerDeleteEcShards(ctx, commandEnv.option.GrpcDialOption, volumeId, existingLocations[0], copiedShardIds)
if err != nil {
return fmt.Errorf("sourceServerDeleteEcShards %s %d.%v: %v", existingLocations[0], volumeId, copiedShardIds, err)
}
// ask the source volume server to delete the original volume
for _, location := range existingLocations {
err = deleteVolume(ctx, commandEnv.option.GrpcDialOption, volumeId, location.Url)
if err != nil {
return fmt.Errorf("deleteVolume %s volume %d: %v", location.Url, volumeId, err)
}
}
return err
}
func parallelCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.DialOption,
targetServers []*master_pb.DataNodeInfo, allocated []int,
volumeId needle.VolumeId, existingLocation wdclient.Location) (err error) {
targetServers []*master_pb.DataNodeInfo, allocated []uint32,
volumeId needle.VolumeId, collection string, existingLocation wdclient.Location) (actuallyCopied []uint32, err error) {
// parallelize
shardIdChan := make(chan []uint32, len(targetServers))
var wg sync.WaitGroup
startFromShardId := 0
startFromShardId := uint32(0)
for i, server := range targetServers {
if allocated[i] <= 0 {
continue
}
wg.Add(1)
go func(server *master_pb.DataNodeInfo, startFromShardId int, shardCount int) {
go func(server *master_pb.DataNodeInfo, startFromShardId uint32, shardCount uint32) {
defer wg.Done()
copyErr := oneServerCopyEcShardsFromSource(ctx, grpcDialOption, server, startFromShardId, shardCount, volumeId, existingLocation)
copiedShardIds, copyErr := oneServerCopyEcShardsFromSource(ctx, grpcDialOption, server,
startFromShardId, shardCount, volumeId, collection, existingLocation)
if copyErr != nil {
err = copyErr
} else {
shardIdChan <- copiedShardIds
}
}(server, startFromShardId, allocated[i])
startFromShardId += allocated[i]
}
wg.Wait()
close(shardIdChan)
return err
if err != nil {
return nil, err
}
for shardIds := range shardIdChan {
actuallyCopied = append(actuallyCopied, shardIds...)
}
return
}
func oneServerCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.DialOption,
targetServer *master_pb.DataNodeInfo, startFromShardId int, shardCount int,
volumeId needle.VolumeId, existingLocation wdclient.Location) (err error) {
targetServer *master_pb.DataNodeInfo, startFromShardId uint32, shardCount uint32,
volumeId needle.VolumeId, collection string, existingLocation wdclient.Location) (copiedShardIds []uint32, err error) {
if targetServer.Id == existingLocation.Url {
return nil
return nil, nil
}
for shardId := startFromShardId; shardId < startFromShardId+shardCount; shardId++ {
fmt.Printf("copy %d.%d %s => %s\n", volumeId, shardId, existingLocation.Url, targetServer.Id)
copiedShardIds = append(copiedShardIds, shardId)
}
return nil
err = operation.WithVolumeServerClient(targetServer.Id, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, copyErr := volumeServerClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
VolumeId: uint32(volumeId),
Collection: collection,
EcIndexes: copiedShardIds,
SourceDataNode: existingLocation.Url,
})
return copyErr
})
if err != nil {
return
}
return
}
func balancedEcDistribution(servers []*master_pb.DataNodeInfo) (allocated []int) {
freeSlots := make([]int, len(servers))
allocated = make([]int, len(servers))
func sourceServerDeleteEcShards(ctx context.Context, grpcDialOption grpc.DialOption,
volumeId needle.VolumeId, sourceLocation wdclient.Location, toBeDeletedShardIds []uint32) error {
shouldDeleteEcx := len(toBeDeletedShardIds) == erasure_coding.TotalShardsCount
return operation.WithVolumeServerClient(sourceLocation.Url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, deleteErr := volumeServerClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: uint32(volumeId),
EcIndexes: toBeDeletedShardIds,
ShouldDeleteEcx: shouldDeleteEcx,
})
return deleteErr
})
}
func balancedEcDistribution(servers []*master_pb.DataNodeInfo) (allocated []uint32) {
freeSlots := make([]uint32, len(servers))
allocated = make([]uint32, len(servers))
for i, server := range servers {
freeSlots[i] = countFreeShardSlots(server)
}
@ -213,14 +270,14 @@ func eachDataNode(topo *master_pb.TopologyInfo, fn func(*master_pb.DataNodeInfo)
}
}
func countShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count int) {
func countShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count uint32) {
for _, ecShardInfo := range ecShardInfos {
shardBits := erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
count += shardBits.ShardIdCount()
count += uint32(shardBits.ShardIdCount())
}
return
}
func countFreeShardSlots(dn *master_pb.DataNodeInfo) (count int) {
return int(dn.FreeVolumeCount)*10 - countShards(dn.EcShardInfos)
func countFreeShardSlots(dn *master_pb.DataNodeInfo) (count uint32) {
return uint32(dn.FreeVolumeCount)*10 - countShards(dn.EcShardInfos)
}
Write Preview
Loading…
Cancel
Save